From 3860642e0a87c8aef3c4285f3bb4ad473cfd514f Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Mon, 17 Mar 2025 16:33:54 +0800 Subject: [PATCH 001/382] crypto: api - Move alg destroy work from instance to template Commit 9ae4577bc077 ("crypto: api - Use work queue in crypto_destroy_instance") introduced a work struct to free an instance after the last user goes away. Move the delayed work from the instance into its template so that when the template is unregistered it can ensure that all its instances have been freed before returning. Signed-off-by: Herbert Xu --- crypto/algapi.c | 35 +++++++++++++++++++++++++++-------- include/crypto/algapi.h | 5 +++-- 2 files changed, 30 insertions(+), 10 deletions(-) diff --git a/crypto/algapi.c b/crypto/algapi.c index ea9ed9580aa8..22bf80aad82b 100644 --- a/crypto/algapi.c +++ b/crypto/algapi.c @@ -71,12 +71,23 @@ static void crypto_free_instance(struct crypto_instance *inst) static void crypto_destroy_instance_workfn(struct work_struct *w) { - struct crypto_instance *inst = container_of(w, struct crypto_instance, + struct crypto_template *tmpl = container_of(w, struct crypto_template, free_work); - struct crypto_template *tmpl = inst->tmpl; + struct crypto_instance *inst; + struct hlist_node *n; + HLIST_HEAD(list); - crypto_free_instance(inst); - crypto_tmpl_put(tmpl); + down_write(&crypto_alg_sem); + hlist_for_each_entry_safe(inst, n, &tmpl->dead, list) { + if (refcount_read(&inst->alg.cra_refcnt) != -1) + continue; + hlist_del(&inst->list); + hlist_add_head(&inst->list, &list); + } + up_write(&crypto_alg_sem); + + hlist_for_each_entry_safe(inst, n, &list, list) + crypto_free_instance(inst); } static void crypto_destroy_instance(struct crypto_alg *alg) @@ -84,9 +95,10 @@ static void crypto_destroy_instance(struct crypto_alg *alg) struct crypto_instance *inst = container_of(alg, struct crypto_instance, alg); + struct crypto_template *tmpl = inst->tmpl; - INIT_WORK(&inst->free_work, crypto_destroy_instance_workfn); - schedule_work(&inst->free_work); + refcount_set(&alg->cra_refcnt, -1); + schedule_work(&tmpl->free_work); } /* @@ -132,14 +144,17 @@ static void crypto_remove_instance(struct crypto_instance *inst, inst->alg.cra_flags |= CRYPTO_ALG_DEAD; - if (!tmpl || !crypto_tmpl_get(tmpl)) + if (!tmpl) return; - list_move(&inst->alg.cra_list, list); + list_del_init(&inst->alg.cra_list); hlist_del(&inst->list); + hlist_add_head(&inst->list, &tmpl->dead); inst->alg.cra_destroy = crypto_destroy_instance; BUG_ON(!list_empty(&inst->alg.cra_users)); + + crypto_alg_put(&inst->alg); } /* @@ -504,6 +519,8 @@ int crypto_register_template(struct crypto_template *tmpl) struct crypto_template *q; int err = -EEXIST; + INIT_WORK(&tmpl->free_work, crypto_destroy_instance_workfn); + down_write(&crypto_alg_sem); crypto_check_module_sig(tmpl->module); @@ -565,6 +582,8 @@ void crypto_unregister_template(struct crypto_template *tmpl) crypto_free_instance(inst); } crypto_remove_final(&users); + + flush_work(&tmpl->free_work); } EXPORT_SYMBOL_GPL(crypto_unregister_template); diff --git a/include/crypto/algapi.h b/include/crypto/algapi.h index 6e07bbc04089..03b7eca8af9a 100644 --- a/include/crypto/algapi.h +++ b/include/crypto/algapi.h @@ -68,16 +68,17 @@ struct crypto_instance { struct crypto_spawn *spawns; }; - struct work_struct free_work; - void *__ctx[] CRYPTO_MINALIGN_ATTR; }; struct crypto_template { struct list_head list; struct hlist_head instances; + struct hlist_head dead; struct module *module; + struct work_struct free_work; + int (*create)(struct crypto_template *tmpl, struct rtattr **tb); char name[CRYPTO_MAX_ALG_NAME]; From 138804c2c18ca8bd1443dea173b3cc2643995919 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Mon, 17 Mar 2025 16:33:57 +0800 Subject: [PATCH 002/382] crypto: api - Ensure cra_type->destroy is done in process context Move the cra_type->destroy call out of crypto_alg_put and into crypto_unregister_alg and crypto_free_instance. This ensures that it's always done in process context so calls such as flush_work can be done. Signed-off-by: Herbert Xu --- crypto/algapi.c | 13 ++++++++++--- crypto/api.c | 10 ---------- crypto/internal.h | 3 +-- 3 files changed, 11 insertions(+), 15 deletions(-) diff --git a/crypto/algapi.c b/crypto/algapi.c index 22bf80aad82b..5b8a4c787387 100644 --- a/crypto/algapi.c +++ b/crypto/algapi.c @@ -66,7 +66,13 @@ static int crypto_check_alg(struct crypto_alg *alg) static void crypto_free_instance(struct crypto_instance *inst) { - inst->alg.cra_type->free(inst); + struct crypto_alg *alg = &inst->alg; + const struct crypto_type *type; + + type = alg->cra_type; + if (type->destroy) + type->destroy(alg); + type->free(inst); } static void crypto_destroy_instance_workfn(struct work_struct *w) @@ -150,7 +156,6 @@ static void crypto_remove_instance(struct crypto_instance *inst, list_del_init(&inst->alg.cra_list); hlist_del(&inst->list); hlist_add_head(&inst->list, &tmpl->dead); - inst->alg.cra_destroy = crypto_destroy_instance; BUG_ON(!list_empty(&inst->alg.cra_users)); @@ -479,7 +484,8 @@ void crypto_unregister_alg(struct crypto_alg *alg) if (WARN_ON(refcount_read(&alg->cra_refcnt) != 1)) return; - crypto_alg_put(alg); + if (alg->cra_type && alg->cra_type->destroy) + alg->cra_type->destroy(alg); crypto_remove_final(&list); } @@ -637,6 +643,7 @@ int crypto_register_instance(struct crypto_template *tmpl, inst->alg.cra_module = tmpl->module; inst->alg.cra_flags |= CRYPTO_ALG_INSTANCE; + inst->alg.cra_destroy = crypto_destroy_instance; down_write(&crypto_alg_sem); diff --git a/crypto/api.c b/crypto/api.c index 3416e98128a0..2880aa04bb99 100644 --- a/crypto/api.c +++ b/crypto/api.c @@ -703,15 +703,5 @@ void crypto_req_done(void *data, int err) } EXPORT_SYMBOL_GPL(crypto_req_done); -void crypto_destroy_alg(struct crypto_alg *alg) -{ - if (alg->cra_type && alg->cra_type->destroy) - alg->cra_type->destroy(alg); - - if (alg->cra_destroy) - alg->cra_destroy(alg); -} -EXPORT_SYMBOL_GPL(crypto_destroy_alg); - MODULE_DESCRIPTION("Cryptographic core API"); MODULE_LICENSE("GPL"); diff --git a/crypto/internal.h b/crypto/internal.h index 11567ea24fc3..2edefb546ad4 100644 --- a/crypto/internal.h +++ b/crypto/internal.h @@ -128,7 +128,6 @@ void *crypto_create_tfm_node(struct crypto_alg *alg, const struct crypto_type *frontend, int node); void *crypto_clone_tfm(const struct crypto_type *frontend, struct crypto_tfm *otfm); -void crypto_destroy_alg(struct crypto_alg *alg); static inline void *crypto_create_tfm(struct crypto_alg *alg, const struct crypto_type *frontend) @@ -166,7 +165,7 @@ static inline struct crypto_alg *crypto_alg_get(struct crypto_alg *alg) static inline void crypto_alg_put(struct crypto_alg *alg) { if (refcount_dec_and_test(&alg->cra_refcnt)) - crypto_destroy_alg(alg); + alg->cra_destroy(alg); } static inline int crypto_tmpl_get(struct crypto_template *tmpl) From c47e1f4142a3823b6e963e14db295a8a733804b5 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Mon, 17 Mar 2025 16:33:59 +0800 Subject: [PATCH 003/382] crypto: scomp - Allocate per-cpu buffer on first use of each CPU Per-cpu buffers can be wasteful when the number of CPUs is large, especially if the buffer itself is likely to never be used. Reduce such wastage by only allocating them on first use of a particular CPU. On start-up allocate a single buffer on the first possible CPU. For every other CPU a work struct will be scheduled on first use to allocate the buffer for that CPU. Until the allocation succeeds simply use the first CPU's buffer which is protected under a spin lock. Signed-off-by: Herbert Xu --- crypto/scompress.c | 203 ++++++++++++++++++++++------ include/crypto/acompress.h | 6 - include/crypto/internal/acompress.h | 1 - include/crypto/internal/scompress.h | 13 ++ 4 files changed, 173 insertions(+), 50 deletions(-) diff --git a/crypto/scompress.c b/crypto/scompress.c index d435d4b24469..f4e3376ca7d3 100644 --- a/crypto/scompress.c +++ b/crypto/scompress.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -20,7 +21,7 @@ #include #include #include -#include +#include #include #include "compress.h" @@ -44,6 +45,10 @@ static const struct crypto_type crypto_scomp_type; static int scomp_scratch_users; static DEFINE_MUTEX(scomp_lock); +static cpumask_t scomp_scratch_want; +static void scomp_scratch_workfn(struct work_struct *work); +static DECLARE_WORK(scomp_scratch_work, scomp_scratch_workfn); + static int __maybe_unused crypto_scomp_report( struct sk_buff *skb, struct crypto_alg *alg) { @@ -74,36 +79,57 @@ static void crypto_scomp_free_scratches(void) scratch = per_cpu_ptr(&scomp_scratch, i); free_page(scratch->saddr); - vfree(scratch->dst); + kvfree(scratch->dst); scratch->src = NULL; scratch->dst = NULL; } } +static int scomp_alloc_scratch(struct scomp_scratch *scratch, int cpu) +{ + int node = cpu_to_node(cpu); + struct page *page; + void *mem; + + mem = kvmalloc_node(SCOMP_SCRATCH_SIZE, GFP_KERNEL, node); + if (!mem) + return -ENOMEM; + page = alloc_pages_node(node, GFP_KERNEL, 0); + if (!page) { + kvfree(mem); + return -ENOMEM; + } + spin_lock_bh(&scratch->lock); + scratch->src = page_address(page); + scratch->dst = mem; + spin_unlock_bh(&scratch->lock); + return 0; +} + +static void scomp_scratch_workfn(struct work_struct *work) +{ + int cpu; + + for_each_cpu(cpu, &scomp_scratch_want) { + struct scomp_scratch *scratch; + + scratch = per_cpu_ptr(&scomp_scratch, cpu); + if (scratch->src) + continue; + if (scomp_alloc_scratch(scratch, cpu)) + break; + + cpumask_clear_cpu(cpu, &scomp_scratch_want); + } +} + static int crypto_scomp_alloc_scratches(void) { + unsigned int i = cpumask_first(cpu_possible_mask); struct scomp_scratch *scratch; - int i; - for_each_possible_cpu(i) { - struct page *page; - void *mem; - - scratch = per_cpu_ptr(&scomp_scratch, i); - - page = alloc_pages_node(cpu_to_node(i), GFP_KERNEL, 0); - if (!page) - goto error; - scratch->src = page_address(page); - mem = vmalloc_node(SCOMP_SCRATCH_SIZE, cpu_to_node(i)); - if (!mem) - goto error; - scratch->dst = mem; - } - return 0; -error: - crypto_scomp_free_scratches(); - return -ENOMEM; + scratch = per_cpu_ptr(&scomp_scratch, i); + return scomp_alloc_scratch(scratch, i); } static void scomp_free_streams(struct scomp_alg *alg) @@ -115,7 +141,7 @@ static void scomp_free_streams(struct scomp_alg *alg) struct crypto_acomp_stream *ps = per_cpu_ptr(stream, i); if (!ps->ctx) - break; + continue; alg->free_ctx(ps->ctx); } @@ -126,21 +152,26 @@ static void scomp_free_streams(struct scomp_alg *alg) static int scomp_alloc_streams(struct scomp_alg *alg) { struct crypto_acomp_stream __percpu *stream; - int i; + struct crypto_acomp_stream *ps; + unsigned int i; + void *ctx; stream = alloc_percpu(struct crypto_acomp_stream); if (!stream) return -ENOMEM; + ctx = alg->alloc_ctx(); + if (IS_ERR(ctx)) { + free_percpu(stream); + return PTR_ERR(ctx); + } + + i = cpumask_first(cpu_possible_mask); + ps = per_cpu_ptr(stream, i); + ps->ctx = ctx; + for_each_possible_cpu(i) { - struct crypto_acomp_stream *ps = per_cpu_ptr(stream, i); - - ps->ctx = alg->alloc_ctx(); - if (IS_ERR(ps->ctx)) { - scomp_free_streams(alg); - return PTR_ERR(ps->ctx); - } - + ps = per_cpu_ptr(stream, i); spin_lock_init(&ps->lock); } @@ -148,6 +179,33 @@ static int scomp_alloc_streams(struct scomp_alg *alg) return 0; } +static void scomp_stream_workfn(struct work_struct *work) +{ + struct scomp_alg *alg = container_of(work, struct scomp_alg, + stream_work); + struct crypto_acomp_stream __percpu *stream = alg->stream; + int cpu; + + for_each_cpu(cpu, &alg->stream_want) { + struct crypto_acomp_stream *ps; + void *ctx; + + ps = per_cpu_ptr(stream, cpu); + if (ps->ctx) + continue; + + ctx = alg->alloc_ctx(); + if (IS_ERR(ctx)) + break; + + spin_lock_bh(&ps->lock); + ps->ctx = ctx; + spin_unlock_bh(&ps->lock); + + cpumask_clear_cpu(cpu, &alg->stream_want); + } +} + static int crypto_scomp_init_tfm(struct crypto_tfm *tfm) { struct scomp_alg *alg = crypto_scomp_alg(__crypto_scomp_tfm(tfm)); @@ -171,13 +229,67 @@ unlock: return ret; } +static struct scomp_scratch *scomp_lock_scratch_bh(void) __acquires(scratch) +{ + int cpu = raw_smp_processor_id(); + struct scomp_scratch *scratch; + + scratch = per_cpu_ptr(&scomp_scratch, cpu); + spin_lock_bh(&scratch->lock); + if (likely(scratch->src)) + return scratch; + spin_unlock(&scratch->lock); + + cpumask_set_cpu(cpu, &scomp_scratch_want); + schedule_work(&scomp_scratch_work); + + scratch = per_cpu_ptr(&scomp_scratch, cpumask_first(cpu_possible_mask)); + spin_lock(&scratch->lock); + return scratch; +} + +static inline void scomp_unlock_scratch_bh(struct scomp_scratch *scratch) + __releases(scratch) +{ + spin_unlock_bh(&scratch->lock); +} + +static struct crypto_acomp_stream *scomp_lock_stream(struct crypto_scomp *tfm) + __acquires(stream) +{ + struct scomp_alg *alg = crypto_scomp_alg(tfm); + struct crypto_acomp_stream __percpu *stream; + int cpu = raw_smp_processor_id(); + struct crypto_acomp_stream *ps; + + stream = alg->stream; + ps = per_cpu_ptr(stream, cpu); + spin_lock(&ps->lock); + if (likely(ps->ctx)) + return ps; + spin_unlock(&ps->lock); + + cpumask_set_cpu(cpu, &alg->stream_want); + schedule_work(&alg->stream_work); + + ps = per_cpu_ptr(stream, cpumask_first(cpu_possible_mask)); + spin_lock(&ps->lock); + return ps; +} + +static inline void scomp_unlock_stream(struct crypto_acomp_stream *stream) + __releases(stream) +{ + spin_unlock(&stream->lock); +} + static int scomp_acomp_comp_decomp(struct acomp_req *req, int dir) { - struct scomp_scratch *scratch = raw_cpu_ptr(&scomp_scratch); struct crypto_acomp *tfm = crypto_acomp_reqtfm(req); struct crypto_scomp **tfm_ctx = acomp_tfm_ctx(tfm); struct crypto_scomp *scomp = *tfm_ctx; struct crypto_acomp_stream *stream; + struct scomp_scratch *scratch; unsigned int slen = req->slen; unsigned int dlen = req->dlen; struct page *spage, *dpage; @@ -194,6 +306,8 @@ static int scomp_acomp_comp_decomp(struct acomp_req *req, int dir) if (!req->dst || !dlen) return -EINVAL; + scratch = scomp_lock_scratch_bh(); + if (acomp_request_src_isvirt(req)) src = req->svirt; else { @@ -218,6 +332,9 @@ static int scomp_acomp_comp_decomp(struct acomp_req *req, int dir) break; src = kmap_local_page(spage) + soff; } while (0); + + if (src == scratch->src) + memcpy_from_sglist(scratch->src, req->src, 0, slen); } if (acomp_request_dst_isvirt(req)) @@ -250,13 +367,7 @@ static int scomp_acomp_comp_decomp(struct acomp_req *req, int dir) dlen = min(dlen, max); } - spin_lock_bh(&scratch->lock); - - if (src == scratch->src) - memcpy_from_sglist(scratch->src, req->src, 0, slen); - - stream = raw_cpu_ptr(crypto_scomp_alg(scomp)->stream); - spin_lock(&stream->lock); + stream = scomp_lock_stream(scomp); if (dir) ret = crypto_scomp_compress(scomp, src, slen, dst, &dlen, stream->ctx); @@ -267,8 +378,8 @@ static int scomp_acomp_comp_decomp(struct acomp_req *req, int dir) if (dst == scratch->dst) memcpy_to_sglist(req->dst, 0, dst, dlen); - spin_unlock(&stream->lock); - spin_unlock_bh(&scratch->lock); + scomp_unlock_stream(stream); + scomp_unlock_scratch_bh(scratch); req->dlen = dlen; @@ -319,6 +430,7 @@ static void crypto_exit_scomp_ops_async(struct crypto_tfm *tfm) crypto_free_scomp(*ctx); + flush_work(&scomp_scratch_work); mutex_lock(&scomp_lock); if (!--scomp_scratch_users) crypto_scomp_free_scratches(); @@ -352,7 +464,10 @@ int crypto_init_scomp_ops_async(struct crypto_tfm *tfm) static void crypto_scomp_destroy(struct crypto_alg *alg) { - scomp_free_streams(__crypto_scomp_alg(alg)); + struct scomp_alg *scomp = __crypto_scomp_alg(alg); + + cancel_work_sync(&scomp->stream_work); + scomp_free_streams(scomp); } static const struct crypto_type crypto_scomp_type = { @@ -378,6 +493,8 @@ static void scomp_prepare_alg(struct scomp_alg *alg) comp_prepare_alg(&alg->calg); base->cra_flags |= CRYPTO_ALG_REQ_CHAIN; + + INIT_WORK(&alg->stream_work, scomp_stream_workfn); } int crypto_register_scomp(struct scomp_alg *alg) diff --git a/include/crypto/acompress.h b/include/crypto/acompress.h index c497c73baf13..f0e01ff77d92 100644 --- a/include/crypto/acompress.h +++ b/include/crypto/acompress.h @@ -130,14 +130,8 @@ struct crypto_acomp { struct crypto_tfm base; }; -struct crypto_acomp_stream { - spinlock_t lock; - void *ctx; -}; - #define COMP_ALG_COMMON { \ struct crypto_alg base; \ - struct crypto_acomp_stream __percpu *stream; \ } struct comp_alg_common COMP_ALG_COMMON; diff --git a/include/crypto/internal/acompress.h b/include/crypto/internal/acompress.h index aaf59f3236fa..2af690819a83 100644 --- a/include/crypto/internal/acompress.h +++ b/include/crypto/internal/acompress.h @@ -37,7 +37,6 @@ * * @reqsize: Context size for (de)compression requests * @base: Common crypto API algorithm data structure - * @stream: Per-cpu memory for algorithm * @calg: Cmonn algorithm data structure shared with scomp */ struct acomp_alg { diff --git a/include/crypto/internal/scompress.h b/include/crypto/internal/scompress.h index f25aa2ea3b48..fd74e656ffd2 100644 --- a/include/crypto/internal/scompress.h +++ b/include/crypto/internal/scompress.h @@ -11,6 +11,8 @@ #include #include +#include +#include struct acomp_req; @@ -18,6 +20,11 @@ struct crypto_scomp { struct crypto_tfm base; }; +struct crypto_acomp_stream { + spinlock_t lock; + void *ctx; +}; + /** * struct scomp_alg - synchronous compression algorithm * @@ -27,6 +34,8 @@ struct crypto_scomp { * @decompress: Function performs a de-compress operation * @base: Common crypto API algorithm data structure * @stream: Per-cpu memory for algorithm + * @stream_work: Work struct to allocate stream memmory + * @stream_want: CPU mask for allocating stream memory * @calg: Cmonn algorithm data structure shared with acomp */ struct scomp_alg { @@ -39,6 +48,10 @@ struct scomp_alg { unsigned int slen, u8 *dst, unsigned int *dlen, void *ctx); + struct crypto_acomp_stream __percpu *stream; + struct work_struct stream_work; + cpumask_t stream_want; + union { struct COMP_ALG_COMMON; struct comp_alg_common calg; From d348ebc658cdf41b922671e472a6192c2338adb4 Mon Sep 17 00:00:00 2001 From: Abhinaba Rakshit Date: Tue, 18 Mar 2025 15:13:23 +0530 Subject: [PATCH 004/382] dt-bindings: crypto: qcom-qce: document QCS615 crypto engine Document the crypto engine on the QCS615 platform. Acked-by: Rob Herring (Arm) Signed-off-by: Abhinaba Rakshit Signed-off-by: Herbert Xu --- Documentation/devicetree/bindings/crypto/qcom-qce.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/crypto/qcom-qce.yaml b/Documentation/devicetree/bindings/crypto/qcom-qce.yaml index 3f35122f7873..e009cb712fb8 100644 --- a/Documentation/devicetree/bindings/crypto/qcom-qce.yaml +++ b/Documentation/devicetree/bindings/crypto/qcom-qce.yaml @@ -45,6 +45,7 @@ properties: - items: - enum: + - qcom,qcs615-qce - qcom,qcs8300-qce - qcom,sa8775p-qce - qcom,sc7280-qce From 42d9f6c774790d290c175e8775ce9f1366438098 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Wed, 19 Mar 2025 14:04:52 +0800 Subject: [PATCH 005/382] crypto: acomp - Move scomp stream allocation code into acomp Move the dynamic stream allocation code into acomp and make it available as a helper for acomp algorithms. Signed-off-by: Herbert Xu --- crypto/acompress.c | 112 +++++++++++++++++++++++ crypto/scompress.c | 133 +++------------------------- include/crypto/internal/acompress.h | 33 +++++++ include/crypto/internal/scompress.h | 28 ++---- 4 files changed, 166 insertions(+), 140 deletions(-) diff --git a/crypto/acompress.c b/crypto/acompress.c index f7a3fbe5447e..2cf5e9d749cf 100644 --- a/crypto/acompress.c +++ b/crypto/acompress.c @@ -9,13 +9,18 @@ #include #include +#include #include #include #include #include +#include #include #include +#include +#include #include +#include #include #include "compress.h" @@ -434,5 +439,112 @@ void crypto_unregister_acomps(struct acomp_alg *algs, int count) } EXPORT_SYMBOL_GPL(crypto_unregister_acomps); +static void acomp_stream_workfn(struct work_struct *work) +{ + struct crypto_acomp_streams *s = + container_of(work, struct crypto_acomp_streams, stream_work); + struct crypto_acomp_stream __percpu *streams = s->streams; + int cpu; + + for_each_cpu(cpu, &s->stream_want) { + struct crypto_acomp_stream *ps; + void *ctx; + + ps = per_cpu_ptr(streams, cpu); + if (ps->ctx) + continue; + + ctx = s->alloc_ctx(); + if (IS_ERR(ctx)) + break; + + spin_lock_bh(&ps->lock); + ps->ctx = ctx; + spin_unlock_bh(&ps->lock); + + cpumask_clear_cpu(cpu, &s->stream_want); + } +} + +void crypto_acomp_free_streams(struct crypto_acomp_streams *s) +{ + struct crypto_acomp_stream __percpu *streams = s->streams; + void (*free_ctx)(void *); + int i; + + cancel_work_sync(&s->stream_work); + free_ctx = s->free_ctx; + + for_each_possible_cpu(i) { + struct crypto_acomp_stream *ps = per_cpu_ptr(streams, i); + + if (!ps->ctx) + continue; + + free_ctx(ps->ctx); + } + + free_percpu(streams); +} +EXPORT_SYMBOL_GPL(crypto_acomp_free_streams); + +int crypto_acomp_alloc_streams(struct crypto_acomp_streams *s) +{ + struct crypto_acomp_stream __percpu *streams; + struct crypto_acomp_stream *ps; + unsigned int i; + void *ctx; + + if (s->streams) + return 0; + + streams = alloc_percpu(struct crypto_acomp_stream); + if (!streams) + return -ENOMEM; + + ctx = s->alloc_ctx(); + if (IS_ERR(ctx)) { + free_percpu(streams); + return PTR_ERR(ctx); + } + + i = cpumask_first(cpu_possible_mask); + ps = per_cpu_ptr(streams, i); + ps->ctx = ctx; + + for_each_possible_cpu(i) { + ps = per_cpu_ptr(streams, i); + spin_lock_init(&ps->lock); + } + + s->streams = streams; + + INIT_WORK(&s->stream_work, acomp_stream_workfn); + return 0; +} +EXPORT_SYMBOL_GPL(crypto_acomp_alloc_streams); + +struct crypto_acomp_stream *crypto_acomp_lock_stream_bh( + struct crypto_acomp_streams *s) __acquires(stream) +{ + struct crypto_acomp_stream __percpu *streams = s->streams; + int cpu = raw_smp_processor_id(); + struct crypto_acomp_stream *ps; + + ps = per_cpu_ptr(streams, cpu); + spin_lock_bh(&ps->lock); + if (likely(ps->ctx)) + return ps; + spin_unlock(&ps->lock); + + cpumask_set_cpu(cpu, &s->stream_want); + schedule_work(&s->stream_work); + + ps = per_cpu_ptr(streams, cpumask_first(cpu_possible_mask)); + spin_lock(&ps->lock); + return ps; +} +EXPORT_SYMBOL_GPL(crypto_acomp_lock_stream_bh); + MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Asynchronous compression type"); diff --git a/crypto/scompress.c b/crypto/scompress.c index f4e3376ca7d3..5a40605570f5 100644 --- a/crypto/scompress.c +++ b/crypto/scompress.c @@ -7,7 +7,6 @@ * Author: Giovanni Cabiddu */ -#include #include #include #include @@ -132,91 +131,15 @@ static int crypto_scomp_alloc_scratches(void) return scomp_alloc_scratch(scratch, i); } -static void scomp_free_streams(struct scomp_alg *alg) -{ - struct crypto_acomp_stream __percpu *stream = alg->stream; - int i; - - for_each_possible_cpu(i) { - struct crypto_acomp_stream *ps = per_cpu_ptr(stream, i); - - if (!ps->ctx) - continue; - - alg->free_ctx(ps->ctx); - } - - free_percpu(stream); -} - -static int scomp_alloc_streams(struct scomp_alg *alg) -{ - struct crypto_acomp_stream __percpu *stream; - struct crypto_acomp_stream *ps; - unsigned int i; - void *ctx; - - stream = alloc_percpu(struct crypto_acomp_stream); - if (!stream) - return -ENOMEM; - - ctx = alg->alloc_ctx(); - if (IS_ERR(ctx)) { - free_percpu(stream); - return PTR_ERR(ctx); - } - - i = cpumask_first(cpu_possible_mask); - ps = per_cpu_ptr(stream, i); - ps->ctx = ctx; - - for_each_possible_cpu(i) { - ps = per_cpu_ptr(stream, i); - spin_lock_init(&ps->lock); - } - - alg->stream = stream; - return 0; -} - -static void scomp_stream_workfn(struct work_struct *work) -{ - struct scomp_alg *alg = container_of(work, struct scomp_alg, - stream_work); - struct crypto_acomp_stream __percpu *stream = alg->stream; - int cpu; - - for_each_cpu(cpu, &alg->stream_want) { - struct crypto_acomp_stream *ps; - void *ctx; - - ps = per_cpu_ptr(stream, cpu); - if (ps->ctx) - continue; - - ctx = alg->alloc_ctx(); - if (IS_ERR(ctx)) - break; - - spin_lock_bh(&ps->lock); - ps->ctx = ctx; - spin_unlock_bh(&ps->lock); - - cpumask_clear_cpu(cpu, &alg->stream_want); - } -} - static int crypto_scomp_init_tfm(struct crypto_tfm *tfm) { struct scomp_alg *alg = crypto_scomp_alg(__crypto_scomp_tfm(tfm)); int ret = 0; mutex_lock(&scomp_lock); - if (!alg->stream) { - ret = scomp_alloc_streams(alg); - if (ret) - goto unlock; - } + ret = crypto_acomp_alloc_streams(&alg->streams); + if (ret) + goto unlock; if (!scomp_scratch_users) { ret = crypto_scomp_alloc_scratches(); if (ret) @@ -229,13 +152,13 @@ unlock: return ret; } -static struct scomp_scratch *scomp_lock_scratch_bh(void) __acquires(scratch) +static struct scomp_scratch *scomp_lock_scratch(void) __acquires(scratch) { int cpu = raw_smp_processor_id(); struct scomp_scratch *scratch; scratch = per_cpu_ptr(&scomp_scratch, cpu); - spin_lock_bh(&scratch->lock); + spin_lock(&scratch->lock); if (likely(scratch->src)) return scratch; spin_unlock(&scratch->lock); @@ -248,39 +171,10 @@ static struct scomp_scratch *scomp_lock_scratch_bh(void) __acquires(scratch) return scratch; } -static inline void scomp_unlock_scratch_bh(struct scomp_scratch *scratch) +static inline void scomp_unlock_scratch(struct scomp_scratch *scratch) __releases(scratch) { - spin_unlock_bh(&scratch->lock); -} - -static struct crypto_acomp_stream *scomp_lock_stream(struct crypto_scomp *tfm) - __acquires(stream) -{ - struct scomp_alg *alg = crypto_scomp_alg(tfm); - struct crypto_acomp_stream __percpu *stream; - int cpu = raw_smp_processor_id(); - struct crypto_acomp_stream *ps; - - stream = alg->stream; - ps = per_cpu_ptr(stream, cpu); - spin_lock(&ps->lock); - if (likely(ps->ctx)) - return ps; - spin_unlock(&ps->lock); - - cpumask_set_cpu(cpu, &alg->stream_want); - schedule_work(&alg->stream_work); - - ps = per_cpu_ptr(stream, cpumask_first(cpu_possible_mask)); - spin_lock(&ps->lock); - return ps; -} - -static inline void scomp_unlock_stream(struct crypto_acomp_stream *stream) - __releases(stream) -{ - spin_unlock(&stream->lock); + spin_unlock(&scratch->lock); } static int scomp_acomp_comp_decomp(struct acomp_req *req, int dir) @@ -306,7 +200,8 @@ static int scomp_acomp_comp_decomp(struct acomp_req *req, int dir) if (!req->dst || !dlen) return -EINVAL; - scratch = scomp_lock_scratch_bh(); + stream = crypto_acomp_lock_stream_bh(&crypto_scomp_alg(scomp)->streams); + scratch = scomp_lock_scratch(); if (acomp_request_src_isvirt(req)) src = req->svirt; @@ -367,7 +262,6 @@ static int scomp_acomp_comp_decomp(struct acomp_req *req, int dir) dlen = min(dlen, max); } - stream = scomp_lock_stream(scomp); if (dir) ret = crypto_scomp_compress(scomp, src, slen, dst, &dlen, stream->ctx); @@ -378,8 +272,8 @@ static int scomp_acomp_comp_decomp(struct acomp_req *req, int dir) if (dst == scratch->dst) memcpy_to_sglist(req->dst, 0, dst, dlen); - scomp_unlock_stream(stream); - scomp_unlock_scratch_bh(scratch); + scomp_unlock_scratch(scratch); + crypto_acomp_unlock_stream_bh(stream); req->dlen = dlen; @@ -466,8 +360,7 @@ static void crypto_scomp_destroy(struct crypto_alg *alg) { struct scomp_alg *scomp = __crypto_scomp_alg(alg); - cancel_work_sync(&scomp->stream_work); - scomp_free_streams(scomp); + crypto_acomp_free_streams(&scomp->streams); } static const struct crypto_type crypto_scomp_type = { @@ -493,8 +386,6 @@ static void scomp_prepare_alg(struct scomp_alg *alg) comp_prepare_alg(&alg->calg); base->cra_flags |= CRYPTO_ALG_REQ_CHAIN; - - INIT_WORK(&alg->stream_work, scomp_stream_workfn); } int crypto_register_scomp(struct scomp_alg *alg) diff --git a/include/crypto/internal/acompress.h b/include/crypto/internal/acompress.h index 2af690819a83..ee5eff19eaf4 100644 --- a/include/crypto/internal/acompress.h +++ b/include/crypto/internal/acompress.h @@ -11,6 +11,10 @@ #include #include +#include +#include +#include +#include #define ACOMP_REQUEST_ON_STACK(name, tfm) \ char __##name##_req[sizeof(struct acomp_req) + \ @@ -53,6 +57,24 @@ struct acomp_alg { }; }; +struct crypto_acomp_stream { + spinlock_t lock; + void *ctx; +}; + +struct crypto_acomp_streams { + /* These must come first because of struct scomp_alg. */ + void *(*alloc_ctx)(void); + union { + void (*free_ctx)(void *); + void (*cfree_ctx)(const void *); + }; + + struct crypto_acomp_stream __percpu *streams; + struct work_struct stream_work; + cpumask_t stream_want; +}; + /* * Transform internal helpers. */ @@ -157,4 +179,15 @@ static inline bool crypto_acomp_req_chain(struct crypto_acomp *tfm) return crypto_tfm_req_chain(&tfm->base); } +void crypto_acomp_free_streams(struct crypto_acomp_streams *s); +int crypto_acomp_alloc_streams(struct crypto_acomp_streams *s); + +struct crypto_acomp_stream *crypto_acomp_lock_stream_bh( + struct crypto_acomp_streams *s) __acquires(stream); + +static inline void crypto_acomp_unlock_stream_bh( + struct crypto_acomp_stream *stream) __releases(stream) +{ + spin_unlock_bh(&stream->lock); +} #endif diff --git a/include/crypto/internal/scompress.h b/include/crypto/internal/scompress.h index fd74e656ffd2..533d6c16a491 100644 --- a/include/crypto/internal/scompress.h +++ b/include/crypto/internal/scompress.h @@ -9,22 +9,12 @@ #ifndef _CRYPTO_SCOMP_INT_H #define _CRYPTO_SCOMP_INT_H -#include -#include -#include -#include - -struct acomp_req; +#include struct crypto_scomp { struct crypto_tfm base; }; -struct crypto_acomp_stream { - spinlock_t lock; - void *ctx; -}; - /** * struct scomp_alg - synchronous compression algorithm * @@ -33,14 +23,10 @@ struct crypto_acomp_stream { * @compress: Function performs a compress operation * @decompress: Function performs a de-compress operation * @base: Common crypto API algorithm data structure - * @stream: Per-cpu memory for algorithm - * @stream_work: Work struct to allocate stream memmory - * @stream_want: CPU mask for allocating stream memory + * @streams: Per-cpu memory for algorithm * @calg: Cmonn algorithm data structure shared with acomp */ struct scomp_alg { - void *(*alloc_ctx)(void); - void (*free_ctx)(void *ctx); int (*compress)(struct crypto_scomp *tfm, const u8 *src, unsigned int slen, u8 *dst, unsigned int *dlen, void *ctx); @@ -48,9 +34,13 @@ struct scomp_alg { unsigned int slen, u8 *dst, unsigned int *dlen, void *ctx); - struct crypto_acomp_stream __percpu *stream; - struct work_struct stream_work; - cpumask_t stream_want; + union { + struct { + void *(*alloc_ctx)(void); + void (*free_ctx)(void *ctx); + }; + struct crypto_acomp_streams streams; + }; union { struct COMP_ALG_COMMON; From 9c8cf582626ef56632006212b385cfbb6c54f094 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Wed, 19 Mar 2025 14:04:54 +0800 Subject: [PATCH 006/382] crypto: acomp - Add acomp_walk Add acomp_walk which is similar to skcipher_walk but tailored for acomp. Signed-off-by: Herbert Xu --- crypto/acompress.c | 116 ++++++++++++++++++++++++++++ include/crypto/internal/acompress.h | 44 +++++++++++ 2 files changed, 160 insertions(+) diff --git a/crypto/acompress.c b/crypto/acompress.c index 2cf5e9d749cf..70bc1fd044e4 100644 --- a/crypto/acompress.c +++ b/crypto/acompress.c @@ -8,6 +8,7 @@ */ #include +#include #include #include #include @@ -15,6 +16,8 @@ #include #include #include +#include +#include #include #include #include @@ -27,6 +30,14 @@ struct crypto_scomp; +enum { + ACOMP_WALK_SLEEP = 1 << 0, + ACOMP_WALK_SRC_LINEAR = 1 << 1, + ACOMP_WALK_SRC_FOLIO = 1 << 2, + ACOMP_WALK_DST_LINEAR = 1 << 3, + ACOMP_WALK_DST_FOLIO = 1 << 4, +}; + static const struct crypto_type crypto_acomp_type; static void acomp_reqchain_done(void *data, int err); @@ -546,5 +557,110 @@ struct crypto_acomp_stream *crypto_acomp_lock_stream_bh( } EXPORT_SYMBOL_GPL(crypto_acomp_lock_stream_bh); +void acomp_walk_done_src(struct acomp_walk *walk, int used) +{ + walk->slen -= used; + if ((walk->flags & ACOMP_WALK_SRC_LINEAR)) + scatterwalk_advance(&walk->in, used); + else + scatterwalk_done_src(&walk->in, used); + + if ((walk->flags & ACOMP_WALK_SLEEP)) + cond_resched(); +} +EXPORT_SYMBOL_GPL(acomp_walk_done_src); + +void acomp_walk_done_dst(struct acomp_walk *walk, int used) +{ + walk->dlen -= used; + if ((walk->flags & ACOMP_WALK_DST_LINEAR)) + scatterwalk_advance(&walk->out, used); + else + scatterwalk_done_dst(&walk->out, used); + + if ((walk->flags & ACOMP_WALK_SLEEP)) + cond_resched(); +} +EXPORT_SYMBOL_GPL(acomp_walk_done_dst); + +int acomp_walk_next_src(struct acomp_walk *walk) +{ + unsigned int slen = walk->slen; + unsigned int max = UINT_MAX; + + if (!preempt_model_preemptible() && (walk->flags & ACOMP_WALK_SLEEP)) + max = PAGE_SIZE; + if ((walk->flags & ACOMP_WALK_SRC_LINEAR)) { + walk->in.__addr = (void *)(((u8 *)walk->in.sg) + + walk->in.offset); + return min(slen, max); + } + + return slen ? scatterwalk_next(&walk->in, slen) : 0; +} +EXPORT_SYMBOL_GPL(acomp_walk_next_src); + +int acomp_walk_next_dst(struct acomp_walk *walk) +{ + unsigned int dlen = walk->dlen; + unsigned int max = UINT_MAX; + + if (!preempt_model_preemptible() && (walk->flags & ACOMP_WALK_SLEEP)) + max = PAGE_SIZE; + if ((walk->flags & ACOMP_WALK_DST_LINEAR)) { + walk->out.__addr = (void *)(((u8 *)walk->out.sg) + + walk->out.offset); + return min(dlen, max); + } + + return dlen ? scatterwalk_next(&walk->out, dlen) : 0; +} +EXPORT_SYMBOL_GPL(acomp_walk_next_dst); + +int acomp_walk_virt(struct acomp_walk *__restrict walk, + struct acomp_req *__restrict req) +{ + struct scatterlist *src = req->src; + struct scatterlist *dst = req->dst; + + walk->slen = req->slen; + walk->dlen = req->dlen; + + if (!walk->slen || !walk->dlen) + return -EINVAL; + + walk->flags = 0; + if ((req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP)) + walk->flags |= ACOMP_WALK_SLEEP; + if ((req->base.flags & CRYPTO_ACOMP_REQ_SRC_VIRT)) + walk->flags |= ACOMP_WALK_SRC_LINEAR; + else if ((req->base.flags & CRYPTO_ACOMP_REQ_SRC_FOLIO)) { + src = &req->chain.ssg; + sg_init_table(src, 1); + sg_set_folio(src, req->sfolio, walk->slen, req->soff); + } + if ((req->base.flags & CRYPTO_ACOMP_REQ_DST_VIRT)) + walk->flags |= ACOMP_WALK_DST_LINEAR; + else if ((req->base.flags & CRYPTO_ACOMP_REQ_DST_FOLIO)) { + dst = &req->chain.dsg; + sg_init_table(dst, 1); + sg_set_folio(dst, req->dfolio, walk->dlen, req->doff); + } + + if ((walk->flags & ACOMP_WALK_SRC_LINEAR)) { + walk->in.sg = (void *)req->svirt; + walk->in.offset = 0; + } else + scatterwalk_start(&walk->in, src); + if ((walk->flags & ACOMP_WALK_DST_LINEAR)) { + walk->out.sg = (void *)req->dvirt; + walk->out.offset = 0; + } else + scatterwalk_start(&walk->out, dst); + + return 0; +} +EXPORT_SYMBOL_GPL(acomp_walk_virt); + MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Asynchronous compression type"); diff --git a/include/crypto/internal/acompress.h b/include/crypto/internal/acompress.h index ee5eff19eaf4..fbbff9a8a2d9 100644 --- a/include/crypto/internal/acompress.h +++ b/include/crypto/internal/acompress.h @@ -11,6 +11,7 @@ #include #include +#include #include #include #include @@ -75,6 +76,37 @@ struct crypto_acomp_streams { cpumask_t stream_want; }; +struct acomp_walk { + union { + /* Virtual address of the source. */ + struct { + struct { + const void *const addr; + } virt; + } src; + + /* Private field for the API, do not use. */ + struct scatter_walk in; + }; + + union { + /* Virtual address of the destination. */ + struct { + struct { + void *const addr; + } virt; + } dst; + + /* Private field for the API, do not use. */ + struct scatter_walk out; + }; + + unsigned int slen; + unsigned int dlen; + + int flags; +}; + /* * Transform internal helpers. */ @@ -190,4 +222,16 @@ static inline void crypto_acomp_unlock_stream_bh( { spin_unlock_bh(&stream->lock); } + +void acomp_walk_done_src(struct acomp_walk *walk, int used); +void acomp_walk_done_dst(struct acomp_walk *walk, int used); +int acomp_walk_next_src(struct acomp_walk *walk); +int acomp_walk_next_dst(struct acomp_walk *walk); +int acomp_walk_virt(struct acomp_walk *__restrict walk, + struct acomp_req *__restrict req); + +static inline bool acomp_walk_more_src(const struct acomp_walk *walk, int cur) +{ + return walk->slen != cur; +} #endif From 08cabc7d3c8638b078e0ac2f755cead1defafe91 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Wed, 19 Mar 2025 14:04:56 +0800 Subject: [PATCH 007/382] crypto: deflate - Convert to acomp This based on work by Ard Biesheuvel . Convert deflate from scomp to acomp. This removes the need for the caller to linearise the source and destination. Link: https://lore.kernel.org/all/20230718125847.3869700-21-ardb@kernel.org/ Signed-off-by: Herbert Xu --- crypto/deflate.c | 405 ++++++++++++++++++++++++----------------------- 1 file changed, 208 insertions(+), 197 deletions(-) diff --git a/crypto/deflate.c b/crypto/deflate.c index 5c346c544093..bc76c343a0cf 100644 --- a/crypto/deflate.c +++ b/crypto/deflate.c @@ -6,246 +6,255 @@ * by IPCOMP (RFC 3173 & RFC 2394). * * Copyright (c) 2003 James Morris - * - * FIXME: deflate transforms will require up to a total of about 436k of kernel - * memory on i386 (390k for compression, the rest for decompression), as the - * current zlib kernel code uses a worst case pre-allocation system by default. - * This needs to be fixed so that the amount of memory required is properly - * related to the winbits and memlevel parameters. - * - * The default winbits of 11 should suit most packets, and it may be something - * to configure on a per-tfm basis in the future. - * - * Currently, compression history is not maintained between tfm calls, as - * it is not needed for IPCOMP and keeps the code simpler. It can be - * implemented if someone wants it. + * Copyright (c) 2023 Google, LLC. + * Copyright (c) 2025 Herbert Xu */ +#include +#include #include +#include #include -#include +#include +#include +#include +#include +#include #include -#include -#include -#include -#include -#include #define DEFLATE_DEF_LEVEL Z_DEFAULT_COMPRESSION #define DEFLATE_DEF_WINBITS 11 #define DEFLATE_DEF_MEMLEVEL MAX_MEM_LEVEL -struct deflate_ctx { - struct z_stream_s comp_stream; - struct z_stream_s decomp_stream; +struct deflate_stream { + struct z_stream_s stream; + u8 workspace[]; }; -static int deflate_comp_init(struct deflate_ctx *ctx) +static DEFINE_MUTEX(deflate_stream_lock); + +static void *deflate_alloc_stream(void) { - int ret = 0; - struct z_stream_s *stream = &ctx->comp_stream; + size_t size = max(zlib_inflate_workspacesize(), + zlib_deflate_workspacesize(-DEFLATE_DEF_WINBITS, + DEFLATE_DEF_MEMLEVEL)); + struct deflate_stream *ctx; - stream->workspace = vzalloc(zlib_deflate_workspacesize( - -DEFLATE_DEF_WINBITS, MAX_MEM_LEVEL)); - if (!stream->workspace) { - ret = -ENOMEM; - goto out; - } - ret = zlib_deflateInit2(stream, DEFLATE_DEF_LEVEL, Z_DEFLATED, - -DEFLATE_DEF_WINBITS, DEFLATE_DEF_MEMLEVEL, - Z_DEFAULT_STRATEGY); - if (ret != Z_OK) { - ret = -EINVAL; - goto out_free; - } -out: - return ret; -out_free: - vfree(stream->workspace); - goto out; -} - -static int deflate_decomp_init(struct deflate_ctx *ctx) -{ - int ret = 0; - struct z_stream_s *stream = &ctx->decomp_stream; - - stream->workspace = vzalloc(zlib_inflate_workspacesize()); - if (!stream->workspace) { - ret = -ENOMEM; - goto out; - } - ret = zlib_inflateInit2(stream, -DEFLATE_DEF_WINBITS); - if (ret != Z_OK) { - ret = -EINVAL; - goto out_free; - } -out: - return ret; -out_free: - vfree(stream->workspace); - goto out; -} - -static void deflate_comp_exit(struct deflate_ctx *ctx) -{ - zlib_deflateEnd(&ctx->comp_stream); - vfree(ctx->comp_stream.workspace); -} - -static void deflate_decomp_exit(struct deflate_ctx *ctx) -{ - zlib_inflateEnd(&ctx->decomp_stream); - vfree(ctx->decomp_stream.workspace); -} - -static int __deflate_init(void *ctx) -{ - int ret; - - ret = deflate_comp_init(ctx); - if (ret) - goto out; - ret = deflate_decomp_init(ctx); - if (ret) - deflate_comp_exit(ctx); -out: - return ret; -} - -static void *deflate_alloc_ctx(void) -{ - struct deflate_ctx *ctx; - int ret; - - ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); + ctx = kvmalloc(sizeof(*ctx) + size, GFP_KERNEL); if (!ctx) return ERR_PTR(-ENOMEM); - ret = __deflate_init(ctx); - if (ret) { - kfree(ctx); - return ERR_PTR(ret); - } + ctx->stream.workspace = ctx->workspace; return ctx; } -static void __deflate_exit(void *ctx) +static struct crypto_acomp_streams deflate_streams = { + .alloc_ctx = deflate_alloc_stream, + .cfree_ctx = kvfree, +}; + +static int deflate_compress_one(struct acomp_req *req, + struct deflate_stream *ds) { - deflate_comp_exit(ctx); - deflate_decomp_exit(ctx); + struct z_stream_s *stream = &ds->stream; + struct acomp_walk walk; + int ret; + + ret = acomp_walk_virt(&walk, req); + if (ret) + return ret; + + do { + unsigned int dcur; + + dcur = acomp_walk_next_dst(&walk); + if (!dcur) + return -ENOSPC; + + stream->avail_out = dcur; + stream->next_out = walk.dst.virt.addr; + + do { + int flush = Z_FINISH; + unsigned int scur; + + stream->avail_in = 0; + stream->next_in = NULL; + + scur = acomp_walk_next_src(&walk); + if (scur) { + if (acomp_walk_more_src(&walk, scur)) + flush = Z_NO_FLUSH; + stream->avail_in = scur; + stream->next_in = walk.src.virt.addr; + } + + ret = zlib_deflate(stream, flush); + + if (scur) { + scur -= stream->avail_in; + acomp_walk_done_src(&walk, scur); + } + } while (ret == Z_OK && stream->avail_out); + + acomp_walk_done_dst(&walk, dcur); + } while (ret == Z_OK); + + if (ret != Z_STREAM_END) + return -EINVAL; + + req->dlen = stream->total_out; + return 0; } -static void deflate_free_ctx(void *ctx) +static int deflate_compress(struct acomp_req *req) { - __deflate_exit(ctx); - kfree_sensitive(ctx); -} + struct crypto_acomp_stream *s; + struct deflate_stream *ds; + struct acomp_req *r2; + int err; -static int __deflate_compress(const u8 *src, unsigned int slen, - u8 *dst, unsigned int *dlen, void *ctx) -{ - int ret = 0; - struct deflate_ctx *dctx = ctx; - struct z_stream_s *stream = &dctx->comp_stream; + s = crypto_acomp_lock_stream_bh(&deflate_streams); + ds = s->ctx; - ret = zlib_deflateReset(stream); - if (ret != Z_OK) { - ret = -EINVAL; + err = zlib_deflateInit2(&ds->stream, DEFLATE_DEF_LEVEL, Z_DEFLATED, + -DEFLATE_DEF_WINBITS, DEFLATE_DEF_MEMLEVEL, + Z_DEFAULT_STRATEGY); + if (err != Z_OK) { + err = -EINVAL; goto out; } - stream->next_in = (u8 *)src; - stream->avail_in = slen; - stream->next_out = (u8 *)dst; - stream->avail_out = *dlen; + err = deflate_compress_one(req, ds); + req->base.err = err; - ret = zlib_deflate(stream, Z_FINISH); - if (ret != Z_STREAM_END) { - ret = -EINVAL; - goto out; + list_for_each_entry(r2, &req->base.list, base.list) { + zlib_deflateReset(&ds->stream); + r2->base.err = deflate_compress_one(r2, ds); } - ret = 0; - *dlen = stream->total_out; + out: + crypto_acomp_unlock_stream_bh(s); + + return err; +} + +static int deflate_decompress_one(struct acomp_req *req, + struct deflate_stream *ds) +{ + struct z_stream_s *stream = &ds->stream; + bool out_of_space = false; + struct acomp_walk walk; + int ret; + + ret = acomp_walk_virt(&walk, req); + if (ret) + return ret; + + do { + unsigned int scur; + + stream->avail_in = 0; + stream->next_in = NULL; + + scur = acomp_walk_next_src(&walk); + if (scur) { + stream->avail_in = scur; + stream->next_in = walk.src.virt.addr; + } + + do { + unsigned int dcur; + + dcur = acomp_walk_next_dst(&walk); + if (!dcur) { + out_of_space = true; + break; + } + + stream->avail_out = dcur; + stream->next_out = walk.dst.virt.addr; + + ret = zlib_inflate(stream, Z_NO_FLUSH); + + dcur -= stream->avail_out; + acomp_walk_done_dst(&walk, dcur); + } while (ret == Z_OK && stream->avail_in); + + if (scur) + acomp_walk_done_src(&walk, scur); + + if (out_of_space) + return -ENOSPC; + } while (ret == Z_OK); + + if (ret != Z_STREAM_END) + return -EINVAL; + + req->dlen = stream->total_out; + return 0; +} + +static int deflate_decompress(struct acomp_req *req) +{ + struct crypto_acomp_stream *s; + struct deflate_stream *ds; + struct acomp_req *r2; + int err; + + s = crypto_acomp_lock_stream_bh(&deflate_streams); + ds = s->ctx; + + err = zlib_inflateInit2(&ds->stream, -DEFLATE_DEF_WINBITS); + if (err != Z_OK) { + err = -EINVAL; + goto out; + } + + err = deflate_decompress_one(req, ds); + req->base.err = err; + + list_for_each_entry(r2, &req->base.list, base.list) { + zlib_inflateReset(&ds->stream); + r2->base.err = deflate_decompress_one(r2, ds); + } + +out: + crypto_acomp_unlock_stream_bh(s); + + return err; +} + +static int deflate_init(struct crypto_acomp *tfm) +{ + int ret; + + mutex_lock(&deflate_stream_lock); + ret = crypto_acomp_alloc_streams(&deflate_streams); + mutex_unlock(&deflate_stream_lock); + return ret; } -static int deflate_scompress(struct crypto_scomp *tfm, const u8 *src, - unsigned int slen, u8 *dst, unsigned int *dlen, - void *ctx) -{ - return __deflate_compress(src, slen, dst, dlen, ctx); -} - -static int __deflate_decompress(const u8 *src, unsigned int slen, - u8 *dst, unsigned int *dlen, void *ctx) -{ - - int ret = 0; - struct deflate_ctx *dctx = ctx; - struct z_stream_s *stream = &dctx->decomp_stream; - - ret = zlib_inflateReset(stream); - if (ret != Z_OK) { - ret = -EINVAL; - goto out; - } - - stream->next_in = (u8 *)src; - stream->avail_in = slen; - stream->next_out = (u8 *)dst; - stream->avail_out = *dlen; - - ret = zlib_inflate(stream, Z_SYNC_FLUSH); - /* - * Work around a bug in zlib, which sometimes wants to taste an extra - * byte when being used in the (undocumented) raw deflate mode. - * (From USAGI). - */ - if (ret == Z_OK && !stream->avail_in && stream->avail_out) { - u8 zerostuff = 0; - stream->next_in = &zerostuff; - stream->avail_in = 1; - ret = zlib_inflate(stream, Z_FINISH); - } - if (ret != Z_STREAM_END) { - ret = -EINVAL; - goto out; - } - ret = 0; - *dlen = stream->total_out; -out: - return ret; -} - -static int deflate_sdecompress(struct crypto_scomp *tfm, const u8 *src, - unsigned int slen, u8 *dst, unsigned int *dlen, - void *ctx) -{ - return __deflate_decompress(src, slen, dst, dlen, ctx); -} - -static struct scomp_alg scomp = { - .alloc_ctx = deflate_alloc_ctx, - .free_ctx = deflate_free_ctx, - .compress = deflate_scompress, - .decompress = deflate_sdecompress, - .base = { - .cra_name = "deflate", - .cra_driver_name = "deflate-scomp", - .cra_module = THIS_MODULE, - } +static struct acomp_alg acomp = { + .compress = deflate_compress, + .decompress = deflate_decompress, + .init = deflate_init, + .base.cra_name = "deflate", + .base.cra_driver_name = "deflate-generic", + .base.cra_flags = CRYPTO_ALG_REQ_CHAIN, + .base.cra_module = THIS_MODULE, }; static int __init deflate_mod_init(void) { - return crypto_register_scomp(&scomp); + return crypto_register_acomp(&acomp); } static void __exit deflate_mod_fini(void) { - crypto_unregister_scomp(&scomp); + crypto_unregister_acomp(&acomp); + crypto_acomp_free_streams(&deflate_streams); } subsys_initcall(deflate_mod_init); @@ -254,5 +263,7 @@ module_exit(deflate_mod_fini); MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Deflate Compression Algorithm for IPCOMP"); MODULE_AUTHOR("James Morris "); +MODULE_AUTHOR("Ard Biesheuvel "); +MODULE_AUTHOR("Herbert Xu "); MODULE_ALIAS_CRYPTO("deflate"); MODULE_ALIAS_CRYPTO("deflate-generic"); From 7b294e3eab5d537af5065dc50c25b0b57d59acce Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Wed, 19 Mar 2025 17:26:43 +0800 Subject: [PATCH 008/382] crypto: crypto4xx - Fix gcc12 uninitialized warning in crypto4xx_crypt The compiler gcc 12 warns about the IV buffer being uninitialized in crypto4xx_crypt. Silence the warning by using the new gcc 12 access attribute to mark crypto4xx_build_pd. Also fix the IV buffer length as it has been quadrupled (64 instead of 16). Signed-off-by: Herbert Xu --- drivers/crypto/amcc/crypto4xx_alg.c | 4 ++-- drivers/crypto/amcc/crypto4xx_core.c | 2 +- drivers/crypto/amcc/crypto4xx_core.h | 10 ++++++++-- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/drivers/crypto/amcc/crypto4xx_alg.c b/drivers/crypto/amcc/crypto4xx_alg.c index e0af611a95d8..289750f34ccf 100644 --- a/drivers/crypto/amcc/crypto4xx_alg.c +++ b/drivers/crypto/amcc/crypto4xx_alg.c @@ -72,7 +72,7 @@ static inline int crypto4xx_crypt(struct skcipher_request *req, { struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req); struct crypto4xx_ctx *ctx = crypto_skcipher_ctx(cipher); - __le32 iv[AES_IV_SIZE]; + __le32 iv[AES_IV_SIZE / 4]; if (check_blocksize && !IS_ALIGNED(req->cryptlen, AES_BLOCK_SIZE)) return -EINVAL; @@ -429,7 +429,7 @@ static int crypto4xx_crypt_aes_ccm(struct aead_request *req, bool decrypt) struct crypto4xx_ctx *ctx = crypto_tfm_ctx(req->base.tfm); struct crypto4xx_aead_reqctx *rctx = aead_request_ctx(req); struct crypto_aead *aead = crypto_aead_reqtfm(req); - __le32 iv[16]; + __le32 iv[4]; u32 tmp_sa[SA_AES128_CCM_LEN + 4]; struct dynamic_sa_ctl *sa = (struct dynamic_sa_ctl *)tmp_sa; unsigned int len = req->cryptlen; diff --git a/drivers/crypto/amcc/crypto4xx_core.c b/drivers/crypto/amcc/crypto4xx_core.c index ec3ccfa60445..c77d06ddb1ec 100644 --- a/drivers/crypto/amcc/crypto4xx_core.c +++ b/drivers/crypto/amcc/crypto4xx_core.c @@ -676,7 +676,7 @@ int crypto4xx_build_pd(struct crypto_async_request *req, struct scatterlist *src, struct scatterlist *dst, const unsigned int datalen, - const __le32 *iv, const u32 iv_len, + const void *iv, const u32 iv_len, const struct dynamic_sa_ctl *req_sa, const unsigned int sa_len, const unsigned int assoclen, diff --git a/drivers/crypto/amcc/crypto4xx_core.h b/drivers/crypto/amcc/crypto4xx_core.h index 3adcc5e65694..9c56c7ac6e4c 100644 --- a/drivers/crypto/amcc/crypto4xx_core.h +++ b/drivers/crypto/amcc/crypto4xx_core.h @@ -147,6 +147,12 @@ struct crypto4xx_alg { struct crypto4xx_device *dev; }; +#if IS_ENABLED(CONFIG_CC_IS_GCC) && CONFIG_GCC_VERSION >= 120000 +#define BUILD_PD_ACCESS __attribute__((access(read_only, 6, 7))) +#else +#define BUILD_PD_ACCESS +#endif + int crypto4xx_alloc_sa(struct crypto4xx_ctx *ctx, u32 size); void crypto4xx_free_sa(struct crypto4xx_ctx *ctx); int crypto4xx_build_pd(struct crypto_async_request *req, @@ -154,11 +160,11 @@ int crypto4xx_build_pd(struct crypto_async_request *req, struct scatterlist *src, struct scatterlist *dst, const unsigned int datalen, - const __le32 *iv, const u32 iv_len, + const void *iv, const u32 iv_len, const struct dynamic_sa_ctl *sa, const unsigned int sa_len, const unsigned int assoclen, - struct scatterlist *dst_tmp); + struct scatterlist *dst_tmp) BUILD_PD_ACCESS; int crypto4xx_setkey_aes_cbc(struct crypto_skcipher *cipher, const u8 *key, unsigned int keylen); int crypto4xx_setkey_aes_ctr(struct crypto_skcipher *cipher, From 7cc17ea196dda26ab34eea5b9aec19fd760761ad Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Wed, 19 Mar 2025 20:31:44 +0800 Subject: [PATCH 009/382] crypto: cavium - Move cpt and nitrox rules into cavium Makefile Move the cpt and nitrox rules into the cavium Makefile. Signed-off-by: Herbert Xu --- drivers/crypto/Makefile | 4 +--- drivers/crypto/cavium/Makefile | 2 ++ 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/crypto/Makefile b/drivers/crypto/Makefile index c97f0ebc55ec..22eadcc8f4a2 100644 --- a/drivers/crypto/Makefile +++ b/drivers/crypto/Makefile @@ -8,12 +8,9 @@ obj-$(CONFIG_CRYPTO_DEV_ATMEL_TDES) += atmel-tdes.o obj-$(CONFIG_CRYPTO_DEV_ATMEL_I2C) += atmel-i2c.o obj-$(CONFIG_CRYPTO_DEV_ATMEL_ECC) += atmel-ecc.o obj-$(CONFIG_CRYPTO_DEV_ATMEL_SHA204A) += atmel-sha204a.o -obj-$(CONFIG_CRYPTO_DEV_CAVIUM_ZIP) += cavium/ obj-$(CONFIG_CRYPTO_DEV_CCP) += ccp/ obj-$(CONFIG_CRYPTO_DEV_CCREE) += ccree/ obj-$(CONFIG_CRYPTO_DEV_CHELSIO) += chelsio/ -obj-$(CONFIG_CRYPTO_DEV_CPT) += cavium/cpt/ -obj-$(CONFIG_CRYPTO_DEV_NITROX) += cavium/nitrox/ obj-$(CONFIG_CRYPTO_DEV_EXYNOS_RNG) += exynos-rng.o obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_COMMON) += caam/ obj-$(CONFIG_CRYPTO_DEV_GEODE) += geode-aes.o @@ -50,3 +47,4 @@ obj-y += hisilicon/ obj-$(CONFIG_CRYPTO_DEV_AMLOGIC_GXL) += amlogic/ obj-y += intel/ obj-y += starfive/ +obj-y += cavium/ diff --git a/drivers/crypto/cavium/Makefile b/drivers/crypto/cavium/Makefile index 4679c06b611f..6c1297426376 100644 --- a/drivers/crypto/cavium/Makefile +++ b/drivers/crypto/cavium/Makefile @@ -2,4 +2,6 @@ # # Makefile for Cavium crypto device drivers # +obj-$(CONFIG_CRYPTO_DEV_CPT) += cpt/ +obj-$(CONFIG_CRYPTO_DEV_NITROX) += nitrox/ obj-$(CONFIG_CRYPTO_DEV_CAVIUM_ZIP) += zip/ From d462ae23bb3ca84eb82a53a671c915dc093eab43 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Wed, 19 Mar 2025 20:31:47 +0800 Subject: [PATCH 010/382] crypto: cavium/zip - Remove driver Remove cavium/zip as it is obsolete and stands in the way of acomp API work. If this is ever resurrected, please turn it into an acomp driver. Signed-off-by: Herbert Xu --- drivers/crypto/Kconfig | 7 - drivers/crypto/cavium/Makefile | 1 - drivers/crypto/cavium/zip/Makefile | 12 - drivers/crypto/cavium/zip/common.h | 222 ---- drivers/crypto/cavium/zip/zip_crypto.c | 261 ----- drivers/crypto/cavium/zip/zip_crypto.h | 68 -- drivers/crypto/cavium/zip/zip_deflate.c | 200 ---- drivers/crypto/cavium/zip/zip_deflate.h | 62 -- drivers/crypto/cavium/zip/zip_device.c | 202 ---- drivers/crypto/cavium/zip/zip_device.h | 108 -- drivers/crypto/cavium/zip/zip_inflate.c | 223 ---- drivers/crypto/cavium/zip/zip_inflate.h | 62 -- drivers/crypto/cavium/zip/zip_main.c | 603 ---------- drivers/crypto/cavium/zip/zip_main.h | 120 -- drivers/crypto/cavium/zip/zip_mem.c | 114 -- drivers/crypto/cavium/zip/zip_mem.h | 78 -- drivers/crypto/cavium/zip/zip_regs.h | 1347 ----------------------- 17 files changed, 3690 deletions(-) delete mode 100644 drivers/crypto/cavium/zip/Makefile delete mode 100644 drivers/crypto/cavium/zip/common.h delete mode 100644 drivers/crypto/cavium/zip/zip_crypto.c delete mode 100644 drivers/crypto/cavium/zip/zip_crypto.h delete mode 100644 drivers/crypto/cavium/zip/zip_deflate.c delete mode 100644 drivers/crypto/cavium/zip/zip_deflate.h delete mode 100644 drivers/crypto/cavium/zip/zip_device.c delete mode 100644 drivers/crypto/cavium/zip/zip_device.h delete mode 100644 drivers/crypto/cavium/zip/zip_inflate.c delete mode 100644 drivers/crypto/cavium/zip/zip_inflate.h delete mode 100644 drivers/crypto/cavium/zip/zip_main.c delete mode 100644 drivers/crypto/cavium/zip/zip_main.h delete mode 100644 drivers/crypto/cavium/zip/zip_mem.c delete mode 100644 drivers/crypto/cavium/zip/zip_mem.h delete mode 100644 drivers/crypto/cavium/zip/zip_regs.h diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig index 47082782008a..5686369779be 100644 --- a/drivers/crypto/Kconfig +++ b/drivers/crypto/Kconfig @@ -530,13 +530,6 @@ source "drivers/crypto/cavium/nitrox/Kconfig" source "drivers/crypto/marvell/Kconfig" source "drivers/crypto/intel/Kconfig" -config CRYPTO_DEV_CAVIUM_ZIP - tristate "Cavium ZIP driver" - depends on PCI && 64BIT && (ARM64 || COMPILE_TEST) - help - Select this option if you want to enable compression/decompression - acceleration on Cavium's ARM based SoCs - config CRYPTO_DEV_QCE tristate "Qualcomm crypto engine accelerator" depends on ARCH_QCOM || COMPILE_TEST diff --git a/drivers/crypto/cavium/Makefile b/drivers/crypto/cavium/Makefile index 6c1297426376..75227c587ed0 100644 --- a/drivers/crypto/cavium/Makefile +++ b/drivers/crypto/cavium/Makefile @@ -4,4 +4,3 @@ # obj-$(CONFIG_CRYPTO_DEV_CPT) += cpt/ obj-$(CONFIG_CRYPTO_DEV_NITROX) += nitrox/ -obj-$(CONFIG_CRYPTO_DEV_CAVIUM_ZIP) += zip/ diff --git a/drivers/crypto/cavium/zip/Makefile b/drivers/crypto/cavium/zip/Makefile deleted file mode 100644 index 020d189d793d..000000000000 --- a/drivers/crypto/cavium/zip/Makefile +++ /dev/null @@ -1,12 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 -# -# Makefile for Cavium's ZIP Driver. -# - -obj-$(CONFIG_CRYPTO_DEV_CAVIUM_ZIP) += thunderx_zip.o -thunderx_zip-y := zip_main.o \ - zip_device.o \ - zip_crypto.o \ - zip_mem.o \ - zip_deflate.o \ - zip_inflate.o diff --git a/drivers/crypto/cavium/zip/common.h b/drivers/crypto/cavium/zip/common.h deleted file mode 100644 index 54f6fb054119..000000000000 --- a/drivers/crypto/cavium/zip/common.h +++ /dev/null @@ -1,222 +0,0 @@ -/***********************license start************************************ - * Copyright (c) 2003-2017 Cavium, Inc. - * All rights reserved. - * - * License: one of 'Cavium License' or 'GNU General Public License Version 2' - * - * This file is provided under the terms of the Cavium License (see below) - * or under the terms of GNU General Public License, Version 2, as - * published by the Free Software Foundation. When using or redistributing - * this file, you may do so under either license. - * - * Cavium License: Redistribution and use in source and binary forms, with - * or without modification, are permitted provided that the following - * conditions are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials provided - * with the distribution. - * - * * Neither the name of Cavium Inc. nor the names of its contributors may be - * used to endorse or promote products derived from this software without - * specific prior written permission. - * - * This Software, including technical data, may be subject to U.S. export - * control laws, including the U.S. Export Administration Act and its - * associated regulations, and may be subject to export or import - * regulations in other countries. - * - * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" - * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS - * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH - * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY - * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT - * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) - * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A - * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET - * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE - * ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES - * WITH YOU. - ***********************license end**************************************/ - -#ifndef __COMMON_H__ -#define __COMMON_H__ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -/* Device specific zlib function definitions */ -#include "zip_device.h" - -/* ZIP device definitions */ -#include "zip_main.h" - -/* ZIP memory allocation/deallocation related definitions */ -#include "zip_mem.h" - -/* Device specific structure definitions */ -#include "zip_regs.h" - -#define ZIP_ERROR -1 - -#define ZIP_FLUSH_FINISH 4 - -#define RAW_FORMAT 0 /* for rawpipe */ -#define ZLIB_FORMAT 1 /* for zpipe */ -#define GZIP_FORMAT 2 /* for gzpipe */ -#define LZS_FORMAT 3 /* for lzspipe */ - -/* Max number of ZIP devices supported */ -#define MAX_ZIP_DEVICES 2 - -/* Configures the number of zip queues to be used */ -#define ZIP_NUM_QUEUES 2 - -#define DYNAMIC_STOP_EXCESS 1024 - -/* Maximum buffer sizes in direct mode */ -#define MAX_INPUT_BUFFER_SIZE (64 * 1024) -#define MAX_OUTPUT_BUFFER_SIZE (64 * 1024) - -/** - * struct zip_operation - common data structure for comp and decomp operations - * @input: Next input byte is read from here - * @output: Next output byte written here - * @ctx_addr: Inflate context buffer address - * @history: Pointer to the history buffer - * @input_len: Number of bytes available at next_in - * @input_total_len: Total number of input bytes read - * @output_len: Remaining free space at next_out - * @output_total_len: Total number of bytes output so far - * @csum: Checksum value of the uncompressed data - * @flush: Flush flag - * @format: Format (depends on stream's wrap) - * @speed: Speed depends on stream's level - * @ccode: Compression code ( stream's strategy) - * @lzs_flag: Flag for LZS support - * @begin_file: Beginning of file indication for inflate - * @history_len: Size of the history data - * @end_file: Ending of the file indication for inflate - * @compcode: Completion status of the ZIP invocation - * @bytes_read: Input bytes read in current instruction - * @bits_processed: Total bits processed for entire file - * @sizeofptr: To distinguish between ILP32 and LP64 - * @sizeofzops: Optional just for padding - * - * This structure is used to maintain the required meta data for the - * comp and decomp operations. - */ -struct zip_operation { - u8 *input; - u8 *output; - u64 ctx_addr; - u64 history; - - u32 input_len; - u32 input_total_len; - - u32 output_len; - u32 output_total_len; - - u32 csum; - u32 flush; - - u32 format; - u32 speed; - u32 ccode; - u32 lzs_flag; - - u32 begin_file; - u32 history_len; - - u32 end_file; - u32 compcode; - u32 bytes_read; - u32 bits_processed; - - u32 sizeofptr; - u32 sizeofzops; -}; - -static inline int zip_poll_result(union zip_zres_s *result) -{ - int retries = 1000; - - while (!result->s.compcode) { - if (!--retries) { - pr_err("ZIP ERR: request timed out"); - return -ETIMEDOUT; - } - udelay(10); - /* - * Force re-reading of compcode which is updated - * by the ZIP coprocessor. - */ - rmb(); - } - return 0; -} - -/* error messages */ -#define zip_err(fmt, args...) pr_err("ZIP ERR:%s():%d: " \ - fmt "\n", __func__, __LINE__, ## args) - -#ifdef MSG_ENABLE -/* Enable all messages */ -#define zip_msg(fmt, args...) pr_info("ZIP_MSG:" fmt "\n", ## args) -#else -#define zip_msg(fmt, args...) -#endif - -#if defined(ZIP_DEBUG_ENABLE) && defined(MSG_ENABLE) - -#ifdef DEBUG_LEVEL - -#define FILE_NAME (strrchr(__FILE__, '/') ? strrchr(__FILE__, '/') + 1 : \ - strrchr(__FILE__, '\\') ? strrchr(__FILE__, '\\') + 1 : __FILE__) - -#if DEBUG_LEVEL >= 4 - -#define zip_dbg(fmt, args...) pr_info("ZIP DBG: %s: %s() : %d: " \ - fmt "\n", FILE_NAME, __func__, __LINE__, ## args) - -#elif DEBUG_LEVEL >= 3 - -#define zip_dbg(fmt, args...) pr_info("ZIP DBG: %s: %s() : %d: " \ - fmt "\n", FILE_NAME, __func__, __LINE__, ## args) - -#elif DEBUG_LEVEL >= 2 - -#define zip_dbg(fmt, args...) pr_info("ZIP DBG: %s() : %d: " \ - fmt "\n", __func__, __LINE__, ## args) - -#else - -#define zip_dbg(fmt, args...) pr_info("ZIP DBG:" fmt "\n", ## args) - -#endif /* DEBUG LEVEL >=4 */ - -#else - -#define zip_dbg(fmt, args...) pr_info("ZIP DBG:" fmt "\n", ## args) - -#endif /* DEBUG_LEVEL */ -#else - -#define zip_dbg(fmt, args...) - -#endif /* ZIP_DEBUG_ENABLE && MSG_ENABLE*/ - -#endif diff --git a/drivers/crypto/cavium/zip/zip_crypto.c b/drivers/crypto/cavium/zip/zip_crypto.c deleted file mode 100644 index 02e87f2d50db..000000000000 --- a/drivers/crypto/cavium/zip/zip_crypto.c +++ /dev/null @@ -1,261 +0,0 @@ -/***********************license start************************************ - * Copyright (c) 2003-2017 Cavium, Inc. - * All rights reserved. - * - * License: one of 'Cavium License' or 'GNU General Public License Version 2' - * - * This file is provided under the terms of the Cavium License (see below) - * or under the terms of GNU General Public License, Version 2, as - * published by the Free Software Foundation. When using or redistributing - * this file, you may do so under either license. - * - * Cavium License: Redistribution and use in source and binary forms, with - * or without modification, are permitted provided that the following - * conditions are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials provided - * with the distribution. - * - * * Neither the name of Cavium Inc. nor the names of its contributors may be - * used to endorse or promote products derived from this software without - * specific prior written permission. - * - * This Software, including technical data, may be subject to U.S. export - * control laws, including the U.S. Export Administration Act and its - * associated regulations, and may be subject to export or import - * regulations in other countries. - * - * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" - * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS - * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH - * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY - * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT - * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) - * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A - * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET - * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE - * ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES - * WITH YOU. - ***********************license end**************************************/ - -#include "zip_crypto.h" - -static void zip_static_init_zip_ops(struct zip_operation *zip_ops, - int lzs_flag) -{ - zip_ops->flush = ZIP_FLUSH_FINISH; - - /* equivalent to level 6 of opensource zlib */ - zip_ops->speed = 1; - - if (!lzs_flag) { - zip_ops->ccode = 0; /* Auto Huffman */ - zip_ops->lzs_flag = 0; - zip_ops->format = ZLIB_FORMAT; - } else { - zip_ops->ccode = 3; /* LZS Encoding */ - zip_ops->lzs_flag = 1; - zip_ops->format = LZS_FORMAT; - } - zip_ops->begin_file = 1; - zip_ops->history_len = 0; - zip_ops->end_file = 1; - zip_ops->compcode = 0; - zip_ops->csum = 1; /* Adler checksum desired */ -} - -static int zip_ctx_init(struct zip_kernel_ctx *zip_ctx, int lzs_flag) -{ - struct zip_operation *comp_ctx = &zip_ctx->zip_comp; - struct zip_operation *decomp_ctx = &zip_ctx->zip_decomp; - - zip_static_init_zip_ops(comp_ctx, lzs_flag); - zip_static_init_zip_ops(decomp_ctx, lzs_flag); - - comp_ctx->input = zip_data_buf_alloc(MAX_INPUT_BUFFER_SIZE); - if (!comp_ctx->input) - return -ENOMEM; - - comp_ctx->output = zip_data_buf_alloc(MAX_OUTPUT_BUFFER_SIZE); - if (!comp_ctx->output) - goto err_comp_input; - - decomp_ctx->input = zip_data_buf_alloc(MAX_INPUT_BUFFER_SIZE); - if (!decomp_ctx->input) - goto err_comp_output; - - decomp_ctx->output = zip_data_buf_alloc(MAX_OUTPUT_BUFFER_SIZE); - if (!decomp_ctx->output) - goto err_decomp_input; - - return 0; - -err_decomp_input: - zip_data_buf_free(decomp_ctx->input, MAX_INPUT_BUFFER_SIZE); - -err_comp_output: - zip_data_buf_free(comp_ctx->output, MAX_OUTPUT_BUFFER_SIZE); - -err_comp_input: - zip_data_buf_free(comp_ctx->input, MAX_INPUT_BUFFER_SIZE); - - return -ENOMEM; -} - -static void zip_ctx_exit(struct zip_kernel_ctx *zip_ctx) -{ - struct zip_operation *comp_ctx = &zip_ctx->zip_comp; - struct zip_operation *dec_ctx = &zip_ctx->zip_decomp; - - zip_data_buf_free(comp_ctx->input, MAX_INPUT_BUFFER_SIZE); - zip_data_buf_free(comp_ctx->output, MAX_OUTPUT_BUFFER_SIZE); - - zip_data_buf_free(dec_ctx->input, MAX_INPUT_BUFFER_SIZE); - zip_data_buf_free(dec_ctx->output, MAX_OUTPUT_BUFFER_SIZE); -} - -static int zip_compress(const u8 *src, unsigned int slen, - u8 *dst, unsigned int *dlen, - struct zip_kernel_ctx *zip_ctx) -{ - struct zip_operation *zip_ops = NULL; - struct zip_state *zip_state; - struct zip_device *zip = NULL; - int ret; - - if (!zip_ctx || !src || !dst || !dlen) - return -ENOMEM; - - zip = zip_get_device(zip_get_node_id()); - if (!zip) - return -ENODEV; - - zip_state = kzalloc(sizeof(*zip_state), GFP_ATOMIC); - if (!zip_state) - return -ENOMEM; - - zip_ops = &zip_ctx->zip_comp; - - zip_ops->input_len = slen; - zip_ops->output_len = *dlen; - memcpy(zip_ops->input, src, slen); - - ret = zip_deflate(zip_ops, zip_state, zip); - - if (!ret) { - *dlen = zip_ops->output_len; - memcpy(dst, zip_ops->output, *dlen); - } - kfree(zip_state); - return ret; -} - -static int zip_decompress(const u8 *src, unsigned int slen, - u8 *dst, unsigned int *dlen, - struct zip_kernel_ctx *zip_ctx) -{ - struct zip_operation *zip_ops = NULL; - struct zip_state *zip_state; - struct zip_device *zip = NULL; - int ret; - - if (!zip_ctx || !src || !dst || !dlen) - return -ENOMEM; - - zip = zip_get_device(zip_get_node_id()); - if (!zip) - return -ENODEV; - - zip_state = kzalloc(sizeof(*zip_state), GFP_ATOMIC); - if (!zip_state) - return -ENOMEM; - - zip_ops = &zip_ctx->zip_decomp; - memcpy(zip_ops->input, src, slen); - - /* Work around for a bug in zlib which needs an extra bytes sometimes */ - if (zip_ops->ccode != 3) /* Not LZS Encoding */ - zip_ops->input[slen++] = 0; - - zip_ops->input_len = slen; - zip_ops->output_len = *dlen; - - ret = zip_inflate(zip_ops, zip_state, zip); - - if (!ret) { - *dlen = zip_ops->output_len; - memcpy(dst, zip_ops->output, *dlen); - } - kfree(zip_state); - return ret; -} - -/* SCOMP framework start */ -void *zip_alloc_scomp_ctx_deflate(void) -{ - int ret; - struct zip_kernel_ctx *zip_ctx; - - zip_ctx = kzalloc(sizeof(*zip_ctx), GFP_KERNEL); - if (!zip_ctx) - return ERR_PTR(-ENOMEM); - - ret = zip_ctx_init(zip_ctx, 0); - - if (ret) { - kfree_sensitive(zip_ctx); - return ERR_PTR(ret); - } - - return zip_ctx; -} - -void *zip_alloc_scomp_ctx_lzs(void) -{ - int ret; - struct zip_kernel_ctx *zip_ctx; - - zip_ctx = kzalloc(sizeof(*zip_ctx), GFP_KERNEL); - if (!zip_ctx) - return ERR_PTR(-ENOMEM); - - ret = zip_ctx_init(zip_ctx, 1); - - if (ret) { - kfree_sensitive(zip_ctx); - return ERR_PTR(ret); - } - - return zip_ctx; -} - -void zip_free_scomp_ctx(void *ctx) -{ - struct zip_kernel_ctx *zip_ctx = ctx; - - zip_ctx_exit(zip_ctx); - kfree_sensitive(zip_ctx); -} - -int zip_scomp_compress(struct crypto_scomp *tfm, - const u8 *src, unsigned int slen, - u8 *dst, unsigned int *dlen, void *ctx) -{ - struct zip_kernel_ctx *zip_ctx = ctx; - - return zip_compress(src, slen, dst, dlen, zip_ctx); -} - -int zip_scomp_decompress(struct crypto_scomp *tfm, - const u8 *src, unsigned int slen, - u8 *dst, unsigned int *dlen, void *ctx) -{ - struct zip_kernel_ctx *zip_ctx = ctx; - - return zip_decompress(src, slen, dst, dlen, zip_ctx); -} /* SCOMP framework end */ diff --git a/drivers/crypto/cavium/zip/zip_crypto.h b/drivers/crypto/cavium/zip/zip_crypto.h deleted file mode 100644 index 10899ece2d1f..000000000000 --- a/drivers/crypto/cavium/zip/zip_crypto.h +++ /dev/null @@ -1,68 +0,0 @@ -/***********************license start************************************ - * Copyright (c) 2003-2017 Cavium, Inc. - * All rights reserved. - * - * License: one of 'Cavium License' or 'GNU General Public License Version 2' - * - * This file is provided under the terms of the Cavium License (see below) - * or under the terms of GNU General Public License, Version 2, as - * published by the Free Software Foundation. When using or redistributing - * this file, you may do so under either license. - * - * Cavium License: Redistribution and use in source and binary forms, with - * or without modification, are permitted provided that the following - * conditions are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials provided - * with the distribution. - * - * * Neither the name of Cavium Inc. nor the names of its contributors may be - * used to endorse or promote products derived from this software without - * specific prior written permission. - * - * This Software, including technical data, may be subject to U.S. export - * control laws, including the U.S. Export Administration Act and its - * associated regulations, and may be subject to export or import - * regulations in other countries. - * - * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" - * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS - * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH - * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY - * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT - * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) - * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A - * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET - * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE - * ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES - * WITH YOU. - ***********************license end**************************************/ - -#ifndef __ZIP_CRYPTO_H__ -#define __ZIP_CRYPTO_H__ - -#include -#include "common.h" -#include "zip_deflate.h" -#include "zip_inflate.h" - -struct zip_kernel_ctx { - struct zip_operation zip_comp; - struct zip_operation zip_decomp; -}; - -void *zip_alloc_scomp_ctx_deflate(void); -void *zip_alloc_scomp_ctx_lzs(void); -void zip_free_scomp_ctx(void *zip_ctx); -int zip_scomp_compress(struct crypto_scomp *tfm, - const u8 *src, unsigned int slen, - u8 *dst, unsigned int *dlen, void *ctx); -int zip_scomp_decompress(struct crypto_scomp *tfm, - const u8 *src, unsigned int slen, - u8 *dst, unsigned int *dlen, void *ctx); -#endif diff --git a/drivers/crypto/cavium/zip/zip_deflate.c b/drivers/crypto/cavium/zip/zip_deflate.c deleted file mode 100644 index d7133f857d67..000000000000 --- a/drivers/crypto/cavium/zip/zip_deflate.c +++ /dev/null @@ -1,200 +0,0 @@ -/***********************license start************************************ - * Copyright (c) 2003-2017 Cavium, Inc. - * All rights reserved. - * - * License: one of 'Cavium License' or 'GNU General Public License Version 2' - * - * This file is provided under the terms of the Cavium License (see below) - * or under the terms of GNU General Public License, Version 2, as - * published by the Free Software Foundation. When using or redistributing - * this file, you may do so under either license. - * - * Cavium License: Redistribution and use in source and binary forms, with - * or without modification, are permitted provided that the following - * conditions are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials provided - * with the distribution. - * - * * Neither the name of Cavium Inc. nor the names of its contributors may be - * used to endorse or promote products derived from this software without - * specific prior written permission. - * - * This Software, including technical data, may be subject to U.S. export - * control laws, including the U.S. Export Administration Act and its - * associated regulations, and may be subject to export or import - * regulations in other countries. - * - * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" - * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS - * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH - * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY - * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT - * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) - * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A - * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET - * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE - * ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES - * WITH YOU. - ***********************license end**************************************/ - -#include -#include - -#include "common.h" -#include "zip_deflate.h" - -/* Prepares the deflate zip command */ -static int prepare_zip_command(struct zip_operation *zip_ops, - struct zip_state *s, union zip_inst_s *zip_cmd) -{ - union zip_zres_s *result_ptr = &s->result; - - memset(zip_cmd, 0, sizeof(s->zip_cmd)); - memset(result_ptr, 0, sizeof(s->result)); - - /* IWORD #0 */ - /* History gather */ - zip_cmd->s.hg = 0; - /* compression enable = 1 for deflate */ - zip_cmd->s.ce = 1; - /* sf (sync flush) */ - zip_cmd->s.sf = 1; - /* ef (end of file) */ - if (zip_ops->flush == ZIP_FLUSH_FINISH) { - zip_cmd->s.ef = 1; - zip_cmd->s.sf = 0; - } - - zip_cmd->s.cc = zip_ops->ccode; - /* ss (compression speed/storage) */ - zip_cmd->s.ss = zip_ops->speed; - - /* IWORD #1 */ - /* adler checksum */ - zip_cmd->s.adlercrc32 = zip_ops->csum; - zip_cmd->s.historylength = zip_ops->history_len; - zip_cmd->s.dg = 0; - - /* IWORD # 6 and 7 - compression input/history pointer */ - zip_cmd->s.inp_ptr_addr.s.addr = __pa(zip_ops->input); - zip_cmd->s.inp_ptr_ctl.s.length = (zip_ops->input_len + - zip_ops->history_len); - zip_cmd->s.ds = 0; - - /* IWORD # 8 and 9 - Output pointer */ - zip_cmd->s.out_ptr_addr.s.addr = __pa(zip_ops->output); - zip_cmd->s.out_ptr_ctl.s.length = zip_ops->output_len; - /* maximum number of output-stream bytes that can be written */ - zip_cmd->s.totaloutputlength = zip_ops->output_len; - - /* IWORD # 10 and 11 - Result pointer */ - zip_cmd->s.res_ptr_addr.s.addr = __pa(result_ptr); - /* Clearing completion code */ - result_ptr->s.compcode = 0; - - return 0; -} - -/** - * zip_deflate - API to offload deflate operation to hardware - * @zip_ops: Pointer to zip operation structure - * @s: Pointer to the structure representing zip state - * @zip_dev: Pointer to zip device structure - * - * This function prepares the zip deflate command and submits it to the zip - * engine for processing. - * - * Return: 0 if successful or error code - */ -int zip_deflate(struct zip_operation *zip_ops, struct zip_state *s, - struct zip_device *zip_dev) -{ - union zip_inst_s *zip_cmd = &s->zip_cmd; - union zip_zres_s *result_ptr = &s->result; - u32 queue; - - /* Prepares zip command based on the input parameters */ - prepare_zip_command(zip_ops, s, zip_cmd); - - atomic64_add(zip_ops->input_len, &zip_dev->stats.comp_in_bytes); - /* Loads zip command into command queues and rings door bell */ - queue = zip_load_instr(zip_cmd, zip_dev); - - /* Stats update for compression requests submitted */ - atomic64_inc(&zip_dev->stats.comp_req_submit); - - /* Wait for completion or error */ - zip_poll_result(result_ptr); - - /* Stats update for compression requests completed */ - atomic64_inc(&zip_dev->stats.comp_req_complete); - - zip_ops->compcode = result_ptr->s.compcode; - switch (zip_ops->compcode) { - case ZIP_CMD_NOTDONE: - zip_dbg("Zip instruction not yet completed"); - return ZIP_ERROR; - - case ZIP_CMD_SUCCESS: - zip_dbg("Zip instruction completed successfully"); - zip_update_cmd_bufs(zip_dev, queue); - break; - - case ZIP_CMD_DTRUNC: - zip_dbg("Output Truncate error"); - /* Returning ZIP_ERROR to avoid copy to user */ - return ZIP_ERROR; - - default: - zip_err("Zip instruction failed. Code:%d", zip_ops->compcode); - return ZIP_ERROR; - } - - /* Update the CRC depending on the format */ - switch (zip_ops->format) { - case RAW_FORMAT: - zip_dbg("RAW Format: %d ", zip_ops->format); - /* Get checksum from engine, need to feed it again */ - zip_ops->csum = result_ptr->s.adler32; - break; - - case ZLIB_FORMAT: - zip_dbg("ZLIB Format: %d ", zip_ops->format); - zip_ops->csum = result_ptr->s.adler32; - break; - - case GZIP_FORMAT: - zip_dbg("GZIP Format: %d ", zip_ops->format); - zip_ops->csum = result_ptr->s.crc32; - break; - - case LZS_FORMAT: - zip_dbg("LZS Format: %d ", zip_ops->format); - break; - - default: - zip_err("Unknown Format:%d\n", zip_ops->format); - } - - atomic64_add(result_ptr->s.totalbyteswritten, - &zip_dev->stats.comp_out_bytes); - - /* Update output_len */ - if (zip_ops->output_len < result_ptr->s.totalbyteswritten) { - /* Dynamic stop && strm->output_len < zipconstants[onfsize] */ - zip_err("output_len (%d) < total bytes written(%d)\n", - zip_ops->output_len, result_ptr->s.totalbyteswritten); - zip_ops->output_len = 0; - - } else { - zip_ops->output_len = result_ptr->s.totalbyteswritten; - } - - return 0; -} diff --git a/drivers/crypto/cavium/zip/zip_deflate.h b/drivers/crypto/cavium/zip/zip_deflate.h deleted file mode 100644 index 1d32e76edc4d..000000000000 --- a/drivers/crypto/cavium/zip/zip_deflate.h +++ /dev/null @@ -1,62 +0,0 @@ -/***********************license start************************************ - * Copyright (c) 2003-2017 Cavium, Inc. - * All rights reserved. - * - * License: one of 'Cavium License' or 'GNU General Public License Version 2' - * - * This file is provided under the terms of the Cavium License (see below) - * or under the terms of GNU General Public License, Version 2, as - * published by the Free Software Foundation. When using or redistributing - * this file, you may do so under either license. - * - * Cavium License: Redistribution and use in source and binary forms, with - * or without modification, are permitted provided that the following - * conditions are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials provided - * with the distribution. - * - * * Neither the name of Cavium Inc. nor the names of its contributors may be - * used to endorse or promote products derived from this software without - * specific prior written permission. - * - * This Software, including technical data, may be subject to U.S. export - * control laws, including the U.S. Export Administration Act and its - * associated regulations, and may be subject to export or import - * regulations in other countries. - * - * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" - * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS - * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH - * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY - * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT - * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) - * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A - * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET - * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE - * ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES - * WITH YOU. - ***********************license end**************************************/ - -#ifndef __ZIP_DEFLATE_H__ -#define __ZIP_DEFLATE_H__ - -/** - * zip_deflate - API to offload deflate operation to hardware - * @zip_ops: Pointer to zip operation structure - * @s: Pointer to the structure representing zip state - * @zip_dev: Pointer to the structure representing zip device - * - * This function prepares the zip deflate command and submits it to the zip - * engine by ringing the doorbell. - * - * Return: 0 if successful or error code - */ -int zip_deflate(struct zip_operation *zip_ops, struct zip_state *s, - struct zip_device *zip_dev); -#endif diff --git a/drivers/crypto/cavium/zip/zip_device.c b/drivers/crypto/cavium/zip/zip_device.c deleted file mode 100644 index f174ec29ed69..000000000000 --- a/drivers/crypto/cavium/zip/zip_device.c +++ /dev/null @@ -1,202 +0,0 @@ -/***********************license start************************************ - * Copyright (c) 2003-2017 Cavium, Inc. - * All rights reserved. - * - * License: one of 'Cavium License' or 'GNU General Public License Version 2' - * - * This file is provided under the terms of the Cavium License (see below) - * or under the terms of GNU General Public License, Version 2, as - * published by the Free Software Foundation. When using or redistributing - * this file, you may do so under either license. - * - * Cavium License: Redistribution and use in source and binary forms, with - * or without modification, are permitted provided that the following - * conditions are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials provided - * with the distribution. - * - * * Neither the name of Cavium Inc. nor the names of its contributors may be - * used to endorse or promote products derived from this software without - * specific prior written permission. - * - * This Software, including technical data, may be subject to U.S. export - * control laws, including the U.S. Export Administration Act and its - * associated regulations, and may be subject to export or import - * regulations in other countries. - * - * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" - * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS - * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH - * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY - * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT - * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) - * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A - * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET - * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE - * ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES - * WITH YOU. - ***********************license end**************************************/ - -#include "common.h" -#include "zip_deflate.h" - -/** - * zip_cmd_queue_consumed - Calculates the space consumed in the command queue. - * - * @zip_dev: Pointer to zip device structure - * @queue: Queue number - * - * Return: Bytes consumed in the command queue buffer. - */ -static inline u32 zip_cmd_queue_consumed(struct zip_device *zip_dev, int queue) -{ - return ((zip_dev->iq[queue].sw_head - zip_dev->iq[queue].sw_tail) * - sizeof(u64 *)); -} - -/** - * zip_load_instr - Submits the instruction into the ZIP command queue - * @instr: Pointer to the instruction to be submitted - * @zip_dev: Pointer to ZIP device structure to which the instruction is to - * be submitted - * - * This function copies the ZIP instruction to the command queue and rings the - * doorbell to notify the engine of the instruction submission. The command - * queue is maintained in a circular fashion. When there is space for exactly - * one instruction in the queue, next chunk pointer of the queue is made to - * point to the head of the queue, thus maintaining a circular queue. - * - * Return: Queue number to which the instruction was submitted - */ -u32 zip_load_instr(union zip_inst_s *instr, - struct zip_device *zip_dev) -{ - union zip_quex_doorbell dbell; - u32 queue = 0; - u32 consumed = 0; - u64 *ncb_ptr = NULL; - union zip_nptr_s ncp; - - /* - * Distribute the instructions between the enabled queues based on - * the CPU id. - */ - if (raw_smp_processor_id() % 2 == 0) - queue = 0; - else - queue = 1; - - zip_dbg("CPU Core: %d Queue number:%d", raw_smp_processor_id(), queue); - - /* Take cmd buffer lock */ - spin_lock(&zip_dev->iq[queue].lock); - - /* - * Command Queue implementation - * 1. If there is place for new instructions, push the cmd at sw_head. - * 2. If there is place for exactly one instruction, push the new cmd - * at the sw_head. Make sw_head point to the sw_tail to make it - * circular. Write sw_head's physical address to the "Next-Chunk - * Buffer Ptr" to make it cmd_hw_tail. - * 3. Ring the door bell. - */ - zip_dbg("sw_head : %lx", zip_dev->iq[queue].sw_head); - zip_dbg("sw_tail : %lx", zip_dev->iq[queue].sw_tail); - - consumed = zip_cmd_queue_consumed(zip_dev, queue); - /* Check if there is space to push just one cmd */ - if ((consumed + 128) == (ZIP_CMD_QBUF_SIZE - 8)) { - zip_dbg("Cmd queue space available for single command"); - /* Space for one cmd, pust it and make it circular queue */ - memcpy((u8 *)zip_dev->iq[queue].sw_head, (u8 *)instr, - sizeof(union zip_inst_s)); - zip_dev->iq[queue].sw_head += 16; /* 16 64_bit words = 128B */ - - /* Now, point the "Next-Chunk Buffer Ptr" to sw_head */ - ncb_ptr = zip_dev->iq[queue].sw_head; - - zip_dbg("ncb addr :0x%lx sw_head addr :0x%lx", - ncb_ptr, zip_dev->iq[queue].sw_head - 16); - - /* Using Circular command queue */ - zip_dev->iq[queue].sw_head = zip_dev->iq[queue].sw_tail; - /* Mark this buffer for free */ - zip_dev->iq[queue].free_flag = 1; - - /* Write new chunk buffer address at "Next-Chunk Buffer Ptr" */ - ncp.u_reg64 = 0ull; - ncp.s.addr = __pa(zip_dev->iq[queue].sw_head); - *ncb_ptr = ncp.u_reg64; - zip_dbg("*ncb_ptr :0x%lx sw_head[phys] :0x%lx", - *ncb_ptr, __pa(zip_dev->iq[queue].sw_head)); - - zip_dev->iq[queue].pend_cnt++; - - } else { - zip_dbg("Enough space is available for commands"); - /* Push this cmd to cmd queue buffer */ - memcpy((u8 *)zip_dev->iq[queue].sw_head, (u8 *)instr, - sizeof(union zip_inst_s)); - zip_dev->iq[queue].sw_head += 16; /* 16 64_bit words = 128B */ - - zip_dev->iq[queue].pend_cnt++; - } - zip_dbg("sw_head :0x%lx sw_tail :0x%lx hw_tail :0x%lx", - zip_dev->iq[queue].sw_head, zip_dev->iq[queue].sw_tail, - zip_dev->iq[queue].hw_tail); - - zip_dbg(" Pushed the new cmd : pend_cnt : %d", - zip_dev->iq[queue].pend_cnt); - - /* Ring the doorbell */ - dbell.u_reg64 = 0ull; - dbell.s.dbell_cnt = 1; - zip_reg_write(dbell.u_reg64, - (zip_dev->reg_base + ZIP_QUEX_DOORBELL(queue))); - - /* Unlock cmd buffer lock */ - spin_unlock(&zip_dev->iq[queue].lock); - - return queue; -} - -/** - * zip_update_cmd_bufs - Updates the queue statistics after posting the - * instruction - * @zip_dev: Pointer to zip device structure - * @queue: Queue number - */ -void zip_update_cmd_bufs(struct zip_device *zip_dev, u32 queue) -{ - /* Take cmd buffer lock */ - spin_lock(&zip_dev->iq[queue].lock); - - /* Check if the previous buffer can be freed */ - if (zip_dev->iq[queue].free_flag == 1) { - zip_dbg("Free flag. Free cmd buffer, adjust sw head and tail"); - /* Reset the free flag */ - zip_dev->iq[queue].free_flag = 0; - - /* Point the hw_tail to start of the new chunk buffer */ - zip_dev->iq[queue].hw_tail = zip_dev->iq[queue].sw_head; - } else { - zip_dbg("Free flag not set. increment hw tail"); - zip_dev->iq[queue].hw_tail += 16; /* 16 64_bit words = 128B */ - } - - zip_dev->iq[queue].done_cnt++; - zip_dev->iq[queue].pend_cnt--; - - zip_dbg("sw_head :0x%lx sw_tail :0x%lx hw_tail :0x%lx", - zip_dev->iq[queue].sw_head, zip_dev->iq[queue].sw_tail, - zip_dev->iq[queue].hw_tail); - zip_dbg(" Got CC : pend_cnt : %d\n", zip_dev->iq[queue].pend_cnt); - - spin_unlock(&zip_dev->iq[queue].lock); -} diff --git a/drivers/crypto/cavium/zip/zip_device.h b/drivers/crypto/cavium/zip/zip_device.h deleted file mode 100644 index 9e18b3b93d38..000000000000 --- a/drivers/crypto/cavium/zip/zip_device.h +++ /dev/null @@ -1,108 +0,0 @@ -/***********************license start************************************ - * Copyright (c) 2003-2017 Cavium, Inc. - * All rights reserved. - * - * License: one of 'Cavium License' or 'GNU General Public License Version 2' - * - * This file is provided under the terms of the Cavium License (see below) - * or under the terms of GNU General Public License, Version 2, as - * published by the Free Software Foundation. When using or redistributing - * this file, you may do so under either license. - * - * Cavium License: Redistribution and use in source and binary forms, with - * or without modification, are permitted provided that the following - * conditions are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials provided - * with the distribution. - * - * * Neither the name of Cavium Inc. nor the names of its contributors may be - * used to endorse or promote products derived from this software without - * specific prior written permission. - * - * This Software, including technical data, may be subject to U.S. export - * control laws, including the U.S. Export Administration Act and its - * associated regulations, and may be subject to export or import - * regulations in other countries. - * - * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" - * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS - * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH - * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY - * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT - * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) - * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A - * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET - * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE - * ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES - * WITH YOU. - ***********************license end**************************************/ - -#ifndef __ZIP_DEVICE_H__ -#define __ZIP_DEVICE_H__ - -#include -#include "zip_main.h" - -struct sg_info { - /* - * Pointer to the input data when scatter_gather == 0 and - * pointer to the input gather list buffer when scatter_gather == 1 - */ - union zip_zptr_s *gather; - - /* - * Pointer to the output data when scatter_gather == 0 and - * pointer to the output scatter list buffer when scatter_gather == 1 - */ - union zip_zptr_s *scatter; - - /* - * Holds size of the output buffer pointed by scatter list - * when scatter_gather == 1 - */ - u64 scatter_buf_size; - - /* for gather data */ - u64 gather_enable; - - /* for scatter data */ - u64 scatter_enable; - - /* Number of gather list pointers for gather data */ - u32 gbuf_cnt; - - /* Number of scatter list pointers for scatter data */ - u32 sbuf_cnt; - - /* Buffers allocation state */ - u8 alloc_state; -}; - -/** - * struct zip_state - Structure representing the required information related - * to a command - * @zip_cmd: Pointer to zip instruction structure - * @result: Pointer to zip result structure - * @ctx: Context pointer for inflate - * @history: Decompression history pointer - * @sginfo: Scatter-gather info structure - */ -struct zip_state { - union zip_inst_s zip_cmd; - union zip_zres_s result; - union zip_zptr_s *ctx; - union zip_zptr_s *history; - struct sg_info sginfo; -}; - -#define ZIP_CONTEXT_SIZE 2048 -#define ZIP_INFLATE_HISTORY_SIZE 32768 -#define ZIP_DEFLATE_HISTORY_SIZE 32768 - -#endif diff --git a/drivers/crypto/cavium/zip/zip_inflate.c b/drivers/crypto/cavium/zip/zip_inflate.c deleted file mode 100644 index 7e0d73e2f89e..000000000000 --- a/drivers/crypto/cavium/zip/zip_inflate.c +++ /dev/null @@ -1,223 +0,0 @@ -/***********************license start************************************ - * Copyright (c) 2003-2017 Cavium, Inc. - * All rights reserved. - * - * License: one of 'Cavium License' or 'GNU General Public License Version 2' - * - * This file is provided under the terms of the Cavium License (see below) - * or under the terms of GNU General Public License, Version 2, as - * published by the Free Software Foundation. When using or redistributing - * this file, you may do so under either license. - * - * Cavium License: Redistribution and use in source and binary forms, with - * or without modification, are permitted provided that the following - * conditions are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials provided - * with the distribution. - * - * * Neither the name of Cavium Inc. nor the names of its contributors may be - * used to endorse or promote products derived from this software without - * specific prior written permission. - * - * This Software, including technical data, may be subject to U.S. export - * control laws, including the U.S. Export Administration Act and its - * associated regulations, and may be subject to export or import - * regulations in other countries. - * - * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" - * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS - * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH - * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY - * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT - * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) - * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A - * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET - * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE - * ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES - * WITH YOU. - ***********************license end**************************************/ - -#include -#include - -#include "common.h" -#include "zip_inflate.h" - -static int prepare_inflate_zcmd(struct zip_operation *zip_ops, - struct zip_state *s, union zip_inst_s *zip_cmd) -{ - union zip_zres_s *result_ptr = &s->result; - - memset(zip_cmd, 0, sizeof(s->zip_cmd)); - memset(result_ptr, 0, sizeof(s->result)); - - /* IWORD#0 */ - - /* Decompression History Gather list - no gather list */ - zip_cmd->s.hg = 0; - /* For decompression, CE must be 0x0. */ - zip_cmd->s.ce = 0; - /* For decompression, SS must be 0x0. */ - zip_cmd->s.ss = 0; - /* For decompression, SF should always be set. */ - zip_cmd->s.sf = 1; - - /* Begin File */ - if (zip_ops->begin_file == 0) - zip_cmd->s.bf = 0; - else - zip_cmd->s.bf = 1; - - zip_cmd->s.ef = 1; - /* 0: for Deflate decompression, 3: for LZS decompression */ - zip_cmd->s.cc = zip_ops->ccode; - - /* IWORD #1*/ - - /* adler checksum */ - zip_cmd->s.adlercrc32 = zip_ops->csum; - - /* - * HISTORYLENGTH must be 0x0 for any ZIP decompress operation. - * History data is added to a decompression operation via IWORD3. - */ - zip_cmd->s.historylength = 0; - zip_cmd->s.ds = 0; - - /* IWORD # 8 and 9 - Output pointer */ - zip_cmd->s.out_ptr_addr.s.addr = __pa(zip_ops->output); - zip_cmd->s.out_ptr_ctl.s.length = zip_ops->output_len; - - /* Maximum number of output-stream bytes that can be written */ - zip_cmd->s.totaloutputlength = zip_ops->output_len; - - zip_dbg("Data Direct Input case "); - - /* IWORD # 6 and 7 - input pointer */ - zip_cmd->s.dg = 0; - zip_cmd->s.inp_ptr_addr.s.addr = __pa((u8 *)zip_ops->input); - zip_cmd->s.inp_ptr_ctl.s.length = zip_ops->input_len; - - /* IWORD # 10 and 11 - Result pointer */ - zip_cmd->s.res_ptr_addr.s.addr = __pa(result_ptr); - - /* Clearing completion code */ - result_ptr->s.compcode = 0; - - /* Returning 0 for time being.*/ - return 0; -} - -/** - * zip_inflate - API to offload inflate operation to hardware - * @zip_ops: Pointer to zip operation structure - * @s: Pointer to the structure representing zip state - * @zip_dev: Pointer to zip device structure - * - * This function prepares the zip inflate command and submits it to the zip - * engine for processing. - * - * Return: 0 if successful or error code - */ -int zip_inflate(struct zip_operation *zip_ops, struct zip_state *s, - struct zip_device *zip_dev) -{ - union zip_inst_s *zip_cmd = &s->zip_cmd; - union zip_zres_s *result_ptr = &s->result; - u32 queue; - - /* Prepare inflate zip command */ - prepare_inflate_zcmd(zip_ops, s, zip_cmd); - - atomic64_add(zip_ops->input_len, &zip_dev->stats.decomp_in_bytes); - - /* Load inflate command to zip queue and ring the doorbell */ - queue = zip_load_instr(zip_cmd, zip_dev); - - /* Decompression requests submitted stats update */ - atomic64_inc(&zip_dev->stats.decomp_req_submit); - - /* Wait for completion or error */ - zip_poll_result(result_ptr); - - /* Decompression requests completed stats update */ - atomic64_inc(&zip_dev->stats.decomp_req_complete); - - zip_ops->compcode = result_ptr->s.compcode; - switch (zip_ops->compcode) { - case ZIP_CMD_NOTDONE: - zip_dbg("Zip Instruction not yet completed\n"); - return ZIP_ERROR; - - case ZIP_CMD_SUCCESS: - zip_dbg("Zip Instruction completed successfully\n"); - break; - - case ZIP_CMD_DYNAMIC_STOP: - zip_dbg(" Dynamic stop Initiated\n"); - break; - - default: - zip_dbg("Instruction failed. Code = %d\n", zip_ops->compcode); - atomic64_inc(&zip_dev->stats.decomp_bad_reqs); - zip_update_cmd_bufs(zip_dev, queue); - return ZIP_ERROR; - } - - zip_update_cmd_bufs(zip_dev, queue); - - if ((zip_ops->ccode == 3) && (zip_ops->flush == 4) && - (zip_ops->compcode != ZIP_CMD_DYNAMIC_STOP)) - result_ptr->s.ef = 1; - - zip_ops->csum = result_ptr->s.adler32; - - atomic64_add(result_ptr->s.totalbyteswritten, - &zip_dev->stats.decomp_out_bytes); - - if (zip_ops->output_len < result_ptr->s.totalbyteswritten) { - zip_err("output_len (%d) < total bytes written (%d)\n", - zip_ops->output_len, result_ptr->s.totalbyteswritten); - zip_ops->output_len = 0; - } else { - zip_ops->output_len = result_ptr->s.totalbyteswritten; - } - - zip_ops->bytes_read = result_ptr->s.totalbytesread; - zip_ops->bits_processed = result_ptr->s.totalbitsprocessed; - zip_ops->end_file = result_ptr->s.ef; - if (zip_ops->end_file) { - switch (zip_ops->format) { - case RAW_FORMAT: - zip_dbg("RAW Format: %d ", zip_ops->format); - /* Get checksum from engine */ - zip_ops->csum = result_ptr->s.adler32; - break; - - case ZLIB_FORMAT: - zip_dbg("ZLIB Format: %d ", zip_ops->format); - zip_ops->csum = result_ptr->s.adler32; - break; - - case GZIP_FORMAT: - zip_dbg("GZIP Format: %d ", zip_ops->format); - zip_ops->csum = result_ptr->s.crc32; - break; - - case LZS_FORMAT: - zip_dbg("LZS Format: %d ", zip_ops->format); - break; - - default: - zip_err("Format error:%d\n", zip_ops->format); - } - } - - return 0; -} diff --git a/drivers/crypto/cavium/zip/zip_inflate.h b/drivers/crypto/cavium/zip/zip_inflate.h deleted file mode 100644 index 6b20f179978e..000000000000 --- a/drivers/crypto/cavium/zip/zip_inflate.h +++ /dev/null @@ -1,62 +0,0 @@ -/***********************license start************************************ - * Copyright (c) 2003-2017 Cavium, Inc. - * All rights reserved. - * - * License: one of 'Cavium License' or 'GNU General Public License Version 2' - * - * This file is provided under the terms of the Cavium License (see below) - * or under the terms of GNU General Public License, Version 2, as - * published by the Free Software Foundation. When using or redistributing - * this file, you may do so under either license. - * - * Cavium License: Redistribution and use in source and binary forms, with - * or without modification, are permitted provided that the following - * conditions are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials provided - * with the distribution. - * - * * Neither the name of Cavium Inc. nor the names of its contributors may be - * used to endorse or promote products derived from this software without - * specific prior written permission. - * - * This Software, including technical data, may be subject to U.S. export - * control laws, including the U.S. Export Administration Act and its - * associated regulations, and may be subject to export or import - * regulations in other countries. - * - * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" - * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS - * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH - * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY - * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT - * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) - * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A - * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET - * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE - * ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES - * WITH YOU. - ***********************license end**************************************/ - -#ifndef __ZIP_INFLATE_H__ -#define __ZIP_INFLATE_H__ - -/** - * zip_inflate - API to offload inflate operation to hardware - * @zip_ops: Pointer to zip operation structure - * @s: Pointer to the structure representing zip state - * @zip_dev: Pointer to the structure representing zip device - * - * This function prepares the zip inflate command and submits it to the zip - * engine for processing. - * - * Return: 0 if successful or error code - */ -int zip_inflate(struct zip_operation *zip_ops, struct zip_state *s, - struct zip_device *zip_dev); -#endif diff --git a/drivers/crypto/cavium/zip/zip_main.c b/drivers/crypto/cavium/zip/zip_main.c deleted file mode 100644 index abd58de4343d..000000000000 --- a/drivers/crypto/cavium/zip/zip_main.c +++ /dev/null @@ -1,603 +0,0 @@ -/***********************license start************************************ - * Copyright (c) 2003-2017 Cavium, Inc. - * All rights reserved. - * - * License: one of 'Cavium License' or 'GNU General Public License Version 2' - * - * This file is provided under the terms of the Cavium License (see below) - * or under the terms of GNU General Public License, Version 2, as - * published by the Free Software Foundation. When using or redistributing - * this file, you may do so under either license. - * - * Cavium License: Redistribution and use in source and binary forms, with - * or without modification, are permitted provided that the following - * conditions are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials provided - * with the distribution. - * - * * Neither the name of Cavium Inc. nor the names of its contributors may be - * used to endorse or promote products derived from this software without - * specific prior written permission. - * - * This Software, including technical data, may be subject to U.S. export - * control laws, including the U.S. Export Administration Act and its - * associated regulations, and may be subject to export or import - * regulations in other countries. - * - * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" - * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS - * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH - * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY - * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT - * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) - * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A - * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET - * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE - * ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES - * WITH YOU. - ***********************license end**************************************/ - -#include "common.h" -#include "zip_crypto.h" - -#define DRV_NAME "ThunderX-ZIP" - -static struct zip_device *zip_dev[MAX_ZIP_DEVICES]; - -static const struct pci_device_id zip_id_table[] = { - { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_THUNDERX_ZIP) }, - { 0, } -}; - -static void zip_debugfs_init(void); -static void zip_debugfs_exit(void); -static int zip_register_compression_device(void); -static void zip_unregister_compression_device(void); - -void zip_reg_write(u64 val, u64 __iomem *addr) -{ - writeq(val, addr); -} - -u64 zip_reg_read(u64 __iomem *addr) -{ - return readq(addr); -} - -/* - * Allocates new ZIP device structure - * Returns zip_device pointer or NULL if cannot allocate memory for zip_device - */ -static struct zip_device *zip_alloc_device(struct pci_dev *pdev) -{ - struct zip_device *zip = NULL; - int idx; - - for (idx = 0; idx < MAX_ZIP_DEVICES; idx++) { - if (!zip_dev[idx]) - break; - } - - /* To ensure that the index is within the limit */ - if (idx < MAX_ZIP_DEVICES) - zip = devm_kzalloc(&pdev->dev, sizeof(*zip), GFP_KERNEL); - - if (!zip) - return NULL; - - zip_dev[idx] = zip; - zip->index = idx; - return zip; -} - -/** - * zip_get_device - Get ZIP device based on node id of cpu - * - * @node: Node id of the current cpu - * Return: Pointer to Zip device structure - */ -struct zip_device *zip_get_device(int node) -{ - if ((node < MAX_ZIP_DEVICES) && (node >= 0)) - return zip_dev[node]; - - zip_err("ZIP device not found for node id %d\n", node); - return NULL; -} - -/** - * zip_get_node_id - Get the node id of the current cpu - * - * Return: Node id of the current cpu - */ -int zip_get_node_id(void) -{ - return cpu_to_node(raw_smp_processor_id()); -} - -/* Initializes the ZIP h/w sub-system */ -static int zip_init_hw(struct zip_device *zip) -{ - union zip_cmd_ctl cmd_ctl; - union zip_constants constants; - union zip_que_ena que_ena; - union zip_quex_map que_map; - union zip_que_pri que_pri; - - union zip_quex_sbuf_addr que_sbuf_addr; - union zip_quex_sbuf_ctl que_sbuf_ctl; - - int q = 0; - - /* Enable the ZIP Engine(Core) Clock */ - cmd_ctl.u_reg64 = zip_reg_read(zip->reg_base + ZIP_CMD_CTL); - cmd_ctl.s.forceclk = 1; - zip_reg_write(cmd_ctl.u_reg64 & 0xFF, (zip->reg_base + ZIP_CMD_CTL)); - - zip_msg("ZIP_CMD_CTL : 0x%016llx", - zip_reg_read(zip->reg_base + ZIP_CMD_CTL)); - - constants.u_reg64 = zip_reg_read(zip->reg_base + ZIP_CONSTANTS); - zip->depth = constants.s.depth; - zip->onfsize = constants.s.onfsize; - zip->ctxsize = constants.s.ctxsize; - - zip_msg("depth: 0x%016llx , onfsize : 0x%016llx , ctxsize : 0x%016llx", - zip->depth, zip->onfsize, zip->ctxsize); - - /* - * Program ZIP_QUE(0..7)_SBUF_ADDR and ZIP_QUE(0..7)_SBUF_CTL to - * have the correct buffer pointer and size configured for each - * instruction queue. - */ - for (q = 0; q < ZIP_NUM_QUEUES; q++) { - que_sbuf_ctl.u_reg64 = 0ull; - que_sbuf_ctl.s.size = (ZIP_CMD_QBUF_SIZE / sizeof(u64)); - que_sbuf_ctl.s.inst_be = 0; - que_sbuf_ctl.s.stream_id = 0; - zip_reg_write(que_sbuf_ctl.u_reg64, - (zip->reg_base + ZIP_QUEX_SBUF_CTL(q))); - - zip_msg("QUEX_SBUF_CTL[%d]: 0x%016llx", q, - zip_reg_read(zip->reg_base + ZIP_QUEX_SBUF_CTL(q))); - } - - for (q = 0; q < ZIP_NUM_QUEUES; q++) { - memset(&zip->iq[q], 0x0, sizeof(struct zip_iq)); - - spin_lock_init(&zip->iq[q].lock); - - if (zip_cmd_qbuf_alloc(zip, q)) { - while (q != 0) { - q--; - zip_cmd_qbuf_free(zip, q); - } - return -ENOMEM; - } - - /* Initialize tail ptr to head */ - zip->iq[q].sw_tail = zip->iq[q].sw_head; - zip->iq[q].hw_tail = zip->iq[q].sw_head; - - /* Write the physical addr to register */ - que_sbuf_addr.u_reg64 = 0ull; - que_sbuf_addr.s.ptr = (__pa(zip->iq[q].sw_head) >> - ZIP_128B_ALIGN); - - zip_msg("QUE[%d]_PTR(PHYS): 0x%016llx", q, - (u64)que_sbuf_addr.s.ptr); - - zip_reg_write(que_sbuf_addr.u_reg64, - (zip->reg_base + ZIP_QUEX_SBUF_ADDR(q))); - - zip_msg("QUEX_SBUF_ADDR[%d]: 0x%016llx", q, - zip_reg_read(zip->reg_base + ZIP_QUEX_SBUF_ADDR(q))); - - zip_dbg("sw_head :0x%lx sw_tail :0x%lx hw_tail :0x%lx", - zip->iq[q].sw_head, zip->iq[q].sw_tail, - zip->iq[q].hw_tail); - zip_dbg("sw_head phy addr : 0x%lx", que_sbuf_addr.s.ptr); - } - - /* - * Queue-to-ZIP core mapping - * If a queue is not mapped to a particular core, it is equivalent to - * the ZIP core being disabled. - */ - que_ena.u_reg64 = 0x0ull; - /* Enabling queues based on ZIP_NUM_QUEUES */ - for (q = 0; q < ZIP_NUM_QUEUES; q++) - que_ena.s.ena |= (0x1 << q); - zip_reg_write(que_ena.u_reg64, (zip->reg_base + ZIP_QUE_ENA)); - - zip_msg("QUE_ENA : 0x%016llx", - zip_reg_read(zip->reg_base + ZIP_QUE_ENA)); - - for (q = 0; q < ZIP_NUM_QUEUES; q++) { - que_map.u_reg64 = 0ull; - /* Mapping each queue to two ZIP cores */ - que_map.s.zce = 0x3; - zip_reg_write(que_map.u_reg64, - (zip->reg_base + ZIP_QUEX_MAP(q))); - - zip_msg("QUE_MAP(%d) : 0x%016llx", q, - zip_reg_read(zip->reg_base + ZIP_QUEX_MAP(q))); - } - - que_pri.u_reg64 = 0ull; - for (q = 0; q < ZIP_NUM_QUEUES; q++) - que_pri.s.pri |= (0x1 << q); /* Higher Priority RR */ - zip_reg_write(que_pri.u_reg64, (zip->reg_base + ZIP_QUE_PRI)); - - zip_msg("QUE_PRI %016llx", zip_reg_read(zip->reg_base + ZIP_QUE_PRI)); - - return 0; -} - -static void zip_reset(struct zip_device *zip) -{ - union zip_cmd_ctl cmd_ctl; - - cmd_ctl.u_reg64 = 0x0ull; - cmd_ctl.s.reset = 1; /* Forces ZIP cores to do reset */ - zip_reg_write(cmd_ctl.u_reg64, (zip->reg_base + ZIP_CMD_CTL)); -} - -static int zip_probe(struct pci_dev *pdev, const struct pci_device_id *ent) -{ - struct device *dev = &pdev->dev; - struct zip_device *zip = NULL; - int err; - - zip = zip_alloc_device(pdev); - if (!zip) - return -ENOMEM; - - dev_info(dev, "Found ZIP device %d %x:%x on Node %d\n", zip->index, - pdev->vendor, pdev->device, dev_to_node(dev)); - - pci_set_drvdata(pdev, zip); - zip->pdev = pdev; - - err = pci_enable_device(pdev); - if (err) { - dev_err(dev, "Failed to enable PCI device"); - goto err_free_device; - } - - err = pci_request_regions(pdev, DRV_NAME); - if (err) { - dev_err(dev, "PCI request regions failed 0x%x", err); - goto err_disable_device; - } - - err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(48)); - if (err) { - dev_err(dev, "Unable to get usable 48-bit DMA configuration\n"); - goto err_release_regions; - } - - /* MAP configuration registers */ - zip->reg_base = pci_ioremap_bar(pdev, PCI_CFG_ZIP_PF_BAR0); - if (!zip->reg_base) { - dev_err(dev, "ZIP: Cannot map BAR0 CSR memory space, aborting"); - err = -ENOMEM; - goto err_release_regions; - } - - /* Initialize ZIP Hardware */ - err = zip_init_hw(zip); - if (err) - goto err_release_regions; - - /* Register with the Kernel Crypto Interface */ - err = zip_register_compression_device(); - if (err < 0) { - zip_err("ZIP: Kernel Crypto Registration failed\n"); - goto err_register; - } - - /* comp-decomp statistics are handled with debugfs interface */ - zip_debugfs_init(); - - return 0; - -err_register: - zip_reset(zip); - -err_release_regions: - if (zip->reg_base) - iounmap(zip->reg_base); - pci_release_regions(pdev); - -err_disable_device: - pci_disable_device(pdev); - -err_free_device: - pci_set_drvdata(pdev, NULL); - - /* Remove zip_dev from zip_device list, free the zip_device memory */ - zip_dev[zip->index] = NULL; - devm_kfree(dev, zip); - - return err; -} - -static void zip_remove(struct pci_dev *pdev) -{ - struct zip_device *zip = pci_get_drvdata(pdev); - int q = 0; - - if (!zip) - return; - - zip_debugfs_exit(); - - zip_unregister_compression_device(); - - if (zip->reg_base) { - zip_reset(zip); - iounmap(zip->reg_base); - } - - pci_release_regions(pdev); - pci_disable_device(pdev); - - /* - * Free Command Queue buffers. This free should be called for all - * the enabled Queues. - */ - for (q = 0; q < ZIP_NUM_QUEUES; q++) - zip_cmd_qbuf_free(zip, q); - - pci_set_drvdata(pdev, NULL); - /* remove zip device from zip device list */ - zip_dev[zip->index] = NULL; -} - -/* PCI Sub-System Interface */ -static struct pci_driver zip_driver = { - .name = DRV_NAME, - .id_table = zip_id_table, - .probe = zip_probe, - .remove = zip_remove, -}; - -/* Kernel Crypto Subsystem Interface */ - -static struct scomp_alg zip_scomp_deflate = { - .alloc_ctx = zip_alloc_scomp_ctx_deflate, - .free_ctx = zip_free_scomp_ctx, - .compress = zip_scomp_compress, - .decompress = zip_scomp_decompress, - .base = { - .cra_name = "deflate", - .cra_driver_name = "deflate-scomp-cavium", - .cra_module = THIS_MODULE, - .cra_priority = 300, - } -}; - -static struct scomp_alg zip_scomp_lzs = { - .alloc_ctx = zip_alloc_scomp_ctx_lzs, - .free_ctx = zip_free_scomp_ctx, - .compress = zip_scomp_compress, - .decompress = zip_scomp_decompress, - .base = { - .cra_name = "lzs", - .cra_driver_name = "lzs-scomp-cavium", - .cra_module = THIS_MODULE, - .cra_priority = 300, - } -}; - -static int zip_register_compression_device(void) -{ - int ret; - - ret = crypto_register_scomp(&zip_scomp_deflate); - if (ret < 0) { - zip_err("Deflate scomp algorithm registration failed\n"); - return ret; - } - - ret = crypto_register_scomp(&zip_scomp_lzs); - if (ret < 0) { - zip_err("LZS scomp algorithm registration failed\n"); - goto err_unregister_scomp_deflate; - } - - return ret; - -err_unregister_scomp_deflate: - crypto_unregister_scomp(&zip_scomp_deflate); - - return ret; -} - -static void zip_unregister_compression_device(void) -{ - crypto_unregister_scomp(&zip_scomp_deflate); - crypto_unregister_scomp(&zip_scomp_lzs); -} - -/* - * debugfs functions - */ -#ifdef CONFIG_DEBUG_FS -#include - -/* Displays ZIP device statistics */ -static int zip_stats_show(struct seq_file *s, void *unused) -{ - u64 val = 0ull; - u64 avg_chunk = 0ull, avg_cr = 0ull; - u32 q = 0; - - int index = 0; - struct zip_device *zip; - struct zip_stats *st; - - for (index = 0; index < MAX_ZIP_DEVICES; index++) { - u64 pending = 0; - - if (zip_dev[index]) { - zip = zip_dev[index]; - st = &zip->stats; - - /* Get all the pending requests */ - for (q = 0; q < ZIP_NUM_QUEUES; q++) { - val = zip_reg_read((zip->reg_base + - ZIP_DBG_QUEX_STA(q))); - pending += val >> 32 & 0xffffff; - } - - val = atomic64_read(&st->comp_req_complete); - avg_chunk = (val) ? atomic64_read(&st->comp_in_bytes) / val : 0; - - val = atomic64_read(&st->comp_out_bytes); - avg_cr = (val) ? atomic64_read(&st->comp_in_bytes) / val : 0; - seq_printf(s, " ZIP Device %d Stats\n" - "-----------------------------------\n" - "Comp Req Submitted : \t%lld\n" - "Comp Req Completed : \t%lld\n" - "Compress In Bytes : \t%lld\n" - "Compressed Out Bytes : \t%lld\n" - "Average Chunk size : \t%llu\n" - "Average Compression ratio : \t%llu\n" - "Decomp Req Submitted : \t%lld\n" - "Decomp Req Completed : \t%lld\n" - "Decompress In Bytes : \t%lld\n" - "Decompressed Out Bytes : \t%lld\n" - "Decompress Bad requests : \t%lld\n" - "Pending Req : \t%lld\n" - "---------------------------------\n", - index, - (u64)atomic64_read(&st->comp_req_submit), - (u64)atomic64_read(&st->comp_req_complete), - (u64)atomic64_read(&st->comp_in_bytes), - (u64)atomic64_read(&st->comp_out_bytes), - avg_chunk, - avg_cr, - (u64)atomic64_read(&st->decomp_req_submit), - (u64)atomic64_read(&st->decomp_req_complete), - (u64)atomic64_read(&st->decomp_in_bytes), - (u64)atomic64_read(&st->decomp_out_bytes), - (u64)atomic64_read(&st->decomp_bad_reqs), - pending); - } - } - return 0; -} - -/* Clears stats data */ -static int zip_clear_show(struct seq_file *s, void *unused) -{ - int index = 0; - - for (index = 0; index < MAX_ZIP_DEVICES; index++) { - if (zip_dev[index]) { - memset(&zip_dev[index]->stats, 0, - sizeof(struct zip_stats)); - seq_printf(s, "Cleared stats for zip %d\n", index); - } - } - - return 0; -} - -static struct zip_registers zipregs[64] = { - {"ZIP_CMD_CTL ", 0x0000ull}, - {"ZIP_THROTTLE ", 0x0010ull}, - {"ZIP_CONSTANTS ", 0x00A0ull}, - {"ZIP_QUE0_MAP ", 0x1400ull}, - {"ZIP_QUE1_MAP ", 0x1408ull}, - {"ZIP_QUE_ENA ", 0x0500ull}, - {"ZIP_QUE_PRI ", 0x0508ull}, - {"ZIP_QUE0_DONE ", 0x2000ull}, - {"ZIP_QUE1_DONE ", 0x2008ull}, - {"ZIP_QUE0_DOORBELL ", 0x4000ull}, - {"ZIP_QUE1_DOORBELL ", 0x4008ull}, - {"ZIP_QUE0_SBUF_ADDR ", 0x1000ull}, - {"ZIP_QUE1_SBUF_ADDR ", 0x1008ull}, - {"ZIP_QUE0_SBUF_CTL ", 0x1200ull}, - {"ZIP_QUE1_SBUF_CTL ", 0x1208ull}, - { NULL, 0} -}; - -/* Prints registers' contents */ -static int zip_regs_show(struct seq_file *s, void *unused) -{ - u64 val = 0; - int i = 0, index = 0; - - for (index = 0; index < MAX_ZIP_DEVICES; index++) { - if (zip_dev[index]) { - seq_printf(s, "--------------------------------\n" - " ZIP Device %d Registers\n" - "--------------------------------\n", - index); - - i = 0; - - while (zipregs[i].reg_name) { - val = zip_reg_read((zip_dev[index]->reg_base + - zipregs[i].reg_offset)); - seq_printf(s, "%s: 0x%016llx\n", - zipregs[i].reg_name, val); - i++; - } - } - } - return 0; -} - -DEFINE_SHOW_ATTRIBUTE(zip_stats); -DEFINE_SHOW_ATTRIBUTE(zip_clear); -DEFINE_SHOW_ATTRIBUTE(zip_regs); - -/* Root directory for thunderx_zip debugfs entry */ -static struct dentry *zip_debugfs_root; - -static void zip_debugfs_init(void) -{ - if (!debugfs_initialized()) - return; - - zip_debugfs_root = debugfs_create_dir("thunderx_zip", NULL); - - /* Creating files for entries inside thunderx_zip directory */ - debugfs_create_file("zip_stats", 0444, zip_debugfs_root, NULL, - &zip_stats_fops); - - debugfs_create_file("zip_clear", 0444, zip_debugfs_root, NULL, - &zip_clear_fops); - - debugfs_create_file("zip_regs", 0444, zip_debugfs_root, NULL, - &zip_regs_fops); - -} - -static void zip_debugfs_exit(void) -{ - debugfs_remove_recursive(zip_debugfs_root); -} - -#else -static void __init zip_debugfs_init(void) { } -static void __exit zip_debugfs_exit(void) { } -#endif -/* debugfs - end */ - -module_pci_driver(zip_driver); - -MODULE_AUTHOR("Cavium Inc"); -MODULE_DESCRIPTION("Cavium Inc ThunderX ZIP Driver"); -MODULE_LICENSE("GPL v2"); -MODULE_DEVICE_TABLE(pci, zip_id_table); diff --git a/drivers/crypto/cavium/zip/zip_main.h b/drivers/crypto/cavium/zip/zip_main.h deleted file mode 100644 index e1e4fa92ce80..000000000000 --- a/drivers/crypto/cavium/zip/zip_main.h +++ /dev/null @@ -1,120 +0,0 @@ -/***********************license start************************************ - * Copyright (c) 2003-2017 Cavium, Inc. - * All rights reserved. - * - * License: one of 'Cavium License' or 'GNU General Public License Version 2' - * - * This file is provided under the terms of the Cavium License (see below) - * or under the terms of GNU General Public License, Version 2, as - * published by the Free Software Foundation. When using or redistributing - * this file, you may do so under either license. - * - * Cavium License: Redistribution and use in source and binary forms, with - * or without modification, are permitted provided that the following - * conditions are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials provided - * with the distribution. - * - * * Neither the name of Cavium Inc. nor the names of its contributors may be - * used to endorse or promote products derived from this software without - * specific prior written permission. - * - * This Software, including technical data, may be subject to U.S. export - * control laws, including the U.S. Export Administration Act and its - * associated regulations, and may be subject to export or import - * regulations in other countries. - * - * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" - * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS - * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH - * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY - * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT - * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) - * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A - * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET - * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE - * ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES - * WITH YOU. - ***********************license end**************************************/ - -#ifndef __ZIP_MAIN_H__ -#define __ZIP_MAIN_H__ - -#include "zip_device.h" -#include "zip_regs.h" - -/* PCI device IDs */ -#define PCI_DEVICE_ID_THUNDERX_ZIP 0xA01A - -/* ZIP device BARs */ -#define PCI_CFG_ZIP_PF_BAR0 0 /* Base addr for normal regs */ - -/* Maximum available zip queues */ -#define ZIP_MAX_NUM_QUEUES 8 - -#define ZIP_128B_ALIGN 7 - -/* Command queue buffer size */ -#define ZIP_CMD_QBUF_SIZE (8064 + 8) - -struct zip_registers { - char *reg_name; - u64 reg_offset; -}; - -/* ZIP Compression - Decompression stats */ -struct zip_stats { - atomic64_t comp_req_submit; - atomic64_t comp_req_complete; - atomic64_t decomp_req_submit; - atomic64_t decomp_req_complete; - atomic64_t comp_in_bytes; - atomic64_t comp_out_bytes; - atomic64_t decomp_in_bytes; - atomic64_t decomp_out_bytes; - atomic64_t decomp_bad_reqs; -}; - -/* ZIP Instruction Queue */ -struct zip_iq { - u64 *sw_head; - u64 *sw_tail; - u64 *hw_tail; - u64 done_cnt; - u64 pend_cnt; - u64 free_flag; - - /* ZIP IQ lock */ - spinlock_t lock; -}; - -/* ZIP Device */ -struct zip_device { - u32 index; - void __iomem *reg_base; - struct pci_dev *pdev; - - /* Different ZIP Constants */ - u64 depth; - u64 onfsize; - u64 ctxsize; - - struct zip_iq iq[ZIP_MAX_NUM_QUEUES]; - struct zip_stats stats; -}; - -/* Prototypes */ -struct zip_device *zip_get_device(int node_id); -int zip_get_node_id(void); -void zip_reg_write(u64 val, u64 __iomem *addr); -u64 zip_reg_read(u64 __iomem *addr); -void zip_update_cmd_bufs(struct zip_device *zip_dev, u32 queue); -u32 zip_load_instr(union zip_inst_s *instr, struct zip_device *zip_dev); - -#endif /* ZIP_MAIN_H */ diff --git a/drivers/crypto/cavium/zip/zip_mem.c b/drivers/crypto/cavium/zip/zip_mem.c deleted file mode 100644 index b3e0843a9169..000000000000 --- a/drivers/crypto/cavium/zip/zip_mem.c +++ /dev/null @@ -1,114 +0,0 @@ -/***********************license start************************************ - * Copyright (c) 2003-2017 Cavium, Inc. - * All rights reserved. - * - * License: one of 'Cavium License' or 'GNU General Public License Version 2' - * - * This file is provided under the terms of the Cavium License (see below) - * or under the terms of GNU General Public License, Version 2, as - * published by the Free Software Foundation. When using or redistributing - * this file, you may do so under either license. - * - * Cavium License: Redistribution and use in source and binary forms, with - * or without modification, are permitted provided that the following - * conditions are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials provided - * with the distribution. - * - * * Neither the name of Cavium Inc. nor the names of its contributors may be - * used to endorse or promote products derived from this software without - * specific prior written permission. - * - * This Software, including technical data, may be subject to U.S. export - * control laws, including the U.S. Export Administration Act and its - * associated regulations, and may be subject to export or import - * regulations in other countries. - * - * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" - * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS - * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH - * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY - * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT - * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) - * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A - * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET - * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE - * ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES - * WITH YOU. - ***********************license end**************************************/ - -#include -#include - -#include "common.h" - -/** - * zip_cmd_qbuf_alloc - Allocates a cmd buffer for ZIP Instruction Queue - * @zip: Pointer to zip device structure - * @q: Queue number to allocate bufffer to - * Return: 0 if successful, -ENOMEM otherwise - */ -int zip_cmd_qbuf_alloc(struct zip_device *zip, int q) -{ - zip->iq[q].sw_head = (u64 *)__get_free_pages((GFP_KERNEL | GFP_DMA), - get_order(ZIP_CMD_QBUF_SIZE)); - - if (!zip->iq[q].sw_head) - return -ENOMEM; - - memset(zip->iq[q].sw_head, 0, ZIP_CMD_QBUF_SIZE); - - zip_dbg("cmd_qbuf_alloc[%d] Success : %p\n", q, zip->iq[q].sw_head); - return 0; -} - -/** - * zip_cmd_qbuf_free - Frees the cmd Queue buffer - * @zip: Pointer to zip device structure - * @q: Queue number to free buffer of - */ -void zip_cmd_qbuf_free(struct zip_device *zip, int q) -{ - zip_dbg("Freeing cmd_qbuf 0x%lx\n", zip->iq[q].sw_tail); - - free_pages((u64)zip->iq[q].sw_tail, get_order(ZIP_CMD_QBUF_SIZE)); -} - -/** - * zip_data_buf_alloc - Allocates memory for a data bufffer - * @size: Size of the buffer to allocate - * Returns: Pointer to the buffer allocated - */ -u8 *zip_data_buf_alloc(u64 size) -{ - u8 *ptr; - - ptr = (u8 *)__get_free_pages((GFP_KERNEL | GFP_DMA), - get_order(size)); - - if (!ptr) - return NULL; - - memset(ptr, 0, size); - - zip_dbg("Data buffer allocation success\n"); - return ptr; -} - -/** - * zip_data_buf_free - Frees the memory of a data buffer - * @ptr: Pointer to the buffer - * @size: Buffer size - */ -void zip_data_buf_free(u8 *ptr, u64 size) -{ - zip_dbg("Freeing data buffer 0x%lx\n", ptr); - - free_pages((u64)ptr, get_order(size)); -} diff --git a/drivers/crypto/cavium/zip/zip_mem.h b/drivers/crypto/cavium/zip/zip_mem.h deleted file mode 100644 index f8f2f08c4a5c..000000000000 --- a/drivers/crypto/cavium/zip/zip_mem.h +++ /dev/null @@ -1,78 +0,0 @@ -/***********************license start************************************ - * Copyright (c) 2003-2017 Cavium, Inc. - * All rights reserved. - * - * License: one of 'Cavium License' or 'GNU General Public License Version 2' - * - * This file is provided under the terms of the Cavium License (see below) - * or under the terms of GNU General Public License, Version 2, as - * published by the Free Software Foundation. When using or redistributing - * this file, you may do so under either license. - * - * Cavium License: Redistribution and use in source and binary forms, with - * or without modification, are permitted provided that the following - * conditions are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials provided - * with the distribution. - * - * * Neither the name of Cavium Inc. nor the names of its contributors may be - * used to endorse or promote products derived from this software without - * specific prior written permission. - * - * This Software, including technical data, may be subject to U.S. export - * control laws, including the U.S. Export Administration Act and its - * associated regulations, and may be subject to export or import - * regulations in other countries. - * - * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" - * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS - * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH - * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY - * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT - * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) - * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A - * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET - * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE - * ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES - * WITH YOU. - ***********************license end**************************************/ - -#ifndef __ZIP_MEM_H__ -#define __ZIP_MEM_H__ - -/** - * zip_cmd_qbuf_free - Frees the cmd Queue buffer - * @zip: Pointer to zip device structure - * @q: Queue nmber to free buffer of - */ -void zip_cmd_qbuf_free(struct zip_device *zip, int q); - -/** - * zip_cmd_qbuf_alloc - Allocates a Chunk/cmd buffer for ZIP Inst(cmd) Queue - * @zip: Pointer to zip device structure - * @q: Queue number to allocate bufffer to - * Return: 0 if successful, 1 otherwise - */ -int zip_cmd_qbuf_alloc(struct zip_device *zip, int q); - -/** - * zip_data_buf_alloc - Allocates memory for a data bufffer - * @size: Size of the buffer to allocate - * Returns: Pointer to the buffer allocated - */ -u8 *zip_data_buf_alloc(u64 size); - -/** - * zip_data_buf_free - Frees the memory of a data buffer - * @ptr: Pointer to the buffer - * @size: Buffer size - */ -void zip_data_buf_free(u8 *ptr, u64 size); - -#endif diff --git a/drivers/crypto/cavium/zip/zip_regs.h b/drivers/crypto/cavium/zip/zip_regs.h deleted file mode 100644 index 874e0236c87e..000000000000 --- a/drivers/crypto/cavium/zip/zip_regs.h +++ /dev/null @@ -1,1347 +0,0 @@ -/***********************license start************************************ - * Copyright (c) 2003-2017 Cavium, Inc. - * All rights reserved. - * - * License: one of 'Cavium License' or 'GNU General Public License Version 2' - * - * This file is provided under the terms of the Cavium License (see below) - * or under the terms of GNU General Public License, Version 2, as - * published by the Free Software Foundation. When using or redistributing - * this file, you may do so under either license. - * - * Cavium License: Redistribution and use in source and binary forms, with - * or without modification, are permitted provided that the following - * conditions are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials provided - * with the distribution. - * - * * Neither the name of Cavium Inc. nor the names of its contributors may be - * used to endorse or promote products derived from this software without - * specific prior written permission. - * - * This Software, including technical data, may be subject to U.S. export - * control laws, including the U.S. Export Administration Act and its - * associated regulations, and may be subject to export or import - * regulations in other countries. - * - * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" - * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS - * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH - * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY - * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT - * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) - * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A - * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET - * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE - * ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES - * WITH YOU. - ***********************license end**************************************/ - -#ifndef __ZIP_REGS_H__ -#define __ZIP_REGS_H__ - -/* - * Configuration and status register (CSR) address and type definitions for - * Cavium ZIP. - */ - -#include - -/* ZIP invocation result completion status codes */ -#define ZIP_CMD_NOTDONE 0x0 - -/* Successful completion. */ -#define ZIP_CMD_SUCCESS 0x1 - -/* Output truncated */ -#define ZIP_CMD_DTRUNC 0x2 - -/* Dynamic Stop */ -#define ZIP_CMD_DYNAMIC_STOP 0x3 - -/* Uncompress ran out of input data when IWORD0[EF] was set */ -#define ZIP_CMD_ITRUNC 0x4 - -/* Uncompress found the reserved block type 3 */ -#define ZIP_CMD_RBLOCK 0x5 - -/* - * Uncompress found LEN != ZIP_CMD_NLEN in an uncompressed block in the input. - */ -#define ZIP_CMD_NLEN 0x6 - -/* Uncompress found a bad code in the main Huffman codes. */ -#define ZIP_CMD_BADCODE 0x7 - -/* Uncompress found a bad code in the 19 Huffman codes encoding lengths. */ -#define ZIP_CMD_BADCODE2 0x8 - -/* Compress found a zero-length input. */ -#define ZIP_CMD_ZERO_LEN 0x9 - -/* The compress or decompress encountered an internal parity error. */ -#define ZIP_CMD_PARITY 0xA - -/* - * Uncompress found a string identifier that precedes the uncompressed data and - * decompression history. - */ -#define ZIP_CMD_FATAL 0xB - -/** - * enum zip_int_vec_e - ZIP MSI-X Vector Enumeration, enumerates the MSI-X - * interrupt vectors. - */ -enum zip_int_vec_e { - ZIP_INT_VEC_E_ECCE = 0x10, - ZIP_INT_VEC_E_FIFE = 0x11, - ZIP_INT_VEC_E_QUE0_DONE = 0x0, - ZIP_INT_VEC_E_QUE0_ERR = 0x8, - ZIP_INT_VEC_E_QUE1_DONE = 0x1, - ZIP_INT_VEC_E_QUE1_ERR = 0x9, - ZIP_INT_VEC_E_QUE2_DONE = 0x2, - ZIP_INT_VEC_E_QUE2_ERR = 0xa, - ZIP_INT_VEC_E_QUE3_DONE = 0x3, - ZIP_INT_VEC_E_QUE3_ERR = 0xb, - ZIP_INT_VEC_E_QUE4_DONE = 0x4, - ZIP_INT_VEC_E_QUE4_ERR = 0xc, - ZIP_INT_VEC_E_QUE5_DONE = 0x5, - ZIP_INT_VEC_E_QUE5_ERR = 0xd, - ZIP_INT_VEC_E_QUE6_DONE = 0x6, - ZIP_INT_VEC_E_QUE6_ERR = 0xe, - ZIP_INT_VEC_E_QUE7_DONE = 0x7, - ZIP_INT_VEC_E_QUE7_ERR = 0xf, - ZIP_INT_VEC_E_ENUM_LAST = 0x12, -}; - -/** - * union zip_zptr_addr_s - ZIP Generic Pointer Structure for ADDR. - * - * It is the generic format of pointers in ZIP_INST_S. - */ -union zip_zptr_addr_s { - u64 u_reg64; - struct { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_49_63 : 15; - u64 addr : 49; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 addr : 49; - u64 reserved_49_63 : 15; -#endif - } s; - -}; - -/** - * union zip_zptr_ctl_s - ZIP Generic Pointer Structure for CTL. - * - * It is the generic format of pointers in ZIP_INST_S. - */ -union zip_zptr_ctl_s { - u64 u_reg64; - struct { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_112_127 : 16; - u64 length : 16; - u64 reserved_67_95 : 29; - u64 fw : 1; - u64 nc : 1; - u64 data_be : 1; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 data_be : 1; - u64 nc : 1; - u64 fw : 1; - u64 reserved_67_95 : 29; - u64 length : 16; - u64 reserved_112_127 : 16; -#endif - } s; -}; - -/** - * union zip_inst_s - ZIP Instruction Structure. - * Each ZIP instruction has 16 words (they are called IWORD0 to IWORD15 within - * the structure). - */ -union zip_inst_s { - u64 u_reg64[16]; - struct { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 doneint : 1; - u64 reserved_56_62 : 7; - u64 totaloutputlength : 24; - u64 reserved_27_31 : 5; - u64 exn : 3; - u64 reserved_23_23 : 1; - u64 exbits : 7; - u64 reserved_12_15 : 4; - u64 sf : 1; - u64 ss : 2; - u64 cc : 2; - u64 ef : 1; - u64 bf : 1; - u64 ce : 1; - u64 reserved_3_3 : 1; - u64 ds : 1; - u64 dg : 1; - u64 hg : 1; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 hg : 1; - u64 dg : 1; - u64 ds : 1; - u64 reserved_3_3 : 1; - u64 ce : 1; - u64 bf : 1; - u64 ef : 1; - u64 cc : 2; - u64 ss : 2; - u64 sf : 1; - u64 reserved_12_15 : 4; - u64 exbits : 7; - u64 reserved_23_23 : 1; - u64 exn : 3; - u64 reserved_27_31 : 5; - u64 totaloutputlength : 24; - u64 reserved_56_62 : 7; - u64 doneint : 1; -#endif -#if defined(__BIG_ENDIAN_BITFIELD) - u64 historylength : 16; - u64 reserved_96_111 : 16; - u64 adlercrc32 : 32; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 adlercrc32 : 32; - u64 reserved_96_111 : 16; - u64 historylength : 16; -#endif - union zip_zptr_addr_s ctx_ptr_addr; - union zip_zptr_ctl_s ctx_ptr_ctl; - union zip_zptr_addr_s his_ptr_addr; - union zip_zptr_ctl_s his_ptr_ctl; - union zip_zptr_addr_s inp_ptr_addr; - union zip_zptr_ctl_s inp_ptr_ctl; - union zip_zptr_addr_s out_ptr_addr; - union zip_zptr_ctl_s out_ptr_ctl; - union zip_zptr_addr_s res_ptr_addr; - union zip_zptr_ctl_s res_ptr_ctl; -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_817_831 : 15; - u64 wq_ptr : 49; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 wq_ptr : 49; - u64 reserved_817_831 : 15; -#endif -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_882_895 : 14; - u64 tt : 2; - u64 reserved_874_879 : 6; - u64 grp : 10; - u64 tag : 32; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 tag : 32; - u64 grp : 10; - u64 reserved_874_879 : 6; - u64 tt : 2; - u64 reserved_882_895 : 14; -#endif -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_896_959 : 64; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 reserved_896_959 : 64; -#endif -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_960_1023 : 64; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 reserved_960_1023 : 64; -#endif - } s; -}; - -/** - * union zip_nptr_s - ZIP Instruction Next-Chunk-Buffer Pointer (NPTR) - * Structure - * - * ZIP_NPTR structure is used to chain all the zip instruction buffers - * together. ZIP instruction buffers are managed (allocated and released) by - * the software. - */ -union zip_nptr_s { - u64 u_reg64; - struct { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_49_63 : 15; - u64 addr : 49; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 addr : 49; - u64 reserved_49_63 : 15; -#endif - } s; -}; - -/** - * union zip_zptr_s - ZIP Generic Pointer Structure. - * - * It is the generic format of pointers in ZIP_INST_S. - */ -union zip_zptr_s { - u64 u_reg64[2]; - struct { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_49_63 : 15; - u64 addr : 49; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 addr : 49; - u64 reserved_49_63 : 15; -#endif -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_112_127 : 16; - u64 length : 16; - u64 reserved_67_95 : 29; - u64 fw : 1; - u64 nc : 1; - u64 data_be : 1; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 data_be : 1; - u64 nc : 1; - u64 fw : 1; - u64 reserved_67_95 : 29; - u64 length : 16; - u64 reserved_112_127 : 16; -#endif - } s; -}; - -/** - * union zip_zres_s - ZIP Result Structure - * - * The ZIP coprocessor writes the result structure after it completes the - * invocation. The result structure is exactly 24 bytes, and each invocation of - * the ZIP coprocessor produces exactly one result structure. - */ -union zip_zres_s { - u64 u_reg64[3]; - struct { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 crc32 : 32; - u64 adler32 : 32; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 adler32 : 32; - u64 crc32 : 32; -#endif -#if defined(__BIG_ENDIAN_BITFIELD) - u64 totalbyteswritten : 32; - u64 totalbytesread : 32; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 totalbytesread : 32; - u64 totalbyteswritten : 32; -#endif -#if defined(__BIG_ENDIAN_BITFIELD) - u64 totalbitsprocessed : 32; - u64 doneint : 1; - u64 reserved_155_158 : 4; - u64 exn : 3; - u64 reserved_151_151 : 1; - u64 exbits : 7; - u64 reserved_137_143 : 7; - u64 ef : 1; - - volatile u64 compcode : 8; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - - volatile u64 compcode : 8; - u64 ef : 1; - u64 reserved_137_143 : 7; - u64 exbits : 7; - u64 reserved_151_151 : 1; - u64 exn : 3; - u64 reserved_155_158 : 4; - u64 doneint : 1; - u64 totalbitsprocessed : 32; -#endif - } s; -}; - -/** - * union zip_cmd_ctl - Structure representing the register that controls - * clock and reset. - */ -union zip_cmd_ctl { - u64 u_reg64; - struct zip_cmd_ctl_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_2_63 : 62; - u64 forceclk : 1; - u64 reset : 1; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 reset : 1; - u64 forceclk : 1; - u64 reserved_2_63 : 62; -#endif - } s; -}; - -#define ZIP_CMD_CTL 0x0ull - -/** - * union zip_constants - Data structure representing the register that contains - * all of the current implementation-related parameters of the zip core in this - * chip. - */ -union zip_constants { - u64 u_reg64; - struct zip_constants_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 nexec : 8; - u64 reserved_49_55 : 7; - u64 syncflush_capable : 1; - u64 depth : 16; - u64 onfsize : 12; - u64 ctxsize : 12; - u64 reserved_1_7 : 7; - u64 disabled : 1; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 disabled : 1; - u64 reserved_1_7 : 7; - u64 ctxsize : 12; - u64 onfsize : 12; - u64 depth : 16; - u64 syncflush_capable : 1; - u64 reserved_49_55 : 7; - u64 nexec : 8; -#endif - } s; -}; - -#define ZIP_CONSTANTS 0x00A0ull - -/** - * union zip_corex_bist_status - Represents registers which have the BIST - * status of memories in zip cores. - * - * Each bit is the BIST result of an individual memory - * (per bit, 0 = pass and 1 = fail). - */ -union zip_corex_bist_status { - u64 u_reg64; - struct zip_corex_bist_status_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_53_63 : 11; - u64 bstatus : 53; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 bstatus : 53; - u64 reserved_53_63 : 11; -#endif - } s; -}; - -static inline u64 ZIP_COREX_BIST_STATUS(u64 param1) -{ - if (param1 <= 1) - return 0x0520ull + (param1 & 1) * 0x8ull; - pr_err("ZIP_COREX_BIST_STATUS: %llu\n", param1); - return 0; -} - -/** - * union zip_ctl_bist_status - Represents register that has the BIST status of - * memories in ZIP_CTL (instruction buffer, G/S pointer FIFO, input data - * buffer, output data buffers). - * - * Each bit is the BIST result of an individual memory - * (per bit, 0 = pass and 1 = fail). - */ -union zip_ctl_bist_status { - u64 u_reg64; - struct zip_ctl_bist_status_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_9_63 : 55; - u64 bstatus : 9; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 bstatus : 9; - u64 reserved_9_63 : 55; -#endif - } s; -}; - -#define ZIP_CTL_BIST_STATUS 0x0510ull - -/** - * union zip_ctl_cfg - Represents the register that controls the behavior of - * the ZIP DMA engines. - * - * It is recommended to keep default values for normal operation. Changing the - * values of the fields may be useful for diagnostics. - */ -union zip_ctl_cfg { - u64 u_reg64; - struct zip_ctl_cfg_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_52_63 : 12; - u64 ildf : 4; - u64 reserved_36_47 : 12; - u64 drtf : 4; - u64 reserved_27_31 : 5; - u64 stcf : 3; - u64 reserved_19_23 : 5; - u64 ldf : 3; - u64 reserved_2_15 : 14; - u64 busy : 1; - u64 reserved_0_0 : 1; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 reserved_0_0 : 1; - u64 busy : 1; - u64 reserved_2_15 : 14; - u64 ldf : 3; - u64 reserved_19_23 : 5; - u64 stcf : 3; - u64 reserved_27_31 : 5; - u64 drtf : 4; - u64 reserved_36_47 : 12; - u64 ildf : 4; - u64 reserved_52_63 : 12; -#endif - } s; -}; - -#define ZIP_CTL_CFG 0x0560ull - -/** - * union zip_dbg_corex_inst - Represents the registers that reflect the status - * of the current instruction that the ZIP core is executing or has executed. - * - * These registers are only for debug use. - */ -union zip_dbg_corex_inst { - u64 u_reg64; - struct zip_dbg_corex_inst_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 busy : 1; - u64 reserved_35_62 : 28; - u64 qid : 3; - u64 iid : 32; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 iid : 32; - u64 qid : 3; - u64 reserved_35_62 : 28; - u64 busy : 1; -#endif - } s; -}; - -static inline u64 ZIP_DBG_COREX_INST(u64 param1) -{ - if (param1 <= 1) - return 0x0640ull + (param1 & 1) * 0x8ull; - pr_err("ZIP_DBG_COREX_INST: %llu\n", param1); - return 0; -} - -/** - * union zip_dbg_corex_sta - Represents registers that reflect the status of - * the zip cores. - * - * They are for debug use only. - */ -union zip_dbg_corex_sta { - u64 u_reg64; - struct zip_dbg_corex_sta_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 busy : 1; - u64 reserved_37_62 : 26; - u64 ist : 5; - u64 nie : 32; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 nie : 32; - u64 ist : 5; - u64 reserved_37_62 : 26; - u64 busy : 1; -#endif - } s; -}; - -static inline u64 ZIP_DBG_COREX_STA(u64 param1) -{ - if (param1 <= 1) - return 0x0680ull + (param1 & 1) * 0x8ull; - pr_err("ZIP_DBG_COREX_STA: %llu\n", param1); - return 0; -} - -/** - * union zip_dbg_quex_sta - Represets registers that reflect status of the zip - * instruction queues. - * - * They are for debug use only. - */ -union zip_dbg_quex_sta { - u64 u_reg64; - struct zip_dbg_quex_sta_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 busy : 1; - u64 reserved_56_62 : 7; - u64 rqwc : 24; - u64 nii : 32; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 nii : 32; - u64 rqwc : 24; - u64 reserved_56_62 : 7; - u64 busy : 1; -#endif - } s; -}; - -static inline u64 ZIP_DBG_QUEX_STA(u64 param1) -{ - if (param1 <= 7) - return 0x1800ull + (param1 & 7) * 0x8ull; - pr_err("ZIP_DBG_QUEX_STA: %llu\n", param1); - return 0; -} - -/** - * union zip_ecc_ctl - Represents the register that enables ECC for each - * individual internal memory that requires ECC. - * - * For debug purpose, it can also flip one or two bits in the ECC data. - */ -union zip_ecc_ctl { - u64 u_reg64; - struct zip_ecc_ctl_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_19_63 : 45; - u64 vmem_cdis : 1; - u64 vmem_fs : 2; - u64 reserved_15_15 : 1; - u64 idf1_cdis : 1; - u64 idf1_fs : 2; - u64 reserved_11_11 : 1; - u64 idf0_cdis : 1; - u64 idf0_fs : 2; - u64 reserved_7_7 : 1; - u64 gspf_cdis : 1; - u64 gspf_fs : 2; - u64 reserved_3_3 : 1; - u64 iqf_cdis : 1; - u64 iqf_fs : 2; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 iqf_fs : 2; - u64 iqf_cdis : 1; - u64 reserved_3_3 : 1; - u64 gspf_fs : 2; - u64 gspf_cdis : 1; - u64 reserved_7_7 : 1; - u64 idf0_fs : 2; - u64 idf0_cdis : 1; - u64 reserved_11_11 : 1; - u64 idf1_fs : 2; - u64 idf1_cdis : 1; - u64 reserved_15_15 : 1; - u64 vmem_fs : 2; - u64 vmem_cdis : 1; - u64 reserved_19_63 : 45; -#endif - } s; -}; - -#define ZIP_ECC_CTL 0x0568ull - -/* NCB - zip_ecce_ena_w1c */ -union zip_ecce_ena_w1c { - u64 u_reg64; - struct zip_ecce_ena_w1c_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_37_63 : 27; - u64 dbe : 5; - u64 reserved_5_31 : 27; - u64 sbe : 5; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 sbe : 5; - u64 reserved_5_31 : 27; - u64 dbe : 5; - u64 reserved_37_63 : 27; -#endif - } s; -}; - -#define ZIP_ECCE_ENA_W1C 0x0598ull - -/* NCB - zip_ecce_ena_w1s */ -union zip_ecce_ena_w1s { - u64 u_reg64; - struct zip_ecce_ena_w1s_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_37_63 : 27; - u64 dbe : 5; - u64 reserved_5_31 : 27; - u64 sbe : 5; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 sbe : 5; - u64 reserved_5_31 : 27; - u64 dbe : 5; - u64 reserved_37_63 : 27; -#endif - } s; -}; - -#define ZIP_ECCE_ENA_W1S 0x0590ull - -/** - * union zip_ecce_int - Represents the register that contains the status of the - * ECC interrupt sources. - */ -union zip_ecce_int { - u64 u_reg64; - struct zip_ecce_int_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_37_63 : 27; - u64 dbe : 5; - u64 reserved_5_31 : 27; - u64 sbe : 5; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 sbe : 5; - u64 reserved_5_31 : 27; - u64 dbe : 5; - u64 reserved_37_63 : 27; -#endif - } s; -}; - -#define ZIP_ECCE_INT 0x0580ull - -/* NCB - zip_ecce_int_w1s */ -union zip_ecce_int_w1s { - u64 u_reg64; - struct zip_ecce_int_w1s_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_37_63 : 27; - u64 dbe : 5; - u64 reserved_5_31 : 27; - u64 sbe : 5; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 sbe : 5; - u64 reserved_5_31 : 27; - u64 dbe : 5; - u64 reserved_37_63 : 27; -#endif - } s; -}; - -#define ZIP_ECCE_INT_W1S 0x0588ull - -/* NCB - zip_fife_ena_w1c */ -union zip_fife_ena_w1c { - u64 u_reg64; - struct zip_fife_ena_w1c_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_42_63 : 22; - u64 asserts : 42; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 asserts : 42; - u64 reserved_42_63 : 22; -#endif - } s; -}; - -#define ZIP_FIFE_ENA_W1C 0x0090ull - -/* NCB - zip_fife_ena_w1s */ -union zip_fife_ena_w1s { - u64 u_reg64; - struct zip_fife_ena_w1s_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_42_63 : 22; - u64 asserts : 42; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 asserts : 42; - u64 reserved_42_63 : 22; -#endif - } s; -}; - -#define ZIP_FIFE_ENA_W1S 0x0088ull - -/* NCB - zip_fife_int */ -union zip_fife_int { - u64 u_reg64; - struct zip_fife_int_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_42_63 : 22; - u64 asserts : 42; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 asserts : 42; - u64 reserved_42_63 : 22; -#endif - } s; -}; - -#define ZIP_FIFE_INT 0x0078ull - -/* NCB - zip_fife_int_w1s */ -union zip_fife_int_w1s { - u64 u_reg64; - struct zip_fife_int_w1s_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_42_63 : 22; - u64 asserts : 42; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 asserts : 42; - u64 reserved_42_63 : 22; -#endif - } s; -}; - -#define ZIP_FIFE_INT_W1S 0x0080ull - -/** - * union zip_msix_pbax - Represents the register that is the MSI-X PBA table - * - * The bit number is indexed by the ZIP_INT_VEC_E enumeration. - */ -union zip_msix_pbax { - u64 u_reg64; - struct zip_msix_pbax_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 pend : 64; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 pend : 64; -#endif - } s; -}; - -static inline u64 ZIP_MSIX_PBAX(u64 param1) -{ - if (param1 == 0) - return 0x0000838000FF0000ull; - pr_err("ZIP_MSIX_PBAX: %llu\n", param1); - return 0; -} - -/** - * union zip_msix_vecx_addr - Represents the register that is the MSI-X vector - * table, indexed by the ZIP_INT_VEC_E enumeration. - */ -union zip_msix_vecx_addr { - u64 u_reg64; - struct zip_msix_vecx_addr_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_49_63 : 15; - u64 addr : 47; - u64 reserved_1_1 : 1; - u64 secvec : 1; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 secvec : 1; - u64 reserved_1_1 : 1; - u64 addr : 47; - u64 reserved_49_63 : 15; -#endif - } s; -}; - -static inline u64 ZIP_MSIX_VECX_ADDR(u64 param1) -{ - if (param1 <= 17) - return 0x0000838000F00000ull + (param1 & 31) * 0x10ull; - pr_err("ZIP_MSIX_VECX_ADDR: %llu\n", param1); - return 0; -} - -/** - * union zip_msix_vecx_ctl - Represents the register that is the MSI-X vector - * table, indexed by the ZIP_INT_VEC_E enumeration. - */ -union zip_msix_vecx_ctl { - u64 u_reg64; - struct zip_msix_vecx_ctl_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_33_63 : 31; - u64 mask : 1; - u64 reserved_20_31 : 12; - u64 data : 20; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 data : 20; - u64 reserved_20_31 : 12; - u64 mask : 1; - u64 reserved_33_63 : 31; -#endif - } s; -}; - -static inline u64 ZIP_MSIX_VECX_CTL(u64 param1) -{ - if (param1 <= 17) - return 0x0000838000F00008ull + (param1 & 31) * 0x10ull; - pr_err("ZIP_MSIX_VECX_CTL: %llu\n", param1); - return 0; -} - -/** - * union zip_quex_done - Represents the registers that contain the per-queue - * instruction done count. - */ -union zip_quex_done { - u64 u_reg64; - struct zip_quex_done_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_20_63 : 44; - u64 done : 20; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 done : 20; - u64 reserved_20_63 : 44; -#endif - } s; -}; - -static inline u64 ZIP_QUEX_DONE(u64 param1) -{ - if (param1 <= 7) - return 0x2000ull + (param1 & 7) * 0x8ull; - pr_err("ZIP_QUEX_DONE: %llu\n", param1); - return 0; -} - -/** - * union zip_quex_done_ack - Represents the registers on write to which will - * decrement the per-queue instructiona done count. - */ -union zip_quex_done_ack { - u64 u_reg64; - struct zip_quex_done_ack_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_20_63 : 44; - u64 done_ack : 20; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 done_ack : 20; - u64 reserved_20_63 : 44; -#endif - } s; -}; - -static inline u64 ZIP_QUEX_DONE_ACK(u64 param1) -{ - if (param1 <= 7) - return 0x2200ull + (param1 & 7) * 0x8ull; - pr_err("ZIP_QUEX_DONE_ACK: %llu\n", param1); - return 0; -} - -/** - * union zip_quex_done_ena_w1c - Represents the register which when written - * 1 to will disable the DONEINT interrupt for the queue. - */ -union zip_quex_done_ena_w1c { - u64 u_reg64; - struct zip_quex_done_ena_w1c_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_1_63 : 63; - u64 done_ena : 1; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 done_ena : 1; - u64 reserved_1_63 : 63; -#endif - } s; -}; - -static inline u64 ZIP_QUEX_DONE_ENA_W1C(u64 param1) -{ - if (param1 <= 7) - return 0x2600ull + (param1 & 7) * 0x8ull; - pr_err("ZIP_QUEX_DONE_ENA_W1C: %llu\n", param1); - return 0; -} - -/** - * union zip_quex_done_ena_w1s - Represents the register that when written 1 to - * will enable the DONEINT interrupt for the queue. - */ -union zip_quex_done_ena_w1s { - u64 u_reg64; - struct zip_quex_done_ena_w1s_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_1_63 : 63; - u64 done_ena : 1; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 done_ena : 1; - u64 reserved_1_63 : 63; -#endif - } s; -}; - -static inline u64 ZIP_QUEX_DONE_ENA_W1S(u64 param1) -{ - if (param1 <= 7) - return 0x2400ull + (param1 & 7) * 0x8ull; - pr_err("ZIP_QUEX_DONE_ENA_W1S: %llu\n", param1); - return 0; -} - -/** - * union zip_quex_done_wait - Represents the register that specifies the per - * queue interrupt coalescing settings. - */ -union zip_quex_done_wait { - u64 u_reg64; - struct zip_quex_done_wait_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_48_63 : 16; - u64 time_wait : 16; - u64 reserved_20_31 : 12; - u64 num_wait : 20; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 num_wait : 20; - u64 reserved_20_31 : 12; - u64 time_wait : 16; - u64 reserved_48_63 : 16; -#endif - } s; -}; - -static inline u64 ZIP_QUEX_DONE_WAIT(u64 param1) -{ - if (param1 <= 7) - return 0x2800ull + (param1 & 7) * 0x8ull; - pr_err("ZIP_QUEX_DONE_WAIT: %llu\n", param1); - return 0; -} - -/** - * union zip_quex_doorbell - Represents doorbell registers for the ZIP - * instruction queues. - */ -union zip_quex_doorbell { - u64 u_reg64; - struct zip_quex_doorbell_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_20_63 : 44; - u64 dbell_cnt : 20; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 dbell_cnt : 20; - u64 reserved_20_63 : 44; -#endif - } s; -}; - -static inline u64 ZIP_QUEX_DOORBELL(u64 param1) -{ - if (param1 <= 7) - return 0x4000ull + (param1 & 7) * 0x8ull; - pr_err("ZIP_QUEX_DOORBELL: %llu\n", param1); - return 0; -} - -union zip_quex_err_ena_w1c { - u64 u_reg64; - struct zip_quex_err_ena_w1c_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_5_63 : 59; - u64 mdbe : 1; - u64 nwrp : 1; - u64 nrrp : 1; - u64 irde : 1; - u64 dovf : 1; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 dovf : 1; - u64 irde : 1; - u64 nrrp : 1; - u64 nwrp : 1; - u64 mdbe : 1; - u64 reserved_5_63 : 59; -#endif - } s; -}; - -static inline u64 ZIP_QUEX_ERR_ENA_W1C(u64 param1) -{ - if (param1 <= 7) - return 0x3600ull + (param1 & 7) * 0x8ull; - pr_err("ZIP_QUEX_ERR_ENA_W1C: %llu\n", param1); - return 0; -} - -union zip_quex_err_ena_w1s { - u64 u_reg64; - struct zip_quex_err_ena_w1s_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_5_63 : 59; - u64 mdbe : 1; - u64 nwrp : 1; - u64 nrrp : 1; - u64 irde : 1; - u64 dovf : 1; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 dovf : 1; - u64 irde : 1; - u64 nrrp : 1; - u64 nwrp : 1; - u64 mdbe : 1; - u64 reserved_5_63 : 59; -#endif - } s; -}; - -static inline u64 ZIP_QUEX_ERR_ENA_W1S(u64 param1) -{ - if (param1 <= 7) - return 0x3400ull + (param1 & 7) * 0x8ull; - pr_err("ZIP_QUEX_ERR_ENA_W1S: %llu\n", param1); - return 0; -} - -/** - * union zip_quex_err_int - Represents registers that contain the per-queue - * error interrupts. - */ -union zip_quex_err_int { - u64 u_reg64; - struct zip_quex_err_int_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_5_63 : 59; - u64 mdbe : 1; - u64 nwrp : 1; - u64 nrrp : 1; - u64 irde : 1; - u64 dovf : 1; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 dovf : 1; - u64 irde : 1; - u64 nrrp : 1; - u64 nwrp : 1; - u64 mdbe : 1; - u64 reserved_5_63 : 59; -#endif - } s; -}; - -static inline u64 ZIP_QUEX_ERR_INT(u64 param1) -{ - if (param1 <= 7) - return 0x3000ull + (param1 & 7) * 0x8ull; - pr_err("ZIP_QUEX_ERR_INT: %llu\n", param1); - return 0; -} - -/* NCB - zip_que#_err_int_w1s */ -union zip_quex_err_int_w1s { - u64 u_reg64; - struct zip_quex_err_int_w1s_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_5_63 : 59; - u64 mdbe : 1; - u64 nwrp : 1; - u64 nrrp : 1; - u64 irde : 1; - u64 dovf : 1; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 dovf : 1; - u64 irde : 1; - u64 nrrp : 1; - u64 nwrp : 1; - u64 mdbe : 1; - u64 reserved_5_63 : 59; -#endif - } s; -}; - -static inline u64 ZIP_QUEX_ERR_INT_W1S(u64 param1) -{ - if (param1 <= 7) - return 0x3200ull + (param1 & 7) * 0x8ull; - pr_err("ZIP_QUEX_ERR_INT_W1S: %llu\n", param1); - return 0; -} - -/** - * union zip_quex_gcfg - Represents the registers that reflect status of the - * zip instruction queues,debug use only. - */ -union zip_quex_gcfg { - u64 u_reg64; - struct zip_quex_gcfg_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_4_63 : 60; - u64 iqb_ldwb : 1; - u64 cbw_sty : 1; - u64 l2ld_cmd : 2; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 l2ld_cmd : 2; - u64 cbw_sty : 1; - u64 iqb_ldwb : 1; - u64 reserved_4_63 : 60; -#endif - } s; -}; - -static inline u64 ZIP_QUEX_GCFG(u64 param1) -{ - if (param1 <= 7) - return 0x1A00ull + (param1 & 7) * 0x8ull; - pr_err("ZIP_QUEX_GCFG: %llu\n", param1); - return 0; -} - -/** - * union zip_quex_map - Represents the registers that control how each - * instruction queue maps to zip cores. - */ -union zip_quex_map { - u64 u_reg64; - struct zip_quex_map_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_2_63 : 62; - u64 zce : 2; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 zce : 2; - u64 reserved_2_63 : 62; -#endif - } s; -}; - -static inline u64 ZIP_QUEX_MAP(u64 param1) -{ - if (param1 <= 7) - return 0x1400ull + (param1 & 7) * 0x8ull; - pr_err("ZIP_QUEX_MAP: %llu\n", param1); - return 0; -} - -/** - * union zip_quex_sbuf_addr - Represents the registers that set the buffer - * parameters for the instruction queues. - * - * When quiescent (i.e. outstanding doorbell count is 0), it is safe to rewrite - * this register to effectively reset the command buffer state machine. - * These registers must be programmed after SW programs the corresponding - * ZIP_QUE(0..7)_SBUF_CTL. - */ -union zip_quex_sbuf_addr { - u64 u_reg64; - struct zip_quex_sbuf_addr_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_49_63 : 15; - u64 ptr : 42; - u64 off : 7; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 off : 7; - u64 ptr : 42; - u64 reserved_49_63 : 15; -#endif - } s; -}; - -static inline u64 ZIP_QUEX_SBUF_ADDR(u64 param1) -{ - if (param1 <= 7) - return 0x1000ull + (param1 & 7) * 0x8ull; - pr_err("ZIP_QUEX_SBUF_ADDR: %llu\n", param1); - return 0; -} - -/** - * union zip_quex_sbuf_ctl - Represents the registers that set the buffer - * parameters for the instruction queues. - * - * When quiescent (i.e. outstanding doorbell count is 0), it is safe to rewrite - * this register to effectively reset the command buffer state machine. - * These registers must be programmed before SW programs the corresponding - * ZIP_QUE(0..7)_SBUF_ADDR. - */ -union zip_quex_sbuf_ctl { - u64 u_reg64; - struct zip_quex_sbuf_ctl_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_45_63 : 19; - u64 size : 13; - u64 inst_be : 1; - u64 reserved_24_30 : 7; - u64 stream_id : 8; - u64 reserved_12_15 : 4; - u64 aura : 12; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 aura : 12; - u64 reserved_12_15 : 4; - u64 stream_id : 8; - u64 reserved_24_30 : 7; - u64 inst_be : 1; - u64 size : 13; - u64 reserved_45_63 : 19; -#endif - } s; -}; - -static inline u64 ZIP_QUEX_SBUF_CTL(u64 param1) -{ - if (param1 <= 7) - return 0x1200ull + (param1 & 7) * 0x8ull; - pr_err("ZIP_QUEX_SBUF_CTL: %llu\n", param1); - return 0; -} - -/** - * union zip_que_ena - Represents queue enable register - * - * If a queue is disabled, ZIP_CTL stops fetching instructions from the queue. - */ -union zip_que_ena { - u64 u_reg64; - struct zip_que_ena_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_8_63 : 56; - u64 ena : 8; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 ena : 8; - u64 reserved_8_63 : 56; -#endif - } s; -}; - -#define ZIP_QUE_ENA 0x0500ull - -/** - * union zip_que_pri - Represents the register that defines the priority - * between instruction queues. - */ -union zip_que_pri { - u64 u_reg64; - struct zip_que_pri_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_8_63 : 56; - u64 pri : 8; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 pri : 8; - u64 reserved_8_63 : 56; -#endif - } s; -}; - -#define ZIP_QUE_PRI 0x0508ull - -/** - * union zip_throttle - Represents the register that controls the maximum - * number of in-flight X2I data fetch transactions. - * - * Writing 0 to this register causes the ZIP module to temporarily suspend NCB - * accesses; it is not recommended for normal operation, but may be useful for - * diagnostics. - */ -union zip_throttle { - u64 u_reg64; - struct zip_throttle_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_6_63 : 58; - u64 ld_infl : 6; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 ld_infl : 6; - u64 reserved_6_63 : 58; -#endif - } s; -}; - -#define ZIP_THROTTLE 0x0010ull - -#endif /* _CSRS_ZIP__ */ From a7b1d0c5f1ca05cabf7a119a86d81a1511dd273a Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Wed, 19 Mar 2025 20:31:49 +0800 Subject: [PATCH 011/382] crypto: scomp - Drop the dst scratch buffer As deflate has been converted over to acomp, and cavium zip has been removed, there are no longer any scomp algorithms that can be used by IPsec. Since IPsec was the only user of the dst scratch buffer, remove it. Signed-off-by: Herbert Xu --- crypto/scompress.c | 98 +++++++++++++++++++++------------------------- 1 file changed, 44 insertions(+), 54 deletions(-) diff --git a/crypto/scompress.c b/crypto/scompress.c index 5a40605570f5..4a2b3933aa95 100644 --- a/crypto/scompress.c +++ b/crypto/scompress.c @@ -25,15 +25,12 @@ #include "compress.h" -#define SCOMP_SCRATCH_SIZE 65400 - struct scomp_scratch { spinlock_t lock; union { void *src; unsigned long saddr; }; - void *dst; }; static DEFINE_PER_CPU(struct scomp_scratch, scomp_scratch) = { @@ -78,9 +75,7 @@ static void crypto_scomp_free_scratches(void) scratch = per_cpu_ptr(&scomp_scratch, i); free_page(scratch->saddr); - kvfree(scratch->dst); scratch->src = NULL; - scratch->dst = NULL; } } @@ -88,19 +83,12 @@ static int scomp_alloc_scratch(struct scomp_scratch *scratch, int cpu) { int node = cpu_to_node(cpu); struct page *page; - void *mem; - mem = kvmalloc_node(SCOMP_SCRATCH_SIZE, GFP_KERNEL, node); - if (!mem) - return -ENOMEM; page = alloc_pages_node(node, GFP_KERNEL, 0); - if (!page) { - kvfree(mem); + if (!page) return -ENOMEM; - } spin_lock_bh(&scratch->lock); scratch->src = page_address(page); - scratch->dst = mem; spin_unlock_bh(&scratch->lock); return 0; } @@ -181,6 +169,8 @@ static int scomp_acomp_comp_decomp(struct acomp_req *req, int dir) { struct crypto_acomp *tfm = crypto_acomp_reqtfm(req); struct crypto_scomp **tfm_ctx = acomp_tfm_ctx(tfm); + bool src_isvirt = acomp_request_src_isvirt(req); + bool dst_isvirt = acomp_request_dst_isvirt(req); struct crypto_scomp *scomp = *tfm_ctx; struct crypto_acomp_stream *stream; struct scomp_scratch *scratch; @@ -200,13 +190,33 @@ static int scomp_acomp_comp_decomp(struct acomp_req *req, int dir) if (!req->dst || !dlen) return -EINVAL; - stream = crypto_acomp_lock_stream_bh(&crypto_scomp_alg(scomp)->streams); - scratch = scomp_lock_scratch(); + if (dst_isvirt) + dst = req->dvirt; + else { + if (acomp_request_dst_isfolio(req)) { + dpage = folio_page(req->dfolio, 0); + doff = req->doff; + } else if (dlen <= req->dst->length) { + dpage = sg_page(req->dst); + doff = req->dst->offset; + } else + return -ENOSYS; - if (acomp_request_src_isvirt(req)) + dpage = nth_page(dpage, doff / PAGE_SIZE); + doff = offset_in_page(doff); + + n = dlen / PAGE_SIZE; + n += (offset_in_page(dlen) + doff - 1) / PAGE_SIZE; + if (PageHighMem(dpage + n) && + size_add(doff, dlen) > PAGE_SIZE) + return -ENOSYS; + dst = kmap_local_page(dpage) + doff; + } + + if (src_isvirt) src = req->svirt; else { - src = scratch->src; + src = NULL; do { if (acomp_request_src_isfolio(req)) { spage = folio_page(req->sfolio, 0); @@ -227,57 +237,39 @@ static int scomp_acomp_comp_decomp(struct acomp_req *req, int dir) break; src = kmap_local_page(spage) + soff; } while (0); - - if (src == scratch->src) - memcpy_from_sglist(scratch->src, req->src, 0, slen); } - if (acomp_request_dst_isvirt(req)) - dst = req->dvirt; - else { - unsigned int max = SCOMP_SCRATCH_SIZE; + stream = crypto_acomp_lock_stream_bh(&crypto_scomp_alg(scomp)->streams); - dst = scratch->dst; - do { - if (acomp_request_dst_isfolio(req)) { - dpage = folio_page(req->dfolio, 0); - doff = req->doff; - } else if (dlen <= req->dst->length) { - dpage = sg_page(req->dst); - doff = req->dst->offset; - } else - break; + if (!src_isvirt && !src) { + const u8 *src; - dpage = nth_page(dpage, doff / PAGE_SIZE); - doff = offset_in_page(doff); + scratch = scomp_lock_scratch(); + src = scratch->src; + memcpy_from_sglist(scratch->src, req->src, 0, slen); - n = dlen / PAGE_SIZE; - n += (offset_in_page(dlen) + doff - 1) / PAGE_SIZE; - if (PageHighMem(dpage + n) && - size_add(doff, dlen) > PAGE_SIZE) - break; - dst = kmap_local_page(dpage) + doff; - max = dlen; - } while (0); - dlen = min(dlen, max); - } + if (dir) + ret = crypto_scomp_compress(scomp, src, slen, + dst, &dlen, stream->ctx); + else + ret = crypto_scomp_decompress(scomp, src, slen, + dst, &dlen, stream->ctx); - if (dir) + scomp_unlock_scratch(scratch); + } else if (dir) ret = crypto_scomp_compress(scomp, src, slen, dst, &dlen, stream->ctx); else ret = crypto_scomp_decompress(scomp, src, slen, dst, &dlen, stream->ctx); - if (dst == scratch->dst) - memcpy_to_sglist(req->dst, 0, dst, dlen); - - scomp_unlock_scratch(scratch); crypto_acomp_unlock_stream_bh(stream); req->dlen = dlen; - if (!acomp_request_dst_isvirt(req) && dst != scratch->dst) { + if (!src_isvirt && src) + kunmap_local(src); + if (!dst_isvirt) { kunmap_local(dst); dlen += doff; for (;;) { @@ -288,8 +280,6 @@ static int scomp_acomp_comp_decomp(struct acomp_req *req, int dir) dpage = nth_page(dpage, 1); } } - if (!acomp_request_src_isvirt(req) && src != scratch->src) - kunmap_local(src); return ret; } From 184e56e77c06a7eef68a021e9d4b11a11a8ab096 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Mon, 24 Mar 2025 11:57:32 +0800 Subject: [PATCH 012/382] crypto: iaa - Move compression CRC into request object Rather than passing around a CRC between the functions, embed it into the acomp_request context. Signed-off-by: Herbert Xu --- drivers/crypto/intel/iaa/iaa_crypto_main.c | 28 ++++++++++------------ 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/drivers/crypto/intel/iaa/iaa_crypto_main.c b/drivers/crypto/intel/iaa/iaa_crypto_main.c index 09d9589f2d68..4240a2e3d375 100644 --- a/drivers/crypto/intel/iaa/iaa_crypto_main.c +++ b/drivers/crypto/intel/iaa/iaa_crypto_main.c @@ -1020,8 +1020,7 @@ static int iaa_remap_for_verify(struct device *dev, struct iaa_wq *iaa_wq, static int iaa_compress_verify(struct crypto_tfm *tfm, struct acomp_req *req, struct idxd_wq *wq, dma_addr_t src_addr, unsigned int slen, - dma_addr_t dst_addr, unsigned int *dlen, - u32 compression_crc); + dma_addr_t dst_addr, unsigned int *dlen); static void iaa_desc_complete(struct idxd_desc *idxd_desc, enum idxd_complete_type comp_type, @@ -1087,10 +1086,10 @@ static void iaa_desc_complete(struct idxd_desc *idxd_desc, } if (ctx->compress && compression_ctx->verify_compress) { + u32 *compression_crc = acomp_request_ctx(ctx->req); dma_addr_t src_addr, dst_addr; - u32 compression_crc; - compression_crc = idxd_desc->iax_completion->crc; + *compression_crc = idxd_desc->iax_completion->crc; ret = iaa_remap_for_verify(dev, iaa_wq, ctx->req, &src_addr, &dst_addr); if (ret) { @@ -1100,8 +1099,7 @@ static void iaa_desc_complete(struct idxd_desc *idxd_desc, } ret = iaa_compress_verify(ctx->tfm, ctx->req, iaa_wq->wq, src_addr, - ctx->req->slen, dst_addr, &ctx->req->dlen, - compression_crc); + ctx->req->slen, dst_addr, &ctx->req->dlen); if (ret) { dev_dbg(dev, "%s: compress verify failed ret=%d\n", __func__, ret); err = -EIO; @@ -1130,11 +1128,11 @@ out: static int iaa_compress(struct crypto_tfm *tfm, struct acomp_req *req, struct idxd_wq *wq, dma_addr_t src_addr, unsigned int slen, - dma_addr_t dst_addr, unsigned int *dlen, - u32 *compression_crc) + dma_addr_t dst_addr, unsigned int *dlen) { struct iaa_device_compression_mode *active_compression_mode; struct iaa_compression_ctx *ctx = crypto_tfm_ctx(tfm); + u32 *compression_crc = acomp_request_ctx(req); struct iaa_device *iaa_device; struct idxd_desc *idxd_desc; struct iax_hw_desc *desc; @@ -1282,11 +1280,11 @@ out: static int iaa_compress_verify(struct crypto_tfm *tfm, struct acomp_req *req, struct idxd_wq *wq, dma_addr_t src_addr, unsigned int slen, - dma_addr_t dst_addr, unsigned int *dlen, - u32 compression_crc) + dma_addr_t dst_addr, unsigned int *dlen) { struct iaa_device_compression_mode *active_compression_mode; struct iaa_compression_ctx *ctx = crypto_tfm_ctx(tfm); + u32 *compression_crc = acomp_request_ctx(req); struct iaa_device *iaa_device; struct idxd_desc *idxd_desc; struct iax_hw_desc *desc; @@ -1346,10 +1344,10 @@ static int iaa_compress_verify(struct crypto_tfm *tfm, struct acomp_req *req, goto err; } - if (compression_crc != idxd_desc->iax_completion->crc) { + if (*compression_crc != idxd_desc->iax_completion->crc) { ret = -EINVAL; dev_dbg(dev, "(verify) iaa comp/decomp crc mismatch:" - " comp=0x%x, decomp=0x%x\n", compression_crc, + " comp=0x%x, decomp=0x%x\n", *compression_crc, idxd_desc->iax_completion->crc); print_hex_dump(KERN_INFO, "cmp-rec: ", DUMP_PREFIX_OFFSET, 8, 1, idxd_desc->iax_completion, 64, 0); @@ -1496,7 +1494,6 @@ static int iaa_comp_acompress(struct acomp_req *req) dma_addr_t src_addr, dst_addr; int nr_sgs, cpu, ret = 0; struct iaa_wq *iaa_wq; - u32 compression_crc; struct idxd_wq *wq; struct device *dev; @@ -1557,7 +1554,7 @@ static int iaa_comp_acompress(struct acomp_req *req) req->dst, req->dlen, sg_dma_len(req->dst)); ret = iaa_compress(tfm, req, wq, src_addr, req->slen, dst_addr, - &req->dlen, &compression_crc); + &req->dlen); if (ret == -EINPROGRESS) return ret; @@ -1569,7 +1566,7 @@ static int iaa_comp_acompress(struct acomp_req *req) } ret = iaa_compress_verify(tfm, req, wq, src_addr, req->slen, - dst_addr, &req->dlen, compression_crc); + dst_addr, &req->dlen); if (ret) dev_dbg(dev, "asynchronous compress verification failed ret=%d\n", ret); @@ -1694,6 +1691,7 @@ static struct acomp_alg iaa_acomp_fixed_deflate = { .init = iaa_comp_init_fixed, .compress = iaa_comp_acompress, .decompress = iaa_comp_adecompress, + .reqsize = sizeof(u32), .base = { .cra_name = "deflate", .cra_driver_name = "deflate-iaa", From cc98d8ce934b99789d30421957fd6a20fffb1c22 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Mon, 24 Mar 2025 12:04:18 +0800 Subject: [PATCH 013/382] crypto: iaa - Do not clobber req->base.data The req->base.data field is for the user and must not be touched by the driver, unless you save it first. The iaa driver doesn't seem to be using the req->base.data value so just remove the assignment. Fixes: 09646c98d0bf ("crypto: iaa - Add irq support for the crypto async interface") Signed-off-by: Herbert Xu --- drivers/crypto/intel/iaa/iaa_crypto_main.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/drivers/crypto/intel/iaa/iaa_crypto_main.c b/drivers/crypto/intel/iaa/iaa_crypto_main.c index 4240a2e3d375..773aab8a8830 100644 --- a/drivers/crypto/intel/iaa/iaa_crypto_main.c +++ b/drivers/crypto/intel/iaa/iaa_crypto_main.c @@ -1185,8 +1185,7 @@ static int iaa_compress(struct crypto_tfm *tfm, struct acomp_req *req, " src_addr %llx, dst_addr %llx\n", __func__, active_compression_mode->name, src_addr, dst_addr); - } else if (ctx->async_mode) - req->base.data = idxd_desc; + } dev_dbg(dev, "%s: compression mode %s," " desc->src1_addr %llx, desc->src1_size %d," @@ -1423,8 +1422,7 @@ static int iaa_decompress(struct crypto_tfm *tfm, struct acomp_req *req, " src_addr %llx, dst_addr %llx\n", __func__, active_compression_mode->name, src_addr, dst_addr); - } else if (ctx->async_mode && !disable_async) - req->base.data = idxd_desc; + } dev_dbg(dev, "%s: decompression mode %s," " desc->src1_addr %llx, desc->src1_size %d," From 39ccd0e54f164ea3c40327d9163205a8449047c2 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Mon, 24 Mar 2025 12:08:52 +0800 Subject: [PATCH 014/382] crypto: iaa - Remove unused disable_async argument from iaa_decompress Remove the disable_async field left over after the NULL dst removal. Signed-off-by: Herbert Xu --- drivers/crypto/intel/iaa/iaa_crypto_main.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/drivers/crypto/intel/iaa/iaa_crypto_main.c b/drivers/crypto/intel/iaa/iaa_crypto_main.c index 773aab8a8830..aa63df16f20b 100644 --- a/drivers/crypto/intel/iaa/iaa_crypto_main.c +++ b/drivers/crypto/intel/iaa/iaa_crypto_main.c @@ -1366,8 +1366,7 @@ err: static int iaa_decompress(struct crypto_tfm *tfm, struct acomp_req *req, struct idxd_wq *wq, dma_addr_t src_addr, unsigned int slen, - dma_addr_t dst_addr, unsigned int *dlen, - bool disable_async) + dma_addr_t dst_addr, unsigned int *dlen) { struct iaa_device_compression_mode *active_compression_mode; struct iaa_compression_ctx *ctx = crypto_tfm_ctx(tfm); @@ -1409,7 +1408,7 @@ static int iaa_decompress(struct crypto_tfm *tfm, struct acomp_req *req, desc->src1_size = slen; desc->completion_addr = idxd_desc->compl_dma; - if (ctx->use_irq && !disable_async) { + if (ctx->use_irq) { desc->flags |= IDXD_OP_FLAG_RCI; idxd_desc->crypto.req = req; @@ -1442,7 +1441,7 @@ static int iaa_decompress(struct crypto_tfm *tfm, struct acomp_req *req, update_total_decomp_calls(); update_wq_decomp_calls(wq); - if (ctx->async_mode && !disable_async) { + if (ctx->async_mode) { ret = -EINPROGRESS; dev_dbg(dev, "%s: returning -EINPROGRESS\n", __func__); goto out; @@ -1470,7 +1469,7 @@ static int iaa_decompress(struct crypto_tfm *tfm, struct acomp_req *req, *dlen = req->dlen; - if (!ctx->async_mode || disable_async) + if (!ctx->async_mode) idxd_free_desc(wq, idxd_desc); /* Update stats */ @@ -1650,7 +1649,7 @@ static int iaa_comp_adecompress(struct acomp_req *req) req->dst, req->dlen, sg_dma_len(req->dst)); ret = iaa_decompress(tfm, req, wq, src_addr, req->slen, - dst_addr, &req->dlen, false); + dst_addr, &req->dlen); if (ret == -EINPROGRESS) return ret; From 6131e119f5799bec55535530fa2fc44f5c7be1f1 Mon Sep 17 00:00:00 2001 From: Ashish Kalra Date: Mon, 24 Mar 2025 21:13:57 +0000 Subject: [PATCH 015/382] crypto: ccp - Abort doing SEV INIT if SNP INIT fails If SNP host support (SYSCFG.SNPEn) is set, then the RMP table must be initialized before calling SEV INIT. In other words, if SNP_INIT(_EX) is not issued or fails then SEV INIT will fail if SNP host support (SYSCFG.SNPEn) is enabled. Signed-off-by: Ashish Kalra Signed-off-by: Herbert Xu --- drivers/crypto/ccp/sev-dev.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/drivers/crypto/ccp/sev-dev.c b/drivers/crypto/ccp/sev-dev.c index 2e87ca0e292a..a0e3de94704e 100644 --- a/drivers/crypto/ccp/sev-dev.c +++ b/drivers/crypto/ccp/sev-dev.c @@ -1112,7 +1112,7 @@ static int __sev_snp_init_locked(int *error) if (!sev_version_greater_or_equal(SNP_MIN_API_MAJOR, SNP_MIN_API_MINOR)) { dev_dbg(sev->dev, "SEV-SNP support requires firmware version >= %d:%d\n", SNP_MIN_API_MAJOR, SNP_MIN_API_MINOR); - return 0; + return -EOPNOTSUPP; } /* SNP_INIT requires MSR_VM_HSAVE_PA to be cleared on all CPUs. */ @@ -1325,12 +1325,9 @@ static int _sev_platform_init_locked(struct sev_platform_init_args *args) */ rc = __sev_snp_init_locked(&args->error); if (rc && rc != -ENODEV) { - /* - * Don't abort the probe if SNP INIT failed, - * continue to initialize the legacy SEV firmware. - */ dev_err(sev->dev, "SEV-SNP: failed to INIT rc %d, error %#x\n", rc, args->error); + return rc; } /* Defer legacy SEV/SEV-ES support if allowed by caller/module. */ From 9770b428b1a28360663f1f5e524ee458b4cf454b Mon Sep 17 00:00:00 2001 From: Ashish Kalra Date: Mon, 24 Mar 2025 21:14:13 +0000 Subject: [PATCH 016/382] crypto: ccp - Move dev_info/err messages for SEV/SNP init and shutdown Move dev_info and dev_err messages related to SEV/SNP initialization and shutdown into __sev_platform_init_locked(), __sev_snp_init_locked() and __sev_platform_shutdown_locked(), __sev_snp_shutdown_locked() so that they don't need to be issued from callers. This allows both _sev_platform_init_locked() and various SEV/SNP ioctls to call __sev_platform_init_locked(), __sev_snp_init_locked() and __sev_platform_shutdown_locked(), __sev_snp_shutdown_locked() for implicit SEV/SNP initialization and shutdown without additionally printing any errors/success messages. Reviewed-by: Tom Lendacky Signed-off-by: Ashish Kalra Signed-off-by: Herbert Xu --- drivers/crypto/ccp/sev-dev.c | 49 ++++++++++++++++++++++++------------ 1 file changed, 33 insertions(+), 16 deletions(-) diff --git a/drivers/crypto/ccp/sev-dev.c b/drivers/crypto/ccp/sev-dev.c index a0e3de94704e..ccd7cc4b36d1 100644 --- a/drivers/crypto/ccp/sev-dev.c +++ b/drivers/crypto/ccp/sev-dev.c @@ -1176,21 +1176,31 @@ static int __sev_snp_init_locked(int *error) wbinvd_on_all_cpus(); rc = __sev_do_cmd_locked(cmd, arg, error); - if (rc) + if (rc) { + dev_err(sev->dev, "SEV-SNP: %s failed rc %d, error %#x\n", + cmd == SEV_CMD_SNP_INIT_EX ? "SNP_INIT_EX" : "SNP_INIT", + rc, *error); return rc; + } /* Prepare for first SNP guest launch after INIT. */ wbinvd_on_all_cpus(); rc = __sev_do_cmd_locked(SEV_CMD_SNP_DF_FLUSH, NULL, error); - if (rc) + if (rc) { + dev_err(sev->dev, "SEV-SNP: SNP_DF_FLUSH failed rc %d, error %#x\n", + rc, *error); return rc; + } sev->snp_initialized = true; dev_dbg(sev->dev, "SEV-SNP firmware initialized\n"); + dev_info(sev->dev, "SEV-SNP API:%d.%d build:%d\n", sev->api_major, + sev->api_minor, sev->build); + sev_es_tmr_size = SNP_TMR_SIZE; - return rc; + return 0; } static void __sev_platform_init_handle_tmr(struct sev_device *sev) @@ -1287,16 +1297,22 @@ static int __sev_platform_init_locked(int *error) if (error) *error = psp_ret; - if (rc) + if (rc) { + dev_err(sev->dev, "SEV: %s failed %#x, rc %d\n", + sev_init_ex_buffer ? "INIT_EX" : "INIT", psp_ret, rc); return rc; + } sev->state = SEV_STATE_INIT; /* Prepare for first SEV guest launch after INIT */ wbinvd_on_all_cpus(); rc = __sev_do_cmd_locked(SEV_CMD_DF_FLUSH, NULL, error); - if (rc) + if (rc) { + dev_err(sev->dev, "SEV: DF_FLUSH failed %#x, rc %d\n", + *error, rc); return rc; + } dev_dbg(sev->dev, "SEV firmware initialized\n"); @@ -1324,11 +1340,8 @@ static int _sev_platform_init_locked(struct sev_platform_init_args *args) * so perform SEV-SNP initialization at probe time. */ rc = __sev_snp_init_locked(&args->error); - if (rc && rc != -ENODEV) { - dev_err(sev->dev, "SEV-SNP: failed to INIT rc %d, error %#x\n", - rc, args->error); + if (rc && rc != -ENODEV) return rc; - } /* Defer legacy SEV/SEV-ES support if allowed by caller/module. */ if (args->probe && !psp_init_on_probe) @@ -1364,8 +1377,11 @@ static int __sev_platform_shutdown_locked(int *error) return 0; ret = __sev_do_cmd_locked(SEV_CMD_SHUTDOWN, NULL, error); - if (ret) + if (ret) { + dev_err(sev->dev, "SEV: failed to SHUTDOWN error %#x, rc %d\n", + *error, ret); return ret; + } sev->state = SEV_STATE_UNINIT; dev_dbg(sev->dev, "SEV firmware shutdown\n"); @@ -1679,9 +1695,12 @@ static int __sev_snp_shutdown_locked(int *error, bool panic) ret = __sev_do_cmd_locked(SEV_CMD_SNP_SHUTDOWN_EX, &data, error); /* SHUTDOWN may require DF_FLUSH */ if (*error == SEV_RET_DFFLUSH_REQUIRED) { - ret = __sev_do_cmd_locked(SEV_CMD_SNP_DF_FLUSH, NULL, NULL); + int dfflush_error; + + ret = __sev_do_cmd_locked(SEV_CMD_SNP_DF_FLUSH, NULL, &dfflush_error); if (ret) { - dev_err(sev->dev, "SEV-SNP DF_FLUSH failed\n"); + dev_err(sev->dev, "SEV-SNP DF_FLUSH failed, ret = %d, error = %#x\n", + ret, dfflush_error); return ret; } /* reissue the shutdown command */ @@ -1689,7 +1708,8 @@ static int __sev_snp_shutdown_locked(int *error, bool panic) error); } if (ret) { - dev_err(sev->dev, "SEV-SNP firmware shutdown failed\n"); + dev_err(sev->dev, "SEV-SNP firmware shutdown failed, rc %d, error %#x\n", + ret, *error); return ret; } @@ -2419,9 +2439,6 @@ void sev_pci_init(void) dev_err(sev->dev, "SEV: failed to INIT error %#x, rc %d\n", args.error, rc); - dev_info(sev->dev, "SEV%s API:%d.%d build:%d\n", sev->snp_initialized ? - "-SNP" : "", sev->api_major, sev->api_minor, sev->build); - atomic_notifier_chain_register(&panic_notifier_list, &snp_panic_notifier); return; From ceac7fb89e8da465aec3ac3c20477f912f5c3a6c Mon Sep 17 00:00:00 2001 From: Ashish Kalra Date: Mon, 24 Mar 2025 21:14:32 +0000 Subject: [PATCH 017/382] crypto: ccp - Ensure implicit SEV/SNP init and shutdown in ioctls Modify the behavior of implicit SEV initialization in some of the SEV ioctls to do both SEV initialization and shutdown and add implicit SNP initialization and shutdown to some of the SNP ioctls so that the change of SEV/SNP platform initialization not being done during PSP driver probe time does not break userspace tools such as sevtool, etc. Prior to this patch, SEV has always been initialized before these ioctls as SEV initialization is done as part of PSP module probe, but now with SEV initialization being moved to KVM module load instead of PSP driver probe, the implied SEV INIT actually makes sense and gets used and additionally to maintain SEV platform state consistency before and after the ioctl SEV shutdown needs to be done after the firmware call. It is important to do SEV Shutdown here with the SEV/SNP initialization moving to KVM, an implicit SEV INIT here as part of the SEV ioctls not followed with SEV Shutdown will cause SEV to remain in INIT state and then a future SNP INIT in KVM module load will fail. Also ensure that for these SEV ioctls both implicit SNP and SEV INIT is done followed by both SEV and SNP shutdown as RMP table must be initialized before calling SEV INIT if SNP host support is enabled. Similarly, prior to this patch, SNP has always been initialized before these ioctls as SNP initialization is done as part of PSP module probe, therefore, to keep a consistent behavior, SNP init needs to be done here implicitly as part of these ioctls followed with SNP shutdown before returning from the ioctl to maintain the consistent platform state before and after the ioctl. Suggested-by: Tom Lendacky Reviewed-by: Tom Lendacky Signed-off-by: Ashish Kalra Signed-off-by: Herbert Xu --- drivers/crypto/ccp/sev-dev.c | 142 +++++++++++++++++++++++++++++------ 1 file changed, 119 insertions(+), 23 deletions(-) diff --git a/drivers/crypto/ccp/sev-dev.c b/drivers/crypto/ccp/sev-dev.c index ccd7cc4b36d1..5bd3df377370 100644 --- a/drivers/crypto/ccp/sev-dev.c +++ b/drivers/crypto/ccp/sev-dev.c @@ -109,6 +109,8 @@ static void *sev_init_ex_buffer; */ static struct sev_data_range_list *snp_range_list; +static void __sev_firmware_shutdown(struct sev_device *sev, bool panic); + static inline bool sev_version_greater_or_equal(u8 maj, u8 min) { struct sev_device *sev = psp_master->sev_data; @@ -1402,6 +1404,37 @@ static int sev_get_platform_state(int *state, int *error) return rc; } +static int sev_move_to_init_state(struct sev_issue_cmd *argp, bool *shutdown_required) +{ + struct sev_platform_init_args init_args = {0}; + int rc; + + rc = _sev_platform_init_locked(&init_args); + if (rc) { + argp->error = SEV_RET_INVALID_PLATFORM_STATE; + return rc; + } + + *shutdown_required = true; + + return 0; +} + +static int snp_move_to_init_state(struct sev_issue_cmd *argp, bool *shutdown_required) +{ + int error, rc; + + rc = __sev_snp_init_locked(&error); + if (rc) { + argp->error = SEV_RET_INVALID_PLATFORM_STATE; + return rc; + } + + *shutdown_required = true; + + return 0; +} + static int sev_ioctl_do_reset(struct sev_issue_cmd *argp, bool writable) { int state, rc; @@ -1454,24 +1487,31 @@ static int sev_ioctl_do_platform_status(struct sev_issue_cmd *argp) static int sev_ioctl_do_pek_pdh_gen(int cmd, struct sev_issue_cmd *argp, bool writable) { struct sev_device *sev = psp_master->sev_data; + bool shutdown_required = false; int rc; if (!writable) return -EPERM; if (sev->state == SEV_STATE_UNINIT) { - rc = __sev_platform_init_locked(&argp->error); + rc = sev_move_to_init_state(argp, &shutdown_required); if (rc) return rc; } - return __sev_do_cmd_locked(cmd, NULL, &argp->error); + rc = __sev_do_cmd_locked(cmd, NULL, &argp->error); + + if (shutdown_required) + __sev_firmware_shutdown(sev, false); + + return rc; } static int sev_ioctl_do_pek_csr(struct sev_issue_cmd *argp, bool writable) { struct sev_device *sev = psp_master->sev_data; struct sev_user_data_pek_csr input; + bool shutdown_required = false; struct sev_data_pek_csr data; void __user *input_address; void *blob = NULL; @@ -1503,7 +1543,7 @@ static int sev_ioctl_do_pek_csr(struct sev_issue_cmd *argp, bool writable) cmd: if (sev->state == SEV_STATE_UNINIT) { - ret = __sev_platform_init_locked(&argp->error); + ret = sev_move_to_init_state(argp, &shutdown_required); if (ret) goto e_free_blob; } @@ -1524,6 +1564,9 @@ cmd: } e_free_blob: + if (shutdown_required) + __sev_firmware_shutdown(sev, false); + kfree(blob); return ret; } @@ -1743,6 +1786,7 @@ static int sev_ioctl_do_pek_import(struct sev_issue_cmd *argp, bool writable) struct sev_device *sev = psp_master->sev_data; struct sev_user_data_pek_cert_import input; struct sev_data_pek_cert_import data; + bool shutdown_required = false; void *pek_blob, *oca_blob; int ret; @@ -1773,7 +1817,7 @@ static int sev_ioctl_do_pek_import(struct sev_issue_cmd *argp, bool writable) /* If platform is not in INIT state then transition it to INIT */ if (sev->state != SEV_STATE_INIT) { - ret = __sev_platform_init_locked(&argp->error); + ret = sev_move_to_init_state(argp, &shutdown_required); if (ret) goto e_free_oca; } @@ -1781,6 +1825,9 @@ static int sev_ioctl_do_pek_import(struct sev_issue_cmd *argp, bool writable) ret = __sev_do_cmd_locked(SEV_CMD_PEK_CERT_IMPORT, &data, &argp->error); e_free_oca: + if (shutdown_required) + __sev_firmware_shutdown(sev, false); + kfree(oca_blob); e_free_pek: kfree(pek_blob); @@ -1897,18 +1944,9 @@ static int sev_ioctl_do_pdh_export(struct sev_issue_cmd *argp, bool writable) struct sev_data_pdh_cert_export data; void __user *input_cert_chain_address; void __user *input_pdh_cert_address; + bool shutdown_required = false; int ret; - /* If platform is not in INIT state then transition it to INIT. */ - if (sev->state != SEV_STATE_INIT) { - if (!writable) - return -EPERM; - - ret = __sev_platform_init_locked(&argp->error); - if (ret) - return ret; - } - if (copy_from_user(&input, (void __user *)argp->data, sizeof(input))) return -EFAULT; @@ -1948,6 +1986,17 @@ static int sev_ioctl_do_pdh_export(struct sev_issue_cmd *argp, bool writable) data.cert_chain_len = input.cert_chain_len; cmd: + /* If platform is not in INIT state then transition it to INIT. */ + if (sev->state != SEV_STATE_INIT) { + if (!writable) { + ret = -EPERM; + goto e_free_cert; + } + ret = sev_move_to_init_state(argp, &shutdown_required); + if (ret) + goto e_free_cert; + } + ret = __sev_do_cmd_locked(SEV_CMD_PDH_CERT_EXPORT, &data, &argp->error); /* If we query the length, FW responded with expected data. */ @@ -1974,6 +2023,9 @@ cmd: } e_free_cert: + if (shutdown_required) + __sev_firmware_shutdown(sev, false); + kfree(cert_blob); e_free_pdh: kfree(pdh_blob); @@ -1983,12 +2035,13 @@ e_free_pdh: static int sev_ioctl_do_snp_platform_status(struct sev_issue_cmd *argp) { struct sev_device *sev = psp_master->sev_data; + bool shutdown_required = false; struct sev_data_snp_addr buf; struct page *status_page; + int ret, error; void *data; - int ret; - if (!sev->snp_initialized || !argp->data) + if (!argp->data) return -EINVAL; status_page = alloc_page(GFP_KERNEL_ACCOUNT); @@ -1997,6 +2050,12 @@ static int sev_ioctl_do_snp_platform_status(struct sev_issue_cmd *argp) data = page_address(status_page); + if (!sev->snp_initialized) { + ret = snp_move_to_init_state(argp, &shutdown_required); + if (ret) + goto cleanup; + } + /* * Firmware expects status page to be in firmware-owned state, otherwise * it will report firmware error code INVALID_PAGE_STATE (0x1A). @@ -2025,6 +2084,9 @@ static int sev_ioctl_do_snp_platform_status(struct sev_issue_cmd *argp) ret = -EFAULT; cleanup: + if (shutdown_required) + __sev_snp_shutdown_locked(&error, false); + __free_pages(status_page, 0); return ret; } @@ -2033,21 +2095,33 @@ static int sev_ioctl_do_snp_commit(struct sev_issue_cmd *argp) { struct sev_device *sev = psp_master->sev_data; struct sev_data_snp_commit buf; + bool shutdown_required = false; + int ret, error; - if (!sev->snp_initialized) - return -EINVAL; + if (!sev->snp_initialized) { + ret = snp_move_to_init_state(argp, &shutdown_required); + if (ret) + return ret; + } buf.len = sizeof(buf); - return __sev_do_cmd_locked(SEV_CMD_SNP_COMMIT, &buf, &argp->error); + ret = __sev_do_cmd_locked(SEV_CMD_SNP_COMMIT, &buf, &argp->error); + + if (shutdown_required) + __sev_snp_shutdown_locked(&error, false); + + return ret; } static int sev_ioctl_do_snp_set_config(struct sev_issue_cmd *argp, bool writable) { struct sev_device *sev = psp_master->sev_data; struct sev_user_data_snp_config config; + bool shutdown_required = false; + int ret, error; - if (!sev->snp_initialized || !argp->data) + if (!argp->data) return -EINVAL; if (!writable) @@ -2056,17 +2130,29 @@ static int sev_ioctl_do_snp_set_config(struct sev_issue_cmd *argp, bool writable if (copy_from_user(&config, (void __user *)argp->data, sizeof(config))) return -EFAULT; - return __sev_do_cmd_locked(SEV_CMD_SNP_CONFIG, &config, &argp->error); + if (!sev->snp_initialized) { + ret = snp_move_to_init_state(argp, &shutdown_required); + if (ret) + return ret; + } + + ret = __sev_do_cmd_locked(SEV_CMD_SNP_CONFIG, &config, &argp->error); + + if (shutdown_required) + __sev_snp_shutdown_locked(&error, false); + + return ret; } static int sev_ioctl_do_snp_vlek_load(struct sev_issue_cmd *argp, bool writable) { struct sev_device *sev = psp_master->sev_data; struct sev_user_data_snp_vlek_load input; + bool shutdown_required = false; + int ret, error; void *blob; - int ret; - if (!sev->snp_initialized || !argp->data) + if (!argp->data) return -EINVAL; if (!writable) @@ -2085,8 +2171,18 @@ static int sev_ioctl_do_snp_vlek_load(struct sev_issue_cmd *argp, bool writable) input.vlek_wrapped_address = __psp_pa(blob); + if (!sev->snp_initialized) { + ret = snp_move_to_init_state(argp, &shutdown_required); + if (ret) + goto cleanup; + } + ret = __sev_do_cmd_locked(SEV_CMD_SNP_VLEK_LOAD, &input, &argp->error); + if (shutdown_required) + __sev_snp_shutdown_locked(&error, false); + +cleanup: kfree(blob); return ret; From 65a895a44e641fe47f3419b064288d1d713024a5 Mon Sep 17 00:00:00 2001 From: Ashish Kalra Date: Mon, 24 Mar 2025 21:14:47 +0000 Subject: [PATCH 018/382] crypto: ccp - Reset TMR size at SNP Shutdown Implicit SNP initialization as part of some SNP ioctls modify TMR size to be SNP compliant which followed by SNP shutdown will leave the TMR size modified and then subsequently cause SEV only initialization to fail, hence, reset TMR size to default at SNP Shutdown. Acked-by: Dionna Glaze Reviewed-by: Tom Lendacky Signed-off-by: Ashish Kalra Signed-off-by: Herbert Xu --- drivers/crypto/ccp/sev-dev.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/crypto/ccp/sev-dev.c b/drivers/crypto/ccp/sev-dev.c index 5bd3df377370..08a6160f0072 100644 --- a/drivers/crypto/ccp/sev-dev.c +++ b/drivers/crypto/ccp/sev-dev.c @@ -1778,6 +1778,9 @@ static int __sev_snp_shutdown_locked(int *error, bool panic) sev->snp_initialized = false; dev_dbg(sev->dev, "SEV-SNP firmware shutdown\n"); + /* Reset TMR size back to default */ + sev_es_tmr_size = SEV_TMR_SIZE; + return ret; } From 19860c3274fb209e36ebcba562dc5da301a53cb3 Mon Sep 17 00:00:00 2001 From: Ashish Kalra Date: Mon, 24 Mar 2025 21:15:02 +0000 Subject: [PATCH 019/382] crypto: ccp - Register SNP panic notifier only if SNP is enabled Currently, the SNP panic notifier is registered on module initialization regardless of whether SNP is being enabled or initialized. Instead, register the SNP panic notifier only when SNP is actually initialized and unregister the notifier when SNP is shutdown. Reviewed-by: Dionna Glaze Reviewed-by: Alexey Kardashevskiy Reviewed-by: Tom Lendacky Signed-off-by: Ashish Kalra Signed-off-by: Herbert Xu --- drivers/crypto/ccp/sev-dev.c | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/drivers/crypto/ccp/sev-dev.c b/drivers/crypto/ccp/sev-dev.c index 08a6160f0072..6fdbb3bf44b5 100644 --- a/drivers/crypto/ccp/sev-dev.c +++ b/drivers/crypto/ccp/sev-dev.c @@ -111,6 +111,13 @@ static struct sev_data_range_list *snp_range_list; static void __sev_firmware_shutdown(struct sev_device *sev, bool panic); +static int snp_shutdown_on_panic(struct notifier_block *nb, + unsigned long reason, void *arg); + +static struct notifier_block snp_panic_notifier = { + .notifier_call = snp_shutdown_on_panic, +}; + static inline bool sev_version_greater_or_equal(u8 maj, u8 min) { struct sev_device *sev = psp_master->sev_data; @@ -1200,6 +1207,9 @@ static int __sev_snp_init_locked(int *error) dev_info(sev->dev, "SEV-SNP API:%d.%d build:%d\n", sev->api_major, sev->api_minor, sev->build); + atomic_notifier_chain_register(&panic_notifier_list, + &snp_panic_notifier); + sev_es_tmr_size = SNP_TMR_SIZE; return 0; @@ -1778,6 +1788,9 @@ static int __sev_snp_shutdown_locked(int *error, bool panic) sev->snp_initialized = false; dev_dbg(sev->dev, "SEV-SNP firmware shutdown\n"); + atomic_notifier_chain_unregister(&panic_notifier_list, + &snp_panic_notifier); + /* Reset TMR size back to default */ sev_es_tmr_size = SEV_TMR_SIZE; @@ -2489,10 +2502,6 @@ static int snp_shutdown_on_panic(struct notifier_block *nb, return NOTIFY_DONE; } -static struct notifier_block snp_panic_notifier = { - .notifier_call = snp_shutdown_on_panic, -}; - int sev_issue_cmd_external_user(struct file *filep, unsigned int cmd, void *data, int *error) { @@ -2538,8 +2547,6 @@ void sev_pci_init(void) dev_err(sev->dev, "SEV: failed to INIT error %#x, rc %d\n", args.error, rc); - atomic_notifier_chain_register(&panic_notifier_list, - &snp_panic_notifier); return; err: @@ -2556,7 +2563,4 @@ void sev_pci_exit(void) return; sev_firmware_shutdown(sev); - - atomic_notifier_chain_unregister(&panic_notifier_list, - &snp_panic_notifier); } From f7b86e0e75bc234751cb7a82d888083a57ef28b2 Mon Sep 17 00:00:00 2001 From: Ashish Kalra Date: Mon, 24 Mar 2025 21:15:17 +0000 Subject: [PATCH 020/382] crypto: ccp - Add new SEV/SNP platform shutdown API Add new API interface to do SEV/SNP platform shutdown when KVM module is unloaded. Reviewed-by: Dionna Glaze Reviewed-by: Tom Lendacky Signed-off-by: Ashish Kalra Signed-off-by: Herbert Xu --- drivers/crypto/ccp/sev-dev.c | 9 +++++++++ include/linux/psp-sev.h | 3 +++ 2 files changed, 12 insertions(+) diff --git a/drivers/crypto/ccp/sev-dev.c b/drivers/crypto/ccp/sev-dev.c index 6fdbb3bf44b5..671347702ae7 100644 --- a/drivers/crypto/ccp/sev-dev.c +++ b/drivers/crypto/ccp/sev-dev.c @@ -2468,6 +2468,15 @@ static void sev_firmware_shutdown(struct sev_device *sev) mutex_unlock(&sev_cmd_mutex); } +void sev_platform_shutdown(void) +{ + if (!psp_master || !psp_master->sev_data) + return; + + sev_firmware_shutdown(psp_master->sev_data); +} +EXPORT_SYMBOL_GPL(sev_platform_shutdown); + void sev_dev_destroy(struct psp_device *psp) { struct sev_device *sev = psp->sev_data; diff --git a/include/linux/psp-sev.h b/include/linux/psp-sev.h index f3cad182d4ef..0b3a36bdaa90 100644 --- a/include/linux/psp-sev.h +++ b/include/linux/psp-sev.h @@ -954,6 +954,7 @@ int sev_do_cmd(int cmd, void *data, int *psp_ret); void *psp_copy_user_blob(u64 uaddr, u32 len); void *snp_alloc_firmware_page(gfp_t mask); void snp_free_firmware_page(void *addr); +void sev_platform_shutdown(void); #else /* !CONFIG_CRYPTO_DEV_SP_PSP */ @@ -988,6 +989,8 @@ static inline void *snp_alloc_firmware_page(gfp_t mask) static inline void snp_free_firmware_page(void *addr) { } +static inline void sev_platform_shutdown(void) { } + #endif /* CONFIG_CRYPTO_DEV_SP_PSP */ #endif /* __PSP_SEV_H__ */ From 26013326a6bad4f6c5a8f367026867b77c5e1ff2 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Wed, 26 Mar 2025 18:05:38 +0800 Subject: [PATCH 021/382] crypto: sa2ul - Use proper helpers to setup request Rather than setting up a request by hand, use the correct API helpers to setup the new request. This is because the API helpers will setup chaining. Also change the fallback allocation to explicitly request for a sync algorithm as this driver will crash if given an async one. Signed-off-by: Herbert Xu Tested-by: Manorit Chawdhry Signed-off-by: Herbert Xu --- drivers/crypto/sa2ul.c | 63 +++++++++++------------------------------- 1 file changed, 16 insertions(+), 47 deletions(-) diff --git a/drivers/crypto/sa2ul.c b/drivers/crypto/sa2ul.c index 091612b066f1..fdc0b2486069 100644 --- a/drivers/crypto/sa2ul.c +++ b/drivers/crypto/sa2ul.c @@ -1415,22 +1415,13 @@ static int sa_sha_run(struct ahash_request *req) (auth_len >= SA_UNSAFE_DATA_SZ_MIN && auth_len <= SA_UNSAFE_DATA_SZ_MAX)) { struct ahash_request *subreq = &rctx->fallback_req; - int ret = 0; + int ret; ahash_request_set_tfm(subreq, ctx->fallback.ahash); - subreq->base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; + ahash_request_set_callback(subreq, req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, NULL, NULL); + ahash_request_set_crypt(subreq, req->src, req->result, auth_len); - crypto_ahash_init(subreq); - - subreq->nbytes = auth_len; - subreq->src = req->src; - subreq->result = req->result; - - ret |= crypto_ahash_update(subreq); - - subreq->nbytes = 0; - - ret |= crypto_ahash_final(subreq); + ret = crypto_ahash_digest(subreq); return ret; } @@ -1502,8 +1493,7 @@ static int sa_sha_cra_init_alg(struct crypto_tfm *tfm, const char *alg_base) return ret; if (alg_base) { - ctx->shash = crypto_alloc_shash(alg_base, 0, - CRYPTO_ALG_NEED_FALLBACK); + ctx->shash = crypto_alloc_shash(alg_base, 0, 0); if (IS_ERR(ctx->shash)) { dev_err(sa_k3_dev, "base driver %s couldn't be loaded\n", alg_base); @@ -1511,8 +1501,7 @@ static int sa_sha_cra_init_alg(struct crypto_tfm *tfm, const char *alg_base) } /* for fallback */ ctx->fallback.ahash = - crypto_alloc_ahash(alg_base, 0, - CRYPTO_ALG_NEED_FALLBACK); + crypto_alloc_ahash(alg_base, 0, CRYPTO_ALG_ASYNC); if (IS_ERR(ctx->fallback.ahash)) { dev_err(ctx->dev_data->dev, "Could not load fallback driver\n"); @@ -1546,54 +1535,38 @@ static int sa_sha_init(struct ahash_request *req) crypto_ahash_digestsize(tfm), rctx); ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash); - rctx->fallback_req.base.flags = - req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; + ahash_request_set_callback(&rctx->fallback_req, req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, NULL, NULL); + ahash_request_set_crypt(&rctx->fallback_req, NULL, NULL, 0); return crypto_ahash_init(&rctx->fallback_req); } static int sa_sha_update(struct ahash_request *req) { - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); struct sa_sha_req_ctx *rctx = ahash_request_ctx(req); - struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm); - ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash); - rctx->fallback_req.base.flags = - req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; - rctx->fallback_req.nbytes = req->nbytes; - rctx->fallback_req.src = req->src; + ahash_request_set_callback(&rctx->fallback_req, req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, NULL, NULL); + ahash_request_set_crypt(&rctx->fallback_req, req->src, NULL, req->nbytes); return crypto_ahash_update(&rctx->fallback_req); } static int sa_sha_final(struct ahash_request *req) { - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); struct sa_sha_req_ctx *rctx = ahash_request_ctx(req); - struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm); - ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash); - rctx->fallback_req.base.flags = - req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; - rctx->fallback_req.result = req->result; + ahash_request_set_callback(&rctx->fallback_req, req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, NULL, NULL); + ahash_request_set_crypt(&rctx->fallback_req, NULL, req->result, 0); return crypto_ahash_final(&rctx->fallback_req); } static int sa_sha_finup(struct ahash_request *req) { - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); struct sa_sha_req_ctx *rctx = ahash_request_ctx(req); - struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm); - ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash); - rctx->fallback_req.base.flags = - req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; - - rctx->fallback_req.nbytes = req->nbytes; - rctx->fallback_req.src = req->src; - rctx->fallback_req.result = req->result; + ahash_request_set_callback(&rctx->fallback_req, req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, NULL, NULL); + ahash_request_set_crypt(&rctx->fallback_req, req->src, req->result, req->nbytes); return crypto_ahash_finup(&rctx->fallback_req); } @@ -1605,8 +1578,7 @@ static int sa_sha_import(struct ahash_request *req, const void *in) struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash); - rctx->fallback_req.base.flags = req->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; + ahash_request_set_callback(&rctx->fallback_req, req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, NULL, NULL); return crypto_ahash_import(&rctx->fallback_req, in); } @@ -1614,12 +1586,9 @@ static int sa_sha_import(struct ahash_request *req, const void *in) static int sa_sha_export(struct ahash_request *req, void *out) { struct sa_sha_req_ctx *rctx = ahash_request_ctx(req); - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm); struct ahash_request *subreq = &rctx->fallback_req; - ahash_request_set_tfm(subreq, ctx->fallback.ahash); - subreq->base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; + ahash_request_set_callback(subreq, req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, NULL, NULL); return crypto_ahash_export(subreq, out); } From 845bc952024dbf482c7434daeac66f764642d52d Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Wed, 26 Mar 2025 15:59:46 +0000 Subject: [PATCH 022/382] crypto: qat - add shutdown handler to qat_4xxx During a warm reset via kexec, the system bypasses the driver removal sequence, meaning that the remove() callback is not invoked. If a QAT device is not shutdown properly, the device driver will fail to load in a newly rebooted kernel. This might result in output like the following after the kexec reboot: 4xxx 0000:01:00.0: Failed to power up the device 4xxx 0000:01:00.0: Failed to initialize device 4xxx 0000:01:00.0: Resetting device qat_dev0 4xxx 0000:01:00.0: probe with driver 4xxx failed with error -14 Implement the shutdown() handler that hooks into the reboot notifier list. This brings down the QAT device and ensures it is shut down properly. Cc: Fixes: 8c8268166e83 ("crypto: qat - add qat_4xxx driver") Link: https://lore.kernel.org/all/Z-DGQrhRj9niR9iZ@gondor.apana.org.au/ Reported-by: Randy Wright Closes: https://issues.redhat.com/browse/RHEL-84366 Reviewed-by: Ahsan Atta Reviewed-by: Andy Shevchenko Signed-off-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/intel/qat/qat_4xxx/adf_drv.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/drivers/crypto/intel/qat/qat_4xxx/adf_drv.c b/drivers/crypto/intel/qat/qat_4xxx/adf_drv.c index 5537a9991e4e..1ac415ef3c31 100644 --- a/drivers/crypto/intel/qat/qat_4xxx/adf_drv.c +++ b/drivers/crypto/intel/qat/qat_4xxx/adf_drv.c @@ -188,11 +188,19 @@ static void adf_remove(struct pci_dev *pdev) adf_cleanup_accel(accel_dev); } +static void adf_shutdown(struct pci_dev *pdev) +{ + struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); + + adf_dev_down(accel_dev); +} + static struct pci_driver adf_driver = { .id_table = adf_pci_tbl, .name = ADF_4XXX_DEVICE_NAME, .probe = adf_probe, .remove = adf_remove, + .shutdown = adf_shutdown, .sriov_configure = adf_sriov_configure, .err_handler = &adf_err_handler, }; From 097143f23a1164bfd1b6f70279d229be44da2e30 Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Wed, 26 Mar 2025 15:59:47 +0000 Subject: [PATCH 023/382] crypto: qat - add shutdown handler to qat_420xx During a warm reset via kexec, the system bypasses the driver removal sequence, meaning that the remove() callback is not invoked. If a QAT device is not shutdown properly, the device driver will fail to load in a newly rebooted kernel. This might result in output like the following after the kexec reboot: 420xx 0000:01:00.0: Failed to power up the device 420xx 0000:01:00.0: Failed to initialize device 420xx 0000:01:00.0: Resetting device qat_dev0 420xx 0000:01:00.0: probe with driver 420xx failed with error -14 Implement the shutdown() handler that hooks into the reboot notifier list. This brings down the QAT device and ensures it is shut down properly. Cc: Fixes: fcf60f4bcf54 ("crypto: qat - add support for 420xx devices") Reviewed-by: Ahsan Atta Reviewed-by: Andy Shevchenko Signed-off-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/intel/qat/qat_420xx/adf_drv.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/drivers/crypto/intel/qat/qat_420xx/adf_drv.c b/drivers/crypto/intel/qat/qat_420xx/adf_drv.c index 8084aa0f7f41..b4731f02deb8 100644 --- a/drivers/crypto/intel/qat/qat_420xx/adf_drv.c +++ b/drivers/crypto/intel/qat/qat_420xx/adf_drv.c @@ -186,11 +186,19 @@ static void adf_remove(struct pci_dev *pdev) adf_cleanup_accel(accel_dev); } +static void adf_shutdown(struct pci_dev *pdev) +{ + struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); + + adf_dev_down(accel_dev); +} + static struct pci_driver adf_driver = { .id_table = adf_pci_tbl, .name = ADF_420XX_DEVICE_NAME, .probe = adf_probe, .remove = adf_remove, + .shutdown = adf_shutdown, .sriov_configure = adf_sriov_configure, .err_handler = &adf_err_handler, }; From 0c4a53fe7affb965e7ddf61f8fe8fa151d1174c3 Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Wed, 26 Mar 2025 15:59:48 +0000 Subject: [PATCH 024/382] crypto: qat - remove redundant prototypes in qat_dh895xcc Move the definition of the adf_driver structure and remove the redundant prototypes for the functions adf_probe() and adf_remove() in the qat_dh895xxcc driver. Also move the pci_device_id table close to where it is used and drop the inner comma as it is not required. This does not introduce any functional change. Signed-off-by: Giovanni Cabiddu Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- .../crypto/intel/qat/qat_dh895xcc/adf_drv.c | 33 +++++++++---------- 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/drivers/crypto/intel/qat/qat_dh895xcc/adf_drv.c b/drivers/crypto/intel/qat/qat_dh895xcc/adf_drv.c index 07e9d7e52861..730147404ceb 100644 --- a/drivers/crypto/intel/qat/qat_dh895xcc/adf_drv.c +++ b/drivers/crypto/intel/qat/qat_dh895xcc/adf_drv.c @@ -19,24 +19,6 @@ #include #include "adf_dh895xcc_hw_data.h" -static const struct pci_device_id adf_pci_tbl[] = { - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QAT_DH895XCC), }, - { } -}; -MODULE_DEVICE_TABLE(pci, adf_pci_tbl); - -static int adf_probe(struct pci_dev *dev, const struct pci_device_id *ent); -static void adf_remove(struct pci_dev *dev); - -static struct pci_driver adf_driver = { - .id_table = adf_pci_tbl, - .name = ADF_DH895XCC_DEVICE_NAME, - .probe = adf_probe, - .remove = adf_remove, - .sriov_configure = adf_sriov_configure, - .err_handler = &adf_err_handler, -}; - static void adf_cleanup_pci_dev(struct adf_accel_dev *accel_dev) { pci_release_regions(accel_dev->accel_pci_dev.pci_dev); @@ -227,6 +209,21 @@ static void adf_remove(struct pci_dev *pdev) kfree(accel_dev); } +static const struct pci_device_id adf_pci_tbl[] = { + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QAT_DH895XCC) }, + { } +}; +MODULE_DEVICE_TABLE(pci, adf_pci_tbl); + +static struct pci_driver adf_driver = { + .id_table = adf_pci_tbl, + .name = ADF_DH895XCC_DEVICE_NAME, + .probe = adf_probe, + .remove = adf_remove, + .sriov_configure = adf_sriov_configure, + .err_handler = &adf_err_handler, +}; + static int __init adfdrv_init(void) { request_module("intel_qat"); From 2c4e8b228733bfbcaf49408fdf94d220f6eb78fc Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Wed, 26 Mar 2025 15:59:49 +0000 Subject: [PATCH 025/382] crypto: qat - add shutdown handler to qat_dh895xcc During a warm reset via kexec, the system bypasses the driver removal sequence, meaning that the remove() callback is not invoked. If a QAT device is not shutdown properly, the device driver will fail to load in a newly rebooted kernel. This might result in output like the following after the kexec reboot: QAT: AE0 is inactive!! QAT: failed to get device out of reset dh895xcc 0000:3f:00.0: qat_hal_clr_reset error dh895xcc 0000:3f:00.0: Failed to init the AEs dh895xcc 0000:3f:00.0: Failed to initialise Acceleration Engine dh895xcc 0000:3f:00.0: Resetting device qat_dev0 dh895xcc 0000:3f:00.0: probe with driver dh895xcc failed with error -14 Implement the shutdown() handler that hooks into the reboot notifier list. This brings down the QAT device and ensures it is shut down properly. Cc: Fixes: 7afa232e76ce ("crypto: qat - Intel(R) QAT DH895xcc accelerator") Reviewed-by: Ahsan Atta Reviewed-by: Andy Shevchenko Signed-off-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/intel/qat/qat_dh895xcc/adf_drv.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/drivers/crypto/intel/qat/qat_dh895xcc/adf_drv.c b/drivers/crypto/intel/qat/qat_dh895xcc/adf_drv.c index 730147404ceb..b59e0cc49e52 100644 --- a/drivers/crypto/intel/qat/qat_dh895xcc/adf_drv.c +++ b/drivers/crypto/intel/qat/qat_dh895xcc/adf_drv.c @@ -209,6 +209,13 @@ static void adf_remove(struct pci_dev *pdev) kfree(accel_dev); } +static void adf_shutdown(struct pci_dev *pdev) +{ + struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); + + adf_dev_down(accel_dev); +} + static const struct pci_device_id adf_pci_tbl[] = { { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QAT_DH895XCC) }, { } @@ -220,6 +227,7 @@ static struct pci_driver adf_driver = { .name = ADF_DH895XCC_DEVICE_NAME, .probe = adf_probe, .remove = adf_remove, + .shutdown = adf_shutdown, .sriov_configure = adf_sriov_configure, .err_handler = &adf_err_handler, }; From 7eb2c73ece5bd866348b3e893e98a744b3aaacf5 Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Wed, 26 Mar 2025 15:59:50 +0000 Subject: [PATCH 026/382] crypto: qat - remove redundant prototypes in qat_c62x Move the definition of the adf_driver structure and remove the redundant prototypes for the functions adf_probe() and adf_remove() in the qat_c62x driver. Also move the pci_device_id table close to where it is used and drop the inner comma as it is not required. This does not introduce any functional change. Signed-off-by: Giovanni Cabiddu Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- drivers/crypto/intel/qat/qat_c62x/adf_drv.c | 33 ++++++++++----------- 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/drivers/crypto/intel/qat/qat_c62x/adf_drv.c b/drivers/crypto/intel/qat/qat_c62x/adf_drv.c index 8a7bdec358d6..0bac717e88d9 100644 --- a/drivers/crypto/intel/qat/qat_c62x/adf_drv.c +++ b/drivers/crypto/intel/qat/qat_c62x/adf_drv.c @@ -19,24 +19,6 @@ #include #include "adf_c62x_hw_data.h" -static const struct pci_device_id adf_pci_tbl[] = { - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QAT_C62X), }, - { } -}; -MODULE_DEVICE_TABLE(pci, adf_pci_tbl); - -static int adf_probe(struct pci_dev *dev, const struct pci_device_id *ent); -static void adf_remove(struct pci_dev *dev); - -static struct pci_driver adf_driver = { - .id_table = adf_pci_tbl, - .name = ADF_C62X_DEVICE_NAME, - .probe = adf_probe, - .remove = adf_remove, - .sriov_configure = adf_sriov_configure, - .err_handler = &adf_err_handler, -}; - static void adf_cleanup_pci_dev(struct adf_accel_dev *accel_dev) { pci_release_regions(accel_dev->accel_pci_dev.pci_dev); @@ -227,6 +209,21 @@ static void adf_remove(struct pci_dev *pdev) kfree(accel_dev); } +static const struct pci_device_id adf_pci_tbl[] = { + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QAT_C62X) }, + { } +}; +MODULE_DEVICE_TABLE(pci, adf_pci_tbl); + +static struct pci_driver adf_driver = { + .id_table = adf_pci_tbl, + .name = ADF_C62X_DEVICE_NAME, + .probe = adf_probe, + .remove = adf_remove, + .sriov_configure = adf_sriov_configure, + .err_handler = &adf_err_handler, +}; + static int __init adfdrv_init(void) { request_module("intel_qat"); From a9a6e9279b2998e2610c70b0dfc80a234f97c76c Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Wed, 26 Mar 2025 15:59:51 +0000 Subject: [PATCH 027/382] crypto: qat - add shutdown handler to qat_c62x During a warm reset via kexec, the system bypasses the driver removal sequence, meaning that the remove() callback is not invoked. If a QAT device is not shutdown properly, the device driver will fail to load in a newly rebooted kernel. This might result in output like the following after the kexec reboot: QAT: AE0 is inactive!! QAT: failed to get device out of reset c6xx 0000:3f:00.0: qat_hal_clr_reset error c6xx 0000:3f:00.0: Failed to init the AEs c6xx 0000:3f:00.0: Failed to initialise Acceleration Engine c6xx 0000:3f:00.0: Resetting device qat_dev0 c6xx 0000:3f:00.0: probe with driver c6xx failed with error -14 Implement the shutdown() handler that hooks into the reboot notifier list. This brings down the QAT device and ensures it is shut down properly. Cc: Fixes: a6dabee6c8ba ("crypto: qat - add support for c62x accel type") Reviewed-by: Ahsan Atta Reviewed-by: Andy Shevchenko Signed-off-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/intel/qat/qat_c62x/adf_drv.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/drivers/crypto/intel/qat/qat_c62x/adf_drv.c b/drivers/crypto/intel/qat/qat_c62x/adf_drv.c index 0bac717e88d9..23ccb72b6ea2 100644 --- a/drivers/crypto/intel/qat/qat_c62x/adf_drv.c +++ b/drivers/crypto/intel/qat/qat_c62x/adf_drv.c @@ -209,6 +209,13 @@ static void adf_remove(struct pci_dev *pdev) kfree(accel_dev); } +static void adf_shutdown(struct pci_dev *pdev) +{ + struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); + + adf_dev_down(accel_dev); +} + static const struct pci_device_id adf_pci_tbl[] = { { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QAT_C62X) }, { } @@ -220,6 +227,7 @@ static struct pci_driver adf_driver = { .name = ADF_C62X_DEVICE_NAME, .probe = adf_probe, .remove = adf_remove, + .shutdown = adf_shutdown, .sriov_configure = adf_sriov_configure, .err_handler = &adf_err_handler, }; From 4fc54f67cb474a25d40c15e8cbdba4e1cdaea61a Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Wed, 26 Mar 2025 15:59:52 +0000 Subject: [PATCH 028/382] crypto: qat - remove redundant prototypes in qat_c3xxx Move the definition of the adf_driver structure and remove the redundant prototypes for the functions adf_probe() and adf_remove() in the qat_c3xxx driver. Also move the pci_device_id table close to where it is used and drop the inner comma as it is not required. This does not introduce any functional change. Signed-off-by: Giovanni Cabiddu Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- drivers/crypto/intel/qat/qat_c3xxx/adf_drv.c | 33 +++++++++----------- 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/drivers/crypto/intel/qat/qat_c3xxx/adf_drv.c b/drivers/crypto/intel/qat/qat_c3xxx/adf_drv.c index b825b35ab4bf..260f34d0d541 100644 --- a/drivers/crypto/intel/qat/qat_c3xxx/adf_drv.c +++ b/drivers/crypto/intel/qat/qat_c3xxx/adf_drv.c @@ -19,24 +19,6 @@ #include #include "adf_c3xxx_hw_data.h" -static const struct pci_device_id adf_pci_tbl[] = { - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QAT_C3XXX), }, - { } -}; -MODULE_DEVICE_TABLE(pci, adf_pci_tbl); - -static int adf_probe(struct pci_dev *dev, const struct pci_device_id *ent); -static void adf_remove(struct pci_dev *dev); - -static struct pci_driver adf_driver = { - .id_table = adf_pci_tbl, - .name = ADF_C3XXX_DEVICE_NAME, - .probe = adf_probe, - .remove = adf_remove, - .sriov_configure = adf_sriov_configure, - .err_handler = &adf_err_handler, -}; - static void adf_cleanup_pci_dev(struct adf_accel_dev *accel_dev) { pci_release_regions(accel_dev->accel_pci_dev.pci_dev); @@ -227,6 +209,21 @@ static void adf_remove(struct pci_dev *pdev) kfree(accel_dev); } +static const struct pci_device_id adf_pci_tbl[] = { + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QAT_C3XXX) }, + { } +}; +MODULE_DEVICE_TABLE(pci, adf_pci_tbl); + +static struct pci_driver adf_driver = { + .id_table = adf_pci_tbl, + .name = ADF_C3XXX_DEVICE_NAME, + .probe = adf_probe, + .remove = adf_remove, + .sriov_configure = adf_sriov_configure, + .err_handler = &adf_err_handler, +}; + static int __init adfdrv_init(void) { request_module("intel_qat"); From 71e0cc1eab584d6f95526a5e8c69ec666ca33e1b Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Wed, 26 Mar 2025 15:59:53 +0000 Subject: [PATCH 029/382] crypto: qat - add shutdown handler to qat_c3xxx During a warm reset via kexec, the system bypasses the driver removal sequence, meaning that the remove() callback is not invoked. If a QAT device is not shutdown properly, the device driver will fail to load in a newly rebooted kernel. This might result in output like the following after the kexec reboot: QAT: AE0 is inactive!! QAT: failed to get device out of reset c3xxx 0000:3f:00.0: qat_hal_clr_reset error c3xxx 0000:3f:00.0: Failed to init the AEs c3xxx 0000:3f:00.0: Failed to initialise Acceleration Engine c3xxx 0000:3f:00.0: Resetting device qat_dev0 c3xxx 0000:3f:00.0: probe with driver c3xxx failed with error -14 Implement the shutdown() handler that hooks into the reboot notifier list. This brings down the QAT device and ensures it is shut down properly. Cc: Fixes: 890c55f4dc0e ("crypto: qat - add support for c3xxx accel type") Reviewed-by: Ahsan Atta Reviewed-by: Andy Shevchenko Signed-off-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/intel/qat/qat_c3xxx/adf_drv.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/drivers/crypto/intel/qat/qat_c3xxx/adf_drv.c b/drivers/crypto/intel/qat/qat_c3xxx/adf_drv.c index 260f34d0d541..bceb5dd8b148 100644 --- a/drivers/crypto/intel/qat/qat_c3xxx/adf_drv.c +++ b/drivers/crypto/intel/qat/qat_c3xxx/adf_drv.c @@ -209,6 +209,13 @@ static void adf_remove(struct pci_dev *pdev) kfree(accel_dev); } +static void adf_shutdown(struct pci_dev *pdev) +{ + struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); + + adf_dev_down(accel_dev); +} + static const struct pci_device_id adf_pci_tbl[] = { { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QAT_C3XXX) }, { } @@ -220,6 +227,7 @@ static struct pci_driver adf_driver = { .name = ADF_C3XXX_DEVICE_NAME, .probe = adf_probe, .remove = adf_remove, + .shutdown = adf_shutdown, .sriov_configure = adf_sriov_configure, .err_handler = &adf_err_handler, }; From e5661d9cf27b6964b6acc0eb8cbe9ce8827e36c8 Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Wed, 26 Mar 2025 16:23:01 +0000 Subject: [PATCH 030/382] crypto: qat - remove initialization in device class The structures adf_hw_device_class_* are static. Remove initialization to zero of the field instance as it is zero by C convention. This does not introduce any functional change. Signed-off-by: Giovanni Cabiddu Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c | 1 - drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c | 1 - drivers/crypto/intel/qat/qat_c3xxx/adf_c3xxx_hw_data.c | 1 - drivers/crypto/intel/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c | 1 - drivers/crypto/intel/qat/qat_c62x/adf_c62x_hw_data.c | 1 - drivers/crypto/intel/qat/qat_c62xvf/adf_c62xvf_hw_data.c | 1 - drivers/crypto/intel/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c | 1 - drivers/crypto/intel/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c | 1 - 8 files changed, 8 deletions(-) diff --git a/drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c b/drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c index 4feeef83f7a3..795f4598400b 100644 --- a/drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c +++ b/drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c @@ -93,7 +93,6 @@ static const struct adf_fw_config adf_fw_dcc_config[] = { static struct adf_hw_device_class adf_420xx_class = { .name = ADF_420XX_DEVICE_NAME, .type = DEV_420XX, - .instances = 0, }; static u32 get_ae_mask(struct adf_hw_device_data *self) diff --git a/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c b/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c index 4eb6ef99efdd..63c7c8e90cca 100644 --- a/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c +++ b/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c @@ -96,7 +96,6 @@ static_assert(ARRAY_SIZE(adf_fw_cy_config) == ARRAY_SIZE(adf_fw_dcc_config)); static struct adf_hw_device_class adf_4xxx_class = { .name = ADF_4XXX_DEVICE_NAME, .type = DEV_4XXX, - .instances = 0, }; static u32 get_ae_mask(struct adf_hw_device_data *self) diff --git a/drivers/crypto/intel/qat/qat_c3xxx/adf_c3xxx_hw_data.c b/drivers/crypto/intel/qat/qat_c3xxx/adf_c3xxx_hw_data.c index e78f7bfd30b8..9425af26d34c 100644 --- a/drivers/crypto/intel/qat/qat_c3xxx/adf_c3xxx_hw_data.c +++ b/drivers/crypto/intel/qat/qat_c3xxx/adf_c3xxx_hw_data.c @@ -22,7 +22,6 @@ static const u32 thrd_to_arb_map[ADF_C3XXX_MAX_ACCELENGINES] = { static struct adf_hw_device_class c3xxx_class = { .name = ADF_C3XXX_DEVICE_NAME, .type = DEV_C3XXX, - .instances = 0 }; static u32 get_accel_mask(struct adf_hw_device_data *self) diff --git a/drivers/crypto/intel/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c b/drivers/crypto/intel/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c index a512ca4efd3f..f73d9a4a9ab7 100644 --- a/drivers/crypto/intel/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c +++ b/drivers/crypto/intel/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c @@ -13,7 +13,6 @@ static struct adf_hw_device_class c3xxxiov_class = { .name = ADF_C3XXXVF_DEVICE_NAME, .type = DEV_C3XXXVF, - .instances = 0 }; static u32 get_accel_mask(struct adf_hw_device_data *self) diff --git a/drivers/crypto/intel/qat/qat_c62x/adf_c62x_hw_data.c b/drivers/crypto/intel/qat/qat_c62x/adf_c62x_hw_data.c index 32ebe09477a8..1a2f36b603fb 100644 --- a/drivers/crypto/intel/qat/qat_c62x/adf_c62x_hw_data.c +++ b/drivers/crypto/intel/qat/qat_c62x/adf_c62x_hw_data.c @@ -22,7 +22,6 @@ static const u32 thrd_to_arb_map[ADF_C62X_MAX_ACCELENGINES] = { static struct adf_hw_device_class c62x_class = { .name = ADF_C62X_DEVICE_NAME, .type = DEV_C62X, - .instances = 0 }; static u32 get_accel_mask(struct adf_hw_device_data *self) diff --git a/drivers/crypto/intel/qat/qat_c62xvf/adf_c62xvf_hw_data.c b/drivers/crypto/intel/qat/qat_c62xvf/adf_c62xvf_hw_data.c index 4aaaaf921734..29e53b41a895 100644 --- a/drivers/crypto/intel/qat/qat_c62xvf/adf_c62xvf_hw_data.c +++ b/drivers/crypto/intel/qat/qat_c62xvf/adf_c62xvf_hw_data.c @@ -13,7 +13,6 @@ static struct adf_hw_device_class c62xiov_class = { .name = ADF_C62XVF_DEVICE_NAME, .type = DEV_C62XVF, - .instances = 0 }; static u32 get_accel_mask(struct adf_hw_device_data *self) diff --git a/drivers/crypto/intel/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c b/drivers/crypto/intel/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c index e48bcf1818cd..bf9e8f34f451 100644 --- a/drivers/crypto/intel/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c +++ b/drivers/crypto/intel/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c @@ -24,7 +24,6 @@ static const u32 thrd_to_arb_map[ADF_DH895XCC_MAX_ACCELENGINES] = { static struct adf_hw_device_class dh895xcc_class = { .name = ADF_DH895XCC_DEVICE_NAME, .type = DEV_DH895XCC, - .instances = 0 }; static u32 get_accel_mask(struct adf_hw_device_data *self) diff --git a/drivers/crypto/intel/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c b/drivers/crypto/intel/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c index f4ee4c2e00da..bc59c1473eef 100644 --- a/drivers/crypto/intel/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c +++ b/drivers/crypto/intel/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c @@ -13,7 +13,6 @@ static struct adf_hw_device_class dh895xcciov_class = { .name = ADF_DH895XCCVF_DEVICE_NAME, .type = DEV_DH895XCCVF, - .instances = 0 }; static u32 get_accel_mask(struct adf_hw_device_data *self) From 1b58e2bb7762c13a5b87c683a8517ad2a1a3cda5 Mon Sep 17 00:00:00 2001 From: Suman Kumar Chakraborty Date: Fri, 28 Mar 2025 10:33:02 +0000 Subject: [PATCH 031/382] crypto: qat - remove BITS_IN_DWORD() The BITS_IN_DWORD() macro, which represents the number of bits in the registers accessed by the firmware loader, is currently defined as 32. For consistency and readability, replace this macro with the existing BITS_PER_TYPE() macro, which serves the same purpose. This does not introduce any functional change. Signed-off-by: Suman Kumar Chakraborty Reviewed-by: Andy Shevchenko Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- .../crypto/intel/qat/qat_common/qat_uclo.c | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/drivers/crypto/intel/qat/qat_common/qat_uclo.c b/drivers/crypto/intel/qat/qat_common/qat_uclo.c index 7678a93c6853..87e247ac1c9a 100644 --- a/drivers/crypto/intel/qat/qat_common/qat_uclo.c +++ b/drivers/crypto/intel/qat/qat_common/qat_uclo.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) /* Copyright(c) 2014 - 2020 Intel Corporation */ #include +#include #include #include #include @@ -1205,7 +1206,6 @@ static int qat_uclo_map_suof(struct icp_qat_fw_loader_handle *handle, } #define ADD_ADDR(high, low) ((((u64)high) << 32) + low) -#define BITS_IN_DWORD 32 static int qat_uclo_auth_fw(struct icp_qat_fw_loader_handle *handle, struct icp_qat_fw_auth_desc *desc) @@ -1223,7 +1223,7 @@ static int qat_uclo_auth_fw(struct icp_qat_fw_loader_handle *handle, fcu_dram_hi_csr = handle->chip_info->fcu_dram_addr_hi; fcu_dram_lo_csr = handle->chip_info->fcu_dram_addr_lo; - SET_CAP_CSR(handle, fcu_dram_hi_csr, (bus_addr >> BITS_IN_DWORD)); + SET_CAP_CSR(handle, fcu_dram_hi_csr, bus_addr >> BITS_PER_TYPE(u32)); SET_CAP_CSR(handle, fcu_dram_lo_csr, bus_addr); SET_CAP_CSR(handle, fcu_ctl_csr, FCU_CTRL_CMD_AUTH); @@ -1438,7 +1438,7 @@ static int qat_uclo_map_auth_fw(struct icp_qat_fw_loader_handle *handle, virt_base = (uintptr_t)img_desc.dram_base_addr_v + simg_offset; bus_addr = img_desc.dram_bus_addr + simg_offset; auth_desc = img_desc.dram_base_addr_v; - auth_desc->css_hdr_high = (unsigned int)(bus_addr >> BITS_IN_DWORD); + auth_desc->css_hdr_high = (unsigned int)(bus_addr >> BITS_PER_TYPE(u32)); auth_desc->css_hdr_low = (unsigned int)bus_addr; virt_addr = virt_base; @@ -1448,7 +1448,7 @@ static int qat_uclo_map_auth_fw(struct icp_qat_fw_loader_handle *handle, sizeof(*css_hdr); virt_addr = virt_addr + sizeof(*css_hdr); - auth_desc->fwsk_pub_high = (unsigned int)(bus_addr >> BITS_IN_DWORD); + auth_desc->fwsk_pub_high = (unsigned int)(bus_addr >> BITS_PER_TYPE(u32)); auth_desc->fwsk_pub_low = (unsigned int)bus_addr; memcpy((void *)(uintptr_t)virt_addr, @@ -1470,7 +1470,7 @@ static int qat_uclo_map_auth_fw(struct icp_qat_fw_loader_handle *handle, auth_desc->fwsk_pub_low) + ICP_QAT_CSS_FWSK_PUB_LEN(handle); virt_addr = virt_addr + ICP_QAT_CSS_FWSK_PUB_LEN(handle); - auth_desc->signature_high = (unsigned int)(bus_addr >> BITS_IN_DWORD); + auth_desc->signature_high = (unsigned int)(bus_addr >> BITS_PER_TYPE(u32)); auth_desc->signature_low = (unsigned int)bus_addr; memcpy((void *)(uintptr_t)virt_addr, @@ -1484,7 +1484,7 @@ static int qat_uclo_map_auth_fw(struct icp_qat_fw_loader_handle *handle, ICP_QAT_CSS_SIGNATURE_LEN(handle); virt_addr += ICP_QAT_CSS_SIGNATURE_LEN(handle); - auth_desc->img_high = (unsigned int)(bus_addr >> BITS_IN_DWORD); + auth_desc->img_high = (unsigned int)(bus_addr >> BITS_PER_TYPE(u32)); auth_desc->img_low = (unsigned int)bus_addr; auth_desc->img_len = size - ICP_QAT_AE_IMG_OFFSET(handle); if (bus_addr + auth_desc->img_len > img_desc.dram_bus_addr + @@ -1507,12 +1507,12 @@ static int qat_uclo_map_auth_fw(struct icp_qat_fw_loader_handle *handle, auth_desc->img_ae_mode_data_low) + sizeof(struct icp_qat_simg_ae_mode); - auth_desc->img_ae_init_data_high = (unsigned int) - (bus_addr >> BITS_IN_DWORD); + auth_desc->img_ae_init_data_high = + (unsigned int)(bus_addr >> BITS_PER_TYPE(u32)); auth_desc->img_ae_init_data_low = (unsigned int)bus_addr; bus_addr += ICP_QAT_SIMG_AE_INIT_SEQ_LEN; - auth_desc->img_ae_insts_high = (unsigned int) - (bus_addr >> BITS_IN_DWORD); + auth_desc->img_ae_insts_high = + (unsigned int)(bus_addr >> BITS_PER_TYPE(u32)); auth_desc->img_ae_insts_low = (unsigned int)bus_addr; virt_addr += sizeof(struct icp_qat_css_hdr); virt_addr += ICP_QAT_CSS_FWSK_PUB_LEN(handle); From 7188186ea657b64bf6a3c62b772a9148bbebb114 Mon Sep 17 00:00:00 2001 From: Ovidiu Panait Date: Sat, 29 Mar 2025 20:01:21 +0200 Subject: [PATCH 032/382] crypto: omap-aes - use dev_groups to register attribute groups Instead of manually adding attribute groups, set dev_groups pointer to have the driver core do it. Signed-off-by: Ovidiu Panait Signed-off-by: Herbert Xu --- drivers/crypto/omap-aes.c | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/drivers/crypto/omap-aes.c b/drivers/crypto/omap-aes.c index 551dd32a8db0..1ecf5f6ac04e 100644 --- a/drivers/crypto/omap-aes.c +++ b/drivers/crypto/omap-aes.c @@ -1086,10 +1086,7 @@ static struct attribute *omap_aes_attrs[] = { &dev_attr_fallback.attr, NULL, }; - -static const struct attribute_group omap_aes_attr_group = { - .attrs = omap_aes_attrs, -}; +ATTRIBUTE_GROUPS(omap_aes); static int omap_aes_probe(struct platform_device *pdev) { @@ -1215,12 +1212,6 @@ static int omap_aes_probe(struct platform_device *pdev) } } - err = sysfs_create_group(&dev->kobj, &omap_aes_attr_group); - if (err) { - dev_err(dev, "could not create sysfs device attrs\n"); - goto err_aead_algs; - } - return 0; err_aead_algs: for (i = dd->pdata->aead_algs_info->registered - 1; i >= 0; i--) { @@ -1277,8 +1268,6 @@ static void omap_aes_remove(struct platform_device *pdev) tasklet_kill(&dd->done_task); omap_aes_dma_cleanup(dd); pm_runtime_disable(dd->dev); - - sysfs_remove_group(&dd->dev->kobj, &omap_aes_attr_group); } #ifdef CONFIG_PM_SLEEP @@ -1304,6 +1293,7 @@ static struct platform_driver omap_aes_driver = { .name = "omap-aes", .pm = &omap_aes_pm_ops, .of_match_table = omap_aes_of_match, + .dev_groups = omap_aes_groups, }, }; From 0a5d5ff90229f51ce16a0fb5da93842b039a2dae Mon Sep 17 00:00:00 2001 From: Ovidiu Panait Date: Sat, 29 Mar 2025 20:01:22 +0200 Subject: [PATCH 033/382] crypto: omap-sham - use dev_groups to register attribute groups Instead of manually adding attribute groups, set dev_groups pointer to have the driver core do it. Signed-off-by: Ovidiu Panait Signed-off-by: Herbert Xu --- drivers/crypto/omap-sham.c | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/drivers/crypto/omap-sham.c b/drivers/crypto/omap-sham.c index 7021481bf027..56f192cb976d 100644 --- a/drivers/crypto/omap-sham.c +++ b/drivers/crypto/omap-sham.c @@ -2039,10 +2039,7 @@ static struct attribute *omap_sham_attrs[] = { &dev_attr_fallback.attr, NULL, }; - -static const struct attribute_group omap_sham_attr_group = { - .attrs = omap_sham_attrs, -}; +ATTRIBUTE_GROUPS(omap_sham); static int omap_sham_probe(struct platform_device *pdev) { @@ -2158,12 +2155,6 @@ static int omap_sham_probe(struct platform_device *pdev) } } - err = sysfs_create_group(&dev->kobj, &omap_sham_attr_group); - if (err) { - dev_err(dev, "could not create sysfs device attrs\n"); - goto err_algs; - } - return 0; err_algs: @@ -2210,8 +2201,6 @@ static void omap_sham_remove(struct platform_device *pdev) if (!dd->polling_mode) dma_release_channel(dd->dma_lch); - - sysfs_remove_group(&dd->dev->kobj, &omap_sham_attr_group); } static struct platform_driver omap_sham_driver = { @@ -2220,6 +2209,7 @@ static struct platform_driver omap_sham_driver = { .driver = { .name = "omap-sham", .of_match_table = omap_sham_of_match, + .dev_groups = omap_sham_groups, }, }; From 5ebc052d3b8233f72cfc0a57afed0cb4b2d2b287 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sun, 30 Mar 2025 09:31:28 +0800 Subject: [PATCH 034/382] MAINTAINERS: Update maintainers for crypto/marvell Remove the entries for Arnaud Ebalard and Boris Brezillon as requested. Link: https://lore.kernel.org/linux-crypto/20f0162643f94509b0928e17afb7efbd@ssi.gouv.fr/ Signed-off-by: Herbert Xu Acked-by: Boris Brezillon Signed-off-by: Herbert Xu --- MAINTAINERS | 2 -- 1 file changed, 2 deletions(-) diff --git a/MAINTAINERS b/MAINTAINERS index 96b827049501..a2604c13f11b 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -14267,8 +14267,6 @@ F: drivers/gpu/drm/armada/ F: include/uapi/drm/armada_drm.h MARVELL CRYPTO DRIVER -M: Boris Brezillon -M: Arnaud Ebalard M: Srujana Challa L: linux-crypto@vger.kernel.org S: Maintained From 7d14fbc5691b39b8ab322c0220713c2ac3298b29 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Tue, 1 Apr 2025 17:24:01 -0700 Subject: [PATCH 035/382] crypto: x86/aes - drop the avx10_256 AES-XTS and AES-CTR code Intel made a late change to the AVX10 specification that removes support for a 256-bit maximum vector length and enumeration of the maximum vector length. AVX10 will imply a maximum vector length of 512 bits. I.e. there won't be any such thing as AVX10/256 or AVX10/512; there will just be AVX10, and it will essentially just consolidate AVX512 features. As a result of this new development, my strategy of providing both *_avx10_256 and *_avx10_512 functions didn't turn out to be that useful. The only remaining motivation for the 256-bit AVX512 / AVX10 functions is to avoid downclocking on older Intel CPUs. But in the case of AES-XTS and AES-CTR, I already wrote *_avx2 code too (primarily to support CPUs without AVX512), which performs almost as well as *_avx10_256. So we should just use that. Therefore, remove the *_avx10_256 AES-XTS and AES-CTR functions and algorithms, and rename the *_avx10_512 AES-XTS and AES-CTR functions and algorithms to *_avx512. Make Ice Lake and Tiger Lake use *_avx2 instead of *_avx10_256 which they previously used. I've left AES-GCM unchanged for now. There is no VAES+AVX2 optimized AES-GCM in the kernel yet, so the path forward for that is not as clear. However, I did write a VAES+AVX2 optimized AES-GCM for BoringSSL. So one option is to port that to the kernel and then do the same cleanup. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/x86/crypto/aes-ctr-avx-x86_64.S | 47 ++++------- arch/x86/crypto/aes-xts-avx-x86_64.S | 118 ++++++++++++--------------- arch/x86/crypto/aesni-intel_glue.c | 30 +++---- 3 files changed, 74 insertions(+), 121 deletions(-) diff --git a/arch/x86/crypto/aes-ctr-avx-x86_64.S b/arch/x86/crypto/aes-ctr-avx-x86_64.S index 1685d8b24b2c..bbbfd80f5a50 100644 --- a/arch/x86/crypto/aes-ctr-avx-x86_64.S +++ b/arch/x86/crypto/aes-ctr-avx-x86_64.S @@ -48,8 +48,7 @@ // using the following sets of CPU features: // - AES-NI && AVX // - VAES && AVX2 -// - VAES && (AVX10/256 || (AVX512BW && AVX512VL)) && BMI2 -// - VAES && (AVX10/512 || (AVX512BW && AVX512VL)) && BMI2 +// - VAES && AVX512BW && AVX512VL && BMI2 // // See the function definitions at the bottom of the file for more information. @@ -76,7 +75,6 @@ .text // Move a vector between memory and a register. -// The register operand must be in the first 16 vector registers. .macro _vmovdqu src, dst .if VL < 64 vmovdqu \src, \dst @@ -86,7 +84,6 @@ .endm // Move a vector between registers. -// The registers must be in the first 16 vector registers. .macro _vmovdqa src, dst .if VL < 64 vmovdqa \src, \dst @@ -96,7 +93,7 @@ .endm // Broadcast a 128-bit value from memory to all 128-bit lanes of a vector -// register. The register operand must be in the first 16 vector registers. +// register. .macro _vbroadcast128 src, dst .if VL == 16 vmovdqu \src, \dst @@ -108,7 +105,6 @@ .endm // XOR two vectors together. -// Any register operands must be in the first 16 vector registers. .macro _vpxor src1, src2, dst .if VL < 64 vpxor \src1, \src2, \dst @@ -199,8 +195,8 @@ // XOR each with the zero-th round key. Also update LE_CTR if !\final. .macro _prepare_2_ctr_vecs is_xctr, i0, i1, final=0 .if \is_xctr - .if USE_AVX10 - _vmovdqa LE_CTR, AESDATA\i0 + .if USE_AVX512 + vmovdqa64 LE_CTR, AESDATA\i0 vpternlogd $0x96, XCTR_IV, RNDKEY0, AESDATA\i0 .else vpxor XCTR_IV, LE_CTR, AESDATA\i0 @@ -208,7 +204,7 @@ .endif vpaddq LE_CTR_INC1, LE_CTR, AESDATA\i1 - .if USE_AVX10 + .if USE_AVX512 vpternlogd $0x96, XCTR_IV, RNDKEY0, AESDATA\i1 .else vpxor XCTR_IV, AESDATA\i1, AESDATA\i1 @@ -481,18 +477,12 @@ .Lxor_tail_partial_vec_0\@: // XOR the remaining 1 <= LEN < VL bytes. It's easy if masked // loads/stores are available; otherwise it's a bit harder... -.if USE_AVX10 - .if VL <= 32 - mov $-1, %eax - bzhi LEN, %eax, %eax - kmovd %eax, %k1 - .else +.if USE_AVX512 mov $-1, %rax bzhi LEN64, %rax, %rax kmovq %rax, %k1 - .endif vmovdqu8 (SRC), AESDATA1{%k1}{z} - _vpxor AESDATA1, AESDATA0, AESDATA0 + vpxord AESDATA1, AESDATA0, AESDATA0 vmovdqu8 AESDATA0, (DST){%k1} .else .if VL == 32 @@ -554,7 +544,7 @@ // eliminates carries. |ctr| is the per-message block counter starting at 1. .set VL, 16 -.set USE_AVX10, 0 +.set USE_AVX512, 0 SYM_TYPED_FUNC_START(aes_ctr64_crypt_aesni_avx) _aes_ctr_crypt 0 SYM_FUNC_END(aes_ctr64_crypt_aesni_avx) @@ -564,7 +554,7 @@ SYM_FUNC_END(aes_xctr_crypt_aesni_avx) #if defined(CONFIG_AS_VAES) && defined(CONFIG_AS_VPCLMULQDQ) .set VL, 32 -.set USE_AVX10, 0 +.set USE_AVX512, 0 SYM_TYPED_FUNC_START(aes_ctr64_crypt_vaes_avx2) _aes_ctr_crypt 0 SYM_FUNC_END(aes_ctr64_crypt_vaes_avx2) @@ -572,21 +562,12 @@ SYM_TYPED_FUNC_START(aes_xctr_crypt_vaes_avx2) _aes_ctr_crypt 1 SYM_FUNC_END(aes_xctr_crypt_vaes_avx2) -.set VL, 32 -.set USE_AVX10, 1 -SYM_TYPED_FUNC_START(aes_ctr64_crypt_vaes_avx10_256) - _aes_ctr_crypt 0 -SYM_FUNC_END(aes_ctr64_crypt_vaes_avx10_256) -SYM_TYPED_FUNC_START(aes_xctr_crypt_vaes_avx10_256) - _aes_ctr_crypt 1 -SYM_FUNC_END(aes_xctr_crypt_vaes_avx10_256) - .set VL, 64 -.set USE_AVX10, 1 -SYM_TYPED_FUNC_START(aes_ctr64_crypt_vaes_avx10_512) +.set USE_AVX512, 1 +SYM_TYPED_FUNC_START(aes_ctr64_crypt_vaes_avx512) _aes_ctr_crypt 0 -SYM_FUNC_END(aes_ctr64_crypt_vaes_avx10_512) -SYM_TYPED_FUNC_START(aes_xctr_crypt_vaes_avx10_512) +SYM_FUNC_END(aes_ctr64_crypt_vaes_avx512) +SYM_TYPED_FUNC_START(aes_xctr_crypt_vaes_avx512) _aes_ctr_crypt 1 -SYM_FUNC_END(aes_xctr_crypt_vaes_avx10_512) +SYM_FUNC_END(aes_xctr_crypt_vaes_avx512) #endif // CONFIG_AS_VAES && CONFIG_AS_VPCLMULQDQ diff --git a/arch/x86/crypto/aes-xts-avx-x86_64.S b/arch/x86/crypto/aes-xts-avx-x86_64.S index 93ba0ddbe009..bbeaccbd1c51 100644 --- a/arch/x86/crypto/aes-xts-avx-x86_64.S +++ b/arch/x86/crypto/aes-xts-avx-x86_64.S @@ -52,32 +52,25 @@ * different code, it uses a macro to generate several implementations that * share similar source code but are targeted at different CPUs, listed below: * - * AES-NI + AVX + * AES-NI && AVX * - 128-bit vectors (1 AES block per vector) * - VEX-coded instructions * - xmm0-xmm15 * - This is for older CPUs that lack VAES but do have AVX. * - * VAES + VPCLMULQDQ + AVX2 + * VAES && VPCLMULQDQ && AVX2 * - 256-bit vectors (2 AES blocks per vector) * - VEX-coded instructions * - ymm0-ymm15 - * - This is for CPUs that have VAES but lack AVX512 or AVX10, - * e.g. Intel's Alder Lake and AMD's Zen 3. + * - This is for CPUs that have VAES but either lack AVX512 (e.g. Intel's + * Alder Lake and AMD's Zen 3) or downclock too eagerly when using zmm + * registers (e.g. Intel's Ice Lake). * - * VAES + VPCLMULQDQ + AVX10/256 + BMI2 - * - 256-bit vectors (2 AES blocks per vector) + * VAES && VPCLMULQDQ && AVX512BW && AVX512VL && BMI2 + * - 512-bit vectors (4 AES blocks per vector) * - EVEX-coded instructions - * - ymm0-ymm31 - * - This is for CPUs that have AVX512 but where using zmm registers causes - * downclocking, and for CPUs that have AVX10/256 but not AVX10/512. - * - By "AVX10/256" we really mean (AVX512BW + AVX512VL) || AVX10/256. - * To avoid confusion with 512-bit, we just write AVX10/256. - * - * VAES + VPCLMULQDQ + AVX10/512 + BMI2 - * - Same as the previous one, but upgrades to 512-bit vectors - * (4 AES blocks per vector) in zmm0-zmm31. - * - This is for CPUs that have good AVX512 or AVX10/512 support. + * - zmm0-zmm31 + * - This is for CPUs that have good AVX512 support. * * This file doesn't have an implementation for AES-NI alone (without AVX), as * the lack of VEX would make all the assembly code different. @@ -109,7 +102,7 @@ // This table contains constants for vpshufb and vpblendvb, used to // handle variable byte shifts and blending during ciphertext stealing - // on CPUs that don't support AVX10-style masking. + // on CPUs that don't support AVX512-style masking. .Lcts_permute_table: .byte 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80 .byte 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80 @@ -138,7 +131,7 @@ .irp i, 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15 _define_Vi \i .endr -.if USE_AVX10 +.if USE_AVX512 .irp i, 16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31 _define_Vi \i .endr @@ -193,7 +186,7 @@ // keys to the *end* of this register range. I.e., AES-128 uses // KEY5-KEY14, AES-192 uses KEY3-KEY14, and AES-256 uses KEY1-KEY14. // (All also use KEY0 for the XOR-only "round" at the beginning.) -.if USE_AVX10 +.if USE_AVX512 .set KEY1_XMM, %xmm16 .set KEY1, V16 .set KEY2_XMM, %xmm17 @@ -227,7 +220,6 @@ .endm // Move a vector between memory and a register. -// The register operand must be in the first 16 vector registers. .macro _vmovdqu src, dst .if VL < 64 vmovdqu \src, \dst @@ -238,9 +230,9 @@ // Broadcast a 128-bit value into a vector. .macro _vbroadcast128 src, dst -.if VL == 16 && !USE_AVX10 +.if VL == 16 vmovdqu \src, \dst -.elseif VL == 32 && !USE_AVX10 +.elseif VL == 32 vbroadcasti128 \src, \dst .else vbroadcasti32x4 \src, \dst @@ -248,7 +240,6 @@ .endm // XOR two vectors together. -// Any register operands must be in the first 16 vector registers. .macro _vpxor src1, src2, dst .if VL < 64 vpxor \src1, \src2, \dst @@ -259,7 +250,7 @@ // XOR three vectors together. .macro _xor3 src1, src2, src3_and_dst -.if USE_AVX10 +.if USE_AVX512 // vpternlogd with immediate 0x96 is a three-argument XOR. vpternlogd $0x96, \src1, \src2, \src3_and_dst .else @@ -274,7 +265,7 @@ vpshufd $0x13, \src, \tmp vpaddq \src, \src, \dst vpsrad $31, \tmp, \tmp -.if USE_AVX10 +.if USE_AVX512 vpternlogd $0x78, GF_POLY_XMM, \tmp, \dst .else vpand GF_POLY_XMM, \tmp, \tmp @@ -337,7 +328,7 @@ vpsllq $1*VL/16, TWEAK0, TWEAK1 vpsllq $2*VL/16, TWEAK0, TWEAK2 vpsllq $3*VL/16, TWEAK0, TWEAK3 -.if USE_AVX10 +.if USE_AVX512 vpternlogd $0x96, V0, V1, TWEAK1 vpternlogd $0x96, V2, V3, TWEAK2 vpternlogd $0x96, V4, V5, TWEAK3 @@ -474,26 +465,26 @@ lea OFFS-16(KEY, KEYLEN64, 4), KEY // If all 32 SIMD registers are available, cache all the round keys. -.if USE_AVX10 +.if USE_AVX512 cmp $24, KEYLEN jl .Laes128\@ je .Laes192\@ - _vbroadcast128 -6*16(KEY), KEY1 - _vbroadcast128 -5*16(KEY), KEY2 + vbroadcasti32x4 -6*16(KEY), KEY1 + vbroadcasti32x4 -5*16(KEY), KEY2 .Laes192\@: - _vbroadcast128 -4*16(KEY), KEY3 - _vbroadcast128 -3*16(KEY), KEY4 + vbroadcasti32x4 -4*16(KEY), KEY3 + vbroadcasti32x4 -3*16(KEY), KEY4 .Laes128\@: - _vbroadcast128 -2*16(KEY), KEY5 - _vbroadcast128 -1*16(KEY), KEY6 - _vbroadcast128 0*16(KEY), KEY7 - _vbroadcast128 1*16(KEY), KEY8 - _vbroadcast128 2*16(KEY), KEY9 - _vbroadcast128 3*16(KEY), KEY10 - _vbroadcast128 4*16(KEY), KEY11 - _vbroadcast128 5*16(KEY), KEY12 - _vbroadcast128 6*16(KEY), KEY13 - _vbroadcast128 7*16(KEY), KEY14 + vbroadcasti32x4 -2*16(KEY), KEY5 + vbroadcasti32x4 -1*16(KEY), KEY6 + vbroadcasti32x4 0*16(KEY), KEY7 + vbroadcasti32x4 1*16(KEY), KEY8 + vbroadcasti32x4 2*16(KEY), KEY9 + vbroadcasti32x4 3*16(KEY), KEY10 + vbroadcasti32x4 4*16(KEY), KEY11 + vbroadcasti32x4 5*16(KEY), KEY12 + vbroadcasti32x4 6*16(KEY), KEY13 + vbroadcasti32x4 7*16(KEY), KEY14 .endif .endm @@ -521,7 +512,7 @@ // using the same key for all block(s). The round key is loaded from the // appropriate register or memory location for round \i. May clobber \tmp. .macro _vaes_1x enc, i, xmm_suffix, data, tmp -.if USE_AVX10 +.if USE_AVX512 _vaes \enc, KEY\i\xmm_suffix, \data .else .ifnb \xmm_suffix @@ -538,7 +529,7 @@ // appropriate register or memory location for round \i. In addition, does two // steps of the computation of the next set of tweaks. May clobber V4 and V5. .macro _vaes_4x enc, i -.if USE_AVX10 +.if USE_AVX512 _tweak_step (2*(\i-5)) _vaes \enc, KEY\i, V0 _vaes \enc, KEY\i, V1 @@ -574,7 +565,7 @@ .irp i, 5,6,7,8,9,10,11,12,13 _vaes_1x \enc, \i, \xmm_suffix, \data, tmp=\tmp .endr -.if USE_AVX10 +.if USE_AVX512 vpxord KEY14\xmm_suffix, \tweak, \tmp .else .ifnb \xmm_suffix @@ -617,11 +608,11 @@ // This is the main loop, en/decrypting 4*VL bytes per iteration. // XOR each source block with its tweak and the zero-th round key. -.if USE_AVX10 - _vmovdqu 0*VL(SRC), V0 - _vmovdqu 1*VL(SRC), V1 - _vmovdqu 2*VL(SRC), V2 - _vmovdqu 3*VL(SRC), V3 +.if USE_AVX512 + vmovdqu8 0*VL(SRC), V0 + vmovdqu8 1*VL(SRC), V1 + vmovdqu8 2*VL(SRC), V2 + vmovdqu8 3*VL(SRC), V3 vpternlogd $0x96, TWEAK0, KEY0, V0 vpternlogd $0x96, TWEAK1, KEY0, V1 vpternlogd $0x96, TWEAK2, KEY0, V2 @@ -654,7 +645,7 @@ // Reduce latency by doing the XOR before the vaesenclast, utilizing the // property vaesenclast(key, a) ^ b == vaesenclast(key ^ b, a) // (and likewise for vaesdeclast). -.if USE_AVX10 +.if USE_AVX512 _tweak_step 18 _tweak_step 19 vpxord TWEAK0, KEY14, V4 @@ -762,7 +753,7 @@ _aes_crypt \enc, _XMM, TWEAK1_XMM, %xmm0, tmp=%xmm1 .endif -.if USE_AVX10 +.if USE_AVX512 // Create a mask that has the first LEN bits set. mov $-1, %r9d bzhi LEN, %r9d, %r9d @@ -811,7 +802,7 @@ // u8 iv[AES_BLOCK_SIZE]); // // Encrypt |iv| using the AES key |tweak_key| to get the first tweak. Assumes -// that the CPU supports AES-NI and AVX, but not necessarily VAES or AVX10. +// that the CPU supports AES-NI and AVX, but not necessarily VAES or AVX512. SYM_TYPED_FUNC_START(aes_xts_encrypt_iv) .set TWEAK_KEY, %rdi .set IV, %rsi @@ -853,7 +844,7 @@ SYM_FUNC_END(aes_xts_encrypt_iv) // multiple of 16, then this function updates |tweak| to contain the next tweak. .set VL, 16 -.set USE_AVX10, 0 +.set USE_AVX512, 0 SYM_TYPED_FUNC_START(aes_xts_encrypt_aesni_avx) _aes_xts_crypt 1 SYM_FUNC_END(aes_xts_encrypt_aesni_avx) @@ -863,7 +854,7 @@ SYM_FUNC_END(aes_xts_decrypt_aesni_avx) #if defined(CONFIG_AS_VAES) && defined(CONFIG_AS_VPCLMULQDQ) .set VL, 32 -.set USE_AVX10, 0 +.set USE_AVX512, 0 SYM_TYPED_FUNC_START(aes_xts_encrypt_vaes_avx2) _aes_xts_crypt 1 SYM_FUNC_END(aes_xts_encrypt_vaes_avx2) @@ -871,21 +862,12 @@ SYM_TYPED_FUNC_START(aes_xts_decrypt_vaes_avx2) _aes_xts_crypt 0 SYM_FUNC_END(aes_xts_decrypt_vaes_avx2) -.set VL, 32 -.set USE_AVX10, 1 -SYM_TYPED_FUNC_START(aes_xts_encrypt_vaes_avx10_256) - _aes_xts_crypt 1 -SYM_FUNC_END(aes_xts_encrypt_vaes_avx10_256) -SYM_TYPED_FUNC_START(aes_xts_decrypt_vaes_avx10_256) - _aes_xts_crypt 0 -SYM_FUNC_END(aes_xts_decrypt_vaes_avx10_256) - .set VL, 64 -.set USE_AVX10, 1 -SYM_TYPED_FUNC_START(aes_xts_encrypt_vaes_avx10_512) +.set USE_AVX512, 1 +SYM_TYPED_FUNC_START(aes_xts_encrypt_vaes_avx512) _aes_xts_crypt 1 -SYM_FUNC_END(aes_xts_encrypt_vaes_avx10_512) -SYM_TYPED_FUNC_START(aes_xts_decrypt_vaes_avx10_512) +SYM_FUNC_END(aes_xts_encrypt_vaes_avx512) +SYM_TYPED_FUNC_START(aes_xts_decrypt_vaes_avx512) _aes_xts_crypt 0 -SYM_FUNC_END(aes_xts_decrypt_vaes_avx10_512) +SYM_FUNC_END(aes_xts_decrypt_vaes_avx512) #endif /* CONFIG_AS_VAES && CONFIG_AS_VPCLMULQDQ */ diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c index bc655d794a95..5bdeda39cef6 100644 --- a/arch/x86/crypto/aesni-intel_glue.c +++ b/arch/x86/crypto/aesni-intel_glue.c @@ -844,8 +844,7 @@ simd_skcipher_algs_##suffix[ARRAY_SIZE(skcipher_algs_##suffix)] DEFINE_AVX_SKCIPHER_ALGS(aesni_avx, "aesni-avx", 500); #if defined(CONFIG_AS_VAES) && defined(CONFIG_AS_VPCLMULQDQ) DEFINE_AVX_SKCIPHER_ALGS(vaes_avx2, "vaes-avx2", 600); -DEFINE_AVX_SKCIPHER_ALGS(vaes_avx10_256, "vaes-avx10_256", 700); -DEFINE_AVX_SKCIPHER_ALGS(vaes_avx10_512, "vaes-avx10_512", 800); +DEFINE_AVX_SKCIPHER_ALGS(vaes_avx512, "vaes-avx512", 800); #endif /* The common part of the x86_64 AES-GCM key struct */ @@ -1592,11 +1591,6 @@ static int __init register_avx_algs(void) XFEATURE_MASK_AVX512, NULL)) return 0; - err = simd_register_skciphers_compat(skcipher_algs_vaes_avx10_256, - ARRAY_SIZE(skcipher_algs_vaes_avx10_256), - simd_skcipher_algs_vaes_avx10_256); - if (err) - return err; err = simd_register_aeads_compat(aes_gcm_algs_vaes_avx10_256, ARRAY_SIZE(aes_gcm_algs_vaes_avx10_256), aes_gcm_simdalgs_vaes_avx10_256); @@ -1606,15 +1600,15 @@ static int __init register_avx_algs(void) if (boot_cpu_has(X86_FEATURE_PREFER_YMM)) { int i; - for (i = 0; i < ARRAY_SIZE(skcipher_algs_vaes_avx10_512); i++) - skcipher_algs_vaes_avx10_512[i].base.cra_priority = 1; + for (i = 0; i < ARRAY_SIZE(skcipher_algs_vaes_avx512); i++) + skcipher_algs_vaes_avx512[i].base.cra_priority = 1; for (i = 0; i < ARRAY_SIZE(aes_gcm_algs_vaes_avx10_512); i++) aes_gcm_algs_vaes_avx10_512[i].base.cra_priority = 1; } - err = simd_register_skciphers_compat(skcipher_algs_vaes_avx10_512, - ARRAY_SIZE(skcipher_algs_vaes_avx10_512), - simd_skcipher_algs_vaes_avx10_512); + err = simd_register_skciphers_compat(skcipher_algs_vaes_avx512, + ARRAY_SIZE(skcipher_algs_vaes_avx512), + simd_skcipher_algs_vaes_avx512); if (err) return err; err = simd_register_aeads_compat(aes_gcm_algs_vaes_avx10_512, @@ -1641,18 +1635,14 @@ static void unregister_avx_algs(void) simd_unregister_skciphers(skcipher_algs_vaes_avx2, ARRAY_SIZE(skcipher_algs_vaes_avx2), simd_skcipher_algs_vaes_avx2); - if (simd_skcipher_algs_vaes_avx10_256[0]) - simd_unregister_skciphers(skcipher_algs_vaes_avx10_256, - ARRAY_SIZE(skcipher_algs_vaes_avx10_256), - simd_skcipher_algs_vaes_avx10_256); if (aes_gcm_simdalgs_vaes_avx10_256[0]) simd_unregister_aeads(aes_gcm_algs_vaes_avx10_256, ARRAY_SIZE(aes_gcm_algs_vaes_avx10_256), aes_gcm_simdalgs_vaes_avx10_256); - if (simd_skcipher_algs_vaes_avx10_512[0]) - simd_unregister_skciphers(skcipher_algs_vaes_avx10_512, - ARRAY_SIZE(skcipher_algs_vaes_avx10_512), - simd_skcipher_algs_vaes_avx10_512); + if (simd_skcipher_algs_vaes_avx512[0]) + simd_unregister_skciphers(skcipher_algs_vaes_avx512, + ARRAY_SIZE(skcipher_algs_vaes_avx512), + simd_skcipher_algs_vaes_avx512); if (aes_gcm_simdalgs_vaes_avx10_512[0]) simd_unregister_aeads(aes_gcm_algs_vaes_avx10_512, ARRAY_SIZE(aes_gcm_algs_vaes_avx10_512), From 3a7dfdbbe333226be65164edcaf7b493f05fbe21 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Tue, 1 Apr 2025 17:24:02 -0700 Subject: [PATCH 036/382] crypto: x86/aegis - stop using the SIMD helper Stop wrapping skcipher and aead algorithms with the crypto SIMD helper (crypto/simd.c). The only purpose of doing so was to work around x86 not always supporting kernel-mode FPU in softirqs. Specifically, if a hardirq interrupted a task context kernel-mode FPU section and then a softirqs were run at the end of that hardirq, those softirqs could not use kernel-mode FPU. This has now been fixed. In combination with the fact that the skcipher and aead APIs only support task and softirq contexts, these can now just use kernel-mode FPU unconditionally on x86. This simplifies the code and improves performance. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/x86/crypto/Kconfig | 1 - arch/x86/crypto/aegis128-aesni-glue.c | 13 ++++--------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/arch/x86/crypto/Kconfig b/arch/x86/crypto/Kconfig index 3d948f10c94c..c15400efac07 100644 --- a/arch/x86/crypto/Kconfig +++ b/arch/x86/crypto/Kconfig @@ -369,7 +369,6 @@ config CRYPTO_AEGIS128_AESNI_SSE2 tristate "AEAD ciphers: AEGIS-128 (AES-NI/SSE4.1)" depends on X86 && 64BIT select CRYPTO_AEAD - select CRYPTO_SIMD help AEGIS-128 AEAD algorithm diff --git a/arch/x86/crypto/aegis128-aesni-glue.c b/arch/x86/crypto/aegis128-aesni-glue.c index 26786e15abac..f1b6d40154e3 100644 --- a/arch/x86/crypto/aegis128-aesni-glue.c +++ b/arch/x86/crypto/aegis128-aesni-glue.c @@ -8,7 +8,6 @@ */ #include -#include #include #include #include @@ -233,21 +232,18 @@ static struct aead_alg crypto_aegis128_aesni_alg = { .chunksize = AEGIS128_BLOCK_SIZE, .base = { - .cra_flags = CRYPTO_ALG_INTERNAL, .cra_blocksize = 1, .cra_ctxsize = sizeof(struct aegis_ctx) + __alignof__(struct aegis_ctx), .cra_priority = 400, - .cra_name = "__aegis128", - .cra_driver_name = "__aegis128-aesni", + .cra_name = "aegis128", + .cra_driver_name = "aegis128-aesni", .cra_module = THIS_MODULE, } }; -static struct simd_aead_alg *simd_alg; - static int __init crypto_aegis128_aesni_module_init(void) { if (!boot_cpu_has(X86_FEATURE_XMM4_1) || @@ -255,13 +251,12 @@ static int __init crypto_aegis128_aesni_module_init(void) !cpu_has_xfeatures(XFEATURE_MASK_SSE, NULL)) return -ENODEV; - return simd_register_aeads_compat(&crypto_aegis128_aesni_alg, 1, - &simd_alg); + return crypto_register_aead(&crypto_aegis128_aesni_alg); } static void __exit crypto_aegis128_aesni_module_exit(void) { - simd_unregister_aeads(&crypto_aegis128_aesni_alg, 1, &simd_alg); + crypto_unregister_aead(&crypto_aegis128_aesni_alg); } module_init(crypto_aegis128_aesni_module_init); From 0ba6ec5b297204cde62bbbde8944219500c6229c Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Tue, 1 Apr 2025 17:24:03 -0700 Subject: [PATCH 037/382] crypto: x86/aes - stop using the SIMD helper Stop wrapping skcipher and aead algorithms with the crypto SIMD helper (crypto/simd.c). The only purpose of doing so was to work around x86 not always supporting kernel-mode FPU in softirqs. Specifically, if a hardirq interrupted a task context kernel-mode FPU section and then a softirqs were run at the end of that hardirq, those softirqs could not use kernel-mode FPU. This has now been fixed. In combination with the fact that the skcipher and aead APIs only support task and softirq contexts, these can now just use kernel-mode FPU unconditionally on x86. This simplifies the code and improves performance. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/x86/crypto/Kconfig | 1 - arch/x86/crypto/aesni-intel_glue.c | 158 +++++++++++------------------ 2 files changed, 59 insertions(+), 100 deletions(-) diff --git a/arch/x86/crypto/Kconfig b/arch/x86/crypto/Kconfig index c15400efac07..d8f9d6279cb2 100644 --- a/arch/x86/crypto/Kconfig +++ b/arch/x86/crypto/Kconfig @@ -23,7 +23,6 @@ config CRYPTO_AES_NI_INTEL select CRYPTO_LIB_GF128MUL select CRYPTO_ALGAPI select CRYPTO_SKCIPHER - select CRYPTO_SIMD help Block cipher: AES cipher algorithms AEAD cipher: AES with GCM diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c index 5bdeda39cef6..061b1ced93c5 100644 --- a/arch/x86/crypto/aesni-intel_glue.c +++ b/arch/x86/crypto/aesni-intel_glue.c @@ -566,10 +566,9 @@ static struct crypto_alg aesni_cipher_alg = { static struct skcipher_alg aesni_skciphers[] = { { .base = { - .cra_name = "__ecb(aes)", - .cra_driver_name = "__ecb-aes-aesni", + .cra_name = "ecb(aes)", + .cra_driver_name = "ecb-aes-aesni", .cra_priority = 400, - .cra_flags = CRYPTO_ALG_INTERNAL, .cra_blocksize = AES_BLOCK_SIZE, .cra_ctxsize = CRYPTO_AES_CTX_SIZE, .cra_module = THIS_MODULE, @@ -581,10 +580,9 @@ static struct skcipher_alg aesni_skciphers[] = { .decrypt = ecb_decrypt, }, { .base = { - .cra_name = "__cbc(aes)", - .cra_driver_name = "__cbc-aes-aesni", + .cra_name = "cbc(aes)", + .cra_driver_name = "cbc-aes-aesni", .cra_priority = 400, - .cra_flags = CRYPTO_ALG_INTERNAL, .cra_blocksize = AES_BLOCK_SIZE, .cra_ctxsize = CRYPTO_AES_CTX_SIZE, .cra_module = THIS_MODULE, @@ -597,10 +595,9 @@ static struct skcipher_alg aesni_skciphers[] = { .decrypt = cbc_decrypt, }, { .base = { - .cra_name = "__cts(cbc(aes))", - .cra_driver_name = "__cts-cbc-aes-aesni", + .cra_name = "cts(cbc(aes))", + .cra_driver_name = "cts-cbc-aes-aesni", .cra_priority = 400, - .cra_flags = CRYPTO_ALG_INTERNAL, .cra_blocksize = AES_BLOCK_SIZE, .cra_ctxsize = CRYPTO_AES_CTX_SIZE, .cra_module = THIS_MODULE, @@ -615,10 +612,9 @@ static struct skcipher_alg aesni_skciphers[] = { #ifdef CONFIG_X86_64 }, { .base = { - .cra_name = "__ctr(aes)", - .cra_driver_name = "__ctr-aes-aesni", + .cra_name = "ctr(aes)", + .cra_driver_name = "ctr-aes-aesni", .cra_priority = 400, - .cra_flags = CRYPTO_ALG_INTERNAL, .cra_blocksize = 1, .cra_ctxsize = CRYPTO_AES_CTX_SIZE, .cra_module = THIS_MODULE, @@ -633,10 +629,9 @@ static struct skcipher_alg aesni_skciphers[] = { #endif }, { .base = { - .cra_name = "__xts(aes)", - .cra_driver_name = "__xts-aes-aesni", + .cra_name = "xts(aes)", + .cra_driver_name = "xts-aes-aesni", .cra_priority = 401, - .cra_flags = CRYPTO_ALG_INTERNAL, .cra_blocksize = AES_BLOCK_SIZE, .cra_ctxsize = XTS_AES_CTX_SIZE, .cra_module = THIS_MODULE, @@ -651,9 +646,6 @@ static struct skcipher_alg aesni_skciphers[] = { } }; -static -struct simd_skcipher_alg *aesni_simd_skciphers[ARRAY_SIZE(aesni_skciphers)]; - #ifdef CONFIG_X86_64 asmlinkage void aes_xts_encrypt_iv(const struct crypto_aes_ctx *tweak_key, u8 iv[AES_BLOCK_SIZE]); @@ -792,10 +784,9 @@ static int xctr_crypt_##suffix(struct skcipher_request *req) \ } \ \ static struct skcipher_alg skcipher_algs_##suffix[] = {{ \ - .base.cra_name = "__xts(aes)", \ - .base.cra_driver_name = "__xts-aes-" driver_name_suffix, \ + .base.cra_name = "xts(aes)", \ + .base.cra_driver_name = "xts-aes-" driver_name_suffix, \ .base.cra_priority = priority, \ - .base.cra_flags = CRYPTO_ALG_INTERNAL, \ .base.cra_blocksize = AES_BLOCK_SIZE, \ .base.cra_ctxsize = XTS_AES_CTX_SIZE, \ .base.cra_module = THIS_MODULE, \ @@ -807,10 +798,9 @@ static struct skcipher_alg skcipher_algs_##suffix[] = {{ \ .encrypt = xts_encrypt_##suffix, \ .decrypt = xts_decrypt_##suffix, \ }, { \ - .base.cra_name = "__ctr(aes)", \ - .base.cra_driver_name = "__ctr-aes-" driver_name_suffix, \ + .base.cra_name = "ctr(aes)", \ + .base.cra_driver_name = "ctr-aes-" driver_name_suffix, \ .base.cra_priority = priority, \ - .base.cra_flags = CRYPTO_ALG_INTERNAL, \ .base.cra_blocksize = 1, \ .base.cra_ctxsize = CRYPTO_AES_CTX_SIZE, \ .base.cra_module = THIS_MODULE, \ @@ -822,10 +812,9 @@ static struct skcipher_alg skcipher_algs_##suffix[] = {{ \ .encrypt = ctr_crypt_##suffix, \ .decrypt = ctr_crypt_##suffix, \ }, { \ - .base.cra_name = "__xctr(aes)", \ - .base.cra_driver_name = "__xctr-aes-" driver_name_suffix, \ + .base.cra_name = "xctr(aes)", \ + .base.cra_driver_name = "xctr-aes-" driver_name_suffix, \ .base.cra_priority = priority, \ - .base.cra_flags = CRYPTO_ALG_INTERNAL, \ .base.cra_blocksize = 1, \ .base.cra_ctxsize = CRYPTO_AES_CTX_SIZE, \ .base.cra_module = THIS_MODULE, \ @@ -836,10 +825,7 @@ static struct skcipher_alg skcipher_algs_##suffix[] = {{ \ .setkey = aesni_skcipher_setkey, \ .encrypt = xctr_crypt_##suffix, \ .decrypt = xctr_crypt_##suffix, \ -}}; \ - \ -static struct simd_skcipher_alg * \ -simd_skcipher_algs_##suffix[ARRAY_SIZE(skcipher_algs_##suffix)] +}} DEFINE_AVX_SKCIPHER_ALGS(aesni_avx, "aesni-avx", 500); #if defined(CONFIG_AS_VAES) && defined(CONFIG_AS_VPCLMULQDQ) @@ -1498,10 +1484,9 @@ static struct aead_alg aes_gcm_algs_##suffix[] = { { \ .chunksize = AES_BLOCK_SIZE, \ .maxauthsize = 16, \ .base = { \ - .cra_name = "__gcm(aes)", \ - .cra_driver_name = "__" generic_driver_name, \ + .cra_name = "gcm(aes)", \ + .cra_driver_name = generic_driver_name, \ .cra_priority = (priority), \ - .cra_flags = CRYPTO_ALG_INTERNAL, \ .cra_blocksize = 1, \ .cra_ctxsize = (ctxsize), \ .cra_module = THIS_MODULE, \ @@ -1515,17 +1500,14 @@ static struct aead_alg aes_gcm_algs_##suffix[] = { { \ .chunksize = AES_BLOCK_SIZE, \ .maxauthsize = 16, \ .base = { \ - .cra_name = "__rfc4106(gcm(aes))", \ - .cra_driver_name = "__" rfc_driver_name, \ + .cra_name = "rfc4106(gcm(aes))", \ + .cra_driver_name = rfc_driver_name, \ .cra_priority = (priority), \ - .cra_flags = CRYPTO_ALG_INTERNAL, \ .cra_blocksize = 1, \ .cra_ctxsize = (ctxsize), \ .cra_module = THIS_MODULE, \ }, \ -} }; \ - \ -static struct simd_aead_alg *aes_gcm_simdalgs_##suffix[2] \ +} } /* aes_gcm_algs_aesni */ DEFINE_GCM_ALGS(aesni, /* no flags */ 0, @@ -1555,14 +1537,12 @@ static int __init register_avx_algs(void) if (!boot_cpu_has(X86_FEATURE_AVX)) return 0; - err = simd_register_skciphers_compat(skcipher_algs_aesni_avx, - ARRAY_SIZE(skcipher_algs_aesni_avx), - simd_skcipher_algs_aesni_avx); + err = crypto_register_skciphers(skcipher_algs_aesni_avx, + ARRAY_SIZE(skcipher_algs_aesni_avx)); if (err) return err; - err = simd_register_aeads_compat(aes_gcm_algs_aesni_avx, - ARRAY_SIZE(aes_gcm_algs_aesni_avx), - aes_gcm_simdalgs_aesni_avx); + err = crypto_register_aeads(aes_gcm_algs_aesni_avx, + ARRAY_SIZE(aes_gcm_algs_aesni_avx)); if (err) return err; /* @@ -1578,9 +1558,8 @@ static int __init register_avx_algs(void) !boot_cpu_has(X86_FEATURE_PCLMULQDQ) || !cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL)) return 0; - err = simd_register_skciphers_compat(skcipher_algs_vaes_avx2, - ARRAY_SIZE(skcipher_algs_vaes_avx2), - simd_skcipher_algs_vaes_avx2); + err = crypto_register_skciphers(skcipher_algs_vaes_avx2, + ARRAY_SIZE(skcipher_algs_vaes_avx2)); if (err) return err; @@ -1591,9 +1570,8 @@ static int __init register_avx_algs(void) XFEATURE_MASK_AVX512, NULL)) return 0; - err = simd_register_aeads_compat(aes_gcm_algs_vaes_avx10_256, - ARRAY_SIZE(aes_gcm_algs_vaes_avx10_256), - aes_gcm_simdalgs_vaes_avx10_256); + err = crypto_register_aeads(aes_gcm_algs_vaes_avx10_256, + ARRAY_SIZE(aes_gcm_algs_vaes_avx10_256)); if (err) return err; @@ -1606,52 +1584,38 @@ static int __init register_avx_algs(void) aes_gcm_algs_vaes_avx10_512[i].base.cra_priority = 1; } - err = simd_register_skciphers_compat(skcipher_algs_vaes_avx512, - ARRAY_SIZE(skcipher_algs_vaes_avx512), - simd_skcipher_algs_vaes_avx512); + err = crypto_register_skciphers(skcipher_algs_vaes_avx512, + ARRAY_SIZE(skcipher_algs_vaes_avx512)); if (err) return err; - err = simd_register_aeads_compat(aes_gcm_algs_vaes_avx10_512, - ARRAY_SIZE(aes_gcm_algs_vaes_avx10_512), - aes_gcm_simdalgs_vaes_avx10_512); + err = crypto_register_aeads(aes_gcm_algs_vaes_avx10_512, + ARRAY_SIZE(aes_gcm_algs_vaes_avx10_512)); if (err) return err; #endif /* CONFIG_AS_VAES && CONFIG_AS_VPCLMULQDQ */ return 0; } +#define unregister_skciphers(A) \ + if (refcount_read(&(A)[0].base.cra_refcnt) != 0) \ + crypto_unregister_skciphers((A), ARRAY_SIZE(A)) +#define unregister_aeads(A) \ + if (refcount_read(&(A)[0].base.cra_refcnt) != 0) \ + crypto_unregister_aeads((A), ARRAY_SIZE(A)) + static void unregister_avx_algs(void) { - if (simd_skcipher_algs_aesni_avx[0]) - simd_unregister_skciphers(skcipher_algs_aesni_avx, - ARRAY_SIZE(skcipher_algs_aesni_avx), - simd_skcipher_algs_aesni_avx); - if (aes_gcm_simdalgs_aesni_avx[0]) - simd_unregister_aeads(aes_gcm_algs_aesni_avx, - ARRAY_SIZE(aes_gcm_algs_aesni_avx), - aes_gcm_simdalgs_aesni_avx); + unregister_skciphers(skcipher_algs_aesni_avx); + unregister_aeads(aes_gcm_algs_aesni_avx); #if defined(CONFIG_AS_VAES) && defined(CONFIG_AS_VPCLMULQDQ) - if (simd_skcipher_algs_vaes_avx2[0]) - simd_unregister_skciphers(skcipher_algs_vaes_avx2, - ARRAY_SIZE(skcipher_algs_vaes_avx2), - simd_skcipher_algs_vaes_avx2); - if (aes_gcm_simdalgs_vaes_avx10_256[0]) - simd_unregister_aeads(aes_gcm_algs_vaes_avx10_256, - ARRAY_SIZE(aes_gcm_algs_vaes_avx10_256), - aes_gcm_simdalgs_vaes_avx10_256); - if (simd_skcipher_algs_vaes_avx512[0]) - simd_unregister_skciphers(skcipher_algs_vaes_avx512, - ARRAY_SIZE(skcipher_algs_vaes_avx512), - simd_skcipher_algs_vaes_avx512); - if (aes_gcm_simdalgs_vaes_avx10_512[0]) - simd_unregister_aeads(aes_gcm_algs_vaes_avx10_512, - ARRAY_SIZE(aes_gcm_algs_vaes_avx10_512), - aes_gcm_simdalgs_vaes_avx10_512); + unregister_skciphers(skcipher_algs_vaes_avx2); + unregister_skciphers(skcipher_algs_vaes_avx512); + unregister_aeads(aes_gcm_algs_vaes_avx10_256); + unregister_aeads(aes_gcm_algs_vaes_avx10_512); #endif } #else /* CONFIG_X86_64 */ static struct aead_alg aes_gcm_algs_aesni[0]; -static struct simd_aead_alg *aes_gcm_simdalgs_aesni[0]; static int __init register_avx_algs(void) { @@ -1680,15 +1644,13 @@ static int __init aesni_init(void) if (err) return err; - err = simd_register_skciphers_compat(aesni_skciphers, - ARRAY_SIZE(aesni_skciphers), - aesni_simd_skciphers); + err = crypto_register_skciphers(aesni_skciphers, + ARRAY_SIZE(aesni_skciphers)); if (err) goto unregister_cipher; - err = simd_register_aeads_compat(aes_gcm_algs_aesni, - ARRAY_SIZE(aes_gcm_algs_aesni), - aes_gcm_simdalgs_aesni); + err = crypto_register_aeads(aes_gcm_algs_aesni, + ARRAY_SIZE(aes_gcm_algs_aesni)); if (err) goto unregister_skciphers; @@ -1700,12 +1662,11 @@ static int __init aesni_init(void) unregister_avx: unregister_avx_algs(); - simd_unregister_aeads(aes_gcm_algs_aesni, - ARRAY_SIZE(aes_gcm_algs_aesni), - aes_gcm_simdalgs_aesni); + crypto_unregister_aeads(aes_gcm_algs_aesni, + ARRAY_SIZE(aes_gcm_algs_aesni)); unregister_skciphers: - simd_unregister_skciphers(aesni_skciphers, ARRAY_SIZE(aesni_skciphers), - aesni_simd_skciphers); + crypto_unregister_skciphers(aesni_skciphers, + ARRAY_SIZE(aesni_skciphers)); unregister_cipher: crypto_unregister_alg(&aesni_cipher_alg); return err; @@ -1713,11 +1674,10 @@ unregister_cipher: static void __exit aesni_exit(void) { - simd_unregister_aeads(aes_gcm_algs_aesni, - ARRAY_SIZE(aes_gcm_algs_aesni), - aes_gcm_simdalgs_aesni); - simd_unregister_skciphers(aesni_skciphers, ARRAY_SIZE(aesni_skciphers), - aesni_simd_skciphers); + crypto_unregister_aeads(aes_gcm_algs_aesni, + ARRAY_SIZE(aes_gcm_algs_aesni)); + crypto_unregister_skciphers(aesni_skciphers, + ARRAY_SIZE(aesni_skciphers)); crypto_unregister_alg(&aesni_cipher_alg); unregister_avx_algs(); } From 6e3379b933bf4f1ceff38f1e5cb4cc1cb43f2c2d Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Tue, 1 Apr 2025 17:24:04 -0700 Subject: [PATCH 038/382] crypto: x86/aria - stop using the SIMD helper Stop wrapping skcipher and aead algorithms with the crypto SIMD helper (crypto/simd.c). The only purpose of doing so was to work around x86 not always supporting kernel-mode FPU in softirqs. Specifically, if a hardirq interrupted a task context kernel-mode FPU section and then a softirqs were run at the end of that hardirq, those softirqs could not use kernel-mode FPU. This has now been fixed. In combination with the fact that the skcipher and aead APIs only support task and softirq contexts, these can now just use kernel-mode FPU unconditionally on x86. This simplifies the code and improves performance. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/x86/crypto/Kconfig | 3 --- arch/x86/crypto/aria_aesni_avx2_glue.c | 22 +++++++--------------- arch/x86/crypto/aria_aesni_avx_glue.c | 20 ++++++-------------- arch/x86/crypto/aria_gfni_avx512_glue.c | 22 +++++++--------------- 4 files changed, 20 insertions(+), 47 deletions(-) diff --git a/arch/x86/crypto/Kconfig b/arch/x86/crypto/Kconfig index d8f9d6279cb2..ad00d53ab83d 100644 --- a/arch/x86/crypto/Kconfig +++ b/arch/x86/crypto/Kconfig @@ -296,7 +296,6 @@ config CRYPTO_ARIA_AESNI_AVX_X86_64 tristate "Ciphers: ARIA with modes: ECB, CTR (AES-NI/AVX/GFNI)" depends on X86 && 64BIT select CRYPTO_SKCIPHER - select CRYPTO_SIMD select CRYPTO_ALGAPI select CRYPTO_ARIA help @@ -314,7 +313,6 @@ config CRYPTO_ARIA_AESNI_AVX2_X86_64 tristate "Ciphers: ARIA with modes: ECB, CTR (AES-NI/AVX2/GFNI)" depends on X86 && 64BIT select CRYPTO_SKCIPHER - select CRYPTO_SIMD select CRYPTO_ALGAPI select CRYPTO_ARIA select CRYPTO_ARIA_AESNI_AVX_X86_64 @@ -333,7 +331,6 @@ config CRYPTO_ARIA_GFNI_AVX512_X86_64 tristate "Ciphers: ARIA with modes: ECB, CTR (AVX512/GFNI)" depends on X86 && 64BIT && AS_AVX512 && AS_GFNI select CRYPTO_SKCIPHER - select CRYPTO_SIMD select CRYPTO_ALGAPI select CRYPTO_ARIA select CRYPTO_ARIA_AESNI_AVX_X86_64 diff --git a/arch/x86/crypto/aria_aesni_avx2_glue.c b/arch/x86/crypto/aria_aesni_avx2_glue.c index 87a11804fc77..b4bddcd58457 100644 --- a/arch/x86/crypto/aria_aesni_avx2_glue.c +++ b/arch/x86/crypto/aria_aesni_avx2_glue.c @@ -6,7 +6,6 @@ */ #include -#include #include #include #include @@ -165,10 +164,9 @@ static int aria_avx2_init_tfm(struct crypto_skcipher *tfm) static struct skcipher_alg aria_algs[] = { { - .base.cra_name = "__ecb(aria)", - .base.cra_driver_name = "__ecb-aria-avx2", + .base.cra_name = "ecb(aria)", + .base.cra_driver_name = "ecb-aria-avx2", .base.cra_priority = 500, - .base.cra_flags = CRYPTO_ALG_INTERNAL, .base.cra_blocksize = ARIA_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct aria_ctx), .base.cra_module = THIS_MODULE, @@ -178,11 +176,10 @@ static struct skcipher_alg aria_algs[] = { .encrypt = aria_avx2_ecb_encrypt, .decrypt = aria_avx2_ecb_decrypt, }, { - .base.cra_name = "__ctr(aria)", - .base.cra_driver_name = "__ctr-aria-avx2", + .base.cra_name = "ctr(aria)", + .base.cra_driver_name = "ctr-aria-avx2", .base.cra_priority = 500, - .base.cra_flags = CRYPTO_ALG_INTERNAL | - CRYPTO_ALG_SKCIPHER_REQSIZE_LARGE, + .base.cra_flags = CRYPTO_ALG_SKCIPHER_REQSIZE_LARGE, .base.cra_blocksize = 1, .base.cra_ctxsize = sizeof(struct aria_ctx), .base.cra_module = THIS_MODULE, @@ -197,8 +194,6 @@ static struct skcipher_alg aria_algs[] = { } }; -static struct simd_skcipher_alg *aria_simd_algs[ARRAY_SIZE(aria_algs)]; - static int __init aria_avx2_init(void) { const char *feature_name; @@ -233,15 +228,12 @@ static int __init aria_avx2_init(void) aria_ops.aria_ctr_crypt_32way = aria_aesni_avx2_ctr_crypt_32way; } - return simd_register_skciphers_compat(aria_algs, - ARRAY_SIZE(aria_algs), - aria_simd_algs); + return crypto_register_skciphers(aria_algs, ARRAY_SIZE(aria_algs)); } static void __exit aria_avx2_exit(void) { - simd_unregister_skciphers(aria_algs, ARRAY_SIZE(aria_algs), - aria_simd_algs); + crypto_unregister_skciphers(aria_algs, ARRAY_SIZE(aria_algs)); } module_init(aria_avx2_init); diff --git a/arch/x86/crypto/aria_aesni_avx_glue.c b/arch/x86/crypto/aria_aesni_avx_glue.c index 4e1516b76669..ab9b38d05332 100644 --- a/arch/x86/crypto/aria_aesni_avx_glue.c +++ b/arch/x86/crypto/aria_aesni_avx_glue.c @@ -6,7 +6,6 @@ */ #include -#include #include #include #include @@ -152,10 +151,9 @@ static int aria_avx_init_tfm(struct crypto_skcipher *tfm) static struct skcipher_alg aria_algs[] = { { - .base.cra_name = "__ecb(aria)", - .base.cra_driver_name = "__ecb-aria-avx", + .base.cra_name = "ecb(aria)", + .base.cra_driver_name = "ecb-aria-avx", .base.cra_priority = 400, - .base.cra_flags = CRYPTO_ALG_INTERNAL, .base.cra_blocksize = ARIA_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct aria_ctx), .base.cra_module = THIS_MODULE, @@ -165,10 +163,9 @@ static struct skcipher_alg aria_algs[] = { .encrypt = aria_avx_ecb_encrypt, .decrypt = aria_avx_ecb_decrypt, }, { - .base.cra_name = "__ctr(aria)", - .base.cra_driver_name = "__ctr-aria-avx", + .base.cra_name = "ctr(aria)", + .base.cra_driver_name = "ctr-aria-avx", .base.cra_priority = 400, - .base.cra_flags = CRYPTO_ALG_INTERNAL, .base.cra_blocksize = 1, .base.cra_ctxsize = sizeof(struct aria_ctx), .base.cra_module = THIS_MODULE, @@ -184,8 +181,6 @@ static struct skcipher_alg aria_algs[] = { } }; -static struct simd_skcipher_alg *aria_simd_algs[ARRAY_SIZE(aria_algs)]; - static int __init aria_avx_init(void) { const char *feature_name; @@ -213,15 +208,12 @@ static int __init aria_avx_init(void) aria_ops.aria_ctr_crypt_16way = aria_aesni_avx_ctr_crypt_16way; } - return simd_register_skciphers_compat(aria_algs, - ARRAY_SIZE(aria_algs), - aria_simd_algs); + return crypto_register_skciphers(aria_algs, ARRAY_SIZE(aria_algs)); } static void __exit aria_avx_exit(void) { - simd_unregister_skciphers(aria_algs, ARRAY_SIZE(aria_algs), - aria_simd_algs); + crypto_unregister_skciphers(aria_algs, ARRAY_SIZE(aria_algs)); } module_init(aria_avx_init); diff --git a/arch/x86/crypto/aria_gfni_avx512_glue.c b/arch/x86/crypto/aria_gfni_avx512_glue.c index f4a2208d2638..363cbf4399cc 100644 --- a/arch/x86/crypto/aria_gfni_avx512_glue.c +++ b/arch/x86/crypto/aria_gfni_avx512_glue.c @@ -6,7 +6,6 @@ */ #include -#include #include #include #include @@ -165,10 +164,9 @@ static int aria_avx512_init_tfm(struct crypto_skcipher *tfm) static struct skcipher_alg aria_algs[] = { { - .base.cra_name = "__ecb(aria)", - .base.cra_driver_name = "__ecb-aria-avx512", + .base.cra_name = "ecb(aria)", + .base.cra_driver_name = "ecb-aria-avx512", .base.cra_priority = 600, - .base.cra_flags = CRYPTO_ALG_INTERNAL, .base.cra_blocksize = ARIA_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct aria_ctx), .base.cra_module = THIS_MODULE, @@ -178,11 +176,10 @@ static struct skcipher_alg aria_algs[] = { .encrypt = aria_avx512_ecb_encrypt, .decrypt = aria_avx512_ecb_decrypt, }, { - .base.cra_name = "__ctr(aria)", - .base.cra_driver_name = "__ctr-aria-avx512", + .base.cra_name = "ctr(aria)", + .base.cra_driver_name = "ctr-aria-avx512", .base.cra_priority = 600, - .base.cra_flags = CRYPTO_ALG_INTERNAL | - CRYPTO_ALG_SKCIPHER_REQSIZE_LARGE, + .base.cra_flags = CRYPTO_ALG_SKCIPHER_REQSIZE_LARGE, .base.cra_blocksize = 1, .base.cra_ctxsize = sizeof(struct aria_ctx), .base.cra_module = THIS_MODULE, @@ -197,8 +194,6 @@ static struct skcipher_alg aria_algs[] = { } }; -static struct simd_skcipher_alg *aria_simd_algs[ARRAY_SIZE(aria_algs)]; - static int __init aria_avx512_init(void) { const char *feature_name; @@ -229,15 +224,12 @@ static int __init aria_avx512_init(void) aria_ops.aria_decrypt_64way = aria_gfni_avx512_decrypt_64way; aria_ops.aria_ctr_crypt_64way = aria_gfni_avx512_ctr_crypt_64way; - return simd_register_skciphers_compat(aria_algs, - ARRAY_SIZE(aria_algs), - aria_simd_algs); + return crypto_register_skciphers(aria_algs, ARRAY_SIZE(aria_algs)); } static void __exit aria_avx512_exit(void) { - simd_unregister_skciphers(aria_algs, ARRAY_SIZE(aria_algs), - aria_simd_algs); + crypto_unregister_skciphers(aria_algs, ARRAY_SIZE(aria_algs)); } module_init(aria_avx512_init); From 3e862a87ffbc70f133e297abee17e687323fe5ab Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Tue, 1 Apr 2025 17:24:05 -0700 Subject: [PATCH 039/382] crypto: x86/camellia - stop using the SIMD helper Stop wrapping skcipher and aead algorithms with the crypto SIMD helper (crypto/simd.c). The only purpose of doing so was to work around x86 not always supporting kernel-mode FPU in softirqs. Specifically, if a hardirq interrupted a task context kernel-mode FPU section and then a softirqs were run at the end of that hardirq, those softirqs could not use kernel-mode FPU. This has now been fixed. In combination with the fact that the skcipher and aead APIs only support task and softirq contexts, these can now just use kernel-mode FPU unconditionally on x86. This simplifies the code and improves performance. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/x86/crypto/Kconfig | 1 - arch/x86/crypto/camellia_aesni_avx2_glue.c | 21 +++++++-------------- arch/x86/crypto/camellia_aesni_avx_glue.c | 21 +++++++-------------- 3 files changed, 14 insertions(+), 29 deletions(-) diff --git a/arch/x86/crypto/Kconfig b/arch/x86/crypto/Kconfig index ad00d53ab83d..de927df89ccf 100644 --- a/arch/x86/crypto/Kconfig +++ b/arch/x86/crypto/Kconfig @@ -63,7 +63,6 @@ config CRYPTO_CAMELLIA_AESNI_AVX_X86_64 depends on X86 && 64BIT select CRYPTO_SKCIPHER select CRYPTO_CAMELLIA_X86_64 - select CRYPTO_SIMD imply CRYPTO_XTS help Length-preserving ciphers: Camellia with ECB and CBC modes diff --git a/arch/x86/crypto/camellia_aesni_avx2_glue.c b/arch/x86/crypto/camellia_aesni_avx2_glue.c index e7e4d64e9577..2d2f4e16537c 100644 --- a/arch/x86/crypto/camellia_aesni_avx2_glue.c +++ b/arch/x86/crypto/camellia_aesni_avx2_glue.c @@ -6,7 +6,6 @@ */ #include -#include #include #include #include @@ -69,10 +68,9 @@ static int cbc_decrypt(struct skcipher_request *req) static struct skcipher_alg camellia_algs[] = { { - .base.cra_name = "__ecb(camellia)", - .base.cra_driver_name = "__ecb-camellia-aesni-avx2", + .base.cra_name = "ecb(camellia)", + .base.cra_driver_name = "ecb-camellia-aesni-avx2", .base.cra_priority = 500, - .base.cra_flags = CRYPTO_ALG_INTERNAL, .base.cra_blocksize = CAMELLIA_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct camellia_ctx), .base.cra_module = THIS_MODULE, @@ -82,10 +80,9 @@ static struct skcipher_alg camellia_algs[] = { .encrypt = ecb_encrypt, .decrypt = ecb_decrypt, }, { - .base.cra_name = "__cbc(camellia)", - .base.cra_driver_name = "__cbc-camellia-aesni-avx2", + .base.cra_name = "cbc(camellia)", + .base.cra_driver_name = "cbc-camellia-aesni-avx2", .base.cra_priority = 500, - .base.cra_flags = CRYPTO_ALG_INTERNAL, .base.cra_blocksize = CAMELLIA_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct camellia_ctx), .base.cra_module = THIS_MODULE, @@ -98,8 +95,6 @@ static struct skcipher_alg camellia_algs[] = { }, }; -static struct simd_skcipher_alg *camellia_simd_algs[ARRAY_SIZE(camellia_algs)]; - static int __init camellia_aesni_init(void) { const char *feature_name; @@ -118,15 +113,13 @@ static int __init camellia_aesni_init(void) return -ENODEV; } - return simd_register_skciphers_compat(camellia_algs, - ARRAY_SIZE(camellia_algs), - camellia_simd_algs); + return crypto_register_skciphers(camellia_algs, + ARRAY_SIZE(camellia_algs)); } static void __exit camellia_aesni_fini(void) { - simd_unregister_skciphers(camellia_algs, ARRAY_SIZE(camellia_algs), - camellia_simd_algs); + crypto_unregister_skciphers(camellia_algs, ARRAY_SIZE(camellia_algs)); } module_init(camellia_aesni_init); diff --git a/arch/x86/crypto/camellia_aesni_avx_glue.c b/arch/x86/crypto/camellia_aesni_avx_glue.c index c7ccf63e741e..a7d162388142 100644 --- a/arch/x86/crypto/camellia_aesni_avx_glue.c +++ b/arch/x86/crypto/camellia_aesni_avx_glue.c @@ -6,7 +6,6 @@ */ #include -#include #include #include #include @@ -69,10 +68,9 @@ static int cbc_decrypt(struct skcipher_request *req) static struct skcipher_alg camellia_algs[] = { { - .base.cra_name = "__ecb(camellia)", - .base.cra_driver_name = "__ecb-camellia-aesni", + .base.cra_name = "ecb(camellia)", + .base.cra_driver_name = "ecb-camellia-aesni", .base.cra_priority = 400, - .base.cra_flags = CRYPTO_ALG_INTERNAL, .base.cra_blocksize = CAMELLIA_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct camellia_ctx), .base.cra_module = THIS_MODULE, @@ -82,10 +80,9 @@ static struct skcipher_alg camellia_algs[] = { .encrypt = ecb_encrypt, .decrypt = ecb_decrypt, }, { - .base.cra_name = "__cbc(camellia)", - .base.cra_driver_name = "__cbc-camellia-aesni", + .base.cra_name = "cbc(camellia)", + .base.cra_driver_name = "cbc-camellia-aesni", .base.cra_priority = 400, - .base.cra_flags = CRYPTO_ALG_INTERNAL, .base.cra_blocksize = CAMELLIA_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct camellia_ctx), .base.cra_module = THIS_MODULE, @@ -98,8 +95,6 @@ static struct skcipher_alg camellia_algs[] = { } }; -static struct simd_skcipher_alg *camellia_simd_algs[ARRAY_SIZE(camellia_algs)]; - static int __init camellia_aesni_init(void) { const char *feature_name; @@ -117,15 +112,13 @@ static int __init camellia_aesni_init(void) return -ENODEV; } - return simd_register_skciphers_compat(camellia_algs, - ARRAY_SIZE(camellia_algs), - camellia_simd_algs); + return crypto_register_skciphers(camellia_algs, + ARRAY_SIZE(camellia_algs)); } static void __exit camellia_aesni_fini(void) { - simd_unregister_skciphers(camellia_algs, ARRAY_SIZE(camellia_algs), - camellia_simd_algs); + crypto_unregister_skciphers(camellia_algs, ARRAY_SIZE(camellia_algs)); } module_init(camellia_aesni_init); From ca6d0e8ed88d694c20e7d95d96fa83ff78376575 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Tue, 1 Apr 2025 17:24:06 -0700 Subject: [PATCH 040/382] crypto: x86/cast - stop using the SIMD helper Stop wrapping skcipher and aead algorithms with the crypto SIMD helper (crypto/simd.c). The only purpose of doing so was to work around x86 not always supporting kernel-mode FPU in softirqs. Specifically, if a hardirq interrupted a task context kernel-mode FPU section and then a softirqs were run at the end of that hardirq, those softirqs could not use kernel-mode FPU. This has now been fixed. In combination with the fact that the skcipher and aead APIs only support task and softirq contexts, these can now just use kernel-mode FPU unconditionally on x86. This simplifies the code and improves performance. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/x86/crypto/Kconfig | 2 -- arch/x86/crypto/cast5_avx_glue.c | 21 +++++++-------------- arch/x86/crypto/cast6_avx_glue.c | 20 ++++++-------------- 3 files changed, 13 insertions(+), 30 deletions(-) diff --git a/arch/x86/crypto/Kconfig b/arch/x86/crypto/Kconfig index de927df89ccf..55800d1ce668 100644 --- a/arch/x86/crypto/Kconfig +++ b/arch/x86/crypto/Kconfig @@ -88,7 +88,6 @@ config CRYPTO_CAST5_AVX_X86_64 select CRYPTO_SKCIPHER select CRYPTO_CAST5 select CRYPTO_CAST_COMMON - select CRYPTO_SIMD imply CRYPTO_CTR help Length-preserving ciphers: CAST5 (CAST-128) cipher algorithm @@ -105,7 +104,6 @@ config CRYPTO_CAST6_AVX_X86_64 select CRYPTO_SKCIPHER select CRYPTO_CAST6 select CRYPTO_CAST_COMMON - select CRYPTO_SIMD imply CRYPTO_XTS imply CRYPTO_CTR help diff --git a/arch/x86/crypto/cast5_avx_glue.c b/arch/x86/crypto/cast5_avx_glue.c index 3976a87f92ad..3aca04d43b34 100644 --- a/arch/x86/crypto/cast5_avx_glue.c +++ b/arch/x86/crypto/cast5_avx_glue.c @@ -8,7 +8,6 @@ #include #include -#include #include #include #include @@ -64,10 +63,9 @@ static int cbc_decrypt(struct skcipher_request *req) static struct skcipher_alg cast5_algs[] = { { - .base.cra_name = "__ecb(cast5)", - .base.cra_driver_name = "__ecb-cast5-avx", + .base.cra_name = "ecb(cast5)", + .base.cra_driver_name = "ecb-cast5-avx", .base.cra_priority = 200, - .base.cra_flags = CRYPTO_ALG_INTERNAL, .base.cra_blocksize = CAST5_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct cast5_ctx), .base.cra_module = THIS_MODULE, @@ -77,10 +75,9 @@ static struct skcipher_alg cast5_algs[] = { .encrypt = ecb_encrypt, .decrypt = ecb_decrypt, }, { - .base.cra_name = "__cbc(cast5)", - .base.cra_driver_name = "__cbc-cast5-avx", + .base.cra_name = "cbc(cast5)", + .base.cra_driver_name = "cbc-cast5-avx", .base.cra_priority = 200, - .base.cra_flags = CRYPTO_ALG_INTERNAL, .base.cra_blocksize = CAST5_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct cast5_ctx), .base.cra_module = THIS_MODULE, @@ -93,8 +90,6 @@ static struct skcipher_alg cast5_algs[] = { } }; -static struct simd_skcipher_alg *cast5_simd_algs[ARRAY_SIZE(cast5_algs)]; - static int __init cast5_init(void) { const char *feature_name; @@ -105,15 +100,13 @@ static int __init cast5_init(void) return -ENODEV; } - return simd_register_skciphers_compat(cast5_algs, - ARRAY_SIZE(cast5_algs), - cast5_simd_algs); + return crypto_register_skciphers(cast5_algs, + ARRAY_SIZE(cast5_algs)); } static void __exit cast5_exit(void) { - simd_unregister_skciphers(cast5_algs, ARRAY_SIZE(cast5_algs), - cast5_simd_algs); + crypto_unregister_skciphers(cast5_algs, ARRAY_SIZE(cast5_algs)); } module_init(cast5_init); diff --git a/arch/x86/crypto/cast6_avx_glue.c b/arch/x86/crypto/cast6_avx_glue.c index 7e2aea372349..c4dd28c30303 100644 --- a/arch/x86/crypto/cast6_avx_glue.c +++ b/arch/x86/crypto/cast6_avx_glue.c @@ -14,7 +14,6 @@ #include #include #include -#include #include "ecb_cbc_helpers.h" @@ -64,10 +63,9 @@ static int cbc_decrypt(struct skcipher_request *req) static struct skcipher_alg cast6_algs[] = { { - .base.cra_name = "__ecb(cast6)", - .base.cra_driver_name = "__ecb-cast6-avx", + .base.cra_name = "ecb(cast6)", + .base.cra_driver_name = "ecb-cast6-avx", .base.cra_priority = 200, - .base.cra_flags = CRYPTO_ALG_INTERNAL, .base.cra_blocksize = CAST6_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct cast6_ctx), .base.cra_module = THIS_MODULE, @@ -77,10 +75,9 @@ static struct skcipher_alg cast6_algs[] = { .encrypt = ecb_encrypt, .decrypt = ecb_decrypt, }, { - .base.cra_name = "__cbc(cast6)", - .base.cra_driver_name = "__cbc-cast6-avx", + .base.cra_name = "cbc(cast6)", + .base.cra_driver_name = "cbc-cast6-avx", .base.cra_priority = 200, - .base.cra_flags = CRYPTO_ALG_INTERNAL, .base.cra_blocksize = CAST6_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct cast6_ctx), .base.cra_module = THIS_MODULE, @@ -93,8 +90,6 @@ static struct skcipher_alg cast6_algs[] = { }, }; -static struct simd_skcipher_alg *cast6_simd_algs[ARRAY_SIZE(cast6_algs)]; - static int __init cast6_init(void) { const char *feature_name; @@ -105,15 +100,12 @@ static int __init cast6_init(void) return -ENODEV; } - return simd_register_skciphers_compat(cast6_algs, - ARRAY_SIZE(cast6_algs), - cast6_simd_algs); + return crypto_register_skciphers(cast6_algs, ARRAY_SIZE(cast6_algs)); } static void __exit cast6_exit(void) { - simd_unregister_skciphers(cast6_algs, ARRAY_SIZE(cast6_algs), - cast6_simd_algs); + crypto_unregister_skciphers(cast6_algs, ARRAY_SIZE(cast6_algs)); } module_init(cast6_init); From cc01d2840f2900c40e05fc10e082b348893172c4 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Tue, 1 Apr 2025 17:24:07 -0700 Subject: [PATCH 041/382] crypto: x86/serpent - stop using the SIMD helper Stop wrapping skcipher and aead algorithms with the crypto SIMD helper (crypto/simd.c). The only purpose of doing so was to work around x86 not always supporting kernel-mode FPU in softirqs. Specifically, if a hardirq interrupted a task context kernel-mode FPU section and then a softirqs were run at the end of that hardirq, those softirqs could not use kernel-mode FPU. This has now been fixed. In combination with the fact that the skcipher and aead APIs only support task and softirq contexts, these can now just use kernel-mode FPU unconditionally on x86. This simplifies the code and improves performance. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/x86/crypto/Kconfig | 3 --- arch/x86/crypto/serpent_avx2_glue.c | 21 +++++++-------------- arch/x86/crypto/serpent_avx_glue.c | 21 +++++++-------------- arch/x86/crypto/serpent_sse2_glue.c | 21 +++++++-------------- 4 files changed, 21 insertions(+), 45 deletions(-) diff --git a/arch/x86/crypto/Kconfig b/arch/x86/crypto/Kconfig index 55800d1ce668..51c74a496126 100644 --- a/arch/x86/crypto/Kconfig +++ b/arch/x86/crypto/Kconfig @@ -134,7 +134,6 @@ config CRYPTO_SERPENT_SSE2_X86_64 depends on X86 && 64BIT select CRYPTO_SKCIPHER select CRYPTO_SERPENT - select CRYPTO_SIMD imply CRYPTO_CTR help Length-preserving ciphers: Serpent cipher algorithm @@ -150,7 +149,6 @@ config CRYPTO_SERPENT_SSE2_586 depends on X86 && !64BIT select CRYPTO_SKCIPHER select CRYPTO_SERPENT - select CRYPTO_SIMD imply CRYPTO_CTR help Length-preserving ciphers: Serpent cipher algorithm @@ -166,7 +164,6 @@ config CRYPTO_SERPENT_AVX_X86_64 depends on X86 && 64BIT select CRYPTO_SKCIPHER select CRYPTO_SERPENT - select CRYPTO_SIMD imply CRYPTO_XTS imply CRYPTO_CTR help diff --git a/arch/x86/crypto/serpent_avx2_glue.c b/arch/x86/crypto/serpent_avx2_glue.c index 347e97f4b713..f5f2121b7956 100644 --- a/arch/x86/crypto/serpent_avx2_glue.c +++ b/arch/x86/crypto/serpent_avx2_glue.c @@ -10,7 +10,6 @@ #include #include #include -#include #include #include "serpent-avx.h" @@ -65,10 +64,9 @@ static int cbc_decrypt(struct skcipher_request *req) static struct skcipher_alg serpent_algs[] = { { - .base.cra_name = "__ecb(serpent)", - .base.cra_driver_name = "__ecb-serpent-avx2", + .base.cra_name = "ecb(serpent)", + .base.cra_driver_name = "ecb-serpent-avx2", .base.cra_priority = 600, - .base.cra_flags = CRYPTO_ALG_INTERNAL, .base.cra_blocksize = SERPENT_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct serpent_ctx), .base.cra_module = THIS_MODULE, @@ -78,10 +76,9 @@ static struct skcipher_alg serpent_algs[] = { .encrypt = ecb_encrypt, .decrypt = ecb_decrypt, }, { - .base.cra_name = "__cbc(serpent)", - .base.cra_driver_name = "__cbc-serpent-avx2", + .base.cra_name = "cbc(serpent)", + .base.cra_driver_name = "cbc-serpent-avx2", .base.cra_priority = 600, - .base.cra_flags = CRYPTO_ALG_INTERNAL, .base.cra_blocksize = SERPENT_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct serpent_ctx), .base.cra_module = THIS_MODULE, @@ -94,8 +91,6 @@ static struct skcipher_alg serpent_algs[] = { }, }; -static struct simd_skcipher_alg *serpent_simd_algs[ARRAY_SIZE(serpent_algs)]; - static int __init serpent_avx2_init(void) { const char *feature_name; @@ -110,15 +105,13 @@ static int __init serpent_avx2_init(void) return -ENODEV; } - return simd_register_skciphers_compat(serpent_algs, - ARRAY_SIZE(serpent_algs), - serpent_simd_algs); + return crypto_register_skciphers(serpent_algs, + ARRAY_SIZE(serpent_algs)); } static void __exit serpent_avx2_fini(void) { - simd_unregister_skciphers(serpent_algs, ARRAY_SIZE(serpent_algs), - serpent_simd_algs); + crypto_unregister_skciphers(serpent_algs, ARRAY_SIZE(serpent_algs)); } module_init(serpent_avx2_init); diff --git a/arch/x86/crypto/serpent_avx_glue.c b/arch/x86/crypto/serpent_avx_glue.c index 6c248e1ea4ef..e640abc1cb8a 100644 --- a/arch/x86/crypto/serpent_avx_glue.c +++ b/arch/x86/crypto/serpent_avx_glue.c @@ -13,7 +13,6 @@ #include #include #include -#include #include #include "serpent-avx.h" @@ -71,10 +70,9 @@ static int cbc_decrypt(struct skcipher_request *req) static struct skcipher_alg serpent_algs[] = { { - .base.cra_name = "__ecb(serpent)", - .base.cra_driver_name = "__ecb-serpent-avx", + .base.cra_name = "ecb(serpent)", + .base.cra_driver_name = "ecb-serpent-avx", .base.cra_priority = 500, - .base.cra_flags = CRYPTO_ALG_INTERNAL, .base.cra_blocksize = SERPENT_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct serpent_ctx), .base.cra_module = THIS_MODULE, @@ -84,10 +82,9 @@ static struct skcipher_alg serpent_algs[] = { .encrypt = ecb_encrypt, .decrypt = ecb_decrypt, }, { - .base.cra_name = "__cbc(serpent)", - .base.cra_driver_name = "__cbc-serpent-avx", + .base.cra_name = "cbc(serpent)", + .base.cra_driver_name = "cbc-serpent-avx", .base.cra_priority = 500, - .base.cra_flags = CRYPTO_ALG_INTERNAL, .base.cra_blocksize = SERPENT_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct serpent_ctx), .base.cra_module = THIS_MODULE, @@ -100,8 +97,6 @@ static struct skcipher_alg serpent_algs[] = { }, }; -static struct simd_skcipher_alg *serpent_simd_algs[ARRAY_SIZE(serpent_algs)]; - static int __init serpent_init(void) { const char *feature_name; @@ -112,15 +107,13 @@ static int __init serpent_init(void) return -ENODEV; } - return simd_register_skciphers_compat(serpent_algs, - ARRAY_SIZE(serpent_algs), - serpent_simd_algs); + return crypto_register_skciphers(serpent_algs, + ARRAY_SIZE(serpent_algs)); } static void __exit serpent_exit(void) { - simd_unregister_skciphers(serpent_algs, ARRAY_SIZE(serpent_algs), - serpent_simd_algs); + crypto_unregister_skciphers(serpent_algs, ARRAY_SIZE(serpent_algs)); } module_init(serpent_init); diff --git a/arch/x86/crypto/serpent_sse2_glue.c b/arch/x86/crypto/serpent_sse2_glue.c index d78f37e9b2cf..80ee17ec21b4 100644 --- a/arch/x86/crypto/serpent_sse2_glue.c +++ b/arch/x86/crypto/serpent_sse2_glue.c @@ -18,7 +18,6 @@ #include #include #include -#include #include #include "serpent-sse2.h" @@ -74,10 +73,9 @@ static int cbc_decrypt(struct skcipher_request *req) static struct skcipher_alg serpent_algs[] = { { - .base.cra_name = "__ecb(serpent)", - .base.cra_driver_name = "__ecb-serpent-sse2", + .base.cra_name = "ecb(serpent)", + .base.cra_driver_name = "ecb-serpent-sse2", .base.cra_priority = 400, - .base.cra_flags = CRYPTO_ALG_INTERNAL, .base.cra_blocksize = SERPENT_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct serpent_ctx), .base.cra_module = THIS_MODULE, @@ -87,10 +85,9 @@ static struct skcipher_alg serpent_algs[] = { .encrypt = ecb_encrypt, .decrypt = ecb_decrypt, }, { - .base.cra_name = "__cbc(serpent)", - .base.cra_driver_name = "__cbc-serpent-sse2", + .base.cra_name = "cbc(serpent)", + .base.cra_driver_name = "cbc-serpent-sse2", .base.cra_priority = 400, - .base.cra_flags = CRYPTO_ALG_INTERNAL, .base.cra_blocksize = SERPENT_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct serpent_ctx), .base.cra_module = THIS_MODULE, @@ -103,8 +100,6 @@ static struct skcipher_alg serpent_algs[] = { }, }; -static struct simd_skcipher_alg *serpent_simd_algs[ARRAY_SIZE(serpent_algs)]; - static int __init serpent_sse2_init(void) { if (!boot_cpu_has(X86_FEATURE_XMM2)) { @@ -112,15 +107,13 @@ static int __init serpent_sse2_init(void) return -ENODEV; } - return simd_register_skciphers_compat(serpent_algs, - ARRAY_SIZE(serpent_algs), - serpent_simd_algs); + return crypto_register_skciphers(serpent_algs, + ARRAY_SIZE(serpent_algs)); } static void __exit serpent_sse2_exit(void) { - simd_unregister_skciphers(serpent_algs, ARRAY_SIZE(serpent_algs), - serpent_simd_algs); + crypto_unregister_skciphers(serpent_algs, ARRAY_SIZE(serpent_algs)); } module_init(serpent_sse2_init); From 982b72cd00b502b30abe8d48d07d7512204e1d73 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Tue, 1 Apr 2025 17:24:08 -0700 Subject: [PATCH 042/382] crypto: x86/sm4 - stop using the SIMD helper Stop wrapping skcipher and aead algorithms with the crypto SIMD helper (crypto/simd.c). The only purpose of doing so was to work around x86 not always supporting kernel-mode FPU in softirqs. Specifically, if a hardirq interrupted a task context kernel-mode FPU section and then a softirqs were run at the end of that hardirq, those softirqs could not use kernel-mode FPU. This has now been fixed. In combination with the fact that the skcipher and aead APIs only support task and softirq contexts, these can now just use kernel-mode FPU unconditionally on x86. This simplifies the code and improves performance. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/x86/crypto/Kconfig | 2 -- arch/x86/crypto/sm4_aesni_avx2_glue.c | 31 ++++++++++----------------- arch/x86/crypto/sm4_aesni_avx_glue.c | 31 ++++++++++----------------- 3 files changed, 22 insertions(+), 42 deletions(-) diff --git a/arch/x86/crypto/Kconfig b/arch/x86/crypto/Kconfig index 51c74a496126..afc1a05e663d 100644 --- a/arch/x86/crypto/Kconfig +++ b/arch/x86/crypto/Kconfig @@ -192,7 +192,6 @@ config CRYPTO_SM4_AESNI_AVX_X86_64 tristate "Ciphers: SM4 with modes: ECB, CBC, CTR (AES-NI/AVX)" depends on X86 && 64BIT select CRYPTO_SKCIPHER - select CRYPTO_SIMD select CRYPTO_ALGAPI select CRYPTO_SM4 help @@ -213,7 +212,6 @@ config CRYPTO_SM4_AESNI_AVX2_X86_64 tristate "Ciphers: SM4 with modes: ECB, CBC, CTR (AES-NI/AVX2)" depends on X86 && 64BIT select CRYPTO_SKCIPHER - select CRYPTO_SIMD select CRYPTO_ALGAPI select CRYPTO_SM4 select CRYPTO_SM4_AESNI_AVX_X86_64 diff --git a/arch/x86/crypto/sm4_aesni_avx2_glue.c b/arch/x86/crypto/sm4_aesni_avx2_glue.c index 1148fd4cd57f..fec0ab7a63dd 100644 --- a/arch/x86/crypto/sm4_aesni_avx2_glue.c +++ b/arch/x86/crypto/sm4_aesni_avx2_glue.c @@ -8,11 +8,10 @@ * Copyright (c) 2021 Tianjia Zhang */ +#include #include #include #include -#include -#include #include #include #include "sm4-avx.h" @@ -48,10 +47,9 @@ static int ctr_crypt(struct skcipher_request *req) static struct skcipher_alg sm4_aesni_avx2_skciphers[] = { { .base = { - .cra_name = "__ecb(sm4)", - .cra_driver_name = "__ecb-sm4-aesni-avx2", + .cra_name = "ecb(sm4)", + .cra_driver_name = "ecb-sm4-aesni-avx2", .cra_priority = 500, - .cra_flags = CRYPTO_ALG_INTERNAL, .cra_blocksize = SM4_BLOCK_SIZE, .cra_ctxsize = sizeof(struct sm4_ctx), .cra_module = THIS_MODULE, @@ -64,10 +62,9 @@ static struct skcipher_alg sm4_aesni_avx2_skciphers[] = { .decrypt = sm4_avx_ecb_decrypt, }, { .base = { - .cra_name = "__cbc(sm4)", - .cra_driver_name = "__cbc-sm4-aesni-avx2", + .cra_name = "cbc(sm4)", + .cra_driver_name = "cbc-sm4-aesni-avx2", .cra_priority = 500, - .cra_flags = CRYPTO_ALG_INTERNAL, .cra_blocksize = SM4_BLOCK_SIZE, .cra_ctxsize = sizeof(struct sm4_ctx), .cra_module = THIS_MODULE, @@ -81,10 +78,9 @@ static struct skcipher_alg sm4_aesni_avx2_skciphers[] = { .decrypt = cbc_decrypt, }, { .base = { - .cra_name = "__ctr(sm4)", - .cra_driver_name = "__ctr-sm4-aesni-avx2", + .cra_name = "ctr(sm4)", + .cra_driver_name = "ctr-sm4-aesni-avx2", .cra_priority = 500, - .cra_flags = CRYPTO_ALG_INTERNAL, .cra_blocksize = 1, .cra_ctxsize = sizeof(struct sm4_ctx), .cra_module = THIS_MODULE, @@ -100,9 +96,6 @@ static struct skcipher_alg sm4_aesni_avx2_skciphers[] = { } }; -static struct simd_skcipher_alg * -simd_sm4_aesni_avx2_skciphers[ARRAY_SIZE(sm4_aesni_avx2_skciphers)]; - static int __init sm4_init(void) { const char *feature_name; @@ -121,16 +114,14 @@ static int __init sm4_init(void) return -ENODEV; } - return simd_register_skciphers_compat(sm4_aesni_avx2_skciphers, - ARRAY_SIZE(sm4_aesni_avx2_skciphers), - simd_sm4_aesni_avx2_skciphers); + return crypto_register_skciphers(sm4_aesni_avx2_skciphers, + ARRAY_SIZE(sm4_aesni_avx2_skciphers)); } static void __exit sm4_exit(void) { - simd_unregister_skciphers(sm4_aesni_avx2_skciphers, - ARRAY_SIZE(sm4_aesni_avx2_skciphers), - simd_sm4_aesni_avx2_skciphers); + crypto_unregister_skciphers(sm4_aesni_avx2_skciphers, + ARRAY_SIZE(sm4_aesni_avx2_skciphers)); } module_init(sm4_init); diff --git a/arch/x86/crypto/sm4_aesni_avx_glue.c b/arch/x86/crypto/sm4_aesni_avx_glue.c index 85b4ca78b47b..72867fc49ce8 100644 --- a/arch/x86/crypto/sm4_aesni_avx_glue.c +++ b/arch/x86/crypto/sm4_aesni_avx_glue.c @@ -8,11 +8,10 @@ * Copyright (c) 2021 Tianjia Zhang */ +#include #include #include #include -#include -#include #include #include #include "sm4-avx.h" @@ -263,10 +262,9 @@ static int ctr_crypt(struct skcipher_request *req) static struct skcipher_alg sm4_aesni_avx_skciphers[] = { { .base = { - .cra_name = "__ecb(sm4)", - .cra_driver_name = "__ecb-sm4-aesni-avx", + .cra_name = "ecb(sm4)", + .cra_driver_name = "ecb-sm4-aesni-avx", .cra_priority = 400, - .cra_flags = CRYPTO_ALG_INTERNAL, .cra_blocksize = SM4_BLOCK_SIZE, .cra_ctxsize = sizeof(struct sm4_ctx), .cra_module = THIS_MODULE, @@ -279,10 +277,9 @@ static struct skcipher_alg sm4_aesni_avx_skciphers[] = { .decrypt = sm4_avx_ecb_decrypt, }, { .base = { - .cra_name = "__cbc(sm4)", - .cra_driver_name = "__cbc-sm4-aesni-avx", + .cra_name = "cbc(sm4)", + .cra_driver_name = "cbc-sm4-aesni-avx", .cra_priority = 400, - .cra_flags = CRYPTO_ALG_INTERNAL, .cra_blocksize = SM4_BLOCK_SIZE, .cra_ctxsize = sizeof(struct sm4_ctx), .cra_module = THIS_MODULE, @@ -296,10 +293,9 @@ static struct skcipher_alg sm4_aesni_avx_skciphers[] = { .decrypt = cbc_decrypt, }, { .base = { - .cra_name = "__ctr(sm4)", - .cra_driver_name = "__ctr-sm4-aesni-avx", + .cra_name = "ctr(sm4)", + .cra_driver_name = "ctr-sm4-aesni-avx", .cra_priority = 400, - .cra_flags = CRYPTO_ALG_INTERNAL, .cra_blocksize = 1, .cra_ctxsize = sizeof(struct sm4_ctx), .cra_module = THIS_MODULE, @@ -315,9 +311,6 @@ static struct skcipher_alg sm4_aesni_avx_skciphers[] = { } }; -static struct simd_skcipher_alg * -simd_sm4_aesni_avx_skciphers[ARRAY_SIZE(sm4_aesni_avx_skciphers)]; - static int __init sm4_init(void) { const char *feature_name; @@ -335,16 +328,14 @@ static int __init sm4_init(void) return -ENODEV; } - return simd_register_skciphers_compat(sm4_aesni_avx_skciphers, - ARRAY_SIZE(sm4_aesni_avx_skciphers), - simd_sm4_aesni_avx_skciphers); + return crypto_register_skciphers(sm4_aesni_avx_skciphers, + ARRAY_SIZE(sm4_aesni_avx_skciphers)); } static void __exit sm4_exit(void) { - simd_unregister_skciphers(sm4_aesni_avx_skciphers, - ARRAY_SIZE(sm4_aesni_avx_skciphers), - simd_sm4_aesni_avx_skciphers); + crypto_unregister_skciphers(sm4_aesni_avx_skciphers, + ARRAY_SIZE(sm4_aesni_avx_skciphers)); } module_init(sm4_init); From bda5cd6e29e82ff0071f801aed75d2480d74703f Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Tue, 1 Apr 2025 17:24:09 -0700 Subject: [PATCH 043/382] crypto: x86/twofish - stop using the SIMD helper Stop wrapping skcipher and aead algorithms with the crypto SIMD helper (crypto/simd.c). The only purpose of doing so was to work around x86 not always supporting kernel-mode FPU in softirqs. Specifically, if a hardirq interrupted a task context kernel-mode FPU section and then a softirqs were run at the end of that hardirq, those softirqs could not use kernel-mode FPU. This has now been fixed. In combination with the fact that the skcipher and aead APIs only support task and softirq contexts, these can now just use kernel-mode FPU unconditionally on x86. This simplifies the code and improves performance. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/x86/crypto/Kconfig | 1 - arch/x86/crypto/twofish_avx_glue.c | 21 +++++++-------------- 2 files changed, 7 insertions(+), 15 deletions(-) diff --git a/arch/x86/crypto/Kconfig b/arch/x86/crypto/Kconfig index afc1a05e663d..f9e46e83440f 100644 --- a/arch/x86/crypto/Kconfig +++ b/arch/x86/crypto/Kconfig @@ -270,7 +270,6 @@ config CRYPTO_TWOFISH_AVX_X86_64 tristate "Ciphers: Twofish with modes: ECB, CBC (AVX)" depends on X86 && 64BIT select CRYPTO_SKCIPHER - select CRYPTO_SIMD select CRYPTO_TWOFISH_COMMON select CRYPTO_TWOFISH_X86_64 select CRYPTO_TWOFISH_X86_64_3WAY diff --git a/arch/x86/crypto/twofish_avx_glue.c b/arch/x86/crypto/twofish_avx_glue.c index 3eb3440b477a..9e20db013750 100644 --- a/arch/x86/crypto/twofish_avx_glue.c +++ b/arch/x86/crypto/twofish_avx_glue.c @@ -13,7 +13,6 @@ #include #include #include -#include #include #include "twofish.h" @@ -74,10 +73,9 @@ static int cbc_decrypt(struct skcipher_request *req) static struct skcipher_alg twofish_algs[] = { { - .base.cra_name = "__ecb(twofish)", - .base.cra_driver_name = "__ecb-twofish-avx", + .base.cra_name = "ecb(twofish)", + .base.cra_driver_name = "ecb-twofish-avx", .base.cra_priority = 400, - .base.cra_flags = CRYPTO_ALG_INTERNAL, .base.cra_blocksize = TF_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct twofish_ctx), .base.cra_module = THIS_MODULE, @@ -87,10 +85,9 @@ static struct skcipher_alg twofish_algs[] = { .encrypt = ecb_encrypt, .decrypt = ecb_decrypt, }, { - .base.cra_name = "__cbc(twofish)", - .base.cra_driver_name = "__cbc-twofish-avx", + .base.cra_name = "cbc(twofish)", + .base.cra_driver_name = "cbc-twofish-avx", .base.cra_priority = 400, - .base.cra_flags = CRYPTO_ALG_INTERNAL, .base.cra_blocksize = TF_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct twofish_ctx), .base.cra_module = THIS_MODULE, @@ -103,8 +100,6 @@ static struct skcipher_alg twofish_algs[] = { }, }; -static struct simd_skcipher_alg *twofish_simd_algs[ARRAY_SIZE(twofish_algs)]; - static int __init twofish_init(void) { const char *feature_name; @@ -114,15 +109,13 @@ static int __init twofish_init(void) return -ENODEV; } - return simd_register_skciphers_compat(twofish_algs, - ARRAY_SIZE(twofish_algs), - twofish_simd_algs); + return crypto_register_skciphers(twofish_algs, + ARRAY_SIZE(twofish_algs)); } static void __exit twofish_exit(void) { - simd_unregister_skciphers(twofish_algs, ARRAY_SIZE(twofish_algs), - twofish_simd_algs); + crypto_unregister_skciphers(twofish_algs, ARRAY_SIZE(twofish_algs)); } module_init(twofish_init); From 83366bcc7cb9bb6570363bef7cf05948cb00fe4e Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Wed, 2 Apr 2025 12:13:47 +0100 Subject: [PATCH 044/382] crypto: eip93 - Make read-only arrays static const Don't populate the read-only arrays sha256_init, sha224_init, sha1_init and md5_init on the stack at run time, instead make them static. Signed-off-by: Colin Ian King Reviewed-by: Antoine Tenart Signed-off-by: Herbert Xu --- .../crypto/inside-secure/eip93/eip93-hash.c | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/drivers/crypto/inside-secure/eip93/eip93-hash.c b/drivers/crypto/inside-secure/eip93/eip93-hash.c index df1b05ac5a57..ac13d90a2b7c 100644 --- a/drivers/crypto/inside-secure/eip93/eip93-hash.c +++ b/drivers/crypto/inside-secure/eip93/eip93-hash.c @@ -97,12 +97,20 @@ void eip93_hash_handle_result(struct crypto_async_request *async, int err) static void eip93_hash_init_sa_state_digest(u32 hash, u8 *digest) { - u32 sha256_init[] = { SHA256_H0, SHA256_H1, SHA256_H2, SHA256_H3, - SHA256_H4, SHA256_H5, SHA256_H6, SHA256_H7 }; - u32 sha224_init[] = { SHA224_H0, SHA224_H1, SHA224_H2, SHA224_H3, - SHA224_H4, SHA224_H5, SHA224_H6, SHA224_H7 }; - u32 sha1_init[] = { SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4 }; - u32 md5_init[] = { MD5_H0, MD5_H1, MD5_H2, MD5_H3 }; + static const u32 sha256_init[] = { + SHA256_H0, SHA256_H1, SHA256_H2, SHA256_H3, + SHA256_H4, SHA256_H5, SHA256_H6, SHA256_H7 + }; + static const u32 sha224_init[] = { + SHA224_H0, SHA224_H1, SHA224_H2, SHA224_H3, + SHA224_H4, SHA224_H5, SHA224_H6, SHA224_H7 + }; + static const u32 sha1_init[] = { + SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4 + }; + static const u32 md5_init[] = { + MD5_H0, MD5_H1, MD5_H2, MD5_H3 + }; /* Init HASH constant */ switch (hash) { From f98ed0dd58d9eabe5d240743fc26d4d22e202a07 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Thu, 3 Apr 2025 10:33:32 +0800 Subject: [PATCH 045/382] crypto: hash - Do not use shash in hard IRQs Update the documentation to be consistent with the fact that shash may not be used in hard IRQs. Reported-by: Eric Biggers Signed-off-by: Herbert Xu --- include/crypto/hash.h | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/include/crypto/hash.h b/include/crypto/hash.h index 2aa83ee0ec98..b987904afdba 100644 --- a/include/crypto/hash.h +++ b/include/crypto/hash.h @@ -834,7 +834,7 @@ static inline void *shash_desc_ctx(struct shash_desc *desc) * cipher handle must point to a keyed message digest cipher in order for this * function to succeed. * - * Context: Any context. + * Context: Softirq or process context. * Return: 0 if the setting of the key was successful; < 0 if an error occurred */ int crypto_shash_setkey(struct crypto_shash *tfm, const u8 *key, @@ -851,7 +851,7 @@ int crypto_shash_setkey(struct crypto_shash *tfm, const u8 *key, * crypto_shash_update and crypto_shash_final. The parameters have the same * meaning as discussed for those separate three functions. * - * Context: Any context. + * Context: Softirq or process context. * Return: 0 if the message digest creation was successful; < 0 if an error * occurred */ @@ -871,7 +871,7 @@ int crypto_shash_digest(struct shash_desc *desc, const u8 *data, * directly, and it allocates a hash descriptor on the stack internally. * Note that this stack allocation may be fairly large. * - * Context: Any context. + * Context: Softirq or process context. * Return: 0 on success; < 0 if an error occurred. */ int crypto_shash_tfm_digest(struct crypto_shash *tfm, const u8 *data, @@ -886,7 +886,7 @@ int crypto_shash_tfm_digest(struct crypto_shash *tfm, const u8 *data, * caller-allocated output buffer out which must have sufficient size (e.g. by * calling crypto_shash_descsize). * - * Context: Any context. + * Context: Softirq or process context. * Return: 0 if the export creation was successful; < 0 if an error occurred */ int crypto_shash_export(struct shash_desc *desc, void *out); @@ -900,7 +900,7 @@ int crypto_shash_export(struct shash_desc *desc, void *out); * the input buffer. That buffer should have been generated with the * crypto_ahash_export function. * - * Context: Any context. + * Context: Softirq or process context. * Return: 0 if the import was successful; < 0 if an error occurred */ int crypto_shash_import(struct shash_desc *desc, const void *in); @@ -913,7 +913,7 @@ int crypto_shash_import(struct shash_desc *desc, const void *in); * operational state handle. Any potentially existing state created by * previous operations is discarded. * - * Context: Any context. + * Context: Softirq or process context. * Return: 0 if the message digest initialization was successful; < 0 if an * error occurred */ @@ -935,7 +935,7 @@ static inline int crypto_shash_init(struct shash_desc *desc) * * Updates the message digest state of the operational state handle. * - * Context: Any context. + * Context: Softirq or process context. * Return: 0 if the message digest update was successful; < 0 if an error * occurred */ @@ -952,7 +952,7 @@ int crypto_shash_update(struct shash_desc *desc, const u8 *data, * into the output buffer. The caller must ensure that the output buffer is * large enough by using crypto_shash_digestsize. * - * Context: Any context. + * Context: Softirq or process context. * Return: 0 if the message digest creation was successful; < 0 if an error * occurred */ @@ -969,7 +969,7 @@ int crypto_shash_final(struct shash_desc *desc, u8 *out); * crypto_shash_update and crypto_shash_final. The parameters have the same * meaning as discussed for those separate functions. * - * Context: Any context. + * Context: Softirq or process context. * Return: 0 if the message digest creation was successful; < 0 if an error * occurred */ From 9b4400215e0e0b3edad010398069fcc51659a8cc Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Thu, 3 Apr 2025 12:14:50 +0800 Subject: [PATCH 046/382] crypto: x86/chacha - Remove SIMD fallback path Get rid of the fallback path as SIMD is now always usable in softirq context. Signed-off-by: Herbert Xu --- arch/x86/crypto/chacha_glue.c | 46 +++++++++-------------------------- 1 file changed, 11 insertions(+), 35 deletions(-) diff --git a/arch/x86/crypto/chacha_glue.c b/arch/x86/crypto/chacha_glue.c index 8bb74a272879..6a3d60cf3192 100644 --- a/arch/x86/crypto/chacha_glue.c +++ b/arch/x86/crypto/chacha_glue.c @@ -6,9 +6,7 @@ * Copyright (C) 2015 Martin Willi */ -#include #include -#include #include #include #include @@ -35,7 +33,6 @@ asmlinkage void chacha_4block_xor_avx512vl(u32 *state, u8 *dst, const u8 *src, asmlinkage void chacha_8block_xor_avx512vl(u32 *state, u8 *dst, const u8 *src, unsigned int len, int nrounds); -static __ro_after_init DEFINE_STATIC_KEY_FALSE(chacha_use_simd); static __ro_after_init DEFINE_STATIC_KEY_FALSE(chacha_use_avx2); static __ro_after_init DEFINE_STATIC_KEY_FALSE(chacha_use_avx512vl); @@ -123,23 +120,15 @@ static void chacha_dosimd(u32 *state, u8 *dst, const u8 *src, void hchacha_block_arch(const u32 *state, u32 *stream, int nrounds) { - if (!static_branch_likely(&chacha_use_simd) || !crypto_simd_usable()) { - hchacha_block_generic(state, stream, nrounds); - } else { - kernel_fpu_begin(); - hchacha_block_ssse3(state, stream, nrounds); - kernel_fpu_end(); - } + kernel_fpu_begin(); + hchacha_block_ssse3(state, stream, nrounds); + kernel_fpu_end(); } EXPORT_SYMBOL(hchacha_block_arch); void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *src, unsigned int bytes, int nrounds) { - if (!static_branch_likely(&chacha_use_simd) || !crypto_simd_usable() || - bytes <= CHACHA_BLOCK_SIZE) - return chacha_crypt_generic(state, dst, src, bytes, nrounds); - do { unsigned int todo = min_t(unsigned int, bytes, SZ_4K); @@ -171,18 +160,11 @@ static int chacha_simd_stream_xor(struct skcipher_request *req, if (nbytes < walk.total) nbytes = round_down(nbytes, walk.stride); - if (!static_branch_likely(&chacha_use_simd) || - !crypto_simd_usable()) { - chacha_crypt_generic(state, walk.dst.virt.addr, - walk.src.virt.addr, nbytes, - ctx->nrounds); - } else { - kernel_fpu_begin(); - chacha_dosimd(state, walk.dst.virt.addr, - walk.src.virt.addr, nbytes, - ctx->nrounds); - kernel_fpu_end(); - } + kernel_fpu_begin(); + chacha_dosimd(state, walk.dst.virt.addr, + walk.src.virt.addr, nbytes, + ctx->nrounds); + kernel_fpu_end(); err = skcipher_walk_done(&walk, walk.nbytes - nbytes); } @@ -207,13 +189,9 @@ static int xchacha_simd(struct skcipher_request *req) chacha_init(state, ctx->key, req->iv); - if (req->cryptlen > CHACHA_BLOCK_SIZE && crypto_simd_usable()) { - kernel_fpu_begin(); - hchacha_block_ssse3(state, subctx.key, ctx->nrounds); - kernel_fpu_end(); - } else { - hchacha_block_generic(state, subctx.key, ctx->nrounds); - } + kernel_fpu_begin(); + hchacha_block_ssse3(state, subctx.key, ctx->nrounds); + kernel_fpu_end(); subctx.nrounds = ctx->nrounds; memcpy(&real_iv[0], req->iv + 24, 8); @@ -275,8 +253,6 @@ static int __init chacha_simd_mod_init(void) if (!boot_cpu_has(X86_FEATURE_SSSE3)) return 0; - static_branch_enable(&chacha_use_simd); - if (boot_cpu_has(X86_FEATURE_AVX) && boot_cpu_has(X86_FEATURE_AVX2) && cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL)) { From e77fe9cce31b5db4a6b3f5fac7d69352612229c5 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Thu, 3 Apr 2025 09:19:55 +0200 Subject: [PATCH 047/382] crypto: arm/aes-ce - stop using the SIMD helper Now that ARM permits use of the NEON unit in softirq context as well as task context, there is no longer a need to rely on the SIMD helper module to construct async skciphers wrapping the sync ones, as the latter can always be called directly. So remove these wrappers and the dependency on the SIMD helper. This permits the use of these algorithms by callers that only support synchronous use. Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- arch/arm/crypto/Kconfig | 1 - arch/arm/crypto/aes-ce-glue.c | 104 ++++------------------------------ 2 files changed, 11 insertions(+), 94 deletions(-) diff --git a/arch/arm/crypto/Kconfig b/arch/arm/crypto/Kconfig index 23e4ea067ddb..2e73200b930a 100644 --- a/arch/arm/crypto/Kconfig +++ b/arch/arm/crypto/Kconfig @@ -200,7 +200,6 @@ config CRYPTO_AES_ARM_CE depends on KERNEL_MODE_NEON select CRYPTO_SKCIPHER select CRYPTO_LIB_AES - select CRYPTO_SIMD help Length-preserving ciphers: AES cipher algorithms (FIPS-197) with block cipher modes: diff --git a/arch/arm/crypto/aes-ce-glue.c b/arch/arm/crypto/aes-ce-glue.c index 1cf61f51e766..00591895d540 100644 --- a/arch/arm/crypto/aes-ce-glue.c +++ b/arch/arm/crypto/aes-ce-glue.c @@ -10,8 +10,6 @@ #include #include #include -#include -#include #include #include #include @@ -418,29 +416,6 @@ static int ctr_encrypt(struct skcipher_request *req) return err; } -static void ctr_encrypt_one(struct crypto_skcipher *tfm, const u8 *src, u8 *dst) -{ - struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); - unsigned long flags; - - /* - * Temporarily disable interrupts to avoid races where - * cachelines are evicted when the CPU is interrupted - * to do something else. - */ - local_irq_save(flags); - aes_encrypt(ctx, dst, src); - local_irq_restore(flags); -} - -static int ctr_encrypt_sync(struct skcipher_request *req) -{ - if (!crypto_simd_usable()) - return crypto_ctr_encrypt_walk(req, ctr_encrypt_one); - - return ctr_encrypt(req); -} - static int xts_encrypt(struct skcipher_request *req) { struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); @@ -586,10 +561,9 @@ static int xts_decrypt(struct skcipher_request *req) } static struct skcipher_alg aes_algs[] = { { - .base.cra_name = "__ecb(aes)", - .base.cra_driver_name = "__ecb-aes-ce", + .base.cra_name = "ecb(aes)", + .base.cra_driver_name = "ecb-aes-ce", .base.cra_priority = 300, - .base.cra_flags = CRYPTO_ALG_INTERNAL, .base.cra_blocksize = AES_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct crypto_aes_ctx), .base.cra_module = THIS_MODULE, @@ -600,10 +574,9 @@ static struct skcipher_alg aes_algs[] = { { .encrypt = ecb_encrypt, .decrypt = ecb_decrypt, }, { - .base.cra_name = "__cbc(aes)", - .base.cra_driver_name = "__cbc-aes-ce", + .base.cra_name = "cbc(aes)", + .base.cra_driver_name = "cbc-aes-ce", .base.cra_priority = 300, - .base.cra_flags = CRYPTO_ALG_INTERNAL, .base.cra_blocksize = AES_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct crypto_aes_ctx), .base.cra_module = THIS_MODULE, @@ -615,10 +588,9 @@ static struct skcipher_alg aes_algs[] = { { .encrypt = cbc_encrypt, .decrypt = cbc_decrypt, }, { - .base.cra_name = "__cts(cbc(aes))", - .base.cra_driver_name = "__cts-cbc-aes-ce", + .base.cra_name = "cts(cbc(aes))", + .base.cra_driver_name = "cts-cbc-aes-ce", .base.cra_priority = 300, - .base.cra_flags = CRYPTO_ALG_INTERNAL, .base.cra_blocksize = AES_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct crypto_aes_ctx), .base.cra_module = THIS_MODULE, @@ -631,10 +603,9 @@ static struct skcipher_alg aes_algs[] = { { .encrypt = cts_cbc_encrypt, .decrypt = cts_cbc_decrypt, }, { - .base.cra_name = "__ctr(aes)", - .base.cra_driver_name = "__ctr-aes-ce", + .base.cra_name = "ctr(aes)", + .base.cra_driver_name = "ctr-aes-ce", .base.cra_priority = 300, - .base.cra_flags = CRYPTO_ALG_INTERNAL, .base.cra_blocksize = 1, .base.cra_ctxsize = sizeof(struct crypto_aes_ctx), .base.cra_module = THIS_MODULE, @@ -647,25 +618,9 @@ static struct skcipher_alg aes_algs[] = { { .encrypt = ctr_encrypt, .decrypt = ctr_encrypt, }, { - .base.cra_name = "ctr(aes)", - .base.cra_driver_name = "ctr-aes-ce-sync", - .base.cra_priority = 300 - 1, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct crypto_aes_ctx), - .base.cra_module = THIS_MODULE, - - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .ivsize = AES_BLOCK_SIZE, - .chunksize = AES_BLOCK_SIZE, - .setkey = ce_aes_setkey, - .encrypt = ctr_encrypt_sync, - .decrypt = ctr_encrypt_sync, -}, { - .base.cra_name = "__xts(aes)", - .base.cra_driver_name = "__xts-aes-ce", + .base.cra_name = "xts(aes)", + .base.cra_driver_name = "xts-aes-ce", .base.cra_priority = 300, - .base.cra_flags = CRYPTO_ALG_INTERNAL, .base.cra_blocksize = AES_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct crypto_aes_xts_ctx), .base.cra_module = THIS_MODULE, @@ -679,51 +634,14 @@ static struct skcipher_alg aes_algs[] = { { .decrypt = xts_decrypt, } }; -static struct simd_skcipher_alg *aes_simd_algs[ARRAY_SIZE(aes_algs)]; - static void aes_exit(void) { - int i; - - for (i = 0; i < ARRAY_SIZE(aes_simd_algs) && aes_simd_algs[i]; i++) - simd_skcipher_free(aes_simd_algs[i]); - crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); } static int __init aes_init(void) { - struct simd_skcipher_alg *simd; - const char *basename; - const char *algname; - const char *drvname; - int err; - int i; - - err = crypto_register_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); - if (err) - return err; - - for (i = 0; i < ARRAY_SIZE(aes_algs); i++) { - if (!(aes_algs[i].base.cra_flags & CRYPTO_ALG_INTERNAL)) - continue; - - algname = aes_algs[i].base.cra_name + 2; - drvname = aes_algs[i].base.cra_driver_name + 2; - basename = aes_algs[i].base.cra_driver_name; - simd = simd_skcipher_create_compat(aes_algs + i, algname, drvname, basename); - err = PTR_ERR(simd); - if (IS_ERR(simd)) - goto unregister_simds; - - aes_simd_algs[i] = simd; - } - - return 0; - -unregister_simds: - aes_exit(); - return err; + return crypto_register_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); } module_cpu_feature_match(AES, aes_init); From 7c79bdf97802290b5dbad043a0c67719797405cc Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Thu, 3 Apr 2025 09:19:56 +0200 Subject: [PATCH 048/382] crypto: arm/aes-neonbs - stop using the SIMD helper Now that ARM permits use of the NEON unit in softirq context as well as task context, there is no longer a need to rely on the SIMD helper module to construct async skciphers wrapping the sync ones, as the latter can always be called directly. So remove these wrappers and the dependency on the SIMD helper. This permits the use of these algorithms by callers that only support synchronous use. Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- arch/arm/crypto/Kconfig | 1 - arch/arm/crypto/aes-neonbs-glue.c | 116 +++--------------------------- 2 files changed, 9 insertions(+), 108 deletions(-) diff --git a/arch/arm/crypto/Kconfig b/arch/arm/crypto/Kconfig index 2e73200b930a..be9c2e19f976 100644 --- a/arch/arm/crypto/Kconfig +++ b/arch/arm/crypto/Kconfig @@ -172,7 +172,6 @@ config CRYPTO_AES_ARM_BS select CRYPTO_AES_ARM select CRYPTO_SKCIPHER select CRYPTO_LIB_AES - select CRYPTO_SIMD help Length-preserving ciphers: AES cipher algorithms (FIPS-197) with block cipher modes: diff --git a/arch/arm/crypto/aes-neonbs-glue.c b/arch/arm/crypto/aes-neonbs-glue.c index f6be80b5938b..95418df97fb4 100644 --- a/arch/arm/crypto/aes-neonbs-glue.c +++ b/arch/arm/crypto/aes-neonbs-glue.c @@ -8,8 +8,6 @@ #include #include #include -#include -#include #include #include #include @@ -59,11 +57,6 @@ struct aesbs_xts_ctx { struct crypto_aes_ctx tweak_key; }; -struct aesbs_ctr_ctx { - struct aesbs_ctx key; /* must be first member */ - struct crypto_aes_ctx fallback; -}; - static int aesbs_setkey(struct crypto_skcipher *tfm, const u8 *in_key, unsigned int key_len) { @@ -200,25 +193,6 @@ static int cbc_decrypt(struct skcipher_request *req) return err; } -static int aesbs_ctr_setkey_sync(struct crypto_skcipher *tfm, const u8 *in_key, - unsigned int key_len) -{ - struct aesbs_ctr_ctx *ctx = crypto_skcipher_ctx(tfm); - int err; - - err = aes_expandkey(&ctx->fallback, in_key, key_len); - if (err) - return err; - - ctx->key.rounds = 6 + key_len / 4; - - kernel_neon_begin(); - aesbs_convert_key(ctx->key.rk, ctx->fallback.key_enc, ctx->key.rounds); - kernel_neon_end(); - - return 0; -} - static int ctr_encrypt(struct skcipher_request *req) { struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); @@ -254,21 +228,6 @@ static int ctr_encrypt(struct skcipher_request *req) return err; } -static void ctr_encrypt_one(struct crypto_skcipher *tfm, const u8 *src, u8 *dst) -{ - struct aesbs_ctr_ctx *ctx = crypto_skcipher_ctx(tfm); - - __aes_arm_encrypt(ctx->fallback.key_enc, ctx->key.rounds, src, dst); -} - -static int ctr_encrypt_sync(struct skcipher_request *req) -{ - if (!crypto_simd_usable()) - return crypto_ctr_encrypt_walk(req, ctr_encrypt_one); - - return ctr_encrypt(req); -} - static int aesbs_xts_setkey(struct crypto_skcipher *tfm, const u8 *in_key, unsigned int key_len) { @@ -374,13 +333,12 @@ static int xts_decrypt(struct skcipher_request *req) } static struct skcipher_alg aes_algs[] = { { - .base.cra_name = "__ecb(aes)", - .base.cra_driver_name = "__ecb-aes-neonbs", + .base.cra_name = "ecb(aes)", + .base.cra_driver_name = "ecb-aes-neonbs", .base.cra_priority = 250, .base.cra_blocksize = AES_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct aesbs_ctx), .base.cra_module = THIS_MODULE, - .base.cra_flags = CRYPTO_ALG_INTERNAL, .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, @@ -389,13 +347,12 @@ static struct skcipher_alg aes_algs[] = { { .encrypt = ecb_encrypt, .decrypt = ecb_decrypt, }, { - .base.cra_name = "__cbc(aes)", - .base.cra_driver_name = "__cbc-aes-neonbs", + .base.cra_name = "cbc(aes)", + .base.cra_driver_name = "cbc-aes-neonbs", .base.cra_priority = 250, .base.cra_blocksize = AES_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct aesbs_cbc_ctx), .base.cra_module = THIS_MODULE, - .base.cra_flags = CRYPTO_ALG_INTERNAL, .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, @@ -405,13 +362,12 @@ static struct skcipher_alg aes_algs[] = { { .encrypt = cbc_encrypt, .decrypt = cbc_decrypt, }, { - .base.cra_name = "__ctr(aes)", - .base.cra_driver_name = "__ctr-aes-neonbs", + .base.cra_name = "ctr(aes)", + .base.cra_driver_name = "ctr-aes-neonbs", .base.cra_priority = 250, .base.cra_blocksize = 1, .base.cra_ctxsize = sizeof(struct aesbs_ctx), .base.cra_module = THIS_MODULE, - .base.cra_flags = CRYPTO_ALG_INTERNAL, .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, @@ -422,29 +378,12 @@ static struct skcipher_alg aes_algs[] = { { .encrypt = ctr_encrypt, .decrypt = ctr_encrypt, }, { - .base.cra_name = "ctr(aes)", - .base.cra_driver_name = "ctr-aes-neonbs-sync", - .base.cra_priority = 250 - 1, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct aesbs_ctr_ctx), - .base.cra_module = THIS_MODULE, - - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .chunksize = AES_BLOCK_SIZE, - .walksize = 8 * AES_BLOCK_SIZE, - .ivsize = AES_BLOCK_SIZE, - .setkey = aesbs_ctr_setkey_sync, - .encrypt = ctr_encrypt_sync, - .decrypt = ctr_encrypt_sync, -}, { - .base.cra_name = "__xts(aes)", - .base.cra_driver_name = "__xts-aes-neonbs", + .base.cra_name = "xts(aes)", + .base.cra_driver_name = "xts-aes-neonbs", .base.cra_priority = 250, .base.cra_blocksize = AES_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct aesbs_xts_ctx), .base.cra_module = THIS_MODULE, - .base.cra_flags = CRYPTO_ALG_INTERNAL, .min_keysize = 2 * AES_MIN_KEY_SIZE, .max_keysize = 2 * AES_MAX_KEY_SIZE, @@ -455,54 +394,17 @@ static struct skcipher_alg aes_algs[] = { { .decrypt = xts_decrypt, } }; -static struct simd_skcipher_alg *aes_simd_algs[ARRAY_SIZE(aes_algs)]; - static void aes_exit(void) { - int i; - - for (i = 0; i < ARRAY_SIZE(aes_simd_algs); i++) - if (aes_simd_algs[i]) - simd_skcipher_free(aes_simd_algs[i]); - crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); } static int __init aes_init(void) { - struct simd_skcipher_alg *simd; - const char *basename; - const char *algname; - const char *drvname; - int err; - int i; - if (!(elf_hwcap & HWCAP_NEON)) return -ENODEV; - err = crypto_register_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); - if (err) - return err; - - for (i = 0; i < ARRAY_SIZE(aes_algs); i++) { - if (!(aes_algs[i].base.cra_flags & CRYPTO_ALG_INTERNAL)) - continue; - - algname = aes_algs[i].base.cra_name + 2; - drvname = aes_algs[i].base.cra_driver_name + 2; - basename = aes_algs[i].base.cra_driver_name; - simd = simd_skcipher_create_compat(aes_algs + i, algname, drvname, basename); - err = PTR_ERR(simd); - if (IS_ERR(simd)) - goto unregister_simds; - - aes_simd_algs[i] = simd; - } - return 0; - -unregister_simds: - aes_exit(); - return err; + return crypto_register_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); } late_initcall(aes_init); From e0f860a1ca32e7206b5cb43812acec9ffe211a52 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Thu, 3 Apr 2025 09:19:57 +0200 Subject: [PATCH 049/382] crypto: ctr - remove unused crypto_ctr_encrypt_walk() crypto_ctr_encrypt_walk() is no longer used so remove it. Note that some existing drivers currently rely on the transitive includes of some other crypto headers so retain those for the time being. Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- include/crypto/ctr.h | 47 -------------------------------------------- 1 file changed, 47 deletions(-) diff --git a/include/crypto/ctr.h b/include/crypto/ctr.h index da1ee73e9ce9..c41685874f00 100644 --- a/include/crypto/ctr.h +++ b/include/crypto/ctr.h @@ -10,56 +10,9 @@ #include #include -#include -#include #define CTR_RFC3686_NONCE_SIZE 4 #define CTR_RFC3686_IV_SIZE 8 #define CTR_RFC3686_BLOCK_SIZE 16 -static inline int crypto_ctr_encrypt_walk(struct skcipher_request *req, - void (*fn)(struct crypto_skcipher *, - const u8 *, u8 *)) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - int blocksize = crypto_skcipher_chunksize(tfm); - u8 buf[MAX_CIPHER_BLOCKSIZE]; - struct skcipher_walk walk; - int err; - - /* avoid integer division due to variable blocksize parameter */ - if (WARN_ON_ONCE(!is_power_of_2(blocksize))) - return -EINVAL; - - err = skcipher_walk_virt(&walk, req, false); - - while (walk.nbytes > 0) { - const u8 *src = walk.src.virt.addr; - u8 *dst = walk.dst.virt.addr; - int nbytes = walk.nbytes; - int tail = 0; - - if (nbytes < walk.total) { - tail = walk.nbytes & (blocksize - 1); - nbytes -= tail; - } - - do { - int bsize = min(nbytes, blocksize); - - fn(tfm, walk.iv, buf); - - crypto_xor_cpy(dst, src, buf, bsize); - crypto_inc(walk.iv, blocksize); - - dst += bsize; - src += bsize; - nbytes -= bsize; - } while (nbytes > 0); - - err = skcipher_walk_done(&walk, tail); - } - return err; -} - #endif /* _CRYPTO_CTR_H */ From d0d9d00b092243b802dad6f23f8b2e8245e1c96b Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Thu, 3 Apr 2025 21:07:28 +0100 Subject: [PATCH 050/382] crypto: qat - switch to standard pattern for PCI IDs Update the names of the defines for PCI IDs to follow the standard naming convention `PCI_DEVICE_ID_`. Also drop the unnecessary inner comma from the pci_device_id tables that use these definitions. This does not introduce any functional change. Signed-off-by: Giovanni Cabiddu Suggested-by: Andy Shevchenko Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- drivers/crypto/intel/qat/qat_420xx/adf_drv.c | 2 +- .../crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c | 4 ++-- drivers/crypto/intel/qat/qat_4xxx/adf_drv.c | 6 +++--- .../intel/qat/qat_common/adf_accel_devices.h | 16 ++++++++-------- drivers/crypto/intel/qat/qat_common/qat_hal.c | 10 +++++----- drivers/crypto/intel/qat/qat_common/qat_uclo.c | 8 ++++---- 6 files changed, 23 insertions(+), 23 deletions(-) diff --git a/drivers/crypto/intel/qat/qat_420xx/adf_drv.c b/drivers/crypto/intel/qat/qat_420xx/adf_drv.c index b4731f02deb8..cfa00daeb4fb 100644 --- a/drivers/crypto/intel/qat/qat_420xx/adf_drv.c +++ b/drivers/crypto/intel/qat/qat_420xx/adf_drv.c @@ -14,7 +14,7 @@ #include "adf_420xx_hw_data.h" static const struct pci_device_id adf_pci_tbl[] = { - { PCI_VDEVICE(INTEL, ADF_420XX_PCI_DEVICE_ID), }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QAT_420XX) }, { } }; MODULE_DEVICE_TABLE(pci, adf_pci_tbl); diff --git a/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c b/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c index 63c7c8e90cca..7d4c366aa8b2 100644 --- a/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c +++ b/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c @@ -421,13 +421,13 @@ void adf_init_hw_data_4xxx(struct adf_hw_device_data *hw_data, u32 dev_id) hw_data->admin_ae_mask = ADF_4XXX_ADMIN_AE_MASK; hw_data->num_rps = ADF_GEN4_MAX_RPS; switch (dev_id) { - case ADF_402XX_PCI_DEVICE_ID: + case PCI_DEVICE_ID_INTEL_QAT_402XX: hw_data->fw_name = ADF_402XX_FW; hw_data->fw_mmp_name = ADF_402XX_MMP; hw_data->uof_get_name = uof_get_name_402xx; hw_data->get_ena_thd_mask = get_ena_thd_mask; break; - case ADF_401XX_PCI_DEVICE_ID: + case PCI_DEVICE_ID_INTEL_QAT_401XX: hw_data->fw_name = ADF_4XXX_FW; hw_data->fw_mmp_name = ADF_4XXX_MMP; hw_data->uof_get_name = uof_get_name_4xxx; diff --git a/drivers/crypto/intel/qat/qat_4xxx/adf_drv.c b/drivers/crypto/intel/qat/qat_4xxx/adf_drv.c index 1ac415ef3c31..c9be5dcddb27 100644 --- a/drivers/crypto/intel/qat/qat_4xxx/adf_drv.c +++ b/drivers/crypto/intel/qat/qat_4xxx/adf_drv.c @@ -14,9 +14,9 @@ #include "adf_4xxx_hw_data.h" static const struct pci_device_id adf_pci_tbl[] = { - { PCI_VDEVICE(INTEL, ADF_4XXX_PCI_DEVICE_ID), }, - { PCI_VDEVICE(INTEL, ADF_401XX_PCI_DEVICE_ID), }, - { PCI_VDEVICE(INTEL, ADF_402XX_PCI_DEVICE_ID), }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QAT_4XXX) }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QAT_401XX) }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QAT_402XX) }, { } }; MODULE_DEVICE_TABLE(pci, adf_pci_tbl); diff --git a/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h b/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h index dc21551153cb..1e301a20c244 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h @@ -25,14 +25,14 @@ #define ADF_C3XXXVF_DEVICE_NAME "c3xxxvf" #define ADF_4XXX_DEVICE_NAME "4xxx" #define ADF_420XX_DEVICE_NAME "420xx" -#define ADF_4XXX_PCI_DEVICE_ID 0x4940 -#define ADF_4XXXIOV_PCI_DEVICE_ID 0x4941 -#define ADF_401XX_PCI_DEVICE_ID 0x4942 -#define ADF_401XXIOV_PCI_DEVICE_ID 0x4943 -#define ADF_402XX_PCI_DEVICE_ID 0x4944 -#define ADF_402XXIOV_PCI_DEVICE_ID 0x4945 -#define ADF_420XX_PCI_DEVICE_ID 0x4946 -#define ADF_420XXIOV_PCI_DEVICE_ID 0x4947 +#define PCI_DEVICE_ID_INTEL_QAT_4XXX 0x4940 +#define PCI_DEVICE_ID_INTEL_QAT_4XXXIOV 0x4941 +#define PCI_DEVICE_ID_INTEL_QAT_401XX 0x4942 +#define PCI_DEVICE_ID_INTEL_QAT_401XXIOV 0x4943 +#define PCI_DEVICE_ID_INTEL_QAT_402XX 0x4944 +#define PCI_DEVICE_ID_INTEL_QAT_402XXIOV 0x4945 +#define PCI_DEVICE_ID_INTEL_QAT_420XX 0x4946 +#define PCI_DEVICE_ID_INTEL_QAT_420XXIOV 0x4947 #define ADF_DEVICE_FUSECTL_OFFSET 0x40 #define ADF_DEVICE_LEGFUSE_OFFSET 0x4C #define ADF_DEVICE_FUSECTL_MASK 0x80000000 diff --git a/drivers/crypto/intel/qat/qat_common/qat_hal.c b/drivers/crypto/intel/qat/qat_common/qat_hal.c index ef8a9cf74f0c..841c1d7d3ffe 100644 --- a/drivers/crypto/intel/qat/qat_common/qat_hal.c +++ b/drivers/crypto/intel/qat/qat_common/qat_hal.c @@ -694,16 +694,16 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, handle->pci_dev = pci_info->pci_dev; switch (handle->pci_dev->device) { - case ADF_4XXX_PCI_DEVICE_ID: - case ADF_401XX_PCI_DEVICE_ID: - case ADF_402XX_PCI_DEVICE_ID: - case ADF_420XX_PCI_DEVICE_ID: + case PCI_DEVICE_ID_INTEL_QAT_4XXX: + case PCI_DEVICE_ID_INTEL_QAT_401XX: + case PCI_DEVICE_ID_INTEL_QAT_402XX: + case PCI_DEVICE_ID_INTEL_QAT_420XX: handle->chip_info->mmp_sram_size = 0; handle->chip_info->nn = false; handle->chip_info->lm2lm3 = true; handle->chip_info->lm_size = ICP_QAT_UCLO_MAX_LMEM_REG_2X; handle->chip_info->icp_rst_csr = ICP_RESET_CPP0; - if (handle->pci_dev->device == ADF_420XX_PCI_DEVICE_ID) + if (handle->pci_dev->device == PCI_DEVICE_ID_INTEL_QAT_420XX) handle->chip_info->icp_rst_mask = 0x100155; else handle->chip_info->icp_rst_mask = 0x100015; diff --git a/drivers/crypto/intel/qat/qat_common/qat_uclo.c b/drivers/crypto/intel/qat/qat_common/qat_uclo.c index 87e247ac1c9a..620300e70238 100644 --- a/drivers/crypto/intel/qat/qat_common/qat_uclo.c +++ b/drivers/crypto/intel/qat/qat_common/qat_uclo.c @@ -732,10 +732,10 @@ qat_uclo_get_dev_type(struct icp_qat_fw_loader_handle *handle) return ICP_QAT_AC_C62X_DEV_TYPE; case PCI_DEVICE_ID_INTEL_QAT_C3XXX: return ICP_QAT_AC_C3XXX_DEV_TYPE; - case ADF_4XXX_PCI_DEVICE_ID: - case ADF_401XX_PCI_DEVICE_ID: - case ADF_402XX_PCI_DEVICE_ID: - case ADF_420XX_PCI_DEVICE_ID: + case PCI_DEVICE_ID_INTEL_QAT_4XXX: + case PCI_DEVICE_ID_INTEL_QAT_401XX: + case PCI_DEVICE_ID_INTEL_QAT_402XX: + case PCI_DEVICE_ID_INTEL_QAT_420XX: return ICP_QAT_AC_4XXX_A_DEV_TYPE; default: pr_err("QAT: unsupported device 0x%x\n", From 984f835009d61316b7a76e547fbe636770c8342d Mon Sep 17 00:00:00 2001 From: Uros Bizjak Date: Fri, 4 Apr 2025 09:41:00 +0200 Subject: [PATCH 051/382] crypto: x86 - Remove CONFIG_AS_SHA1_NI Current minimum required version of binutils is 2.25, which supports SHA-1 instruction mnemonics. Remove check for assembler support of SHA-1 instructions and all relevant macros for conditional compilation. No functional change intended. Signed-off-by: Uros Bizjak Cc: Herbert Xu Cc: "David S. Miller" Cc: Thomas Gleixner Cc: Ingo Molnar Cc: Borislav Petkov Cc: Dave Hansen Cc: "H. Peter Anvin" Reviewed-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/x86/Kconfig.assembler | 5 ----- arch/x86/crypto/Makefile | 3 +-- arch/x86/crypto/sha1_ssse3_glue.c | 10 ---------- 3 files changed, 1 insertion(+), 17 deletions(-) diff --git a/arch/x86/Kconfig.assembler b/arch/x86/Kconfig.assembler index 6d20a6ce0507..70fe87bb1055 100644 --- a/arch/x86/Kconfig.assembler +++ b/arch/x86/Kconfig.assembler @@ -6,11 +6,6 @@ config AS_AVX512 help Supported by binutils >= 2.25 and LLVM integrated assembler -config AS_SHA1_NI - def_bool $(as-instr,sha1msg1 %xmm0$(comma)%xmm1) - help - Supported by binutils >= 2.24 and LLVM integrated assembler - config AS_SHA256_NI def_bool $(as-instr,sha256msg1 %xmm0$(comma)%xmm1) help diff --git a/arch/x86/crypto/Makefile b/arch/x86/crypto/Makefile index 5d19f41bde58..97c1dbc3b7d6 100644 --- a/arch/x86/crypto/Makefile +++ b/arch/x86/crypto/Makefile @@ -56,8 +56,7 @@ aesni-intel-$(CONFIG_64BIT) += aes-gcm-avx10-x86_64.o endif obj-$(CONFIG_CRYPTO_SHA1_SSSE3) += sha1-ssse3.o -sha1-ssse3-y := sha1_avx2_x86_64_asm.o sha1_ssse3_asm.o sha1_ssse3_glue.o -sha1-ssse3-$(CONFIG_AS_SHA1_NI) += sha1_ni_asm.o +sha1-ssse3-y := sha1_avx2_x86_64_asm.o sha1_ssse3_asm.o sha1_ni_asm.o sha1_ssse3_glue.o obj-$(CONFIG_CRYPTO_SHA256_SSSE3) += sha256-ssse3.o sha256-ssse3-y := sha256-ssse3-asm.o sha256-avx-asm.o sha256-avx2-asm.o sha256_ssse3_glue.o diff --git a/arch/x86/crypto/sha1_ssse3_glue.c b/arch/x86/crypto/sha1_ssse3_glue.c index ab8bc54f254d..abb793cbad01 100644 --- a/arch/x86/crypto/sha1_ssse3_glue.c +++ b/arch/x86/crypto/sha1_ssse3_glue.c @@ -28,9 +28,7 @@ #include static const struct x86_cpu_id module_cpu_ids[] = { -#ifdef CONFIG_AS_SHA1_NI X86_MATCH_FEATURE(X86_FEATURE_SHA_NI, NULL), -#endif X86_MATCH_FEATURE(X86_FEATURE_AVX2, NULL), X86_MATCH_FEATURE(X86_FEATURE_AVX, NULL), X86_MATCH_FEATURE(X86_FEATURE_SSSE3, NULL), @@ -256,7 +254,6 @@ static void unregister_sha1_avx2(void) crypto_unregister_shash(&sha1_avx2_alg); } -#ifdef CONFIG_AS_SHA1_NI asmlinkage void sha1_ni_transform(struct sha1_state *digest, const u8 *data, int rounds); @@ -306,11 +303,6 @@ static void unregister_sha1_ni(void) crypto_unregister_shash(&sha1_ni_alg); } -#else -static inline int register_sha1_ni(void) { return 0; } -static inline void unregister_sha1_ni(void) { } -#endif - static int __init sha1_ssse3_mod_init(void) { if (!x86_match_cpu(module_cpu_ids)) @@ -360,6 +352,4 @@ MODULE_ALIAS_CRYPTO("sha1"); MODULE_ALIAS_CRYPTO("sha1-ssse3"); MODULE_ALIAS_CRYPTO("sha1-avx"); MODULE_ALIAS_CRYPTO("sha1-avx2"); -#ifdef CONFIG_AS_SHA1_NI MODULE_ALIAS_CRYPTO("sha1-ni"); -#endif From d032a27e8fe9e22ef8d70dd50ee36d0a5de12198 Mon Sep 17 00:00:00 2001 From: Uros Bizjak Date: Fri, 4 Apr 2025 09:41:01 +0200 Subject: [PATCH 052/382] crypto: x86 - Remove CONFIG_AS_SHA256_NI Current minimum required version of binutils is 2.25, which supports SHA-256 instruction mnemonics. Remove check for assembler support of SHA-256 instructions and all relevant macros for conditional compilation. No functional change intended. Signed-off-by: Uros Bizjak Cc: Herbert Xu Cc: "David S. Miller" Cc: Thomas Gleixner Cc: Ingo Molnar Cc: Borislav Petkov Cc: Dave Hansen Cc: "H. Peter Anvin" Reviewed-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/x86/Kconfig.assembler | 4 ---- arch/x86/crypto/Makefile | 3 +-- arch/x86/crypto/sha256_ssse3_glue.c | 10 ---------- 3 files changed, 1 insertion(+), 16 deletions(-) diff --git a/arch/x86/Kconfig.assembler b/arch/x86/Kconfig.assembler index 70fe87bb1055..4d06fd3c8dfe 100644 --- a/arch/x86/Kconfig.assembler +++ b/arch/x86/Kconfig.assembler @@ -6,10 +6,6 @@ config AS_AVX512 help Supported by binutils >= 2.25 and LLVM integrated assembler -config AS_SHA256_NI - def_bool $(as-instr,sha256msg1 %xmm0$(comma)%xmm1) - help - Supported by binutils >= 2.24 and LLVM integrated assembler config AS_TPAUSE def_bool $(as-instr,tpause %ecx) help diff --git a/arch/x86/crypto/Makefile b/arch/x86/crypto/Makefile index 97c1dbc3b7d6..e06b739176c9 100644 --- a/arch/x86/crypto/Makefile +++ b/arch/x86/crypto/Makefile @@ -59,8 +59,7 @@ obj-$(CONFIG_CRYPTO_SHA1_SSSE3) += sha1-ssse3.o sha1-ssse3-y := sha1_avx2_x86_64_asm.o sha1_ssse3_asm.o sha1_ni_asm.o sha1_ssse3_glue.o obj-$(CONFIG_CRYPTO_SHA256_SSSE3) += sha256-ssse3.o -sha256-ssse3-y := sha256-ssse3-asm.o sha256-avx-asm.o sha256-avx2-asm.o sha256_ssse3_glue.o -sha256-ssse3-$(CONFIG_AS_SHA256_NI) += sha256_ni_asm.o +sha256-ssse3-y := sha256-ssse3-asm.o sha256-avx-asm.o sha256-avx2-asm.o sha256_ni_asm.o sha256_ssse3_glue.o obj-$(CONFIG_CRYPTO_SHA512_SSSE3) += sha512-ssse3.o sha512-ssse3-y := sha512-ssse3-asm.o sha512-avx-asm.o sha512-avx2-asm.o sha512_ssse3_glue.o diff --git a/arch/x86/crypto/sha256_ssse3_glue.c b/arch/x86/crypto/sha256_ssse3_glue.c index e04a43d9f7d5..429a3cefbab4 100644 --- a/arch/x86/crypto/sha256_ssse3_glue.c +++ b/arch/x86/crypto/sha256_ssse3_glue.c @@ -45,9 +45,7 @@ asmlinkage void sha256_transform_ssse3(struct sha256_state *state, const u8 *data, int blocks); static const struct x86_cpu_id module_cpu_ids[] = { -#ifdef CONFIG_AS_SHA256_NI X86_MATCH_FEATURE(X86_FEATURE_SHA_NI, NULL), -#endif X86_MATCH_FEATURE(X86_FEATURE_AVX2, NULL), X86_MATCH_FEATURE(X86_FEATURE_AVX, NULL), X86_MATCH_FEATURE(X86_FEATURE_SSSE3, NULL), @@ -329,7 +327,6 @@ static void unregister_sha256_avx2(void) ARRAY_SIZE(sha256_avx2_algs)); } -#ifdef CONFIG_AS_SHA256_NI asmlinkage void sha256_ni_transform(struct sha256_state *digest, const u8 *data, int rounds); @@ -403,11 +400,6 @@ static void unregister_sha256_ni(void) ARRAY_SIZE(sha256_ni_algs)); } -#else -static inline int register_sha256_ni(void) { return 0; } -static inline void unregister_sha256_ni(void) { } -#endif - static int __init sha256_ssse3_mod_init(void) { if (!x86_match_cpu(module_cpu_ids)) @@ -461,7 +453,5 @@ MODULE_ALIAS_CRYPTO("sha224"); MODULE_ALIAS_CRYPTO("sha224-ssse3"); MODULE_ALIAS_CRYPTO("sha224-avx"); MODULE_ALIAS_CRYPTO("sha224-avx2"); -#ifdef CONFIG_AS_SHA256_NI MODULE_ALIAS_CRYPTO("sha256-ni"); MODULE_ALIAS_CRYPTO("sha224-ni"); -#endif From bc23fe6dc172b09778ce3d75a9157decd153f4ef Mon Sep 17 00:00:00 2001 From: Uros Bizjak Date: Fri, 4 Apr 2025 09:41:02 +0200 Subject: [PATCH 053/382] crypto: x86 - Remove CONFIG_AS_AVX512 handling Current minimum required version of binutils is 2.25, which supports AVX-512 instruction mnemonics. Remove check for assembler support of AVX-512 instructions and all relevant macros for conditional compilation. No functional change intended. Signed-off-by: Uros Bizjak Cc: Herbert Xu Cc: "David S. Miller" Cc: Thomas Gleixner Cc: Ingo Molnar Cc: Borislav Petkov Cc: Dave Hansen Cc: "H. Peter Anvin" Reviewed-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/x86/crypto/Kconfig | 2 +- arch/x86/crypto/Makefile | 3 +-- arch/x86/crypto/blake2s-core.S | 4 ---- arch/x86/crypto/blake2s-glue.c | 6 ++---- arch/x86/crypto/chacha_glue.c | 6 ++---- arch/x86/crypto/poly1305-x86_64-cryptogams.pl | 8 -------- arch/x86/crypto/poly1305_glue.c | 6 +++--- 7 files changed, 9 insertions(+), 26 deletions(-) diff --git a/arch/x86/crypto/Kconfig b/arch/x86/crypto/Kconfig index f9e46e83440f..2fae1674f756 100644 --- a/arch/x86/crypto/Kconfig +++ b/arch/x86/crypto/Kconfig @@ -320,7 +320,7 @@ config CRYPTO_ARIA_AESNI_AVX2_X86_64 config CRYPTO_ARIA_GFNI_AVX512_X86_64 tristate "Ciphers: ARIA with modes: ECB, CTR (AVX512/GFNI)" - depends on X86 && 64BIT && AS_AVX512 && AS_GFNI + depends on X86 && 64BIT && AS_GFNI select CRYPTO_SKCIPHER select CRYPTO_ALGAPI select CRYPTO_ARIA diff --git a/arch/x86/crypto/Makefile b/arch/x86/crypto/Makefile index e06b739176c9..2f22b381f244 100644 --- a/arch/x86/crypto/Makefile +++ b/arch/x86/crypto/Makefile @@ -43,8 +43,7 @@ obj-$(CONFIG_CRYPTO_AEGIS128_AESNI_SSE2) += aegis128-aesni.o aegis128-aesni-y := aegis128-aesni-asm.o aegis128-aesni-glue.o obj-$(CONFIG_CRYPTO_CHACHA20_X86_64) += chacha-x86_64.o -chacha-x86_64-y := chacha-avx2-x86_64.o chacha-ssse3-x86_64.o chacha_glue.o -chacha-x86_64-$(CONFIG_AS_AVX512) += chacha-avx512vl-x86_64.o +chacha-x86_64-y := chacha-avx2-x86_64.o chacha-ssse3-x86_64.o chacha-avx512vl-x86_64.o chacha_glue.o obj-$(CONFIG_CRYPTO_AES_NI_INTEL) += aesni-intel.o aesni-intel-y := aesni-intel_asm.o aesni-intel_glue.o diff --git a/arch/x86/crypto/blake2s-core.S b/arch/x86/crypto/blake2s-core.S index b50b35ff1fdb..ac1c845445a4 100644 --- a/arch/x86/crypto/blake2s-core.S +++ b/arch/x86/crypto/blake2s-core.S @@ -29,7 +29,6 @@ SIGMA: .byte 13, 7, 12, 3, 11, 14, 1, 9, 2, 5, 15, 8, 10, 0, 4, 6 .byte 6, 14, 11, 0, 15, 9, 3, 8, 10, 12, 13, 1, 5, 2, 7, 4 .byte 10, 8, 7, 1, 2, 4, 6, 5, 13, 15, 9, 3, 0, 11, 14, 12 -#ifdef CONFIG_AS_AVX512 .section .rodata.cst64.BLAKE2S_SIGMA2, "aM", @progbits, 640 .align 64 SIGMA2: @@ -43,7 +42,6 @@ SIGMA2: .long 6, 13, 0, 14, 12, 2, 1, 11, 15, 4, 5, 8, 7, 9, 3, 10 .long 15, 5, 4, 13, 10, 7, 3, 11, 12, 2, 0, 6, 9, 8, 1, 14 .long 8, 7, 14, 11, 13, 15, 0, 12, 10, 4, 5, 6, 3, 2, 1, 9 -#endif /* CONFIG_AS_AVX512 */ .text SYM_FUNC_START(blake2s_compress_ssse3) @@ -174,7 +172,6 @@ SYM_FUNC_START(blake2s_compress_ssse3) RET SYM_FUNC_END(blake2s_compress_ssse3) -#ifdef CONFIG_AS_AVX512 SYM_FUNC_START(blake2s_compress_avx512) vmovdqu (%rdi),%xmm0 vmovdqu 0x10(%rdi),%xmm1 @@ -253,4 +250,3 @@ SYM_FUNC_START(blake2s_compress_avx512) vzeroupper RET SYM_FUNC_END(blake2s_compress_avx512) -#endif /* CONFIG_AS_AVX512 */ diff --git a/arch/x86/crypto/blake2s-glue.c b/arch/x86/crypto/blake2s-glue.c index 0313f9673f56..00f84f29cc8c 100644 --- a/arch/x86/crypto/blake2s-glue.c +++ b/arch/x86/crypto/blake2s-glue.c @@ -41,8 +41,7 @@ void blake2s_compress(struct blake2s_state *state, const u8 *block, SZ_4K / BLAKE2S_BLOCK_SIZE); kernel_fpu_begin(); - if (IS_ENABLED(CONFIG_AS_AVX512) && - static_branch_likely(&blake2s_use_avx512)) + if (static_branch_likely(&blake2s_use_avx512)) blake2s_compress_avx512(state, block, blocks, inc); else blake2s_compress_ssse3(state, block, blocks, inc); @@ -59,8 +58,7 @@ static int __init blake2s_mod_init(void) if (boot_cpu_has(X86_FEATURE_SSSE3)) static_branch_enable(&blake2s_use_ssse3); - if (IS_ENABLED(CONFIG_AS_AVX512) && - boot_cpu_has(X86_FEATURE_AVX) && + if (boot_cpu_has(X86_FEATURE_AVX) && boot_cpu_has(X86_FEATURE_AVX2) && boot_cpu_has(X86_FEATURE_AVX512F) && boot_cpu_has(X86_FEATURE_AVX512VL) && diff --git a/arch/x86/crypto/chacha_glue.c b/arch/x86/crypto/chacha_glue.c index 6a3d60cf3192..946c306f60cd 100644 --- a/arch/x86/crypto/chacha_glue.c +++ b/arch/x86/crypto/chacha_glue.c @@ -45,8 +45,7 @@ static unsigned int chacha_advance(unsigned int len, unsigned int maxblocks) static void chacha_dosimd(u32 *state, u8 *dst, const u8 *src, unsigned int bytes, int nrounds) { - if (IS_ENABLED(CONFIG_AS_AVX512) && - static_branch_likely(&chacha_use_avx512vl)) { + if (static_branch_likely(&chacha_use_avx512vl)) { while (bytes >= CHACHA_BLOCK_SIZE * 8) { chacha_8block_xor_avx512vl(state, dst, src, bytes, nrounds); @@ -258,8 +257,7 @@ static int __init chacha_simd_mod_init(void) cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL)) { static_branch_enable(&chacha_use_avx2); - if (IS_ENABLED(CONFIG_AS_AVX512) && - boot_cpu_has(X86_FEATURE_AVX512VL) && + if (boot_cpu_has(X86_FEATURE_AVX512VL) && boot_cpu_has(X86_FEATURE_AVX512BW)) /* kmovq */ static_branch_enable(&chacha_use_avx512vl); } diff --git a/arch/x86/crypto/poly1305-x86_64-cryptogams.pl b/arch/x86/crypto/poly1305-x86_64-cryptogams.pl index b9abcd79c1f4..409ec6955733 100644 --- a/arch/x86/crypto/poly1305-x86_64-cryptogams.pl +++ b/arch/x86/crypto/poly1305-x86_64-cryptogams.pl @@ -2811,18 +2811,10 @@ if ($avx>2) { # reason stack layout is kept identical to poly1305_blocks_avx2. If not # for this tail, we wouldn't have to even allocate stack frame... -if($kernel) { - $code .= "#ifdef CONFIG_AS_AVX512\n"; -} - &declare_function("poly1305_blocks_avx512", 32, 4); poly1305_blocks_avxN(1); &end_function("poly1305_blocks_avx512"); -if ($kernel) { - $code .= "#endif\n"; -} - if (!$kernel && $avx>3) { ######################################################################## # VPMADD52 version using 2^44 radix. diff --git a/arch/x86/crypto/poly1305_glue.c b/arch/x86/crypto/poly1305_glue.c index 08ff4b489f7e..8b5593c46da7 100644 --- a/arch/x86/crypto/poly1305_glue.c +++ b/arch/x86/crypto/poly1305_glue.c @@ -107,7 +107,7 @@ static void poly1305_simd_blocks(void *ctx, const u8 *inp, size_t len, const size_t bytes = min_t(size_t, len, SZ_4K); kernel_fpu_begin(); - if (IS_ENABLED(CONFIG_AS_AVX512) && static_branch_likely(&poly1305_use_avx512)) + if (static_branch_likely(&poly1305_use_avx512)) poly1305_blocks_avx512(ctx, inp, bytes, padbit); else if (static_branch_likely(&poly1305_use_avx2)) poly1305_blocks_avx2(ctx, inp, bytes, padbit); @@ -265,8 +265,8 @@ static int __init poly1305_simd_mod_init(void) if (boot_cpu_has(X86_FEATURE_AVX) && boot_cpu_has(X86_FEATURE_AVX2) && cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL)) static_branch_enable(&poly1305_use_avx2); - if (IS_ENABLED(CONFIG_AS_AVX512) && boot_cpu_has(X86_FEATURE_AVX) && - boot_cpu_has(X86_FEATURE_AVX2) && boot_cpu_has(X86_FEATURE_AVX512F) && + if (boot_cpu_has(X86_FEATURE_AVX) && boot_cpu_has(X86_FEATURE_AVX2) && + boot_cpu_has(X86_FEATURE_AVX512F) && cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM | XFEATURE_MASK_AVX512, NULL) && /* Skylake downclocks unacceptably much when using zmm, but later generations are fast. */ boot_cpu_data.x86_vfm != INTEL_SKYLAKE_X) From 570ef50a15d7caa4d63d66f1e28d967065733f84 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Fri, 4 Apr 2025 21:09:30 -0700 Subject: [PATCH 054/382] crypto: x86/aes-xts - optimize _compute_first_set_of_tweaks for AVX-512 Optimize the AVX-512 version of _compute_first_set_of_tweaks by using vectorized shifts to compute the first vector of tweak blocks, and by using byte-aligned shifts when multiplying by x^8. AES-XTS performance on AMD Ryzen 9 9950X (Zen 5) improves by about 2% for 4096-byte messages or 6% for 512-byte messages. AES-XTS performance on Intel Sapphire Rapids improves by about 1% for 4096-byte messages or 3% for 512-byte messages. Code size decreases by 75 bytes which outweighs the increase in rodata size of 16 bytes. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/x86/crypto/aes-xts-avx-x86_64.S | 90 +++++++++++++++++++--------- 1 file changed, 62 insertions(+), 28 deletions(-) diff --git a/arch/x86/crypto/aes-xts-avx-x86_64.S b/arch/x86/crypto/aes-xts-avx-x86_64.S index bbeaccbd1c51..db79cdf81588 100644 --- a/arch/x86/crypto/aes-xts-avx-x86_64.S +++ b/arch/x86/crypto/aes-xts-avx-x86_64.S @@ -100,6 +100,17 @@ // exists when there's a carry out of the low 64 bits of the tweak. .quad 0x87, 1 + // These are the shift amounts that are needed when multiplying by [x^0, + // x^1, x^2, x^3] to compute the first vector of tweaks when VL=64. + // + // The right shifts by 64 are expected to zeroize the destination. + // 'vpsrlvq' is indeed defined to do that; i.e. it doesn't truncate the + // amount to 64 & 63 = 0 like the 'shr' scalar shift instruction would. +.Lrshift_amounts: + .byte 64, 64, 63, 63, 62, 62, 61, 61 +.Llshift_amounts: + .byte 0, 0, 1, 1, 2, 2, 3, 3 + // This table contains constants for vpshufb and vpblendvb, used to // handle variable byte shifts and blending during ciphertext stealing // on CPUs that don't support AVX512-style masking. @@ -294,52 +305,75 @@ // Given the first XTS tweak at (TWEAK), compute the first set of tweaks and // store them in the vector registers TWEAK0-TWEAK3. Clobbers V0-V5. .macro _compute_first_set_of_tweaks - vmovdqu (TWEAK), TWEAK0_XMM - _vbroadcast128 .Lgf_poly(%rip), GF_POLY .if VL == 16 - // With VL=16, multiplying by x serially is fastest. + vmovdqu (TWEAK), TWEAK0_XMM + vmovdqu .Lgf_poly(%rip), GF_POLY _next_tweak TWEAK0, %xmm0, TWEAK1 _next_tweak TWEAK1, %xmm0, TWEAK2 _next_tweak TWEAK2, %xmm0, TWEAK3 -.else -.if VL == 32 - // Compute the second block of TWEAK0. +.elseif VL == 32 + vmovdqu (TWEAK), TWEAK0_XMM + vbroadcasti128 .Lgf_poly(%rip), GF_POLY + + // Compute the first vector of tweaks. _next_tweak TWEAK0_XMM, %xmm0, %xmm1 vinserti128 $1, %xmm1, TWEAK0, TWEAK0 -.elseif VL == 64 - // Compute the remaining blocks of TWEAK0. - _next_tweak TWEAK0_XMM, %xmm0, %xmm1 - _next_tweak %xmm1, %xmm0, %xmm2 - _next_tweak %xmm2, %xmm0, %xmm3 - vinserti32x4 $1, %xmm1, TWEAK0, TWEAK0 - vinserti32x4 $2, %xmm2, TWEAK0, TWEAK0 - vinserti32x4 $3, %xmm3, TWEAK0, TWEAK0 -.endif - // Compute TWEAK[1-3] from TWEAK0. - vpsrlq $64 - 1*VL/16, TWEAK0, V0 - vpsrlq $64 - 2*VL/16, TWEAK0, V2 - vpsrlq $64 - 3*VL/16, TWEAK0, V4 + + // Compute the next three vectors of tweaks: + // TWEAK1 = TWEAK0 * [x^2, x^2] + // TWEAK2 = TWEAK0 * [x^4, x^4] + // TWEAK3 = TWEAK0 * [x^6, x^6] + vpsrlq $64 - 2, TWEAK0, V0 + vpsrlq $64 - 4, TWEAK0, V2 + vpsrlq $64 - 6, TWEAK0, V4 vpclmulqdq $0x01, GF_POLY, V0, V1 vpclmulqdq $0x01, GF_POLY, V2, V3 vpclmulqdq $0x01, GF_POLY, V4, V5 vpslldq $8, V0, V0 vpslldq $8, V2, V2 vpslldq $8, V4, V4 - vpsllq $1*VL/16, TWEAK0, TWEAK1 - vpsllq $2*VL/16, TWEAK0, TWEAK2 - vpsllq $3*VL/16, TWEAK0, TWEAK3 -.if USE_AVX512 - vpternlogd $0x96, V0, V1, TWEAK1 - vpternlogd $0x96, V2, V3, TWEAK2 - vpternlogd $0x96, V4, V5, TWEAK3 -.else + vpsllq $2, TWEAK0, TWEAK1 + vpsllq $4, TWEAK0, TWEAK2 + vpsllq $6, TWEAK0, TWEAK3 vpxor V0, TWEAK1, TWEAK1 vpxor V2, TWEAK2, TWEAK2 vpxor V4, TWEAK3, TWEAK3 vpxor V1, TWEAK1, TWEAK1 vpxor V3, TWEAK2, TWEAK2 vpxor V5, TWEAK3, TWEAK3 -.endif +.else + vbroadcasti32x4 (TWEAK), TWEAK0 + vbroadcasti32x4 .Lgf_poly(%rip), GF_POLY + + // Compute the first vector of tweaks: + // TWEAK0 = broadcast128(TWEAK) * [x^0, x^1, x^2, x^3] + vpmovzxbq .Lrshift_amounts(%rip), V4 + vpsrlvq V4, TWEAK0, V0 + vpclmulqdq $0x01, GF_POLY, V0, V1 + vpmovzxbq .Llshift_amounts(%rip), V4 + vpslldq $8, V0, V0 + vpsllvq V4, TWEAK0, TWEAK0 + vpternlogd $0x96, V0, V1, TWEAK0 + + // Compute the next three vectors of tweaks: + // TWEAK1 = TWEAK0 * [x^4, x^4, x^4, x^4] + // TWEAK2 = TWEAK0 * [x^8, x^8, x^8, x^8] + // TWEAK3 = TWEAK0 * [x^12, x^12, x^12, x^12] + // x^8 only needs byte-aligned shifts, so optimize accordingly. + vpsrlq $64 - 4, TWEAK0, V0 + vpsrldq $(64 - 8) / 8, TWEAK0, V2 + vpsrlq $64 - 12, TWEAK0, V4 + vpclmulqdq $0x01, GF_POLY, V0, V1 + vpclmulqdq $0x01, GF_POLY, V2, V3 + vpclmulqdq $0x01, GF_POLY, V4, V5 + vpslldq $8, V0, V0 + vpslldq $8, V4, V4 + vpsllq $4, TWEAK0, TWEAK1 + vpslldq $8 / 8, TWEAK0, TWEAK2 + vpsllq $12, TWEAK0, TWEAK3 + vpternlogd $0x96, V0, V1, TWEAK1 + vpxord V3, TWEAK2, TWEAK2 + vpternlogd $0x96, V4, V5, TWEAK3 .endif .endm From ceba0eda831382864dc420fcab5f6079fbb26ba2 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Sat, 5 Apr 2025 11:26:01 -0700 Subject: [PATCH 055/382] crypto: riscv/chacha - implement library instead of skcipher Currently the RISC-V optimized ChaCha20 is only wired up to the crypto_skcipher API, which makes it unavailable to users of the library API. The crypto_skcipher API for ChaCha20 is going to change to be implemented on top of the library API, so the library API needs to be supported. And of course it's needed anyway to serve the library users. Therefore, change the RISC-V ChaCha20 code to implement the library API instead of the crypto_skcipher API. The library functions take the ChaCha state matrix directly (instead of key and IV) and support both ChaCha20 and ChaCha12. To make the RISC-V code work properly for that, change the assembly code to take the state matrix directly and add a nrounds parameter. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/riscv/crypto/Kconfig | 11 +-- arch/riscv/crypto/chacha-riscv64-glue.c | 112 ++++++++---------------- arch/riscv/crypto/chacha-riscv64-zvkb.S | 71 ++++++++------- 3 files changed, 78 insertions(+), 116 deletions(-) diff --git a/arch/riscv/crypto/Kconfig b/arch/riscv/crypto/Kconfig index c67095a3d669..6392e1e11bc9 100644 --- a/arch/riscv/crypto/Kconfig +++ b/arch/riscv/crypto/Kconfig @@ -19,14 +19,11 @@ config CRYPTO_AES_RISCV64 - Zvkg vector crypto extension (XTS) config CRYPTO_CHACHA_RISCV64 - tristate "Ciphers: ChaCha" + tristate depends on 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO - select CRYPTO_SKCIPHER - help - Length-preserving ciphers: ChaCha20 stream cipher algorithm - - Architecture: riscv64 using: - - Zvkb vector crypto extension + select CRYPTO_ARCH_HAVE_LIB_CHACHA + select CRYPTO_LIB_CHACHA_GENERIC + default CRYPTO_LIB_CHACHA_INTERNAL config CRYPTO_GHASH_RISCV64 tristate "Hash functions: GHASH" diff --git a/arch/riscv/crypto/chacha-riscv64-glue.c b/arch/riscv/crypto/chacha-riscv64-glue.c index 10b46f36375a..68caef7a3d50 100644 --- a/arch/riscv/crypto/chacha-riscv64-glue.c +++ b/arch/riscv/crypto/chacha-riscv64-glue.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * ChaCha20 using the RISC-V vector crypto extensions + * ChaCha stream cipher (RISC-V optimized) * * Copyright (C) 2023 SiFive, Inc. * Author: Jerry Shih @@ -8,94 +8,56 @@ #include #include -#include -#include +#include +#include #include #include -asmlinkage void chacha20_zvkb(const u32 key[8], const u8 *in, u8 *out, - size_t len, const u32 iv[4]); +static __ro_after_init DEFINE_STATIC_KEY_FALSE(use_zvkb); -static int riscv64_chacha20_crypt(struct skcipher_request *req) +asmlinkage void chacha_zvkb(u32 state[16], const u8 *in, u8 *out, + size_t nblocks, int nrounds); + +void hchacha_block_arch(const u32 *state, u32 *out, int nrounds) { - u32 iv[CHACHA_IV_SIZE / sizeof(u32)]; - u8 block_buffer[CHACHA_BLOCK_SIZE]; - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - const struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm); - struct skcipher_walk walk; - unsigned int nbytes; - unsigned int tail_bytes; - int err; - - iv[0] = get_unaligned_le32(req->iv); - iv[1] = get_unaligned_le32(req->iv + 4); - iv[2] = get_unaligned_le32(req->iv + 8); - iv[3] = get_unaligned_le32(req->iv + 12); - - err = skcipher_walk_virt(&walk, req, false); - while (walk.nbytes) { - nbytes = walk.nbytes & ~(CHACHA_BLOCK_SIZE - 1); - tail_bytes = walk.nbytes & (CHACHA_BLOCK_SIZE - 1); - kernel_vector_begin(); - if (nbytes) { - chacha20_zvkb(ctx->key, walk.src.virt.addr, - walk.dst.virt.addr, nbytes, iv); - iv[0] += nbytes / CHACHA_BLOCK_SIZE; - } - if (walk.nbytes == walk.total && tail_bytes > 0) { - memcpy(block_buffer, walk.src.virt.addr + nbytes, - tail_bytes); - chacha20_zvkb(ctx->key, block_buffer, block_buffer, - CHACHA_BLOCK_SIZE, iv); - memcpy(walk.dst.virt.addr + nbytes, block_buffer, - tail_bytes); - tail_bytes = 0; - } - kernel_vector_end(); - - err = skcipher_walk_done(&walk, tail_bytes); - } - - return err; + hchacha_block_generic(state, out, nrounds); } +EXPORT_SYMBOL(hchacha_block_arch); -static struct skcipher_alg riscv64_chacha_alg = { - .setkey = chacha20_setkey, - .encrypt = riscv64_chacha20_crypt, - .decrypt = riscv64_chacha20_crypt, - .min_keysize = CHACHA_KEY_SIZE, - .max_keysize = CHACHA_KEY_SIZE, - .ivsize = CHACHA_IV_SIZE, - .chunksize = CHACHA_BLOCK_SIZE, - .walksize = 4 * CHACHA_BLOCK_SIZE, - .base = { - .cra_blocksize = 1, - .cra_ctxsize = sizeof(struct chacha_ctx), - .cra_priority = 300, - .cra_name = "chacha20", - .cra_driver_name = "chacha20-riscv64-zvkb", - .cra_module = THIS_MODULE, - }, -}; +void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *src, unsigned int bytes, + int nrounds) +{ + u8 block_buffer[CHACHA_BLOCK_SIZE]; + unsigned int full_blocks = bytes / CHACHA_BLOCK_SIZE; + unsigned int tail_bytes = bytes % CHACHA_BLOCK_SIZE; + + if (!static_branch_likely(&use_zvkb) || !crypto_simd_usable()) + return chacha_crypt_generic(state, dst, src, bytes, nrounds); + + kernel_vector_begin(); + if (full_blocks) { + chacha_zvkb(state, src, dst, full_blocks, nrounds); + src += full_blocks * CHACHA_BLOCK_SIZE; + dst += full_blocks * CHACHA_BLOCK_SIZE; + } + if (tail_bytes) { + memcpy(block_buffer, src, tail_bytes); + chacha_zvkb(state, block_buffer, block_buffer, 1, nrounds); + memcpy(dst, block_buffer, tail_bytes); + } + kernel_vector_end(); +} +EXPORT_SYMBOL(chacha_crypt_arch); static int __init riscv64_chacha_mod_init(void) { if (riscv_isa_extension_available(NULL, ZVKB) && riscv_vector_vlen() >= 128) - return crypto_register_skcipher(&riscv64_chacha_alg); - - return -ENODEV; + static_branch_enable(&use_zvkb); + return 0; } - -static void __exit riscv64_chacha_mod_exit(void) -{ - crypto_unregister_skcipher(&riscv64_chacha_alg); -} - module_init(riscv64_chacha_mod_init); -module_exit(riscv64_chacha_mod_exit); -MODULE_DESCRIPTION("ChaCha20 (RISC-V accelerated)"); +MODULE_DESCRIPTION("ChaCha stream cipher (RISC-V optimized)"); MODULE_AUTHOR("Jerry Shih "); MODULE_LICENSE("GPL"); -MODULE_ALIAS_CRYPTO("chacha20"); diff --git a/arch/riscv/crypto/chacha-riscv64-zvkb.S b/arch/riscv/crypto/chacha-riscv64-zvkb.S index bf057737ac69..ab4423b3880e 100644 --- a/arch/riscv/crypto/chacha-riscv64-zvkb.S +++ b/arch/riscv/crypto/chacha-riscv64-zvkb.S @@ -46,11 +46,11 @@ .text .option arch, +zvkb -#define KEYP a0 +#define STATEP a0 #define INP a1 #define OUTP a2 -#define LEN a3 -#define IVP a4 +#define NBLOCKS a3 +#define NROUNDS a4 #define CONSTS0 a5 #define CONSTS1 a6 @@ -59,7 +59,7 @@ #define TMP t1 #define VL t2 #define STRIDE t3 -#define NROUNDS t4 +#define ROUND_CTR t4 #define KEY0 s0 #define KEY1 s1 #define KEY2 s2 @@ -132,14 +132,16 @@ vror.vi \b3, \b3, 32 - 7 .endm -// void chacha20_zvkb(const u32 key[8], const u8 *in, u8 *out, size_t len, -// const u32 iv[4]); +// void chacha_zvkb(u32 state[16], const u8 *in, u8 *out, size_t nblocks, +// int nrounds); // -// |len| must be nonzero and a multiple of 64 (CHACHA_BLOCK_SIZE). -// The counter is treated as 32-bit, following the RFC7539 convention. -SYM_FUNC_START(chacha20_zvkb) - srli LEN, LEN, 6 // Bytes to blocks - +// |nblocks| is the number of 64-byte blocks to process, and must be nonzero. +// +// |state| gives the ChaCha state matrix, including the 32-bit counter in +// state[12] following the RFC7539 convention; note that this differs from the +// original Salsa20 paper which uses a 64-bit counter in state[12..13]. The +// updated 32-bit counter is written back to state[12] before returning. +SYM_FUNC_START(chacha_zvkb) addi sp, sp, -96 sd s0, 0(sp) sd s1, 8(sp) @@ -157,26 +159,26 @@ SYM_FUNC_START(chacha20_zvkb) li STRIDE, 64 // Set up the initial state matrix in scalar registers. - li CONSTS0, 0x61707865 // "expa" little endian - li CONSTS1, 0x3320646e // "nd 3" little endian - li CONSTS2, 0x79622d32 // "2-by" little endian - li CONSTS3, 0x6b206574 // "te k" little endian - lw KEY0, 0(KEYP) - lw KEY1, 4(KEYP) - lw KEY2, 8(KEYP) - lw KEY3, 12(KEYP) - lw KEY4, 16(KEYP) - lw KEY5, 20(KEYP) - lw KEY6, 24(KEYP) - lw KEY7, 28(KEYP) - lw COUNTER, 0(IVP) - lw NONCE0, 4(IVP) - lw NONCE1, 8(IVP) - lw NONCE2, 12(IVP) + lw CONSTS0, 0(STATEP) + lw CONSTS1, 4(STATEP) + lw CONSTS2, 8(STATEP) + lw CONSTS3, 12(STATEP) + lw KEY0, 16(STATEP) + lw KEY1, 20(STATEP) + lw KEY2, 24(STATEP) + lw KEY3, 28(STATEP) + lw KEY4, 32(STATEP) + lw KEY5, 36(STATEP) + lw KEY6, 40(STATEP) + lw KEY7, 44(STATEP) + lw COUNTER, 48(STATEP) + lw NONCE0, 52(STATEP) + lw NONCE1, 56(STATEP) + lw NONCE2, 60(STATEP) .Lblock_loop: // Set vl to the number of blocks to process in this iteration. - vsetvli VL, LEN, e32, m1, ta, ma + vsetvli VL, NBLOCKS, e32, m1, ta, ma // Set up the initial state matrix for the next VL blocks in v0-v15. // v{i} holds the i'th 32-bit word of the state matrix for all blocks. @@ -203,16 +205,16 @@ SYM_FUNC_START(chacha20_zvkb) // v{16+i} holds the i'th 32-bit word for all blocks. vlsseg8e32.v v16, (INP), STRIDE - li NROUNDS, 20 + mv ROUND_CTR, NROUNDS .Lnext_doubleround: - addi NROUNDS, NROUNDS, -2 + addi ROUND_CTR, ROUND_CTR, -2 // column round chacha_round v0, v4, v8, v12, v1, v5, v9, v13, \ v2, v6, v10, v14, v3, v7, v11, v15 // diagonal round chacha_round v0, v5, v10, v15, v1, v6, v11, v12, \ v2, v7, v8, v13, v3, v4, v9, v14 - bnez NROUNDS, .Lnext_doubleround + bnez ROUND_CTR, .Lnext_doubleround // Load the second half of the input data for each block into v24-v31. // v{24+i} holds the {8+i}'th 32-bit word for all blocks. @@ -271,12 +273,13 @@ SYM_FUNC_START(chacha20_zvkb) // Update the counter, the remaining number of blocks, and the input and // output pointers according to the number of blocks processed (VL). add COUNTER, COUNTER, VL - sub LEN, LEN, VL + sub NBLOCKS, NBLOCKS, VL slli TMP, VL, 6 add OUTP, OUTP, TMP add INP, INP, TMP - bnez LEN, .Lblock_loop + bnez NBLOCKS, .Lblock_loop + sw COUNTER, 48(STATEP) ld s0, 0(sp) ld s1, 8(sp) ld s2, 16(sp) @@ -291,4 +294,4 @@ SYM_FUNC_START(chacha20_zvkb) ld s11, 88(sp) addi sp, sp, 96 ret -SYM_FUNC_END(chacha20_zvkb) +SYM_FUNC_END(chacha_zvkb) From 4aa6dc909e400b247bd3abf29f805d49a9c140bf Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Sat, 5 Apr 2025 11:26:02 -0700 Subject: [PATCH 056/382] crypto: chacha - centralize the skcipher wrappers for arch code Following the example of the crc32 and crc32c code, make the crypto subsystem register both generic and architecture-optimized chacha20, xchacha20, and xchacha12 skcipher algorithms, all implemented on top of the appropriate library functions. This eliminates the need for every architecture to implement the same skcipher glue code. To register the architecture-optimized skciphers only when architecture-optimized code is actually being used, add a function chacha_is_arch_optimized() and make each arch implement it. Change each architecture's ChaCha module_init function to arch_initcall so that the CPU feature detection is guaranteed to run before chacha_is_arch_optimized() gets called by crypto/chacha.c. In the case of s390, remove the CPU feature based module autoloading, which is no longer needed since the module just gets pulled in via function linkage. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/arm/crypto/chacha-glue.c | 9 +- arch/arm64/crypto/chacha-neon-glue.c | 8 +- arch/mips/crypto/chacha-glue.c | 8 +- arch/powerpc/crypto/chacha-p10-glue.c | 8 +- arch/riscv/crypto/chacha-riscv64-glue.c | 8 +- arch/s390/crypto/chacha-glue.c | 8 +- arch/x86/crypto/chacha_glue.c | 8 +- crypto/Makefile | 3 +- crypto/chacha.c | 227 ++++++++++++++++++++++++ crypto/chacha_generic.c | 139 --------------- include/crypto/chacha.h | 9 + 11 files changed, 288 insertions(+), 147 deletions(-) create mode 100644 crypto/chacha.c delete mode 100644 crypto/chacha_generic.c diff --git a/arch/arm/crypto/chacha-glue.c b/arch/arm/crypto/chacha-glue.c index 50e635512046..e1cb34d31771 100644 --- a/arch/arm/crypto/chacha-glue.c +++ b/arch/arm/crypto/chacha-glue.c @@ -287,6 +287,13 @@ static struct skcipher_alg neon_algs[] = { } }; +bool chacha_is_arch_optimized(void) +{ + /* We always can use at least the ARM scalar implementation. */ + return true; +} +EXPORT_SYMBOL(chacha_is_arch_optimized); + static int __init chacha_simd_mod_init(void) { int err = 0; @@ -333,7 +340,7 @@ static void __exit chacha_simd_mod_fini(void) } } -module_init(chacha_simd_mod_init); +arch_initcall(chacha_simd_mod_init); module_exit(chacha_simd_mod_fini); MODULE_DESCRIPTION("ChaCha and XChaCha stream ciphers (scalar and NEON accelerated)"); diff --git a/arch/arm64/crypto/chacha-neon-glue.c b/arch/arm64/crypto/chacha-neon-glue.c index 229876acfc58..bb9b52321bda 100644 --- a/arch/arm64/crypto/chacha-neon-glue.c +++ b/arch/arm64/crypto/chacha-neon-glue.c @@ -206,6 +206,12 @@ static struct skcipher_alg algs[] = { } }; +bool chacha_is_arch_optimized(void) +{ + return static_key_enabled(&have_neon); +} +EXPORT_SYMBOL(chacha_is_arch_optimized); + static int __init chacha_simd_mod_init(void) { if (!cpu_have_named_feature(ASIMD)) @@ -223,7 +229,7 @@ static void __exit chacha_simd_mod_fini(void) crypto_unregister_skciphers(algs, ARRAY_SIZE(algs)); } -module_init(chacha_simd_mod_init); +arch_initcall(chacha_simd_mod_init); module_exit(chacha_simd_mod_fini); MODULE_DESCRIPTION("ChaCha and XChaCha stream ciphers (NEON accelerated)"); diff --git a/arch/mips/crypto/chacha-glue.c b/arch/mips/crypto/chacha-glue.c index f6fc2e1079a1..64ccaeaeaa1e 100644 --- a/arch/mips/crypto/chacha-glue.c +++ b/arch/mips/crypto/chacha-glue.c @@ -120,6 +120,12 @@ static struct skcipher_alg algs[] = { } }; +bool chacha_is_arch_optimized(void) +{ + return true; +} +EXPORT_SYMBOL(chacha_is_arch_optimized); + static int __init chacha_simd_mod_init(void) { return IS_REACHABLE(CONFIG_CRYPTO_SKCIPHER) ? @@ -132,7 +138,7 @@ static void __exit chacha_simd_mod_fini(void) crypto_unregister_skciphers(algs, ARRAY_SIZE(algs)); } -module_init(chacha_simd_mod_init); +arch_initcall(chacha_simd_mod_init); module_exit(chacha_simd_mod_fini); MODULE_DESCRIPTION("ChaCha and XChaCha stream ciphers (MIPS accelerated)"); diff --git a/arch/powerpc/crypto/chacha-p10-glue.c b/arch/powerpc/crypto/chacha-p10-glue.c index d8796decc1fb..3355305b6c7f 100644 --- a/arch/powerpc/crypto/chacha-p10-glue.c +++ b/arch/powerpc/crypto/chacha-p10-glue.c @@ -189,6 +189,12 @@ static struct skcipher_alg algs[] = { } }; +bool chacha_is_arch_optimized(void) +{ + return static_key_enabled(&have_p10); +} +EXPORT_SYMBOL(chacha_is_arch_optimized); + static int __init chacha_p10_init(void) { if (!cpu_has_feature(CPU_FTR_ARCH_31)) @@ -207,7 +213,7 @@ static void __exit chacha_p10_exit(void) crypto_unregister_skciphers(algs, ARRAY_SIZE(algs)); } -module_init(chacha_p10_init); +arch_initcall(chacha_p10_init); module_exit(chacha_p10_exit); MODULE_DESCRIPTION("ChaCha and XChaCha stream ciphers (P10 accelerated)"); diff --git a/arch/riscv/crypto/chacha-riscv64-glue.c b/arch/riscv/crypto/chacha-riscv64-glue.c index 68caef7a3d50..ccaab0dea383 100644 --- a/arch/riscv/crypto/chacha-riscv64-glue.c +++ b/arch/riscv/crypto/chacha-riscv64-glue.c @@ -49,6 +49,12 @@ void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *src, unsigned int bytes, } EXPORT_SYMBOL(chacha_crypt_arch); +bool chacha_is_arch_optimized(void) +{ + return static_key_enabled(&use_zvkb); +} +EXPORT_SYMBOL(chacha_is_arch_optimized); + static int __init riscv64_chacha_mod_init(void) { if (riscv_isa_extension_available(NULL, ZVKB) && @@ -56,7 +62,7 @@ static int __init riscv64_chacha_mod_init(void) static_branch_enable(&use_zvkb); return 0; } -module_init(riscv64_chacha_mod_init); +arch_initcall(riscv64_chacha_mod_init); MODULE_DESCRIPTION("ChaCha stream cipher (RISC-V optimized)"); MODULE_AUTHOR("Jerry Shih "); diff --git a/arch/s390/crypto/chacha-glue.c b/arch/s390/crypto/chacha-glue.c index 920e9f0941e7..0c68191f2aa4 100644 --- a/arch/s390/crypto/chacha-glue.c +++ b/arch/s390/crypto/chacha-glue.c @@ -103,6 +103,12 @@ static struct skcipher_alg chacha_algs[] = { } }; +bool chacha_is_arch_optimized(void) +{ + return cpu_has_vx(); +} +EXPORT_SYMBOL(chacha_is_arch_optimized); + static int __init chacha_mod_init(void) { return IS_REACHABLE(CONFIG_CRYPTO_SKCIPHER) ? @@ -115,7 +121,7 @@ static void __exit chacha_mod_fini(void) crypto_unregister_skciphers(chacha_algs, ARRAY_SIZE(chacha_algs)); } -module_cpu_feature_match(S390_CPU_FEATURE_VXRS, chacha_mod_init); +arch_initcall(chacha_mod_init); module_exit(chacha_mod_fini); MODULE_DESCRIPTION("ChaCha20 stream cipher"); diff --git a/arch/x86/crypto/chacha_glue.c b/arch/x86/crypto/chacha_glue.c index 946c306f60cd..8858de7b33e3 100644 --- a/arch/x86/crypto/chacha_glue.c +++ b/arch/x86/crypto/chacha_glue.c @@ -247,6 +247,12 @@ static struct skcipher_alg algs[] = { }, }; +bool chacha_is_arch_optimized(void) +{ + return static_key_enabled(&chacha_use_simd); +} +EXPORT_SYMBOL(chacha_is_arch_optimized); + static int __init chacha_simd_mod_init(void) { if (!boot_cpu_has(X86_FEATURE_SSSE3)) @@ -271,7 +277,7 @@ static void __exit chacha_simd_mod_fini(void) crypto_unregister_skciphers(algs, ARRAY_SIZE(algs)); } -module_init(chacha_simd_mod_init); +arch_initcall(chacha_simd_mod_init); module_exit(chacha_simd_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/Makefile b/crypto/Makefile index 0e6ab5ffd3f7..98510a2aa0b1 100644 --- a/crypto/Makefile +++ b/crypto/Makefile @@ -148,7 +148,8 @@ obj-$(CONFIG_CRYPTO_KHAZAD) += khazad.o obj-$(CONFIG_CRYPTO_ANUBIS) += anubis.o obj-$(CONFIG_CRYPTO_SEED) += seed.o obj-$(CONFIG_CRYPTO_ARIA) += aria_generic.o -obj-$(CONFIG_CRYPTO_CHACHA20) += chacha_generic.o +obj-$(CONFIG_CRYPTO_CHACHA20) += chacha.o +CFLAGS_chacha.o += -DARCH=$(ARCH) obj-$(CONFIG_CRYPTO_POLY1305) += poly1305_generic.o obj-$(CONFIG_CRYPTO_DEFLATE) += deflate.o obj-$(CONFIG_CRYPTO_MICHAEL_MIC) += michael_mic.o diff --git a/crypto/chacha.c b/crypto/chacha.c new file mode 100644 index 000000000000..2009038c5e56 --- /dev/null +++ b/crypto/chacha.c @@ -0,0 +1,227 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Crypto API wrappers for the ChaCha20, XChaCha20, and XChaCha12 stream ciphers + * + * Copyright (C) 2015 Martin Willi + * Copyright (C) 2018 Google LLC + */ + +#include +#include +#include +#include +#include + +static int chacha_stream_xor(struct skcipher_request *req, + const struct chacha_ctx *ctx, const u8 *iv, + bool arch) +{ + struct skcipher_walk walk; + u32 state[16]; + int err; + + err = skcipher_walk_virt(&walk, req, false); + + chacha_init(state, ctx->key, iv); + + while (walk.nbytes > 0) { + unsigned int nbytes = walk.nbytes; + + if (nbytes < walk.total) + nbytes = round_down(nbytes, CHACHA_BLOCK_SIZE); + + if (arch) + chacha_crypt(state, walk.dst.virt.addr, + walk.src.virt.addr, nbytes, ctx->nrounds); + else + chacha_crypt_generic(state, walk.dst.virt.addr, + walk.src.virt.addr, nbytes, + ctx->nrounds); + err = skcipher_walk_done(&walk, walk.nbytes - nbytes); + } + + return err; +} + +static int crypto_chacha_crypt_generic(struct skcipher_request *req) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + const struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm); + + return chacha_stream_xor(req, ctx, req->iv, false); +} + +static int crypto_chacha_crypt_arch(struct skcipher_request *req) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + const struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm); + + return chacha_stream_xor(req, ctx, req->iv, true); +} + +static int crypto_xchacha_crypt(struct skcipher_request *req, bool arch) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + const struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm); + struct chacha_ctx subctx; + u32 state[16]; + u8 real_iv[16]; + + /* Compute the subkey given the original key and first 128 nonce bits */ + chacha_init(state, ctx->key, req->iv); + if (arch) + hchacha_block(state, subctx.key, ctx->nrounds); + else + hchacha_block_generic(state, subctx.key, ctx->nrounds); + subctx.nrounds = ctx->nrounds; + + /* Build the real IV */ + memcpy(&real_iv[0], req->iv + 24, 8); /* stream position */ + memcpy(&real_iv[8], req->iv + 16, 8); /* remaining 64 nonce bits */ + + /* Generate the stream and XOR it with the data */ + return chacha_stream_xor(req, &subctx, real_iv, arch); +} + +static int crypto_xchacha_crypt_generic(struct skcipher_request *req) +{ + return crypto_xchacha_crypt(req, false); +} + +static int crypto_xchacha_crypt_arch(struct skcipher_request *req) +{ + return crypto_xchacha_crypt(req, true); +} + +static struct skcipher_alg algs[] = { + { + .base.cra_name = "chacha20", + .base.cra_driver_name = "chacha20-generic", + .base.cra_priority = 100, + .base.cra_blocksize = 1, + .base.cra_ctxsize = sizeof(struct chacha_ctx), + .base.cra_module = THIS_MODULE, + + .min_keysize = CHACHA_KEY_SIZE, + .max_keysize = CHACHA_KEY_SIZE, + .ivsize = CHACHA_IV_SIZE, + .chunksize = CHACHA_BLOCK_SIZE, + .setkey = chacha20_setkey, + .encrypt = crypto_chacha_crypt_generic, + .decrypt = crypto_chacha_crypt_generic, + }, + { + .base.cra_name = "xchacha20", + .base.cra_driver_name = "xchacha20-generic", + .base.cra_priority = 100, + .base.cra_blocksize = 1, + .base.cra_ctxsize = sizeof(struct chacha_ctx), + .base.cra_module = THIS_MODULE, + + .min_keysize = CHACHA_KEY_SIZE, + .max_keysize = CHACHA_KEY_SIZE, + .ivsize = XCHACHA_IV_SIZE, + .chunksize = CHACHA_BLOCK_SIZE, + .setkey = chacha20_setkey, + .encrypt = crypto_xchacha_crypt_generic, + .decrypt = crypto_xchacha_crypt_generic, + }, + { + .base.cra_name = "xchacha12", + .base.cra_driver_name = "xchacha12-generic", + .base.cra_priority = 100, + .base.cra_blocksize = 1, + .base.cra_ctxsize = sizeof(struct chacha_ctx), + .base.cra_module = THIS_MODULE, + + .min_keysize = CHACHA_KEY_SIZE, + .max_keysize = CHACHA_KEY_SIZE, + .ivsize = XCHACHA_IV_SIZE, + .chunksize = CHACHA_BLOCK_SIZE, + .setkey = chacha12_setkey, + .encrypt = crypto_xchacha_crypt_generic, + .decrypt = crypto_xchacha_crypt_generic, + }, + { + .base.cra_name = "chacha20", + .base.cra_driver_name = "chacha20-" __stringify(ARCH), + .base.cra_priority = 300, + .base.cra_blocksize = 1, + .base.cra_ctxsize = sizeof(struct chacha_ctx), + .base.cra_module = THIS_MODULE, + + .min_keysize = CHACHA_KEY_SIZE, + .max_keysize = CHACHA_KEY_SIZE, + .ivsize = CHACHA_IV_SIZE, + .chunksize = CHACHA_BLOCK_SIZE, + .setkey = chacha20_setkey, + .encrypt = crypto_chacha_crypt_arch, + .decrypt = crypto_chacha_crypt_arch, + }, + { + .base.cra_name = "xchacha20", + .base.cra_driver_name = "xchacha20-" __stringify(ARCH), + .base.cra_priority = 300, + .base.cra_blocksize = 1, + .base.cra_ctxsize = sizeof(struct chacha_ctx), + .base.cra_module = THIS_MODULE, + + .min_keysize = CHACHA_KEY_SIZE, + .max_keysize = CHACHA_KEY_SIZE, + .ivsize = XCHACHA_IV_SIZE, + .chunksize = CHACHA_BLOCK_SIZE, + .setkey = chacha20_setkey, + .encrypt = crypto_xchacha_crypt_arch, + .decrypt = crypto_xchacha_crypt_arch, + }, + { + .base.cra_name = "xchacha12", + .base.cra_driver_name = "xchacha12-" __stringify(ARCH), + .base.cra_priority = 300, + .base.cra_blocksize = 1, + .base.cra_ctxsize = sizeof(struct chacha_ctx), + .base.cra_module = THIS_MODULE, + + .min_keysize = CHACHA_KEY_SIZE, + .max_keysize = CHACHA_KEY_SIZE, + .ivsize = XCHACHA_IV_SIZE, + .chunksize = CHACHA_BLOCK_SIZE, + .setkey = chacha12_setkey, + .encrypt = crypto_xchacha_crypt_arch, + .decrypt = crypto_xchacha_crypt_arch, + } +}; + +static unsigned int num_algs; + +static int __init crypto_chacha_mod_init(void) +{ + /* register the arch flavours only if they differ from generic */ + num_algs = ARRAY_SIZE(algs); + BUILD_BUG_ON(ARRAY_SIZE(algs) % 2 != 0); + if (!chacha_is_arch_optimized()) + num_algs /= 2; + + return crypto_register_skciphers(algs, num_algs); +} + +static void __exit crypto_chacha_mod_fini(void) +{ + crypto_unregister_skciphers(algs, num_algs); +} + +subsys_initcall(crypto_chacha_mod_init); +module_exit(crypto_chacha_mod_fini); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Martin Willi "); +MODULE_DESCRIPTION("Crypto API wrappers for the ChaCha20, XChaCha20, and XChaCha12 stream ciphers"); +MODULE_ALIAS_CRYPTO("chacha20"); +MODULE_ALIAS_CRYPTO("chacha20-generic"); +MODULE_ALIAS_CRYPTO("chacha20-" __stringify(ARCH)); +MODULE_ALIAS_CRYPTO("xchacha20"); +MODULE_ALIAS_CRYPTO("xchacha20-generic"); +MODULE_ALIAS_CRYPTO("xchacha20-" __stringify(ARCH)); +MODULE_ALIAS_CRYPTO("xchacha12"); +MODULE_ALIAS_CRYPTO("xchacha12-generic"); +MODULE_ALIAS_CRYPTO("xchacha12-" __stringify(ARCH)); diff --git a/crypto/chacha_generic.c b/crypto/chacha_generic.c deleted file mode 100644 index 1fb9fbd302c6..000000000000 --- a/crypto/chacha_generic.c +++ /dev/null @@ -1,139 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * ChaCha and XChaCha stream ciphers, including ChaCha20 (RFC7539) - * - * Copyright (C) 2015 Martin Willi - * Copyright (C) 2018 Google LLC - */ - -#include -#include -#include -#include -#include - -static int chacha_stream_xor(struct skcipher_request *req, - const struct chacha_ctx *ctx, const u8 *iv) -{ - struct skcipher_walk walk; - u32 state[16]; - int err; - - err = skcipher_walk_virt(&walk, req, false); - - chacha_init(state, ctx->key, iv); - - while (walk.nbytes > 0) { - unsigned int nbytes = walk.nbytes; - - if (nbytes < walk.total) - nbytes = round_down(nbytes, CHACHA_BLOCK_SIZE); - - chacha_crypt_generic(state, walk.dst.virt.addr, - walk.src.virt.addr, nbytes, ctx->nrounds); - err = skcipher_walk_done(&walk, walk.nbytes - nbytes); - } - - return err; -} - -static int crypto_chacha_crypt(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm); - - return chacha_stream_xor(req, ctx, req->iv); -} - -static int crypto_xchacha_crypt(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm); - struct chacha_ctx subctx; - u32 state[16]; - u8 real_iv[16]; - - /* Compute the subkey given the original key and first 128 nonce bits */ - chacha_init(state, ctx->key, req->iv); - hchacha_block_generic(state, subctx.key, ctx->nrounds); - subctx.nrounds = ctx->nrounds; - - /* Build the real IV */ - memcpy(&real_iv[0], req->iv + 24, 8); /* stream position */ - memcpy(&real_iv[8], req->iv + 16, 8); /* remaining 64 nonce bits */ - - /* Generate the stream and XOR it with the data */ - return chacha_stream_xor(req, &subctx, real_iv); -} - -static struct skcipher_alg algs[] = { - { - .base.cra_name = "chacha20", - .base.cra_driver_name = "chacha20-generic", - .base.cra_priority = 100, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct chacha_ctx), - .base.cra_module = THIS_MODULE, - - .min_keysize = CHACHA_KEY_SIZE, - .max_keysize = CHACHA_KEY_SIZE, - .ivsize = CHACHA_IV_SIZE, - .chunksize = CHACHA_BLOCK_SIZE, - .setkey = chacha20_setkey, - .encrypt = crypto_chacha_crypt, - .decrypt = crypto_chacha_crypt, - }, { - .base.cra_name = "xchacha20", - .base.cra_driver_name = "xchacha20-generic", - .base.cra_priority = 100, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct chacha_ctx), - .base.cra_module = THIS_MODULE, - - .min_keysize = CHACHA_KEY_SIZE, - .max_keysize = CHACHA_KEY_SIZE, - .ivsize = XCHACHA_IV_SIZE, - .chunksize = CHACHA_BLOCK_SIZE, - .setkey = chacha20_setkey, - .encrypt = crypto_xchacha_crypt, - .decrypt = crypto_xchacha_crypt, - }, { - .base.cra_name = "xchacha12", - .base.cra_driver_name = "xchacha12-generic", - .base.cra_priority = 100, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct chacha_ctx), - .base.cra_module = THIS_MODULE, - - .min_keysize = CHACHA_KEY_SIZE, - .max_keysize = CHACHA_KEY_SIZE, - .ivsize = XCHACHA_IV_SIZE, - .chunksize = CHACHA_BLOCK_SIZE, - .setkey = chacha12_setkey, - .encrypt = crypto_xchacha_crypt, - .decrypt = crypto_xchacha_crypt, - } -}; - -static int __init chacha_generic_mod_init(void) -{ - return crypto_register_skciphers(algs, ARRAY_SIZE(algs)); -} - -static void __exit chacha_generic_mod_fini(void) -{ - crypto_unregister_skciphers(algs, ARRAY_SIZE(algs)); -} - -subsys_initcall(chacha_generic_mod_init); -module_exit(chacha_generic_mod_fini); - -MODULE_LICENSE("GPL"); -MODULE_AUTHOR("Martin Willi "); -MODULE_DESCRIPTION("ChaCha and XChaCha stream ciphers (generic)"); -MODULE_ALIAS_CRYPTO("chacha20"); -MODULE_ALIAS_CRYPTO("chacha20-generic"); -MODULE_ALIAS_CRYPTO("xchacha20"); -MODULE_ALIAS_CRYPTO("xchacha20-generic"); -MODULE_ALIAS_CRYPTO("xchacha12"); -MODULE_ALIAS_CRYPTO("xchacha12-generic"); diff --git a/include/crypto/chacha.h b/include/crypto/chacha.h index f8cc073bba41..58129e18cc31 100644 --- a/include/crypto/chacha.h +++ b/include/crypto/chacha.h @@ -99,4 +99,13 @@ static inline void chacha20_crypt(u32 *state, u8 *dst, const u8 *src, chacha_crypt(state, dst, src, bytes, 20); } +#if IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_CHACHA) +bool chacha_is_arch_optimized(void); +#else +static inline bool chacha_is_arch_optimized(void) +{ + return false; +} +#endif + #endif /* _CRYPTO_CHACHA_H */ From 08820553f33ae0d9143ff33910dbb5874b742455 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Sat, 5 Apr 2025 11:26:03 -0700 Subject: [PATCH 057/382] crypto: arm/chacha - remove the redundant skcipher algorithms Since crypto/chacha.c now registers chacha20-$(ARCH), xchacha20-$(ARCH), and xchacha12-$(ARCH) skcipher algorithms that use the architecture's ChaCha and HChaCha library functions, individual architectures no longer need to do the same. Therefore, remove the redundant skcipher algorithms and leave just the library functions. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/arm/crypto/Kconfig | 7 - arch/arm/crypto/chacha-glue.c | 242 +---------------------------- arch/arm/crypto/chacha-neon-core.S | 2 +- 3 files changed, 7 insertions(+), 244 deletions(-) diff --git a/arch/arm/crypto/Kconfig b/arch/arm/crypto/Kconfig index be9c2e19f976..332f2430beec 100644 --- a/arch/arm/crypto/Kconfig +++ b/arch/arm/crypto/Kconfig @@ -214,15 +214,8 @@ config CRYPTO_AES_ARM_CE config CRYPTO_CHACHA20_NEON tristate - select CRYPTO_SKCIPHER select CRYPTO_ARCH_HAVE_LIB_CHACHA default CRYPTO_LIB_CHACHA_INTERNAL - help - Length-preserving ciphers: ChaCha20, XChaCha20, and XChaCha12 - stream cipher algorithms - - Architecture: arm using: - - NEON (Advanced SIMD) extensions endmenu diff --git a/arch/arm/crypto/chacha-glue.c b/arch/arm/crypto/chacha-glue.c index e1cb34d31771..3a5c75c95d43 100644 --- a/arch/arm/crypto/chacha-glue.c +++ b/arch/arm/crypto/chacha-glue.c @@ -1,16 +1,13 @@ // SPDX-License-Identifier: GPL-2.0 /* - * ARM NEON accelerated ChaCha and XChaCha stream ciphers, - * including ChaCha20 (RFC7539) + * ChaCha and HChaCha functions (ARM optimized) * * Copyright (C) 2016-2019 Linaro, Ltd. * Copyright (C) 2015 Martin Willi */ -#include -#include +#include #include -#include #include #include #include @@ -100,193 +97,6 @@ void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *src, unsigned int bytes, } EXPORT_SYMBOL(chacha_crypt_arch); -static int chacha_stream_xor(struct skcipher_request *req, - const struct chacha_ctx *ctx, const u8 *iv, - bool neon) -{ - struct skcipher_walk walk; - u32 state[16]; - int err; - - err = skcipher_walk_virt(&walk, req, false); - - chacha_init(state, ctx->key, iv); - - while (walk.nbytes > 0) { - unsigned int nbytes = walk.nbytes; - - if (nbytes < walk.total) - nbytes = round_down(nbytes, walk.stride); - - if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) || !neon) { - chacha_doarm(walk.dst.virt.addr, walk.src.virt.addr, - nbytes, state, ctx->nrounds); - state[12] += DIV_ROUND_UP(nbytes, CHACHA_BLOCK_SIZE); - } else { - kernel_neon_begin(); - chacha_doneon(state, walk.dst.virt.addr, - walk.src.virt.addr, nbytes, ctx->nrounds); - kernel_neon_end(); - } - err = skcipher_walk_done(&walk, walk.nbytes - nbytes); - } - - return err; -} - -static int do_chacha(struct skcipher_request *req, bool neon) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm); - - return chacha_stream_xor(req, ctx, req->iv, neon); -} - -static int chacha_arm(struct skcipher_request *req) -{ - return do_chacha(req, false); -} - -static int chacha_neon(struct skcipher_request *req) -{ - return do_chacha(req, neon_usable()); -} - -static int do_xchacha(struct skcipher_request *req, bool neon) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm); - struct chacha_ctx subctx; - u32 state[16]; - u8 real_iv[16]; - - chacha_init(state, ctx->key, req->iv); - - if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) || !neon) { - hchacha_block_arm(state, subctx.key, ctx->nrounds); - } else { - kernel_neon_begin(); - hchacha_block_neon(state, subctx.key, ctx->nrounds); - kernel_neon_end(); - } - subctx.nrounds = ctx->nrounds; - - memcpy(&real_iv[0], req->iv + 24, 8); - memcpy(&real_iv[8], req->iv + 16, 8); - return chacha_stream_xor(req, &subctx, real_iv, neon); -} - -static int xchacha_arm(struct skcipher_request *req) -{ - return do_xchacha(req, false); -} - -static int xchacha_neon(struct skcipher_request *req) -{ - return do_xchacha(req, neon_usable()); -} - -static struct skcipher_alg arm_algs[] = { - { - .base.cra_name = "chacha20", - .base.cra_driver_name = "chacha20-arm", - .base.cra_priority = 200, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct chacha_ctx), - .base.cra_module = THIS_MODULE, - - .min_keysize = CHACHA_KEY_SIZE, - .max_keysize = CHACHA_KEY_SIZE, - .ivsize = CHACHA_IV_SIZE, - .chunksize = CHACHA_BLOCK_SIZE, - .setkey = chacha20_setkey, - .encrypt = chacha_arm, - .decrypt = chacha_arm, - }, { - .base.cra_name = "xchacha20", - .base.cra_driver_name = "xchacha20-arm", - .base.cra_priority = 200, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct chacha_ctx), - .base.cra_module = THIS_MODULE, - - .min_keysize = CHACHA_KEY_SIZE, - .max_keysize = CHACHA_KEY_SIZE, - .ivsize = XCHACHA_IV_SIZE, - .chunksize = CHACHA_BLOCK_SIZE, - .setkey = chacha20_setkey, - .encrypt = xchacha_arm, - .decrypt = xchacha_arm, - }, { - .base.cra_name = "xchacha12", - .base.cra_driver_name = "xchacha12-arm", - .base.cra_priority = 200, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct chacha_ctx), - .base.cra_module = THIS_MODULE, - - .min_keysize = CHACHA_KEY_SIZE, - .max_keysize = CHACHA_KEY_SIZE, - .ivsize = XCHACHA_IV_SIZE, - .chunksize = CHACHA_BLOCK_SIZE, - .setkey = chacha12_setkey, - .encrypt = xchacha_arm, - .decrypt = xchacha_arm, - }, -}; - -static struct skcipher_alg neon_algs[] = { - { - .base.cra_name = "chacha20", - .base.cra_driver_name = "chacha20-neon", - .base.cra_priority = 300, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct chacha_ctx), - .base.cra_module = THIS_MODULE, - - .min_keysize = CHACHA_KEY_SIZE, - .max_keysize = CHACHA_KEY_SIZE, - .ivsize = CHACHA_IV_SIZE, - .chunksize = CHACHA_BLOCK_SIZE, - .walksize = 4 * CHACHA_BLOCK_SIZE, - .setkey = chacha20_setkey, - .encrypt = chacha_neon, - .decrypt = chacha_neon, - }, { - .base.cra_name = "xchacha20", - .base.cra_driver_name = "xchacha20-neon", - .base.cra_priority = 300, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct chacha_ctx), - .base.cra_module = THIS_MODULE, - - .min_keysize = CHACHA_KEY_SIZE, - .max_keysize = CHACHA_KEY_SIZE, - .ivsize = XCHACHA_IV_SIZE, - .chunksize = CHACHA_BLOCK_SIZE, - .walksize = 4 * CHACHA_BLOCK_SIZE, - .setkey = chacha20_setkey, - .encrypt = xchacha_neon, - .decrypt = xchacha_neon, - }, { - .base.cra_name = "xchacha12", - .base.cra_driver_name = "xchacha12-neon", - .base.cra_priority = 300, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct chacha_ctx), - .base.cra_module = THIS_MODULE, - - .min_keysize = CHACHA_KEY_SIZE, - .max_keysize = CHACHA_KEY_SIZE, - .ivsize = XCHACHA_IV_SIZE, - .chunksize = CHACHA_BLOCK_SIZE, - .walksize = 4 * CHACHA_BLOCK_SIZE, - .setkey = chacha12_setkey, - .encrypt = xchacha_neon, - .decrypt = xchacha_neon, - } -}; - bool chacha_is_arch_optimized(void) { /* We always can use at least the ARM scalar implementation. */ @@ -294,19 +104,9 @@ bool chacha_is_arch_optimized(void) } EXPORT_SYMBOL(chacha_is_arch_optimized); -static int __init chacha_simd_mod_init(void) +static int __init chacha_arm_mod_init(void) { - int err = 0; - - if (IS_REACHABLE(CONFIG_CRYPTO_SKCIPHER)) { - err = crypto_register_skciphers(arm_algs, ARRAY_SIZE(arm_algs)); - if (err) - return err; - } - if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && (elf_hwcap & HWCAP_NEON)) { - int i; - switch (read_cpuid_part()) { case ARM_CPU_PART_CORTEX_A7: case ARM_CPU_PART_CORTEX_A5: @@ -315,45 +115,15 @@ static int __init chacha_simd_mod_init(void) * the NEON implementation but do incredibly with the * scalar one and use less power. */ - for (i = 0; i < ARRAY_SIZE(neon_algs); i++) - neon_algs[i].base.cra_priority = 0; break; default: static_branch_enable(&use_neon); } - - if (IS_REACHABLE(CONFIG_CRYPTO_SKCIPHER)) { - err = crypto_register_skciphers(neon_algs, ARRAY_SIZE(neon_algs)); - if (err) - crypto_unregister_skciphers(arm_algs, ARRAY_SIZE(arm_algs)); - } } - return err; + return 0; } +arch_initcall(chacha_arm_mod_init); -static void __exit chacha_simd_mod_fini(void) -{ - if (IS_REACHABLE(CONFIG_CRYPTO_SKCIPHER)) { - crypto_unregister_skciphers(arm_algs, ARRAY_SIZE(arm_algs)); - if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && (elf_hwcap & HWCAP_NEON)) - crypto_unregister_skciphers(neon_algs, ARRAY_SIZE(neon_algs)); - } -} - -arch_initcall(chacha_simd_mod_init); -module_exit(chacha_simd_mod_fini); - -MODULE_DESCRIPTION("ChaCha and XChaCha stream ciphers (scalar and NEON accelerated)"); +MODULE_DESCRIPTION("ChaCha and HChaCha functions (ARM optimized)"); MODULE_AUTHOR("Ard Biesheuvel "); MODULE_LICENSE("GPL v2"); -MODULE_ALIAS_CRYPTO("chacha20"); -MODULE_ALIAS_CRYPTO("chacha20-arm"); -MODULE_ALIAS_CRYPTO("xchacha20"); -MODULE_ALIAS_CRYPTO("xchacha20-arm"); -MODULE_ALIAS_CRYPTO("xchacha12"); -MODULE_ALIAS_CRYPTO("xchacha12-arm"); -#ifdef CONFIG_KERNEL_MODE_NEON -MODULE_ALIAS_CRYPTO("chacha20-neon"); -MODULE_ALIAS_CRYPTO("xchacha20-neon"); -MODULE_ALIAS_CRYPTO("xchacha12-neon"); -#endif diff --git a/arch/arm/crypto/chacha-neon-core.S b/arch/arm/crypto/chacha-neon-core.S index 13d12f672656..ddd62b6294a5 100644 --- a/arch/arm/crypto/chacha-neon-core.S +++ b/arch/arm/crypto/chacha-neon-core.S @@ -1,5 +1,5 @@ /* - * ChaCha/XChaCha NEON helper functions + * ChaCha/HChaCha NEON helper functions * * Copyright (C) 2016 Linaro, Ltd. * From 8c28abede16c9396eb90356a4e084d5f3e847886 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Sat, 5 Apr 2025 11:26:04 -0700 Subject: [PATCH 058/382] crypto: arm64/chacha - remove the skcipher algorithms Since crypto/chacha.c now registers chacha20-$(ARCH), xchacha20-$(ARCH), and xchacha12-$(ARCH) skcipher algorithms that use the architecture's ChaCha and HChaCha library functions, individual architectures no longer need to do the same. Therefore, remove the redundant skcipher algorithms and leave just the library functions. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/arm64/crypto/Kconfig | 7 -- arch/arm64/crypto/chacha-neon-core.S | 2 +- arch/arm64/crypto/chacha-neon-glue.c | 144 ++------------------------- 3 files changed, 7 insertions(+), 146 deletions(-) diff --git a/arch/arm64/crypto/Kconfig b/arch/arm64/crypto/Kconfig index 3418c8d3c78d..ce655da0fbee 100644 --- a/arch/arm64/crypto/Kconfig +++ b/arch/arm64/crypto/Kconfig @@ -189,16 +189,9 @@ config CRYPTO_AES_ARM64_NEON_BLK config CRYPTO_CHACHA20_NEON tristate depends on KERNEL_MODE_NEON - select CRYPTO_SKCIPHER select CRYPTO_LIB_CHACHA_GENERIC select CRYPTO_ARCH_HAVE_LIB_CHACHA default CRYPTO_LIB_CHACHA_INTERNAL - help - Length-preserving ciphers: ChaCha20, XChaCha20, and XChaCha12 - stream cipher algorithms - - Architecture: arm64 using: - - NEON (Advanced SIMD) extensions config CRYPTO_AES_ARM64_BS tristate "Ciphers: AES, modes: ECB/CBC/CTR/XCTR/XTS modes (bit-sliced NEON)" diff --git a/arch/arm64/crypto/chacha-neon-core.S b/arch/arm64/crypto/chacha-neon-core.S index b70ac76f2610..80079586ecc7 100644 --- a/arch/arm64/crypto/chacha-neon-core.S +++ b/arch/arm64/crypto/chacha-neon-core.S @@ -1,5 +1,5 @@ /* - * ChaCha/XChaCha NEON helper functions + * ChaCha/HChaCha NEON helper functions * * Copyright (C) 2016-2018 Linaro, Ltd. * diff --git a/arch/arm64/crypto/chacha-neon-glue.c b/arch/arm64/crypto/chacha-neon-glue.c index bb9b52321bda..a0c336b28402 100644 --- a/arch/arm64/crypto/chacha-neon-glue.c +++ b/arch/arm64/crypto/chacha-neon-glue.c @@ -1,6 +1,5 @@ /* - * ARM NEON and scalar accelerated ChaCha and XChaCha stream ciphers, - * including ChaCha20 (RFC7539) + * ChaCha and HChaCha functions (ARM64 optimized) * * Copyright (C) 2016 - 2017 Linaro, Ltd. * @@ -19,10 +18,8 @@ * (at your option) any later version. */ -#include -#include +#include #include -#include #include #include #include @@ -95,117 +92,6 @@ void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *src, unsigned int bytes, } EXPORT_SYMBOL(chacha_crypt_arch); -static int chacha_neon_stream_xor(struct skcipher_request *req, - const struct chacha_ctx *ctx, const u8 *iv) -{ - struct skcipher_walk walk; - u32 state[16]; - int err; - - err = skcipher_walk_virt(&walk, req, false); - - chacha_init(state, ctx->key, iv); - - while (walk.nbytes > 0) { - unsigned int nbytes = walk.nbytes; - - if (nbytes < walk.total) - nbytes = rounddown(nbytes, walk.stride); - - if (!static_branch_likely(&have_neon) || - !crypto_simd_usable()) { - chacha_crypt_generic(state, walk.dst.virt.addr, - walk.src.virt.addr, nbytes, - ctx->nrounds); - } else { - kernel_neon_begin(); - chacha_doneon(state, walk.dst.virt.addr, - walk.src.virt.addr, nbytes, ctx->nrounds); - kernel_neon_end(); - } - err = skcipher_walk_done(&walk, walk.nbytes - nbytes); - } - - return err; -} - -static int chacha_neon(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm); - - return chacha_neon_stream_xor(req, ctx, req->iv); -} - -static int xchacha_neon(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm); - struct chacha_ctx subctx; - u32 state[16]; - u8 real_iv[16]; - - chacha_init(state, ctx->key, req->iv); - hchacha_block_arch(state, subctx.key, ctx->nrounds); - subctx.nrounds = ctx->nrounds; - - memcpy(&real_iv[0], req->iv + 24, 8); - memcpy(&real_iv[8], req->iv + 16, 8); - return chacha_neon_stream_xor(req, &subctx, real_iv); -} - -static struct skcipher_alg algs[] = { - { - .base.cra_name = "chacha20", - .base.cra_driver_name = "chacha20-neon", - .base.cra_priority = 300, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct chacha_ctx), - .base.cra_module = THIS_MODULE, - - .min_keysize = CHACHA_KEY_SIZE, - .max_keysize = CHACHA_KEY_SIZE, - .ivsize = CHACHA_IV_SIZE, - .chunksize = CHACHA_BLOCK_SIZE, - .walksize = 5 * CHACHA_BLOCK_SIZE, - .setkey = chacha20_setkey, - .encrypt = chacha_neon, - .decrypt = chacha_neon, - }, { - .base.cra_name = "xchacha20", - .base.cra_driver_name = "xchacha20-neon", - .base.cra_priority = 300, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct chacha_ctx), - .base.cra_module = THIS_MODULE, - - .min_keysize = CHACHA_KEY_SIZE, - .max_keysize = CHACHA_KEY_SIZE, - .ivsize = XCHACHA_IV_SIZE, - .chunksize = CHACHA_BLOCK_SIZE, - .walksize = 5 * CHACHA_BLOCK_SIZE, - .setkey = chacha20_setkey, - .encrypt = xchacha_neon, - .decrypt = xchacha_neon, - }, { - .base.cra_name = "xchacha12", - .base.cra_driver_name = "xchacha12-neon", - .base.cra_priority = 300, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct chacha_ctx), - .base.cra_module = THIS_MODULE, - - .min_keysize = CHACHA_KEY_SIZE, - .max_keysize = CHACHA_KEY_SIZE, - .ivsize = XCHACHA_IV_SIZE, - .chunksize = CHACHA_BLOCK_SIZE, - .walksize = 5 * CHACHA_BLOCK_SIZE, - .setkey = chacha12_setkey, - .encrypt = xchacha_neon, - .decrypt = xchacha_neon, - } -}; - bool chacha_is_arch_optimized(void) { return static_key_enabled(&have_neon); @@ -214,30 +100,12 @@ EXPORT_SYMBOL(chacha_is_arch_optimized); static int __init chacha_simd_mod_init(void) { - if (!cpu_have_named_feature(ASIMD)) - return 0; - - static_branch_enable(&have_neon); - - return IS_REACHABLE(CONFIG_CRYPTO_SKCIPHER) ? - crypto_register_skciphers(algs, ARRAY_SIZE(algs)) : 0; + if (cpu_have_named_feature(ASIMD)) + static_branch_enable(&have_neon); + return 0; } - -static void __exit chacha_simd_mod_fini(void) -{ - if (IS_REACHABLE(CONFIG_CRYPTO_SKCIPHER) && cpu_have_named_feature(ASIMD)) - crypto_unregister_skciphers(algs, ARRAY_SIZE(algs)); -} - arch_initcall(chacha_simd_mod_init); -module_exit(chacha_simd_mod_fini); -MODULE_DESCRIPTION("ChaCha and XChaCha stream ciphers (NEON accelerated)"); +MODULE_DESCRIPTION("ChaCha and HChaCha functions (ARM64 optimized)"); MODULE_AUTHOR("Ard Biesheuvel "); MODULE_LICENSE("GPL v2"); -MODULE_ALIAS_CRYPTO("chacha20"); -MODULE_ALIAS_CRYPTO("chacha20-neon"); -MODULE_ALIAS_CRYPTO("xchacha20"); -MODULE_ALIAS_CRYPTO("xchacha20-neon"); -MODULE_ALIAS_CRYPTO("xchacha12"); -MODULE_ALIAS_CRYPTO("xchacha12-neon"); From 5772a781822dda72523e837f90a777890b090e50 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Sat, 5 Apr 2025 11:26:05 -0700 Subject: [PATCH 059/382] crypto: mips/chacha - remove the skcipher algorithms Since crypto/chacha.c now registers chacha20-$(ARCH), xchacha20-$(ARCH), and xchacha12-$(ARCH) skcipher algorithms that use the architecture's ChaCha and HChaCha library functions, individual architectures no longer need to do the same. Therefore, remove the redundant skcipher algorithms and leave just the library functions. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/mips/crypto/Kconfig | 6 -- arch/mips/crypto/chacha-glue.c | 131 +-------------------------------- 2 files changed, 3 insertions(+), 134 deletions(-) diff --git a/arch/mips/crypto/Kconfig b/arch/mips/crypto/Kconfig index 545fc0e12422..0189686de3a1 100644 --- a/arch/mips/crypto/Kconfig +++ b/arch/mips/crypto/Kconfig @@ -56,13 +56,7 @@ config CRYPTO_SHA512_OCTEON config CRYPTO_CHACHA_MIPS tristate depends on CPU_MIPS32_R2 - select CRYPTO_SKCIPHER select CRYPTO_ARCH_HAVE_LIB_CHACHA default CRYPTO_LIB_CHACHA_INTERNAL - help - Length-preserving ciphers: ChaCha20, XChaCha20, and XChaCha12 - stream cipher algorithms - - Architecture: MIPS32r2 endmenu diff --git a/arch/mips/crypto/chacha-glue.c b/arch/mips/crypto/chacha-glue.c index 64ccaeaeaa1e..334ecb29fb8f 100644 --- a/arch/mips/crypto/chacha-glue.c +++ b/arch/mips/crypto/chacha-glue.c @@ -1,15 +1,11 @@ // SPDX-License-Identifier: GPL-2.0 /* - * MIPS accelerated ChaCha and XChaCha stream ciphers, - * including ChaCha20 (RFC7539) + * ChaCha and HChaCha functions (MIPS optimized) * * Copyright (C) 2019 Linaro, Ltd. */ -#include -#include -#include -#include +#include #include #include @@ -20,133 +16,12 @@ EXPORT_SYMBOL(chacha_crypt_arch); asmlinkage void hchacha_block_arch(const u32 *state, u32 *stream, int nrounds); EXPORT_SYMBOL(hchacha_block_arch); -static int chacha_mips_stream_xor(struct skcipher_request *req, - const struct chacha_ctx *ctx, const u8 *iv) -{ - struct skcipher_walk walk; - u32 state[16]; - int err; - - err = skcipher_walk_virt(&walk, req, false); - - chacha_init(state, ctx->key, iv); - - while (walk.nbytes > 0) { - unsigned int nbytes = walk.nbytes; - - if (nbytes < walk.total) - nbytes = round_down(nbytes, walk.stride); - - chacha_crypt(state, walk.dst.virt.addr, walk.src.virt.addr, - nbytes, ctx->nrounds); - err = skcipher_walk_done(&walk, walk.nbytes - nbytes); - } - - return err; -} - -static int chacha_mips(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm); - - return chacha_mips_stream_xor(req, ctx, req->iv); -} - -static int xchacha_mips(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm); - struct chacha_ctx subctx; - u32 state[16]; - u8 real_iv[16]; - - chacha_init(state, ctx->key, req->iv); - - hchacha_block(state, subctx.key, ctx->nrounds); - subctx.nrounds = ctx->nrounds; - - memcpy(&real_iv[0], req->iv + 24, 8); - memcpy(&real_iv[8], req->iv + 16, 8); - return chacha_mips_stream_xor(req, &subctx, real_iv); -} - -static struct skcipher_alg algs[] = { - { - .base.cra_name = "chacha20", - .base.cra_driver_name = "chacha20-mips", - .base.cra_priority = 200, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct chacha_ctx), - .base.cra_module = THIS_MODULE, - - .min_keysize = CHACHA_KEY_SIZE, - .max_keysize = CHACHA_KEY_SIZE, - .ivsize = CHACHA_IV_SIZE, - .chunksize = CHACHA_BLOCK_SIZE, - .setkey = chacha20_setkey, - .encrypt = chacha_mips, - .decrypt = chacha_mips, - }, { - .base.cra_name = "xchacha20", - .base.cra_driver_name = "xchacha20-mips", - .base.cra_priority = 200, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct chacha_ctx), - .base.cra_module = THIS_MODULE, - - .min_keysize = CHACHA_KEY_SIZE, - .max_keysize = CHACHA_KEY_SIZE, - .ivsize = XCHACHA_IV_SIZE, - .chunksize = CHACHA_BLOCK_SIZE, - .setkey = chacha20_setkey, - .encrypt = xchacha_mips, - .decrypt = xchacha_mips, - }, { - .base.cra_name = "xchacha12", - .base.cra_driver_name = "xchacha12-mips", - .base.cra_priority = 200, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct chacha_ctx), - .base.cra_module = THIS_MODULE, - - .min_keysize = CHACHA_KEY_SIZE, - .max_keysize = CHACHA_KEY_SIZE, - .ivsize = XCHACHA_IV_SIZE, - .chunksize = CHACHA_BLOCK_SIZE, - .setkey = chacha12_setkey, - .encrypt = xchacha_mips, - .decrypt = xchacha_mips, - } -}; - bool chacha_is_arch_optimized(void) { return true; } EXPORT_SYMBOL(chacha_is_arch_optimized); -static int __init chacha_simd_mod_init(void) -{ - return IS_REACHABLE(CONFIG_CRYPTO_SKCIPHER) ? - crypto_register_skciphers(algs, ARRAY_SIZE(algs)) : 0; -} - -static void __exit chacha_simd_mod_fini(void) -{ - if (IS_REACHABLE(CONFIG_CRYPTO_SKCIPHER)) - crypto_unregister_skciphers(algs, ARRAY_SIZE(algs)); -} - -arch_initcall(chacha_simd_mod_init); -module_exit(chacha_simd_mod_fini); - -MODULE_DESCRIPTION("ChaCha and XChaCha stream ciphers (MIPS accelerated)"); +MODULE_DESCRIPTION("ChaCha and HChaCha functions (MIPS optimized)"); MODULE_AUTHOR("Ard Biesheuvel "); MODULE_LICENSE("GPL v2"); -MODULE_ALIAS_CRYPTO("chacha20"); -MODULE_ALIAS_CRYPTO("chacha20-mips"); -MODULE_ALIAS_CRYPTO("xchacha20"); -MODULE_ALIAS_CRYPTO("xchacha20-mips"); -MODULE_ALIAS_CRYPTO("xchacha12"); -MODULE_ALIAS_CRYPTO("xchacha12-mips"); From f7915484c0200e7fe67d25b715408de6dc287047 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Sat, 5 Apr 2025 11:26:06 -0700 Subject: [PATCH 060/382] crypto: powerpc/chacha - remove the skcipher algorithms Since crypto/chacha.c now registers chacha20-$(ARCH), xchacha20-$(ARCH), and xchacha12-$(ARCH) skcipher algorithms that use the architecture's ChaCha and HChaCha library functions, individual architectures no longer need to do the same. Therefore, remove the redundant skcipher algorithms and leave just the library functions. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/powerpc/crypto/Kconfig | 8 -- arch/powerpc/crypto/chacha-p10-glue.c | 145 ++------------------------ 2 files changed, 6 insertions(+), 147 deletions(-) diff --git a/arch/powerpc/crypto/Kconfig b/arch/powerpc/crypto/Kconfig index 370db8192ce6..47dccdd49637 100644 --- a/arch/powerpc/crypto/Kconfig +++ b/arch/powerpc/crypto/Kconfig @@ -95,17 +95,9 @@ config CRYPTO_AES_GCM_P10 config CRYPTO_CHACHA20_P10 tristate depends on PPC64 && CPU_LITTLE_ENDIAN && VSX - select CRYPTO_SKCIPHER select CRYPTO_LIB_CHACHA_GENERIC select CRYPTO_ARCH_HAVE_LIB_CHACHA default CRYPTO_LIB_CHACHA_INTERNAL - help - Length-preserving ciphers: ChaCha20, XChaCha20, and XChaCha12 - stream cipher algorithms - - Architecture: PowerPC64 - - Power10 or later - - Little-endian config CRYPTO_POLY1305_P10 tristate "Hash functions: Poly1305 (P10 or later)" diff --git a/arch/powerpc/crypto/chacha-p10-glue.c b/arch/powerpc/crypto/chacha-p10-glue.c index 3355305b6c7f..9982929573ad 100644 --- a/arch/powerpc/crypto/chacha-p10-glue.c +++ b/arch/powerpc/crypto/chacha-p10-glue.c @@ -1,15 +1,12 @@ // SPDX-License-Identifier: GPL-2.0-or-later /* - * PowerPC P10 (ppc64le) accelerated ChaCha and XChaCha stream ciphers, - * including ChaCha20 (RFC7539) + * ChaCha stream cipher (P10 accelerated) * * Copyright 2023- IBM Corp. All rights reserved. */ -#include -#include +#include #include -#include #include #include #include @@ -78,117 +75,6 @@ void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *src, unsigned int bytes, } EXPORT_SYMBOL(chacha_crypt_arch); -static int chacha_p10_stream_xor(struct skcipher_request *req, - const struct chacha_ctx *ctx, const u8 *iv) -{ - struct skcipher_walk walk; - u32 state[16]; - int err; - - err = skcipher_walk_virt(&walk, req, false); - if (err) - return err; - - chacha_init(state, ctx->key, iv); - - while (walk.nbytes > 0) { - unsigned int nbytes = walk.nbytes; - - if (nbytes < walk.total) - nbytes = rounddown(nbytes, walk.stride); - - if (!crypto_simd_usable()) { - chacha_crypt_generic(state, walk.dst.virt.addr, - walk.src.virt.addr, nbytes, - ctx->nrounds); - } else { - vsx_begin(); - chacha_p10_do_8x(state, walk.dst.virt.addr, - walk.src.virt.addr, nbytes, ctx->nrounds); - vsx_end(); - } - err = skcipher_walk_done(&walk, walk.nbytes - nbytes); - if (err) - break; - } - - return err; -} - -static int chacha_p10(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm); - - return chacha_p10_stream_xor(req, ctx, req->iv); -} - -static int xchacha_p10(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm); - struct chacha_ctx subctx; - u32 state[16]; - u8 real_iv[16]; - - chacha_init(state, ctx->key, req->iv); - hchacha_block_arch(state, subctx.key, ctx->nrounds); - subctx.nrounds = ctx->nrounds; - - memcpy(&real_iv[0], req->iv + 24, 8); - memcpy(&real_iv[8], req->iv + 16, 8); - return chacha_p10_stream_xor(req, &subctx, real_iv); -} - -static struct skcipher_alg algs[] = { - { - .base.cra_name = "chacha20", - .base.cra_driver_name = "chacha20-p10", - .base.cra_priority = 300, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct chacha_ctx), - .base.cra_module = THIS_MODULE, - - .min_keysize = CHACHA_KEY_SIZE, - .max_keysize = CHACHA_KEY_SIZE, - .ivsize = CHACHA_IV_SIZE, - .chunksize = CHACHA_BLOCK_SIZE, - .setkey = chacha20_setkey, - .encrypt = chacha_p10, - .decrypt = chacha_p10, - }, { - .base.cra_name = "xchacha20", - .base.cra_driver_name = "xchacha20-p10", - .base.cra_priority = 300, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct chacha_ctx), - .base.cra_module = THIS_MODULE, - - .min_keysize = CHACHA_KEY_SIZE, - .max_keysize = CHACHA_KEY_SIZE, - .ivsize = XCHACHA_IV_SIZE, - .chunksize = CHACHA_BLOCK_SIZE, - .setkey = chacha20_setkey, - .encrypt = xchacha_p10, - .decrypt = xchacha_p10, - }, { - .base.cra_name = "xchacha12", - .base.cra_driver_name = "xchacha12-p10", - .base.cra_priority = 300, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct chacha_ctx), - .base.cra_module = THIS_MODULE, - - .min_keysize = CHACHA_KEY_SIZE, - .max_keysize = CHACHA_KEY_SIZE, - .ivsize = XCHACHA_IV_SIZE, - .chunksize = CHACHA_BLOCK_SIZE, - .setkey = chacha12_setkey, - .encrypt = xchacha_p10, - .decrypt = xchacha_p10, - } -}; - bool chacha_is_arch_optimized(void) { return static_key_enabled(&have_p10); @@ -197,31 +83,12 @@ EXPORT_SYMBOL(chacha_is_arch_optimized); static int __init chacha_p10_init(void) { - if (!cpu_has_feature(CPU_FTR_ARCH_31)) - return 0; - - static_branch_enable(&have_p10); - - return crypto_register_skciphers(algs, ARRAY_SIZE(algs)); + if (cpu_has_feature(CPU_FTR_ARCH_31)) + static_branch_enable(&have_p10); + return 0; } - -static void __exit chacha_p10_exit(void) -{ - if (!static_branch_likely(&have_p10)) - return; - - crypto_unregister_skciphers(algs, ARRAY_SIZE(algs)); -} - arch_initcall(chacha_p10_init); -module_exit(chacha_p10_exit); -MODULE_DESCRIPTION("ChaCha and XChaCha stream ciphers (P10 accelerated)"); +MODULE_DESCRIPTION("ChaCha stream cipher (P10 accelerated)"); MODULE_AUTHOR("Danny Tsen "); MODULE_LICENSE("GPL v2"); -MODULE_ALIAS_CRYPTO("chacha20"); -MODULE_ALIAS_CRYPTO("chacha20-p10"); -MODULE_ALIAS_CRYPTO("xchacha20"); -MODULE_ALIAS_CRYPTO("xchacha20-p10"); -MODULE_ALIAS_CRYPTO("xchacha12"); -MODULE_ALIAS_CRYPTO("xchacha12-p10"); From efe8ddfaa3361c5080ad10a1226652cc0839ee49 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Sat, 5 Apr 2025 11:26:07 -0700 Subject: [PATCH 061/382] crypto: s390/chacha - remove the skcipher algorithms Since crypto/chacha.c now registers chacha20-$(ARCH), xchacha20-$(ARCH), and xchacha12-$(ARCH) skcipher algorithms that use the architecture's ChaCha and HChaCha library functions, individual architectures no longer need to do the same. Therefore, remove the redundant skcipher algorithms and leave just the library functions. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/s390/crypto/Kconfig | 7 --- arch/s390/crypto/chacha-glue.c | 103 +++++---------------------------- 2 files changed, 14 insertions(+), 96 deletions(-) diff --git a/arch/s390/crypto/Kconfig b/arch/s390/crypto/Kconfig index 8c4db8b64fa2..055b08f259ab 100644 --- a/arch/s390/crypto/Kconfig +++ b/arch/s390/crypto/Kconfig @@ -110,16 +110,9 @@ config CRYPTO_DES_S390 config CRYPTO_CHACHA_S390 tristate depends on S390 - select CRYPTO_SKCIPHER select CRYPTO_LIB_CHACHA_GENERIC select CRYPTO_ARCH_HAVE_LIB_CHACHA default CRYPTO_LIB_CHACHA_INTERNAL - help - Length-preserving cipher: ChaCha20 stream cipher (RFC 7539) - - Architecture: s390 - - It is available as of z13. config CRYPTO_HMAC_S390 tristate "Keyed-hash message authentication code: HMAC" diff --git a/arch/s390/crypto/chacha-glue.c b/arch/s390/crypto/chacha-glue.c index 0c68191f2aa4..b3ffaa555385 100644 --- a/arch/s390/crypto/chacha-glue.c +++ b/arch/s390/crypto/chacha-glue.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 /* - * s390 ChaCha stream cipher. + * ChaCha stream cipher (s390 optimized) * * Copyright IBM Corp. 2021 */ @@ -8,9 +8,7 @@ #define KMSG_COMPONENT "chacha_s390" #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt -#include -#include -#include +#include #include #include #include @@ -18,50 +16,6 @@ #include #include "chacha-s390.h" -static void chacha20_crypt_s390(u32 *state, u8 *dst, const u8 *src, - unsigned int nbytes, const u32 *key, - u32 *counter) -{ - DECLARE_KERNEL_FPU_ONSTACK32(vxstate); - - kernel_fpu_begin(&vxstate, KERNEL_VXR); - chacha20_vx(dst, src, nbytes, key, counter); - kernel_fpu_end(&vxstate, KERNEL_VXR); - - *counter += round_up(nbytes, CHACHA_BLOCK_SIZE) / CHACHA_BLOCK_SIZE; -} - -static int chacha20_s390(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm); - u32 state[CHACHA_STATE_WORDS] __aligned(16); - struct skcipher_walk walk; - unsigned int nbytes; - int rc; - - rc = skcipher_walk_virt(&walk, req, false); - chacha_init(state, ctx->key, req->iv); - - while (walk.nbytes > 0) { - nbytes = walk.nbytes; - if (nbytes < walk.total) - nbytes = round_down(nbytes, walk.stride); - - if (nbytes <= CHACHA_BLOCK_SIZE) { - chacha_crypt_generic(state, walk.dst.virt.addr, - walk.src.virt.addr, nbytes, - ctx->nrounds); - } else { - chacha20_crypt_s390(state, walk.dst.virt.addr, - walk.src.virt.addr, nbytes, - &state[4], &state[12]); - } - rc = skcipher_walk_done(&walk, walk.nbytes - nbytes); - } - return rc; -} - void hchacha_block_arch(const u32 *state, u32 *stream, int nrounds) { /* TODO: implement hchacha_block_arch() in assembly */ @@ -76,55 +30,26 @@ void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *src, * it cannot handle a block of data or less, but otherwise * it can handle data of arbitrary size */ - if (bytes <= CHACHA_BLOCK_SIZE || nrounds != 20 || !cpu_has_vx()) + if (bytes <= CHACHA_BLOCK_SIZE || nrounds != 20 || !cpu_has_vx()) { chacha_crypt_generic(state, dst, src, bytes, nrounds); - else - chacha20_crypt_s390(state, dst, src, bytes, - &state[4], &state[12]); + } else { + DECLARE_KERNEL_FPU_ONSTACK32(vxstate); + + kernel_fpu_begin(&vxstate, KERNEL_VXR); + chacha20_vx(dst, src, bytes, &state[4], &state[12]); + kernel_fpu_end(&vxstate, KERNEL_VXR); + + state[12] += round_up(bytes, CHACHA_BLOCK_SIZE) / + CHACHA_BLOCK_SIZE; + } } EXPORT_SYMBOL(chacha_crypt_arch); -static struct skcipher_alg chacha_algs[] = { - { - .base.cra_name = "chacha20", - .base.cra_driver_name = "chacha20-s390", - .base.cra_priority = 900, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct chacha_ctx), - .base.cra_module = THIS_MODULE, - - .min_keysize = CHACHA_KEY_SIZE, - .max_keysize = CHACHA_KEY_SIZE, - .ivsize = CHACHA_IV_SIZE, - .chunksize = CHACHA_BLOCK_SIZE, - .setkey = chacha20_setkey, - .encrypt = chacha20_s390, - .decrypt = chacha20_s390, - } -}; - bool chacha_is_arch_optimized(void) { return cpu_has_vx(); } EXPORT_SYMBOL(chacha_is_arch_optimized); -static int __init chacha_mod_init(void) -{ - return IS_REACHABLE(CONFIG_CRYPTO_SKCIPHER) ? - crypto_register_skciphers(chacha_algs, ARRAY_SIZE(chacha_algs)) : 0; -} - -static void __exit chacha_mod_fini(void) -{ - if (IS_REACHABLE(CONFIG_CRYPTO_SKCIPHER)) - crypto_unregister_skciphers(chacha_algs, ARRAY_SIZE(chacha_algs)); -} - -arch_initcall(chacha_mod_init); -module_exit(chacha_mod_fini); - -MODULE_DESCRIPTION("ChaCha20 stream cipher"); +MODULE_DESCRIPTION("ChaCha stream cipher (s390 optimized)"); MODULE_LICENSE("GPL v2"); - -MODULE_ALIAS_CRYPTO("chacha20"); From 632ab0978f0895ca2e477b053d9933768cf0339a Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Sat, 5 Apr 2025 11:26:08 -0700 Subject: [PATCH 062/382] crypto: x86/chacha - remove the skcipher algorithms Since crypto/chacha.c now registers chacha20-$(ARCH), xchacha20-$(ARCH), and xchacha12-$(ARCH) skcipher algorithms that use the architecture's ChaCha and HChaCha library functions, individual architectures no longer need to do the same. Therefore, remove the redundant skcipher algorithms and leave just the library functions. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/x86/crypto/Kconfig | 9 --- arch/x86/crypto/chacha_glue.c | 132 ++-------------------------------- 2 files changed, 5 insertions(+), 136 deletions(-) diff --git a/arch/x86/crypto/Kconfig b/arch/x86/crypto/Kconfig index 2fae1674f756..0cec75926380 100644 --- a/arch/x86/crypto/Kconfig +++ b/arch/x86/crypto/Kconfig @@ -339,18 +339,9 @@ config CRYPTO_ARIA_GFNI_AVX512_X86_64 config CRYPTO_CHACHA20_X86_64 tristate depends on X86 && 64BIT - select CRYPTO_SKCIPHER select CRYPTO_LIB_CHACHA_GENERIC select CRYPTO_ARCH_HAVE_LIB_CHACHA default CRYPTO_LIB_CHACHA_INTERNAL - help - Length-preserving ciphers: ChaCha20, XChaCha20, and XChaCha12 - stream cipher algorithms - - Architecture: x86_64 using: - - SSSE3 (Supplemental SSE3) - - AVX2 (Advanced Vector Extensions 2) - - AVX-512VL (Advanced Vector Extensions-512VL) config CRYPTO_AEGIS128_AESNI_SSE2 tristate "AEAD ciphers: AEGIS-128 (AES-NI/SSE4.1)" diff --git a/arch/x86/crypto/chacha_glue.c b/arch/x86/crypto/chacha_glue.c index 8858de7b33e3..b7fd7a1f0e15 100644 --- a/arch/x86/crypto/chacha_glue.c +++ b/arch/x86/crypto/chacha_glue.c @@ -1,13 +1,11 @@ // SPDX-License-Identifier: GPL-2.0-or-later /* - * x64 SIMD accelerated ChaCha and XChaCha stream ciphers, - * including ChaCha20 (RFC7539) + * ChaCha and HChaCha functions (x86_64 optimized) * * Copyright (C) 2015 Martin Willi */ -#include -#include +#include #include #include #include @@ -142,114 +140,9 @@ void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *src, unsigned int bytes, } EXPORT_SYMBOL(chacha_crypt_arch); -static int chacha_simd_stream_xor(struct skcipher_request *req, - const struct chacha_ctx *ctx, const u8 *iv) -{ - u32 state[CHACHA_STATE_WORDS] __aligned(8); - struct skcipher_walk walk; - int err; - - err = skcipher_walk_virt(&walk, req, false); - - chacha_init(state, ctx->key, iv); - - while (walk.nbytes > 0) { - unsigned int nbytes = walk.nbytes; - - if (nbytes < walk.total) - nbytes = round_down(nbytes, walk.stride); - - kernel_fpu_begin(); - chacha_dosimd(state, walk.dst.virt.addr, - walk.src.virt.addr, nbytes, - ctx->nrounds); - kernel_fpu_end(); - err = skcipher_walk_done(&walk, walk.nbytes - nbytes); - } - - return err; -} - -static int chacha_simd(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm); - - return chacha_simd_stream_xor(req, ctx, req->iv); -} - -static int xchacha_simd(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm); - u32 state[CHACHA_STATE_WORDS] __aligned(8); - struct chacha_ctx subctx; - u8 real_iv[16]; - - chacha_init(state, ctx->key, req->iv); - - kernel_fpu_begin(); - hchacha_block_ssse3(state, subctx.key, ctx->nrounds); - kernel_fpu_end(); - subctx.nrounds = ctx->nrounds; - - memcpy(&real_iv[0], req->iv + 24, 8); - memcpy(&real_iv[8], req->iv + 16, 8); - return chacha_simd_stream_xor(req, &subctx, real_iv); -} - -static struct skcipher_alg algs[] = { - { - .base.cra_name = "chacha20", - .base.cra_driver_name = "chacha20-simd", - .base.cra_priority = 300, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct chacha_ctx), - .base.cra_module = THIS_MODULE, - - .min_keysize = CHACHA_KEY_SIZE, - .max_keysize = CHACHA_KEY_SIZE, - .ivsize = CHACHA_IV_SIZE, - .chunksize = CHACHA_BLOCK_SIZE, - .setkey = chacha20_setkey, - .encrypt = chacha_simd, - .decrypt = chacha_simd, - }, { - .base.cra_name = "xchacha20", - .base.cra_driver_name = "xchacha20-simd", - .base.cra_priority = 300, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct chacha_ctx), - .base.cra_module = THIS_MODULE, - - .min_keysize = CHACHA_KEY_SIZE, - .max_keysize = CHACHA_KEY_SIZE, - .ivsize = XCHACHA_IV_SIZE, - .chunksize = CHACHA_BLOCK_SIZE, - .setkey = chacha20_setkey, - .encrypt = xchacha_simd, - .decrypt = xchacha_simd, - }, { - .base.cra_name = "xchacha12", - .base.cra_driver_name = "xchacha12-simd", - .base.cra_priority = 300, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct chacha_ctx), - .base.cra_module = THIS_MODULE, - - .min_keysize = CHACHA_KEY_SIZE, - .max_keysize = CHACHA_KEY_SIZE, - .ivsize = XCHACHA_IV_SIZE, - .chunksize = CHACHA_BLOCK_SIZE, - .setkey = chacha12_setkey, - .encrypt = xchacha_simd, - .decrypt = xchacha_simd, - }, -}; - bool chacha_is_arch_optimized(void) { - return static_key_enabled(&chacha_use_simd); + return true; } EXPORT_SYMBOL(chacha_is_arch_optimized); @@ -267,25 +160,10 @@ static int __init chacha_simd_mod_init(void) boot_cpu_has(X86_FEATURE_AVX512BW)) /* kmovq */ static_branch_enable(&chacha_use_avx512vl); } - return IS_REACHABLE(CONFIG_CRYPTO_SKCIPHER) ? - crypto_register_skciphers(algs, ARRAY_SIZE(algs)) : 0; + return 0; } - -static void __exit chacha_simd_mod_fini(void) -{ - if (IS_REACHABLE(CONFIG_CRYPTO_SKCIPHER) && boot_cpu_has(X86_FEATURE_SSSE3)) - crypto_unregister_skciphers(algs, ARRAY_SIZE(algs)); -} - arch_initcall(chacha_simd_mod_init); -module_exit(chacha_simd_mod_fini); MODULE_LICENSE("GPL"); MODULE_AUTHOR("Martin Willi "); -MODULE_DESCRIPTION("ChaCha and XChaCha stream ciphers (x64 SIMD accelerated)"); -MODULE_ALIAS_CRYPTO("chacha20"); -MODULE_ALIAS_CRYPTO("chacha20-simd"); -MODULE_ALIAS_CRYPTO("xchacha20"); -MODULE_ALIAS_CRYPTO("xchacha20-simd"); -MODULE_ALIAS_CRYPTO("xchacha12"); -MODULE_ALIAS_CRYPTO("xchacha12-simd"); +MODULE_DESCRIPTION("ChaCha and HChaCha functions (x86_64 optimized)"); From d23fce15abd480811098c0bca6d4edeb17824279 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Sat, 5 Apr 2025 11:26:09 -0700 Subject: [PATCH 063/382] crypto: chacha - remove is now included only by crypto/chacha.c, so fold it into there. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/chacha.c | 35 +++++++++++++++++++++++++- include/crypto/internal/chacha.h | 43 -------------------------------- 2 files changed, 34 insertions(+), 44 deletions(-) delete mode 100644 include/crypto/internal/chacha.h diff --git a/crypto/chacha.c b/crypto/chacha.c index 2009038c5e56..5103bc0b2881 100644 --- a/crypto/chacha.c +++ b/crypto/chacha.c @@ -8,10 +8,43 @@ #include #include -#include +#include #include #include +struct chacha_ctx { + u32 key[8]; + int nrounds; +}; + +static int chacha_setkey(struct crypto_skcipher *tfm, + const u8 *key, unsigned int keysize, int nrounds) +{ + struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm); + int i; + + if (keysize != CHACHA_KEY_SIZE) + return -EINVAL; + + for (i = 0; i < ARRAY_SIZE(ctx->key); i++) + ctx->key[i] = get_unaligned_le32(key + i * sizeof(u32)); + + ctx->nrounds = nrounds; + return 0; +} + +static int chacha20_setkey(struct crypto_skcipher *tfm, + const u8 *key, unsigned int keysize) +{ + return chacha_setkey(tfm, key, keysize, 20); +} + +static int chacha12_setkey(struct crypto_skcipher *tfm, + const u8 *key, unsigned int keysize) +{ + return chacha_setkey(tfm, key, keysize, 12); +} + static int chacha_stream_xor(struct skcipher_request *req, const struct chacha_ctx *ctx, const u8 *iv, bool arch) diff --git a/include/crypto/internal/chacha.h b/include/crypto/internal/chacha.h deleted file mode 100644 index b085dc1ac151..000000000000 --- a/include/crypto/internal/chacha.h +++ /dev/null @@ -1,43 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ - -#ifndef _CRYPTO_INTERNAL_CHACHA_H -#define _CRYPTO_INTERNAL_CHACHA_H - -#include -#include -#include - -struct chacha_ctx { - u32 key[8]; - int nrounds; -}; - -static inline int chacha_setkey(struct crypto_skcipher *tfm, const u8 *key, - unsigned int keysize, int nrounds) -{ - struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm); - int i; - - if (keysize != CHACHA_KEY_SIZE) - return -EINVAL; - - for (i = 0; i < ARRAY_SIZE(ctx->key); i++) - ctx->key[i] = get_unaligned_le32(key + i * sizeof(u32)); - - ctx->nrounds = nrounds; - return 0; -} - -static inline int chacha20_setkey(struct crypto_skcipher *tfm, const u8 *key, - unsigned int keysize) -{ - return chacha_setkey(tfm, key, keysize, 20); -} - -static inline int chacha12_setkey(struct crypto_skcipher *tfm, const u8 *key, - unsigned int keysize) -{ - return chacha_setkey(tfm, key, keysize, 12); -} - -#endif /* _CRYPTO_CHACHA_H */ From 6f1d5a3513c2370bbd6115dd430906bc2f4bdc53 Mon Sep 17 00:00:00 2001 From: Ashish Kalra Date: Mon, 24 Mar 2025 21:15:31 +0000 Subject: [PATCH 064/382] KVM: SVM: Add support to initialize SEV/SNP functionality in KVM Move platform initialization of SEV/SNP from CCP driver probe time to KVM module load time so that KVM can do SEV/SNP platform initialization explicitly if it actually wants to use SEV/SNP functionality. Add support for KVM to explicitly call into the CCP driver at load time to initialize SEV/SNP. If required, this behavior can be altered with KVM module parameters to not do SEV/SNP platform initialization at module load time. Additionally, a corresponding SEV/SNP platform shutdown is invoked during KVM module unload time. Continue to support SEV deferred initialization as the user may have the file containing SEV persistent data for SEV INIT_EX available only later after module load/init. Suggested-by: Sean Christopherson Reviewed-by: Tom Lendacky Signed-off-by: Ashish Kalra Signed-off-by: Herbert Xu --- arch/x86/kvm/svm/sev.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index 0bc708ee2788..7be4e1647903 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -2933,6 +2933,7 @@ void __init sev_set_cpu_caps(void) void __init sev_hardware_setup(void) { unsigned int eax, ebx, ecx, edx, sev_asid_count, sev_es_asid_count; + struct sev_platform_init_args init_args = {0}; bool sev_snp_supported = false; bool sev_es_supported = false; bool sev_supported = false; @@ -3059,6 +3060,15 @@ out: sev_supported_vmsa_features = 0; if (sev_es_debug_swap_enabled) sev_supported_vmsa_features |= SVM_SEV_FEAT_DEBUG_SWAP; + + if (!sev_enabled) + return; + + /* + * Do both SNP and SEV initialization at KVM module load. + */ + init_args.probe = true; + sev_platform_init(&init_args); } void sev_hardware_unsetup(void) @@ -3074,6 +3084,8 @@ void sev_hardware_unsetup(void) misc_cg_set_capacity(MISC_CG_RES_SEV, 0); misc_cg_set_capacity(MISC_CG_RES_SEV_ES, 0); + + sev_platform_shutdown(); } int sev_cpu_init(struct svm_cpu_data *sd) From 3f8f0133a5fc9b32d0c308530320c3f2430ba5ab Mon Sep 17 00:00:00 2001 From: Ashish Kalra Date: Mon, 24 Mar 2025 21:15:47 +0000 Subject: [PATCH 065/382] crypto: ccp - Move SEV/SNP Platform initialization to KVM SNP initialization is forced during PSP driver probe purely because SNP can't be initialized if VMs are running. But the only in-tree user of SEV/SNP functionality is KVM, and KVM depends on PSP driver for the same. Forcing SEV/SNP initialization because a hypervisor could be running legacy non-confidential VMs make no sense. This patch removes SEV/SNP initialization from the PSP driver probe time and moves the requirement to initialize SEV/SNP functionality to KVM if it wants to use SEV/SNP. Suggested-by: Sean Christopherson Reviewed-by: Alexey Kardashevskiy Signed-off-by: Ashish Kalra Signed-off-by: Herbert Xu --- drivers/crypto/ccp/sev-dev.c | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/drivers/crypto/ccp/sev-dev.c b/drivers/crypto/ccp/sev-dev.c index 671347702ae7..980b3d296dc6 100644 --- a/drivers/crypto/ccp/sev-dev.c +++ b/drivers/crypto/ccp/sev-dev.c @@ -1347,10 +1347,6 @@ static int _sev_platform_init_locked(struct sev_platform_init_args *args) if (sev->state == SEV_STATE_INIT) return 0; - /* - * Legacy guests cannot be running while SNP_INIT(_EX) is executing, - * so perform SEV-SNP initialization at probe time. - */ rc = __sev_snp_init_locked(&args->error); if (rc && rc != -ENODEV) return rc; @@ -2524,9 +2520,7 @@ EXPORT_SYMBOL_GPL(sev_issue_cmd_external_user); void sev_pci_init(void) { struct sev_device *sev = psp_master->sev_data; - struct sev_platform_init_args args = {0}; u8 api_major, api_minor, build; - int rc; if (!sev) return; @@ -2549,13 +2543,6 @@ void sev_pci_init(void) api_major, api_minor, build, sev->api_major, sev->api_minor, sev->build); - /* Initialize the platform */ - args.probe = true; - rc = sev_platform_init(&args); - if (rc) - dev_err(sev->dev, "SEV: failed to INIT error %#x, rc %d\n", - args.error, rc); - return; err: From ea4dd134ef332bd9e3e734c1ba0a1521f436b678 Mon Sep 17 00:00:00 2001 From: Ovidiu Panait Date: Tue, 1 Apr 2025 22:23:16 +0300 Subject: [PATCH 066/382] crypto: sun8i-ce-hash - fix error handling in sun8i_ce_hash_run() Rework error handling in sun8i_ce_hash_run() to unmap the dma buffers in case of failure. Currently, the dma unmap functions are not called if the function errors out at various points. Fixes: 56f6d5aee88d1 ("crypto: sun8i-ce - support hash algorithms") Signed-off-by: Ovidiu Panait Signed-off-by: Herbert Xu --- .../crypto/allwinner/sun8i-ce/sun8i-ce-hash.c | 34 ++++++++++++------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c index 6072dd9f390b..3f9d79ea01aa 100644 --- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c @@ -343,9 +343,8 @@ int sun8i_ce_hash_run(struct crypto_engine *engine, void *breq) u32 common; u64 byte_count; __le32 *bf; - void *buf = NULL; + void *buf, *result; int j, i, todo; - void *result = NULL; u64 bs; int digestsize; dma_addr_t addr_res, addr_pad; @@ -365,14 +364,14 @@ int sun8i_ce_hash_run(struct crypto_engine *engine, void *breq) buf = kcalloc(2, bs, GFP_KERNEL | GFP_DMA); if (!buf) { err = -ENOMEM; - goto theend; + goto err_out; } bf = (__le32 *)buf; result = kzalloc(digestsize, GFP_KERNEL | GFP_DMA); if (!result) { err = -ENOMEM; - goto theend; + goto err_free_buf; } flow = rctx->flow; @@ -398,7 +397,7 @@ int sun8i_ce_hash_run(struct crypto_engine *engine, void *breq) if (nr_sgs <= 0 || nr_sgs > MAX_SG) { dev_err(ce->dev, "Invalid sg number %d\n", nr_sgs); err = -EINVAL; - goto theend; + goto err_free_result; } len = areq->nbytes; @@ -411,7 +410,7 @@ int sun8i_ce_hash_run(struct crypto_engine *engine, void *breq) if (len > 0) { dev_err(ce->dev, "remaining len %d\n", len); err = -EINVAL; - goto theend; + goto err_unmap_src; } addr_res = dma_map_single(ce->dev, result, digestsize, DMA_FROM_DEVICE); cet->t_dst[0].addr = desc_addr_val_le32(ce, addr_res); @@ -419,7 +418,7 @@ int sun8i_ce_hash_run(struct crypto_engine *engine, void *breq) if (dma_mapping_error(ce->dev, addr_res)) { dev_err(ce->dev, "DMA map dest\n"); err = -EINVAL; - goto theend; + goto err_unmap_src; } byte_count = areq->nbytes; @@ -441,7 +440,7 @@ int sun8i_ce_hash_run(struct crypto_engine *engine, void *breq) } if (!j) { err = -EINVAL; - goto theend; + goto err_unmap_result; } addr_pad = dma_map_single(ce->dev, buf, j * 4, DMA_TO_DEVICE); @@ -450,7 +449,7 @@ int sun8i_ce_hash_run(struct crypto_engine *engine, void *breq) if (dma_mapping_error(ce->dev, addr_pad)) { dev_err(ce->dev, "DMA error on padding SG\n"); err = -EINVAL; - goto theend; + goto err_unmap_result; } if (ce->variant->hash_t_dlen_in_bits) @@ -463,16 +462,25 @@ int sun8i_ce_hash_run(struct crypto_engine *engine, void *breq) err = sun8i_ce_run_task(ce, flow, crypto_ahash_alg_name(tfm)); dma_unmap_single(ce->dev, addr_pad, j * 4, DMA_TO_DEVICE); - dma_unmap_sg(ce->dev, areq->src, ns, DMA_TO_DEVICE); + +err_unmap_result: dma_unmap_single(ce->dev, addr_res, digestsize, DMA_FROM_DEVICE); + if (!err) + memcpy(areq->result, result, algt->alg.hash.base.halg.digestsize); +err_unmap_src: + dma_unmap_sg(ce->dev, areq->src, ns, DMA_TO_DEVICE); - memcpy(areq->result, result, algt->alg.hash.base.halg.digestsize); -theend: - kfree(buf); +err_free_result: kfree(result); + +err_free_buf: + kfree(buf); + +err_out: local_bh_disable(); crypto_finalize_hash_request(engine, breq, err); local_bh_enable(); + return 0; } From d68c4a7c6cda91c2d97c5588cadc043780d010b6 Mon Sep 17 00:00:00 2001 From: Ovidiu Panait Date: Tue, 1 Apr 2025 22:23:17 +0300 Subject: [PATCH 067/382] crypto: sun8i-ce-hash - remove duplicated fallback checks Remove duplicated fallback checks in sun8i_ce_hash_digest(). The same checks are already done previously in sun8i_ce_hash_need_fallback(). Signed-off-by: Ovidiu Panait Signed-off-by: Herbert Xu --- drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c index 3f9d79ea01aa..2c2d285de154 100644 --- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c @@ -244,21 +244,11 @@ int sun8i_ce_hash_digest(struct ahash_request *areq) struct sun8i_ce_alg_template *algt; struct sun8i_ce_dev *ce; struct crypto_engine *engine; - struct scatterlist *sg; - int nr_sgs, e, i; + int e; if (sun8i_ce_hash_need_fallback(areq)) return sun8i_ce_hash_digest_fb(areq); - nr_sgs = sg_nents_for_len(areq->src, areq->nbytes); - if (nr_sgs > MAX_SG - 1) - return sun8i_ce_hash_digest_fb(areq); - - for_each_sg(areq->src, sg, nr_sgs, i) { - if (sg->length % 4 || !IS_ALIGNED(sg->offset, sizeof(u32))) - return sun8i_ce_hash_digest_fb(areq); - } - algt = container_of(alg, struct sun8i_ce_alg_template, alg.hash.base); ce = algt->ce; From b745ab78a8ca8a7c16f0cdca9d03680b07d2607a Mon Sep 17 00:00:00 2001 From: Ovidiu Panait Date: Tue, 1 Apr 2025 22:23:18 +0300 Subject: [PATCH 068/382] crypto: sun8i-ce-hash - factor out debugfs fallback statistics In order to reduce code duplication, factor out to a separate function the codepath that increments debugfs fallback stats. Signed-off-by: Ovidiu Panait Signed-off-by: Herbert Xu --- .../crypto/allwinner/sun8i-ce/sun8i-ce-hash.c | 53 +++++++------------ 1 file changed, 20 insertions(+), 33 deletions(-) diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c index 2c2d285de154..b0959d8218cb 100644 --- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c @@ -23,6 +23,20 @@ #include #include "sun8i-ce.h" +static void sun8i_ce_hash_stat_fb_inc(struct crypto_ahash *tfm) +{ + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) { + struct sun8i_ce_alg_template *algt __maybe_unused; + struct ahash_alg *alg = crypto_ahash_alg(tfm); + + algt = container_of(alg, struct sun8i_ce_alg_template, + alg.hash.base); +#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG + algt->stat_fb++; +#endif + } +} + int sun8i_ce_hash_init_tfm(struct crypto_ahash *tfm) { struct sun8i_ce_hash_tfm_ctx *op = crypto_ahash_ctx(tfm); @@ -113,22 +127,13 @@ int sun8i_ce_hash_final(struct ahash_request *areq) struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); struct sun8i_ce_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); + sun8i_ce_hash_stat_fb_inc(tfm); + ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); rctx->fallback_req.base.flags = areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; rctx->fallback_req.result = areq->result; - if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) { - struct sun8i_ce_alg_template *algt __maybe_unused; - struct ahash_alg *alg = crypto_ahash_alg(tfm); - - algt = container_of(alg, struct sun8i_ce_alg_template, - alg.hash.base); -#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG - algt->stat_fb++; -#endif - } - return crypto_ahash_final(&rctx->fallback_req); } @@ -153,6 +158,8 @@ int sun8i_ce_hash_finup(struct ahash_request *areq) struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); struct sun8i_ce_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); + sun8i_ce_hash_stat_fb_inc(tfm); + ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); rctx->fallback_req.base.flags = areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; @@ -161,17 +168,6 @@ int sun8i_ce_hash_finup(struct ahash_request *areq) rctx->fallback_req.src = areq->src; rctx->fallback_req.result = areq->result; - if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) { - struct sun8i_ce_alg_template *algt __maybe_unused; - struct ahash_alg *alg = crypto_ahash_alg(tfm); - - algt = container_of(alg, struct sun8i_ce_alg_template, - alg.hash.base); -#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG - algt->stat_fb++; -#endif - } - return crypto_ahash_finup(&rctx->fallback_req); } @@ -181,6 +177,8 @@ static int sun8i_ce_hash_digest_fb(struct ahash_request *areq) struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); struct sun8i_ce_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); + sun8i_ce_hash_stat_fb_inc(tfm); + ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); rctx->fallback_req.base.flags = areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; @@ -189,17 +187,6 @@ static int sun8i_ce_hash_digest_fb(struct ahash_request *areq) rctx->fallback_req.src = areq->src; rctx->fallback_req.result = areq->result; - if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) { - struct sun8i_ce_alg_template *algt __maybe_unused; - struct ahash_alg *alg = crypto_ahash_alg(tfm); - - algt = container_of(alg, struct sun8i_ce_alg_template, - alg.hash.base); -#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG - algt->stat_fb++; -#endif - } - return crypto_ahash_digest(&rctx->fallback_req); } From a8632253f39b5f6c1cf8e1c1da382a4d22e61e57 Mon Sep 17 00:00:00 2001 From: Ovidiu Panait Date: Tue, 1 Apr 2025 22:23:19 +0300 Subject: [PATCH 069/382] crypto: sun8i-ce-hash - drop CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG ifdefs 'struct sun8i_ce_alg_template' is always defined, even with CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG disabled, so the ifdef guards are not needed. Make sure the statistics have IS_ENABLED() checks instead. Signed-off-by: Ovidiu Panait Signed-off-by: Herbert Xu --- drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c index b0959d8218cb..530c5181d73c 100644 --- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c @@ -31,9 +31,7 @@ static void sun8i_ce_hash_stat_fb_inc(struct crypto_ahash *tfm) algt = container_of(alg, struct sun8i_ce_alg_template, alg.hash.base); -#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG algt->stat_fb++; -#endif } } @@ -354,9 +352,9 @@ int sun8i_ce_hash_run(struct crypto_engine *engine, void *breq) flow = rctx->flow; chan = &ce->chanlist[flow]; -#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG - algt->stat_req++; -#endif + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) + algt->stat_req++; + dev_dbg(ce->dev, "%s %s len=%d\n", __func__, crypto_tfm_alg_name(areq->base.tfm), areq->nbytes); cet = chan->tl; From 593c76e1aafb550b3ffa697a4369435216f82e4c Mon Sep 17 00:00:00 2001 From: Ovidiu Panait Date: Tue, 1 Apr 2025 22:23:20 +0300 Subject: [PATCH 070/382] crypto: sun8i-ce-hash - add IS_ENABLED() checks to debugfs stats Add IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG) checks before the fallback counter updates to make sure the code is not included when debugfs statistics support is not enabled. Signed-off-by: Ovidiu Panait Signed-off-by: Herbert Xu --- .../crypto/allwinner/sun8i-ce/sun8i-ce-hash.c | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c index 530c5181d73c..f2481e1585ba 100644 --- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c @@ -60,8 +60,10 @@ int sun8i_ce_hash_init_tfm(struct crypto_ahash *tfm) sizeof(struct sun8i_ce_hash_reqctx) + crypto_ahash_reqsize(op->fallback_tfm)); - memcpy(algt->fbname, crypto_ahash_driver_name(op->fallback_tfm), - CRYPTO_MAX_ALG_NAME); + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) + memcpy(algt->fbname, + crypto_ahash_driver_name(op->fallback_tfm), + CRYPTO_MAX_ALG_NAME); err = pm_runtime_get_sync(op->ce->dev); if (err < 0) @@ -198,22 +200,30 @@ static bool sun8i_ce_hash_need_fallback(struct ahash_request *areq) algt = container_of(alg, struct sun8i_ce_alg_template, alg.hash.base); if (areq->nbytes == 0) { - algt->stat_fb_len0++; + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) + algt->stat_fb_len0++; + return true; } /* we need to reserve one SG for padding one */ if (sg_nents_for_len(areq->src, areq->nbytes) > MAX_SG - 1) { - algt->stat_fb_maxsg++; + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) + algt->stat_fb_maxsg++; + return true; } sg = areq->src; while (sg) { if (sg->length % 4) { - algt->stat_fb_srclen++; + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) + algt->stat_fb_srclen++; + return true; } if (!IS_ALIGNED(sg->offset, sizeof(u32))) { - algt->stat_fb_srcali++; + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) + algt->stat_fb_srcali++; + return true; } sg = sg_next(sg); From bfc68ebefb49a3e99cbcfca5caf26182a814a18d Mon Sep 17 00:00:00 2001 From: Ovidiu Panait Date: Tue, 1 Apr 2025 22:23:21 +0300 Subject: [PATCH 071/382] crypto: sun8i-ce-hash - use API helpers to setup fallback request Rather than setting up the fallback request by hand, use ahash_request_set_callback() and ahash_request_set_crypt() API helpers to properly setup the new request. Signed-off-by: Ovidiu Panait Signed-off-by: Herbert Xu --- .../crypto/allwinner/sun8i-ce/sun8i-ce-hash.c | 49 ++++++++++--------- 1 file changed, 27 insertions(+), 22 deletions(-) diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c index f2481e1585ba..ba13fb75c05d 100644 --- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c @@ -92,7 +92,9 @@ int sun8i_ce_hash_init(struct ahash_request *areq) memset(rctx, 0, sizeof(struct sun8i_ce_hash_reqctx)); ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); - rctx->fallback_req.base.flags = areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; + ahash_request_set_callback(&rctx->fallback_req, + areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + areq->base.complete, areq->base.data); return crypto_ahash_init(&rctx->fallback_req); } @@ -104,7 +106,9 @@ int sun8i_ce_hash_export(struct ahash_request *areq, void *out) struct sun8i_ce_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); - rctx->fallback_req.base.flags = areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; + ahash_request_set_callback(&rctx->fallback_req, + areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + areq->base.complete, areq->base.data); return crypto_ahash_export(&rctx->fallback_req, out); } @@ -116,7 +120,9 @@ int sun8i_ce_hash_import(struct ahash_request *areq, const void *in) struct sun8i_ce_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); - rctx->fallback_req.base.flags = areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; + ahash_request_set_callback(&rctx->fallback_req, + areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + areq->base.complete, areq->base.data); return crypto_ahash_import(&rctx->fallback_req, in); } @@ -130,9 +136,10 @@ int sun8i_ce_hash_final(struct ahash_request *areq) sun8i_ce_hash_stat_fb_inc(tfm); ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); - rctx->fallback_req.base.flags = areq->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; - rctx->fallback_req.result = areq->result; + ahash_request_set_callback(&rctx->fallback_req, + areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + areq->base.complete, areq->base.data); + ahash_request_set_crypt(&rctx->fallback_req, NULL, areq->result, 0); return crypto_ahash_final(&rctx->fallback_req); } @@ -144,10 +151,10 @@ int sun8i_ce_hash_update(struct ahash_request *areq) struct sun8i_ce_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); - rctx->fallback_req.base.flags = areq->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; - rctx->fallback_req.nbytes = areq->nbytes; - rctx->fallback_req.src = areq->src; + ahash_request_set_callback(&rctx->fallback_req, + areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + areq->base.complete, areq->base.data); + ahash_request_set_crypt(&rctx->fallback_req, areq->src, NULL, areq->nbytes); return crypto_ahash_update(&rctx->fallback_req); } @@ -161,12 +168,11 @@ int sun8i_ce_hash_finup(struct ahash_request *areq) sun8i_ce_hash_stat_fb_inc(tfm); ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); - rctx->fallback_req.base.flags = areq->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; - - rctx->fallback_req.nbytes = areq->nbytes; - rctx->fallback_req.src = areq->src; - rctx->fallback_req.result = areq->result; + ahash_request_set_callback(&rctx->fallback_req, + areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + areq->base.complete, areq->base.data); + ahash_request_set_crypt(&rctx->fallback_req, areq->src, areq->result, + areq->nbytes); return crypto_ahash_finup(&rctx->fallback_req); } @@ -180,12 +186,11 @@ static int sun8i_ce_hash_digest_fb(struct ahash_request *areq) sun8i_ce_hash_stat_fb_inc(tfm); ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); - rctx->fallback_req.base.flags = areq->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; - - rctx->fallback_req.nbytes = areq->nbytes; - rctx->fallback_req.src = areq->src; - rctx->fallback_req.result = areq->result; + ahash_request_set_callback(&rctx->fallback_req, + areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + areq->base.complete, areq->base.data); + ahash_request_set_crypt(&rctx->fallback_req, areq->src, areq->result, + areq->nbytes); return crypto_ahash_digest(&rctx->fallback_req); } From af7e23c616f523b1c4d1f5f0cd7f9852fa4aa24f Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Mon, 7 Apr 2025 12:57:54 +0800 Subject: [PATCH 072/382] crypto: ccp - Silence may-be-uninitialized warning in sev_ioctl_do_pdh_export MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The recent reordering of code in sev_ioctl_do_pdh_export triggered a false-positive may-be-uninitialized warning from gcc: In file included from ../include/linux/sched/task.h:13, from ../include/linux/sched/signal.h:9, from ../include/linux/rcuwait.h:6, from ../include/linux/percpu-rwsem.h:7, from ../include/linux/fs.h:34, from ../include/linux/compat.h:17, from ../arch/x86/include/asm/ia32.h:7, from ../arch/x86/include/asm/elf.h:10, from ../include/linux/elf.h:6, from ../include/linux/module.h:19, from ../drivers/crypto/ccp/sev-dev.c:11: In function ‘copy_to_user’, inlined from ‘sev_ioctl_do_pdh_export’ at ../drivers/crypto/ccp/sev-dev.c:2036:7, inlined from ‘sev_ioctl’ at ../drivers/crypto/ccp/sev-dev.c:2249:9: ../include/linux/uaccess.h:225:16: warning: ‘input_cert_chain_address’ may be used uninitialized [-Wmaybe-uninitialized] 225 | return _copy_to_user(to, from, n); | ^~~~~~~~~~~~~~~~~~~~~~~~~~ ../drivers/crypto/ccp/sev-dev.c: In function ‘sev_ioctl’: ../drivers/crypto/ccp/sev-dev.c:1961:22: note: ‘input_cert_chain_address’ was declared here 1961 | void __user *input_cert_chain_address; | ^~~~~~~~~~~~~~~~~~~~~~~~ Silence it by moving the initialisation of the variables in question prior to the NULL check. Signed-off-by: Herbert Xu Acked-by: Tom Lendacky Signed-off-by: Herbert Xu --- drivers/crypto/ccp/sev-dev.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/crypto/ccp/sev-dev.c b/drivers/crypto/ccp/sev-dev.c index 980b3d296dc6..19fb51558a7d 100644 --- a/drivers/crypto/ccp/sev-dev.c +++ b/drivers/crypto/ccp/sev-dev.c @@ -1964,15 +1964,15 @@ static int sev_ioctl_do_pdh_export(struct sev_issue_cmd *argp, bool writable) memset(&data, 0, sizeof(data)); + input_pdh_cert_address = (void __user *)input.pdh_cert_address; + input_cert_chain_address = (void __user *)input.cert_chain_address; + /* Userspace wants to query the certificate length. */ if (!input.pdh_cert_address || !input.pdh_cert_len || !input.cert_chain_address) goto cmd; - input_pdh_cert_address = (void __user *)input.pdh_cert_address; - input_cert_chain_address = (void __user *)input.cert_chain_address; - /* Allocate a physically contiguous buffer to store the PDH blob. */ if (input.pdh_cert_len > SEV_FW_BLOB_MAX_SIZE) return -EFAULT; From 3be3f70ee95da03a87d94c4a714ee679a5c7b34d Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Tue, 8 Apr 2025 10:12:20 +0800 Subject: [PATCH 073/382] crypto: x86/chacha - Restore SSSE3 fallback path The chacha_use_simd static branch is required for x86 machines that lack SSSE3 support. Restore it and the generic fallback code. Reported-by: Eric Biggers Fixes: 9b4400215e0e ("crypto: x86/chacha - Remove SIMD fallback path") Signed-off-by: Herbert Xu --- arch/x86/crypto/chacha_glue.c | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/arch/x86/crypto/chacha_glue.c b/arch/x86/crypto/chacha_glue.c index b7fd7a1f0e15..fcc14c006bde 100644 --- a/arch/x86/crypto/chacha_glue.c +++ b/arch/x86/crypto/chacha_glue.c @@ -5,11 +5,12 @@ * Copyright (C) 2015 Martin Willi */ +#include #include +#include #include #include #include -#include asmlinkage void chacha_block_xor_ssse3(u32 *state, u8 *dst, const u8 *src, unsigned int len, int nrounds); @@ -31,6 +32,7 @@ asmlinkage void chacha_4block_xor_avx512vl(u32 *state, u8 *dst, const u8 *src, asmlinkage void chacha_8block_xor_avx512vl(u32 *state, u8 *dst, const u8 *src, unsigned int len, int nrounds); +static __ro_after_init DEFINE_STATIC_KEY_FALSE(chacha_use_simd); static __ro_after_init DEFINE_STATIC_KEY_FALSE(chacha_use_avx2); static __ro_after_init DEFINE_STATIC_KEY_FALSE(chacha_use_avx512vl); @@ -117,15 +119,23 @@ static void chacha_dosimd(u32 *state, u8 *dst, const u8 *src, void hchacha_block_arch(const u32 *state, u32 *stream, int nrounds) { - kernel_fpu_begin(); - hchacha_block_ssse3(state, stream, nrounds); - kernel_fpu_end(); + if (!static_branch_likely(&chacha_use_simd)) { + hchacha_block_generic(state, stream, nrounds); + } else { + kernel_fpu_begin(); + hchacha_block_ssse3(state, stream, nrounds); + kernel_fpu_end(); + } } EXPORT_SYMBOL(hchacha_block_arch); void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *src, unsigned int bytes, int nrounds) { + if (!static_branch_likely(&chacha_use_simd) || + bytes <= CHACHA_BLOCK_SIZE) + return chacha_crypt_generic(state, dst, src, bytes, nrounds); + do { unsigned int todo = min_t(unsigned int, bytes, SZ_4K); @@ -142,7 +152,7 @@ EXPORT_SYMBOL(chacha_crypt_arch); bool chacha_is_arch_optimized(void) { - return true; + return static_key_enabled(&chacha_use_simd); } EXPORT_SYMBOL(chacha_is_arch_optimized); @@ -151,6 +161,8 @@ static int __init chacha_simd_mod_init(void) if (!boot_cpu_has(X86_FEATURE_SSSE3)) return 0; + static_branch_enable(&chacha_use_simd); + if (boot_cpu_has(X86_FEATURE_AVX) && boot_cpu_has(X86_FEATURE_AVX2) && cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL)) { From 5976fe19e240a681b5dd13362c8507e4c66b466b Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sat, 12 Apr 2025 13:36:51 +0800 Subject: [PATCH 074/382] Revert "crypto: testmgr - Add multibuffer acomp testing" This reverts commit 99585c2192cb1ce212876e82ef01d1c98c7f4699. Remove the acomp multibuffer tests so that the interface can be redesigned. Signed-off-by: Herbert Xu --- crypto/testmgr.c | 145 ++++++++++++++++++++--------------------------- 1 file changed, 63 insertions(+), 82 deletions(-) diff --git a/crypto/testmgr.c b/crypto/testmgr.c index abd609d4c8ef..82977ea25db3 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -58,9 +58,6 @@ module_param(fuzz_iterations, uint, 0644); MODULE_PARM_DESC(fuzz_iterations, "number of fuzz test iterations"); #endif -/* Multibuffer is unlimited. Set arbitrary limit for testing. */ -#define MAX_MB_MSGS 16 - #ifdef CONFIG_CRYPTO_MANAGER_DISABLE_TESTS /* a perfect nop */ @@ -3329,48 +3326,27 @@ static int test_acomp(struct crypto_acomp *tfm, int ctcount, int dtcount) { const char *algo = crypto_tfm_alg_driver_name(crypto_acomp_tfm(tfm)); - struct scatterlist *src = NULL, *dst = NULL; - struct acomp_req *reqs[MAX_MB_MSGS] = {}; - char *decomp_out[MAX_MB_MSGS] = {}; - char *output[MAX_MB_MSGS] = {}; - struct crypto_wait wait; - struct acomp_req *req; - int ret = -ENOMEM; unsigned int i; + char *output, *decomp_out; + int ret; + struct scatterlist src, dst; + struct acomp_req *req; + struct crypto_wait wait; - src = kmalloc_array(MAX_MB_MSGS, sizeof(*src), GFP_KERNEL); - if (!src) - goto out; - dst = kmalloc_array(MAX_MB_MSGS, sizeof(*dst), GFP_KERNEL); - if (!dst) - goto out; + output = kmalloc(COMP_BUF_SIZE, GFP_KERNEL); + if (!output) + return -ENOMEM; - for (i = 0; i < MAX_MB_MSGS; i++) { - reqs[i] = acomp_request_alloc(tfm); - if (!reqs[i]) - goto out; - - acomp_request_set_callback(reqs[i], - CRYPTO_TFM_REQ_MAY_SLEEP | - CRYPTO_TFM_REQ_MAY_BACKLOG, - crypto_req_done, &wait); - if (i) - acomp_request_chain(reqs[i], reqs[0]); - - output[i] = kmalloc(COMP_BUF_SIZE, GFP_KERNEL); - if (!output[i]) - goto out; - - decomp_out[i] = kmalloc(COMP_BUF_SIZE, GFP_KERNEL); - if (!decomp_out[i]) - goto out; + decomp_out = kmalloc(COMP_BUF_SIZE, GFP_KERNEL); + if (!decomp_out) { + kfree(output); + return -ENOMEM; } for (i = 0; i < ctcount; i++) { unsigned int dlen = COMP_BUF_SIZE; int ilen = ctemplate[i].inlen; void *input_vec; - int j; input_vec = kmemdup(ctemplate[i].input, ilen, GFP_KERNEL); if (!input_vec) { @@ -3378,61 +3354,70 @@ static int test_acomp(struct crypto_acomp *tfm, goto out; } + memset(output, 0, dlen); crypto_init_wait(&wait); - sg_init_one(src, input_vec, ilen); + sg_init_one(&src, input_vec, ilen); + sg_init_one(&dst, output, dlen); - for (j = 0; j < MAX_MB_MSGS; j++) { - sg_init_one(dst + j, output[j], dlen); - acomp_request_set_params(reqs[j], src, dst + j, ilen, dlen); + req = acomp_request_alloc(tfm); + if (!req) { + pr_err("alg: acomp: request alloc failed for %s\n", + algo); + kfree(input_vec); + ret = -ENOMEM; + goto out; } - req = reqs[0]; + acomp_request_set_params(req, &src, &dst, ilen, dlen); + acomp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, + crypto_req_done, &wait); + ret = crypto_wait_req(crypto_acomp_compress(req), &wait); if (ret) { pr_err("alg: acomp: compression failed on test %d for %s: ret=%d\n", i + 1, algo, -ret); kfree(input_vec); + acomp_request_free(req); goto out; } ilen = req->dlen; dlen = COMP_BUF_SIZE; + sg_init_one(&src, output, ilen); + sg_init_one(&dst, decomp_out, dlen); crypto_init_wait(&wait); - for (j = 0; j < MAX_MB_MSGS; j++) { - sg_init_one(src + j, output[j], ilen); - sg_init_one(dst + j, decomp_out[j], dlen); - acomp_request_set_params(reqs[j], src + j, dst + j, ilen, dlen); + acomp_request_set_params(req, &src, &dst, ilen, dlen); + + ret = crypto_wait_req(crypto_acomp_decompress(req), &wait); + if (ret) { + pr_err("alg: acomp: compression failed on test %d for %s: ret=%d\n", + i + 1, algo, -ret); + kfree(input_vec); + acomp_request_free(req); + goto out; } - crypto_wait_req(crypto_acomp_decompress(req), &wait); - for (j = 0; j < MAX_MB_MSGS; j++) { - ret = reqs[j]->base.err; - if (ret) { - pr_err("alg: acomp: compression failed on test %d (%d) for %s: ret=%d\n", - i + 1, j, algo, -ret); - kfree(input_vec); - goto out; - } + if (req->dlen != ctemplate[i].inlen) { + pr_err("alg: acomp: Compression test %d failed for %s: output len = %d\n", + i + 1, algo, req->dlen); + ret = -EINVAL; + kfree(input_vec); + acomp_request_free(req); + goto out; + } - if (reqs[j]->dlen != ctemplate[i].inlen) { - pr_err("alg: acomp: Compression test %d (%d) failed for %s: output len = %d\n", - i + 1, j, algo, reqs[j]->dlen); - ret = -EINVAL; - kfree(input_vec); - goto out; - } - - if (memcmp(input_vec, decomp_out[j], reqs[j]->dlen)) { - pr_err("alg: acomp: Compression test %d (%d) failed for %s\n", - i + 1, j, algo); - hexdump(output[j], reqs[j]->dlen); - ret = -EINVAL; - kfree(input_vec); - goto out; - } + if (memcmp(input_vec, decomp_out, req->dlen)) { + pr_err("alg: acomp: Compression test %d failed for %s\n", + i + 1, algo); + hexdump(output, req->dlen); + ret = -EINVAL; + kfree(input_vec); + acomp_request_free(req); + goto out; } kfree(input_vec); + acomp_request_free(req); } for (i = 0; i < dtcount; i++) { @@ -3446,9 +3431,10 @@ static int test_acomp(struct crypto_acomp *tfm, goto out; } + memset(output, 0, dlen); crypto_init_wait(&wait); - sg_init_one(src, input_vec, ilen); - sg_init_one(dst, output[0], dlen); + sg_init_one(&src, input_vec, ilen); + sg_init_one(&dst, output, dlen); req = acomp_request_alloc(tfm); if (!req) { @@ -3459,7 +3445,7 @@ static int test_acomp(struct crypto_acomp *tfm, goto out; } - acomp_request_set_params(req, src, dst, ilen, dlen); + acomp_request_set_params(req, &src, &dst, ilen, dlen); acomp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, crypto_req_done, &wait); @@ -3481,10 +3467,10 @@ static int test_acomp(struct crypto_acomp *tfm, goto out; } - if (memcmp(output[0], dtemplate[i].output, req->dlen)) { + if (memcmp(output, dtemplate[i].output, req->dlen)) { pr_err("alg: acomp: Decompression test %d failed for %s\n", i + 1, algo); - hexdump(output[0], req->dlen); + hexdump(output, req->dlen); ret = -EINVAL; kfree(input_vec); acomp_request_free(req); @@ -3498,13 +3484,8 @@ static int test_acomp(struct crypto_acomp *tfm, ret = 0; out: - acomp_request_free(reqs[0]); - for (i = 0; i < MAX_MB_MSGS; i++) { - kfree(output[i]); - kfree(decomp_out[i]); - } - kfree(dst); - kfree(src); + kfree(decomp_out); + kfree(output); return ret; } From 78e2846aa4b29b44368377842d2369e0198022b7 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sat, 12 Apr 2025 13:36:53 +0800 Subject: [PATCH 075/382] crypto: deflate - Remove request chaining Remove request chaining support from deflate. Signed-off-by: Herbert Xu --- crypto/deflate.c | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/crypto/deflate.c b/crypto/deflate.c index bc76c343a0cf..57d7af4dfdfb 100644 --- a/crypto/deflate.c +++ b/crypto/deflate.c @@ -111,7 +111,6 @@ static int deflate_compress(struct acomp_req *req) { struct crypto_acomp_stream *s; struct deflate_stream *ds; - struct acomp_req *r2; int err; s = crypto_acomp_lock_stream_bh(&deflate_streams); @@ -126,12 +125,6 @@ static int deflate_compress(struct acomp_req *req) } err = deflate_compress_one(req, ds); - req->base.err = err; - - list_for_each_entry(r2, &req->base.list, base.list) { - zlib_deflateReset(&ds->stream); - r2->base.err = deflate_compress_one(r2, ds); - } out: crypto_acomp_unlock_stream_bh(s); @@ -199,7 +192,6 @@ static int deflate_decompress(struct acomp_req *req) { struct crypto_acomp_stream *s; struct deflate_stream *ds; - struct acomp_req *r2; int err; s = crypto_acomp_lock_stream_bh(&deflate_streams); @@ -212,12 +204,6 @@ static int deflate_decompress(struct acomp_req *req) } err = deflate_decompress_one(req, ds); - req->base.err = err; - - list_for_each_entry(r2, &req->base.list, base.list) { - zlib_inflateReset(&ds->stream); - r2->base.err = deflate_decompress_one(r2, ds); - } out: crypto_acomp_unlock_stream_bh(s); From 64929fe8c0a43508eee952cf57903a61c52601e7 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sat, 12 Apr 2025 13:36:55 +0800 Subject: [PATCH 076/382] crypto: acomp - Remove request chaining Request chaining requires the user to do too much book keeping. Remove it from acomp. Signed-off-by: Herbert Xu --- crypto/acompress.c | 117 ++++++++-------------------- crypto/scompress.c | 18 +---- include/crypto/acompress.h | 14 ---- include/crypto/internal/acompress.h | 5 -- 4 files changed, 35 insertions(+), 119 deletions(-) diff --git a/crypto/acompress.c b/crypto/acompress.c index 5d0b8b8b84f6..b682a88781f0 100644 --- a/crypto/acompress.c +++ b/crypto/acompress.c @@ -177,7 +177,6 @@ static void acomp_save_req(struct acomp_req *req, crypto_completion_t cplt) state->data = req->base.data; req->base.complete = cplt; req->base.data = state; - state->req0 = req; } static void acomp_restore_req(struct acomp_req *req) @@ -188,23 +187,20 @@ static void acomp_restore_req(struct acomp_req *req) req->base.data = state->data; } -static void acomp_reqchain_virt(struct acomp_req_chain *state, int err) +static void acomp_reqchain_virt(struct acomp_req *req) { - struct acomp_req *req = state->cur; + struct acomp_req_chain *state = &req->chain; unsigned int slen = req->slen; unsigned int dlen = req->dlen; - req->base.err = err; - state = &req->chain; - if (state->flags & CRYPTO_ACOMP_REQ_SRC_VIRT) acomp_request_set_src_dma(req, state->src, slen); else if (state->flags & CRYPTO_ACOMP_REQ_SRC_FOLIO) - acomp_request_set_src_folio(req, state->sfolio, state->soff, slen); + acomp_request_set_src_folio(req, state->sfolio, req->soff, slen); if (state->flags & CRYPTO_ACOMP_REQ_DST_VIRT) acomp_request_set_dst_dma(req, state->dst, dlen); else if (state->flags & CRYPTO_ACOMP_REQ_DST_FOLIO) - acomp_request_set_dst_folio(req, state->dfolio, state->doff, dlen); + acomp_request_set_dst_folio(req, state->dfolio, req->doff, dlen); } static void acomp_virt_to_sg(struct acomp_req *req) @@ -229,7 +225,6 @@ static void acomp_virt_to_sg(struct acomp_req *req) size_t off = req->soff; state->sfolio = folio; - state->soff = off; sg_init_table(&state->ssg, 1); sg_set_page(&state->ssg, folio_page(folio, off / PAGE_SIZE), slen, off % PAGE_SIZE); @@ -249,7 +244,6 @@ static void acomp_virt_to_sg(struct acomp_req *req) size_t off = req->doff; state->dfolio = folio; - state->doff = off; sg_init_table(&state->dsg, 1); sg_set_page(&state->dsg, folio_page(folio, off / PAGE_SIZE), dlen, off % PAGE_SIZE); @@ -257,8 +251,7 @@ static void acomp_virt_to_sg(struct acomp_req *req) } } -static int acomp_do_nondma(struct acomp_req_chain *state, - struct acomp_req *req) +static int acomp_do_nondma(struct acomp_req *req, bool comp) { u32 keep = CRYPTO_ACOMP_REQ_SRC_VIRT | CRYPTO_ACOMP_REQ_SRC_NONDMA | @@ -275,7 +268,7 @@ static int acomp_do_nondma(struct acomp_req_chain *state, fbreq->slen = req->slen; fbreq->dlen = req->dlen; - if (state->op == crypto_acomp_reqtfm(req)->compress) + if (comp) err = crypto_acomp_compress(fbreq); else err = crypto_acomp_decompress(fbreq); @@ -284,114 +277,70 @@ static int acomp_do_nondma(struct acomp_req_chain *state, return err; } -static int acomp_do_one_req(struct acomp_req_chain *state, - struct acomp_req *req) +static int acomp_do_one_req(struct acomp_req *req, bool comp) { - state->cur = req; - if (acomp_request_isnondma(req)) - return acomp_do_nondma(state, req); + return acomp_do_nondma(req, comp); acomp_virt_to_sg(req); - return state->op(req); + return comp ? crypto_acomp_reqtfm(req)->compress(req) : + crypto_acomp_reqtfm(req)->decompress(req); } -static int acomp_reqchain_finish(struct acomp_req *req0, int err, u32 mask) +static int acomp_reqchain_finish(struct acomp_req *req, int err) { - struct acomp_req_chain *state = req0->base.data; - struct acomp_req *req = state->cur; - struct acomp_req *n; - - acomp_reqchain_virt(state, err); - - if (req != req0) - list_add_tail(&req->base.list, &req0->base.list); - - list_for_each_entry_safe(req, n, &state->head, base.list) { - list_del_init(&req->base.list); - - req->base.flags &= mask; - req->base.complete = acomp_reqchain_done; - req->base.data = state; - - err = acomp_do_one_req(state, req); - - if (err == -EINPROGRESS) { - if (!list_empty(&state->head)) - err = -EBUSY; - goto out; - } - - if (err == -EBUSY) - goto out; - - acomp_reqchain_virt(state, err); - list_add_tail(&req->base.list, &req0->base.list); - } - - acomp_restore_req(req0); - -out: + acomp_reqchain_virt(req); + acomp_restore_req(req); return err; } static void acomp_reqchain_done(void *data, int err) { - struct acomp_req_chain *state = data; - crypto_completion_t compl = state->compl; + struct acomp_req *req = data; + crypto_completion_t compl; - data = state->data; + compl = req->chain.compl; + data = req->chain.data; - if (err == -EINPROGRESS) { - if (!list_empty(&state->head)) - return; + if (err == -EINPROGRESS) goto notify; - } - err = acomp_reqchain_finish(state->req0, err, - CRYPTO_TFM_REQ_MAY_BACKLOG); - if (err == -EBUSY) - return; + err = acomp_reqchain_finish(req, err); notify: compl(data, err); } -static int acomp_do_req_chain(struct acomp_req *req, - int (*op)(struct acomp_req *req)) +static int acomp_do_req_chain(struct acomp_req *req, bool comp) { - struct crypto_acomp *tfm = crypto_acomp_reqtfm(req); - struct acomp_req_chain *state; int err; - if (crypto_acomp_req_chain(tfm) || - (!acomp_request_chained(req) && acomp_request_issg(req))) - return op(req); - acomp_save_req(req, acomp_reqchain_done); - state = req->base.data; - state->op = op; - state->src = NULL; - INIT_LIST_HEAD(&state->head); - list_splice_init(&req->base.list, &state->head); - - err = acomp_do_one_req(state, req); + err = acomp_do_one_req(req, comp); if (err == -EBUSY || err == -EINPROGRESS) - return -EBUSY; + return err; - return acomp_reqchain_finish(req, err, ~0); + return acomp_reqchain_finish(req, err); } int crypto_acomp_compress(struct acomp_req *req) { - return acomp_do_req_chain(req, crypto_acomp_reqtfm(req)->compress); + struct crypto_acomp *tfm = crypto_acomp_reqtfm(req); + + if (crypto_acomp_req_chain(tfm) || acomp_request_issg(req)) + crypto_acomp_reqtfm(req)->compress(req); + return acomp_do_req_chain(req, true); } EXPORT_SYMBOL_GPL(crypto_acomp_compress); int crypto_acomp_decompress(struct acomp_req *req) { - return acomp_do_req_chain(req, crypto_acomp_reqtfm(req)->decompress); + struct crypto_acomp *tfm = crypto_acomp_reqtfm(req); + + if (crypto_acomp_req_chain(tfm) || acomp_request_issg(req)) + crypto_acomp_reqtfm(req)->decompress(req); + return acomp_do_req_chain(req, false); } EXPORT_SYMBOL_GPL(crypto_acomp_decompress); diff --git a/crypto/scompress.c b/crypto/scompress.c index 4a2b3933aa95..7ade3f2fee7e 100644 --- a/crypto/scompress.c +++ b/crypto/scompress.c @@ -284,28 +284,14 @@ static int scomp_acomp_comp_decomp(struct acomp_req *req, int dir) return ret; } -static int scomp_acomp_chain(struct acomp_req *req, int dir) -{ - struct acomp_req *r2; - int err; - - err = scomp_acomp_comp_decomp(req, dir); - req->base.err = err; - - list_for_each_entry(r2, &req->base.list, base.list) - r2->base.err = scomp_acomp_comp_decomp(r2, dir); - - return err; -} - static int scomp_acomp_compress(struct acomp_req *req) { - return scomp_acomp_chain(req, 1); + return scomp_acomp_comp_decomp(req, 1); } static int scomp_acomp_decompress(struct acomp_req *req) { - return scomp_acomp_chain(req, 0); + return scomp_acomp_comp_decomp(req, 0); } static void crypto_exit_scomp_ops_async(struct crypto_tfm *tfm) diff --git a/include/crypto/acompress.h b/include/crypto/acompress.h index f0e01ff77d92..080e134df35c 100644 --- a/include/crypto/acompress.h +++ b/include/crypto/acompress.h @@ -52,10 +52,6 @@ struct acomp_req; struct folio; struct acomp_req_chain { - struct list_head head; - struct acomp_req *req0; - struct acomp_req *cur; - int (*op)(struct acomp_req *req); crypto_completion_t compl; void *data; struct scatterlist ssg; @@ -68,8 +64,6 @@ struct acomp_req_chain { u8 *dst; struct folio *dfolio; }; - size_t soff; - size_t doff; u32 flags; }; @@ -343,8 +337,6 @@ static inline void acomp_request_set_callback(struct acomp_req *req, req->base.data = data; req->base.flags &= keep; req->base.flags |= flgs & ~keep; - - crypto_reqchain_init(&req->base); } /** @@ -552,12 +544,6 @@ static inline void acomp_request_set_dst_folio(struct acomp_req *req, req->base.flags |= CRYPTO_ACOMP_REQ_DST_FOLIO; } -static inline void acomp_request_chain(struct acomp_req *req, - struct acomp_req *head) -{ - crypto_request_chain(&req->base, &head->base); -} - /** * crypto_acomp_compress() -- Invoke asynchronous compress operation * diff --git a/include/crypto/internal/acompress.h b/include/crypto/internal/acompress.h index fbbff9a8a2d9..960cdd1f3a57 100644 --- a/include/crypto/internal/acompress.h +++ b/include/crypto/internal/acompress.h @@ -151,11 +151,6 @@ void crypto_unregister_acomp(struct acomp_alg *alg); int crypto_register_acomps(struct acomp_alg *algs, int count); void crypto_unregister_acomps(struct acomp_alg *algs, int count); -static inline bool acomp_request_chained(struct acomp_req *req) -{ - return crypto_request_chained(&req->base); -} - static inline bool acomp_request_issg(struct acomp_req *req) { return !(req->base.flags & (CRYPTO_ACOMP_REQ_SRC_VIRT | From 69e5a1228d43f76f7a6a2e73ddd55c543fac8826 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sat, 12 Apr 2025 13:36:57 +0800 Subject: [PATCH 077/382] Revert "crypto: tcrypt - Restore multibuffer ahash tests" This reverts commit c664f034172705a75f3f8a0c409b9bf95b633093. Remove the multibuffer ahash speed tests again. Signed-off-by: Herbert Xu --- crypto/tcrypt.c | 231 ------------------------------------------------ 1 file changed, 231 deletions(-) diff --git a/crypto/tcrypt.c b/crypto/tcrypt.c index 96f4a66be14c..879fc21dcc16 100644 --- a/crypto/tcrypt.c +++ b/crypto/tcrypt.c @@ -716,207 +716,6 @@ static inline int do_one_ahash_op(struct ahash_request *req, int ret) return crypto_wait_req(ret, wait); } -struct test_mb_ahash_data { - struct scatterlist sg[XBUFSIZE]; - char result[64]; - struct ahash_request *req; - struct crypto_wait wait; - char *xbuf[XBUFSIZE]; -}; - -static inline int do_mult_ahash_op(struct test_mb_ahash_data *data, u32 num_mb, - int *rc) -{ - int i, err; - - /* Fire up a bunch of concurrent requests */ - err = crypto_ahash_digest(data[0].req); - - /* Wait for all requests to finish */ - err = crypto_wait_req(err, &data[0].wait); - if (num_mb < 2) - return err; - - for (i = 0; i < num_mb; i++) { - rc[i] = ahash_request_err(data[i].req); - if (rc[i]) { - pr_info("concurrent request %d error %d\n", i, rc[i]); - err = rc[i]; - } - } - - return err; -} - -static int test_mb_ahash_jiffies(struct test_mb_ahash_data *data, int blen, - int secs, u32 num_mb) -{ - unsigned long start, end; - int bcount; - int ret = 0; - int *rc; - - rc = kcalloc(num_mb, sizeof(*rc), GFP_KERNEL); - if (!rc) - return -ENOMEM; - - for (start = jiffies, end = start + secs * HZ, bcount = 0; - time_before(jiffies, end); bcount++) { - ret = do_mult_ahash_op(data, num_mb, rc); - if (ret) - goto out; - } - - pr_cont("%d operations in %d seconds (%llu bytes)\n", - bcount * num_mb, secs, (u64)bcount * blen * num_mb); - -out: - kfree(rc); - return ret; -} - -static int test_mb_ahash_cycles(struct test_mb_ahash_data *data, int blen, - u32 num_mb) -{ - unsigned long cycles = 0; - int ret = 0; - int i; - int *rc; - - rc = kcalloc(num_mb, sizeof(*rc), GFP_KERNEL); - if (!rc) - return -ENOMEM; - - /* Warm-up run. */ - for (i = 0; i < 4; i++) { - ret = do_mult_ahash_op(data, num_mb, rc); - if (ret) - goto out; - } - - /* The real thing. */ - for (i = 0; i < 8; i++) { - cycles_t start, end; - - start = get_cycles(); - ret = do_mult_ahash_op(data, num_mb, rc); - end = get_cycles(); - - if (ret) - goto out; - - cycles += end - start; - } - - pr_cont("1 operation in %lu cycles (%d bytes)\n", - (cycles + 4) / (8 * num_mb), blen); - -out: - kfree(rc); - return ret; -} - -static void test_mb_ahash_speed(const char *algo, unsigned int secs, - struct hash_speed *speed, u32 num_mb) -{ - struct test_mb_ahash_data *data; - struct crypto_ahash *tfm; - unsigned int i, j, k; - int ret; - - data = kcalloc(num_mb, sizeof(*data), GFP_KERNEL); - if (!data) - return; - - tfm = crypto_alloc_ahash(algo, 0, 0); - if (IS_ERR(tfm)) { - pr_err("failed to load transform for %s: %ld\n", - algo, PTR_ERR(tfm)); - goto free_data; - } - - for (i = 0; i < num_mb; ++i) { - if (testmgr_alloc_buf(data[i].xbuf)) - goto out; - - crypto_init_wait(&data[i].wait); - - data[i].req = ahash_request_alloc(tfm, GFP_KERNEL); - if (!data[i].req) { - pr_err("alg: hash: Failed to allocate request for %s\n", - algo); - goto out; - } - - - if (i) { - ahash_request_set_callback(data[i].req, 0, NULL, NULL); - ahash_request_chain(data[i].req, data[0].req); - } else - ahash_request_set_callback(data[0].req, 0, - crypto_req_done, - &data[0].wait); - - sg_init_table(data[i].sg, XBUFSIZE); - for (j = 0; j < XBUFSIZE; j++) { - sg_set_buf(data[i].sg + j, data[i].xbuf[j], PAGE_SIZE); - memset(data[i].xbuf[j], 0xff, PAGE_SIZE); - } - } - - pr_info("\ntesting speed of multibuffer %s (%s)\n", algo, - get_driver_name(crypto_ahash, tfm)); - - for (i = 0; speed[i].blen != 0; i++) { - /* For some reason this only tests digests. */ - if (speed[i].blen != speed[i].plen) - continue; - - if (speed[i].blen > XBUFSIZE * PAGE_SIZE) { - pr_err("template (%u) too big for tvmem (%lu)\n", - speed[i].blen, XBUFSIZE * PAGE_SIZE); - goto out; - } - - if (klen) - crypto_ahash_setkey(tfm, tvmem[0], klen); - - for (k = 0; k < num_mb; k++) - ahash_request_set_crypt(data[k].req, data[k].sg, - data[k].result, speed[i].blen); - - pr_info("test%3u " - "(%5u byte blocks,%5u bytes per update,%4u updates): ", - i, speed[i].blen, speed[i].plen, - speed[i].blen / speed[i].plen); - - if (secs) { - ret = test_mb_ahash_jiffies(data, speed[i].blen, secs, - num_mb); - cond_resched(); - } else { - ret = test_mb_ahash_cycles(data, speed[i].blen, num_mb); - } - - - if (ret) { - pr_err("At least one hashing failed ret=%d\n", ret); - break; - } - } - -out: - ahash_request_free(data[0].req); - - for (k = 0; k < num_mb; ++k) - testmgr_free_buf(data[k].xbuf); - - crypto_free_ahash(tfm); - -free_data: - kfree(data); -} - static int test_ahash_jiffies_digest(struct ahash_request *req, int blen, char *out, int secs) { @@ -2584,36 +2383,6 @@ static int do_test(const char *alg, u32 type, u32 mask, int m, u32 num_mb) test_ahash_speed("sm3", sec, generic_hash_speed_template); if (mode > 400 && mode < 500) break; fallthrough; - case 450: - test_mb_ahash_speed("sha1", sec, generic_hash_speed_template, - num_mb); - if (mode > 400 && mode < 500) break; - fallthrough; - case 451: - test_mb_ahash_speed("sha256", sec, generic_hash_speed_template, - num_mb); - if (mode > 400 && mode < 500) break; - fallthrough; - case 452: - test_mb_ahash_speed("sha512", sec, generic_hash_speed_template, - num_mb); - if (mode > 400 && mode < 500) break; - fallthrough; - case 453: - test_mb_ahash_speed("sm3", sec, generic_hash_speed_template, - num_mb); - if (mode > 400 && mode < 500) break; - fallthrough; - case 454: - test_mb_ahash_speed("streebog256", sec, - generic_hash_speed_template, num_mb); - if (mode > 400 && mode < 500) break; - fallthrough; - case 455: - test_mb_ahash_speed("streebog512", sec, - generic_hash_speed_template, num_mb); - if (mode > 400 && mode < 500) break; - fallthrough; case 499: break; From 5bb61dc76d11a661c323dee1505b408d18c31565 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sat, 12 Apr 2025 13:37:00 +0800 Subject: [PATCH 078/382] crypto: ahash - Remove request chaining Request chaining requires the user to do too much book keeping. Remove it from ahash. Signed-off-by: Herbert Xu --- crypto/ahash.c | 169 +++++++-------------------------- include/crypto/algapi.h | 5 - include/crypto/hash.h | 12 --- include/crypto/internal/hash.h | 5 - include/linux/crypto.h | 15 --- 5 files changed, 32 insertions(+), 174 deletions(-) diff --git a/crypto/ahash.c b/crypto/ahash.c index 2d9eec2b2b1c..fc59897c234c 100644 --- a/crypto/ahash.c +++ b/crypto/ahash.c @@ -43,10 +43,7 @@ struct crypto_hash_walk { }; struct ahash_save_req_state { - struct list_head head; struct ahash_request *req0; - struct ahash_request *cur; - int (*op)(struct ahash_request *req); crypto_completion_t compl; void *data; struct scatterlist sg; @@ -54,9 +51,9 @@ struct ahash_save_req_state { u8 *page; unsigned int offset; unsigned int nbytes; + bool update; }; -static void ahash_reqchain_done(void *data, int err); static int ahash_save_req(struct ahash_request *req, crypto_completion_t cplt); static void ahash_restore_req(struct ahash_request *req); static void ahash_def_finup_done1(void *data, int err); @@ -313,21 +310,17 @@ int crypto_ahash_setkey(struct crypto_ahash *tfm, const u8 *key, } EXPORT_SYMBOL_GPL(crypto_ahash_setkey); -static bool ahash_request_hasvirt(struct ahash_request *req) -{ - return ahash_request_isvirt(req); -} - static int ahash_reqchain_virt(struct ahash_save_req_state *state, int err, u32 mask) { - struct ahash_request *req = state->cur; + struct ahash_request *req = state->req0; + struct crypto_ahash *tfm; + + tfm = crypto_ahash_reqtfm(req); for (;;) { unsigned len = state->nbytes; - req->base.err = err; - if (!state->offset) break; @@ -346,10 +339,9 @@ static int ahash_reqchain_virt(struct ahash_save_req_state *state, state->offset += len; req->nbytes = len; - err = state->op(req); + err = crypto_ahash_alg(tfm)->update(req); if (err == -EINPROGRESS) { - if (!list_empty(&state->head) || - state->offset < state->nbytes) + if (state->offset < state->nbytes) err = -EBUSY; break; } @@ -365,64 +357,12 @@ static int ahash_reqchain_finish(struct ahash_request *req0, struct ahash_save_req_state *state, int err, u32 mask) { - struct ahash_request *req = state->cur; - struct crypto_ahash *tfm; - struct ahash_request *n; - bool update; u8 *page; err = ahash_reqchain_virt(state, err, mask); if (err == -EINPROGRESS || err == -EBUSY) goto out; - if (req != req0) - list_add_tail(&req->base.list, &req0->base.list); - - tfm = crypto_ahash_reqtfm(req); - update = state->op == crypto_ahash_alg(tfm)->update; - - list_for_each_entry_safe(req, n, &state->head, base.list) { - list_del_init(&req->base.list); - - req->base.flags &= mask; - req->base.complete = ahash_reqchain_done; - req->base.data = state; - state->cur = req; - - if (update && ahash_request_isvirt(req) && req->nbytes) { - unsigned len = req->nbytes; - u8 *result = req->result; - - state->src = req->svirt; - state->nbytes = len; - - len = min(PAGE_SIZE, len); - - memcpy(state->page, req->svirt, len); - state->offset = len; - - ahash_request_set_crypt(req, &state->sg, result, len); - } - - err = state->op(req); - - if (err == -EINPROGRESS) { - if (!list_empty(&state->head) || - state->offset < state->nbytes) - err = -EBUSY; - goto out; - } - - if (err == -EBUSY) - goto out; - - err = ahash_reqchain_virt(state, err, mask); - if (err == -EINPROGRESS || err == -EBUSY) - goto out; - - list_add_tail(&req->base.list, &req0->base.list); - } - page = state->page; if (page) { memset(page, 0, PAGE_SIZE); @@ -442,7 +382,7 @@ static void ahash_reqchain_done(void *data, int err) data = state->data; if (err == -EINPROGRESS) { - if (!list_empty(&state->head) || state->offset < state->nbytes) + if (state->offset < state->nbytes) return; goto notify; } @@ -467,21 +407,14 @@ static int ahash_do_req_chain(struct ahash_request *req, int err; if (crypto_ahash_req_chain(tfm) || - (!ahash_request_chained(req) && - (!update || !ahash_request_isvirt(req)))) + !update || !ahash_request_isvirt(req)) return op(req); - if (update && ahash_request_hasvirt(req)) { - gfp_t gfp; - u32 flags; - - flags = ahash_request_flags(req); - gfp = (flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? - GFP_KERNEL : GFP_ATOMIC; - page = (void *)__get_free_page(gfp); + if (update && ahash_request_isvirt(req)) { + page = (void *)__get_free_page(GFP_ATOMIC); err = -ENOMEM; if (!page) - goto out_set_chain; + goto out; } state = &state0; @@ -493,12 +426,10 @@ static int ahash_do_req_chain(struct ahash_request *req, state = req->base.data; } - state->op = op; - state->cur = req; + state->update = update; state->page = page; state->offset = 0; state->nbytes = 0; - INIT_LIST_HEAD(&state->head); if (page) sg_init_one(&state->sg, page, PAGE_SIZE); @@ -519,16 +450,18 @@ static int ahash_do_req_chain(struct ahash_request *req, } err = op(req); - if (err == -EBUSY || err == -EINPROGRESS) - return -EBUSY; + if (err == -EINPROGRESS || err == -EBUSY) { + if (state->offset < state->nbytes) + err = -EBUSY; + return err; + } return ahash_reqchain_finish(req, state, err, ~0); out_free_page: free_page((unsigned long)page); -out_set_chain: - req->base.err = err; +out: return err; } @@ -536,17 +469,10 @@ int crypto_ahash_init(struct ahash_request *req) { struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - if (likely(tfm->using_shash)) { - int err; - - err = crypto_shash_init(prepare_shash_desc(req, tfm)); - req->base.err = err; - return err; - } - + if (likely(tfm->using_shash)) + return crypto_shash_init(prepare_shash_desc(req, tfm)); if (crypto_ahash_get_flags(tfm) & CRYPTO_TFM_NEED_KEY) return -ENOKEY; - return ahash_do_req_chain(req, crypto_ahash_alg(tfm)->init); } EXPORT_SYMBOL_GPL(crypto_ahash_init); @@ -555,15 +481,11 @@ static int ahash_save_req(struct ahash_request *req, crypto_completion_t cplt) { struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); struct ahash_save_req_state *state; - gfp_t gfp; - u32 flags; if (!ahash_is_async(tfm)) return 0; - flags = ahash_request_flags(req); - gfp = (flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL : GFP_ATOMIC; - state = kmalloc(sizeof(*state), gfp); + state = kmalloc(sizeof(*state), GFP_ATOMIC); if (!state) return -ENOMEM; @@ -596,14 +518,8 @@ int crypto_ahash_update(struct ahash_request *req) { struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - if (likely(tfm->using_shash)) { - int err; - - err = shash_ahash_update(req, ahash_request_ctx(req)); - req->base.err = err; - return err; - } - + if (likely(tfm->using_shash)) + return shash_ahash_update(req, ahash_request_ctx(req)); return ahash_do_req_chain(req, crypto_ahash_alg(tfm)->update); } EXPORT_SYMBOL_GPL(crypto_ahash_update); @@ -612,14 +528,8 @@ int crypto_ahash_final(struct ahash_request *req) { struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - if (likely(tfm->using_shash)) { - int err; - - err = crypto_shash_final(ahash_request_ctx(req), req->result); - req->base.err = err; - return err; - } - + if (likely(tfm->using_shash)) + return crypto_shash_final(ahash_request_ctx(req), req->result); return ahash_do_req_chain(req, crypto_ahash_alg(tfm)->final); } EXPORT_SYMBOL_GPL(crypto_ahash_final); @@ -628,18 +538,11 @@ int crypto_ahash_finup(struct ahash_request *req) { struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - if (likely(tfm->using_shash)) { - int err; - - err = shash_ahash_finup(req, ahash_request_ctx(req)); - req->base.err = err; - return err; - } - + if (likely(tfm->using_shash)) + return shash_ahash_finup(req, ahash_request_ctx(req)); if (!crypto_ahash_alg(tfm)->finup || - (!crypto_ahash_req_chain(tfm) && ahash_request_hasvirt(req))) + (!crypto_ahash_req_chain(tfm) && ahash_request_isvirt(req))) return ahash_def_finup(req); - return ahash_do_req_chain(req, crypto_ahash_alg(tfm)->finup); } EXPORT_SYMBOL_GPL(crypto_ahash_finup); @@ -706,20 +609,12 @@ int crypto_ahash_digest(struct ahash_request *req) { struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - if (likely(tfm->using_shash)) { - int err; - - err = shash_ahash_digest(req, prepare_shash_desc(req, tfm)); - req->base.err = err; - return err; - } - - if (!crypto_ahash_req_chain(tfm) && ahash_request_hasvirt(req)) + if (likely(tfm->using_shash)) + return shash_ahash_digest(req, prepare_shash_desc(req, tfm)); + if (!crypto_ahash_req_chain(tfm) && ahash_request_isvirt(req)) return ahash_def_digest(req); - if (crypto_ahash_get_flags(tfm) & CRYPTO_TFM_NEED_KEY) return -ENOKEY; - return ahash_do_req_chain(req, crypto_ahash_alg(tfm)->digest); } EXPORT_SYMBOL_GPL(crypto_ahash_digest); diff --git a/include/crypto/algapi.h b/include/crypto/algapi.h index 03b7eca8af9a..ede622ecefa8 100644 --- a/include/crypto/algapi.h +++ b/include/crypto/algapi.h @@ -267,11 +267,6 @@ static inline u32 crypto_tfm_alg_type(struct crypto_tfm *tfm) return tfm->__crt_alg->cra_flags & CRYPTO_ALG_TYPE_MASK; } -static inline bool crypto_request_chained(struct crypto_async_request *req) -{ - return !list_empty(&req->list); -} - static inline bool crypto_tfm_req_chain(struct crypto_tfm *tfm) { return tfm->__crt_alg->cra_flags & CRYPTO_ALG_REQ_CHAIN; diff --git a/include/crypto/hash.h b/include/crypto/hash.h index a13e2de3c9b4..113a5835e586 100644 --- a/include/crypto/hash.h +++ b/include/crypto/hash.h @@ -630,7 +630,6 @@ static inline void ahash_request_set_callback(struct ahash_request *req, flags &= ~keep; req->base.flags &= keep; req->base.flags |= flags; - crypto_reqchain_init(&req->base); } /** @@ -679,12 +678,6 @@ static inline void ahash_request_set_virt(struct ahash_request *req, req->base.flags |= CRYPTO_AHASH_REQ_VIRT; } -static inline void ahash_request_chain(struct ahash_request *req, - struct ahash_request *head) -{ - crypto_request_chain(&req->base, &head->base); -} - /** * DOC: Synchronous Message Digest API * @@ -986,11 +979,6 @@ static inline void shash_desc_zero(struct shash_desc *desc) sizeof(*desc) + crypto_shash_descsize(desc->tfm)); } -static inline int ahash_request_err(struct ahash_request *req) -{ - return req->base.err; -} - static inline bool ahash_is_async(struct crypto_ahash *tfm) { return crypto_tfm_is_async(&tfm->base); diff --git a/include/crypto/internal/hash.h b/include/crypto/internal/hash.h index 052ac7924af3..e2a1fac38610 100644 --- a/include/crypto/internal/hash.h +++ b/include/crypto/internal/hash.h @@ -247,11 +247,6 @@ static inline struct crypto_shash *__crypto_shash_cast(struct crypto_tfm *tfm) return container_of(tfm, struct crypto_shash, base); } -static inline bool ahash_request_chained(struct ahash_request *req) -{ - return false; -} - static inline bool ahash_request_isvirt(struct ahash_request *req) { return req->base.flags & CRYPTO_AHASH_REQ_VIRT; diff --git a/include/linux/crypto.h b/include/linux/crypto.h index 1e3809d28abd..dd817f56ff0c 100644 --- a/include/linux/crypto.h +++ b/include/linux/crypto.h @@ -14,7 +14,6 @@ #include #include -#include #include #include #include @@ -179,7 +178,6 @@ struct crypto_async_request { struct crypto_tfm *tfm; u32 flags; - int err; }; /** @@ -473,19 +471,6 @@ static inline unsigned int crypto_tfm_ctx_alignment(void) return __alignof__(tfm->__crt_ctx); } -static inline void crypto_reqchain_init(struct crypto_async_request *req) -{ - req->err = -EINPROGRESS; - INIT_LIST_HEAD(&req->list); -} - -static inline void crypto_request_chain(struct crypto_async_request *req, - struct crypto_async_request *head) -{ - req->err = -EINPROGRESS; - list_add_tail(&req->list, &head->list); -} - static inline bool crypto_tfm_is_async(struct crypto_tfm *tfm) { return tfm->__crt_alg->cra_flags & CRYPTO_ALG_ASYNC; From b93336cd767fc266dcccfa034a1fb32ae1a23564 Mon Sep 17 00:00:00 2001 From: Thorsten Blum Date: Mon, 7 Apr 2025 10:22:47 +0200 Subject: [PATCH 079/382] crypto: x509 - Replace kmalloc() + NUL-termination with kzalloc() Use kzalloc() to zero out the one-element array instead of using kmalloc() followed by a manual NUL-termination. No functional changes intended. Signed-off-by: Thorsten Blum Reviewed-by: Lukas Wunner Signed-off-by: Herbert Xu --- crypto/asymmetric_keys/x509_cert_parser.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crypto/asymmetric_keys/x509_cert_parser.c b/crypto/asymmetric_keys/x509_cert_parser.c index ee2fdab42334..2ffe4ae90bea 100644 --- a/crypto/asymmetric_keys/x509_cert_parser.c +++ b/crypto/asymmetric_keys/x509_cert_parser.c @@ -372,10 +372,9 @@ static int x509_fabricate_name(struct x509_parse_context *ctx, size_t hdrlen, /* Empty name string if no material */ if (!ctx->cn_size && !ctx->o_size && !ctx->email_size) { - buffer = kmalloc(1, GFP_KERNEL); + buffer = kzalloc(1, GFP_KERNEL); if (!buffer) return -ENOMEM; - buffer[0] = 0; goto done; } From 1451e3e561be9ff4e86b911b9367a2223275d16f Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Mon, 7 Apr 2025 18:02:51 +0800 Subject: [PATCH 080/382] crypto: api - Add helpers to manage request flags Add helpers so that the ON_STACK request flag management is not duplicated all over the place. Signed-off-by: Herbert Xu --- include/crypto/algapi.h | 5 +++++ include/linux/crypto.h | 24 ++++++++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/include/crypto/algapi.h b/include/crypto/algapi.h index ede622ecefa8..6999e10ea09e 100644 --- a/include/crypto/algapi.h +++ b/include/crypto/algapi.h @@ -272,4 +272,9 @@ static inline bool crypto_tfm_req_chain(struct crypto_tfm *tfm) return tfm->__crt_alg->cra_flags & CRYPTO_ALG_REQ_CHAIN; } +static inline u32 crypto_request_flags(struct crypto_async_request *req) +{ + return req->flags & ~CRYPTO_TFM_REQ_ON_STACK; +} + #endif /* _CRYPTO_ALGAPI_H */ diff --git a/include/linux/crypto.h b/include/linux/crypto.h index dd817f56ff0c..a387f1547ea0 100644 --- a/include/linux/crypto.h +++ b/include/linux/crypto.h @@ -476,5 +476,29 @@ static inline bool crypto_tfm_is_async(struct crypto_tfm *tfm) return tfm->__crt_alg->cra_flags & CRYPTO_ALG_ASYNC; } +static inline bool crypto_req_on_stack(struct crypto_async_request *req) +{ + return req->flags & CRYPTO_TFM_REQ_ON_STACK; +} + +static inline void crypto_request_set_callback( + struct crypto_async_request *req, u32 flags, + crypto_completion_t compl, void *data) +{ + u32 keep = CRYPTO_TFM_REQ_ON_STACK; + + req->complete = compl; + req->data = data; + req->flags &= keep; + req->flags |= flags & ~keep; +} + +static inline void crypto_request_set_tfm(struct crypto_async_request *req, + struct crypto_tfm *tfm) +{ + req->tfm = tfm; + req->flags &= ~CRYPTO_TFM_REQ_ON_STACK; +} + #endif /* _LINUX_CRYPTO_H */ From b04b395f7a29ed28d3cb27a7b39ac67dfb959fa0 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Mon, 7 Apr 2025 18:02:53 +0800 Subject: [PATCH 081/382] crypto: acomp - Use request flag helpers and add acomp_request_flags Use the newly added request flag helpers to manage the request flags. Also add acomp_request_flags which lets bottom-level users to access the request flags without the bits private to the acomp API. Signed-off-by: Herbert Xu --- include/crypto/acompress.h | 27 ++++++++++++++++----------- include/crypto/internal/acompress.h | 6 ++++++ 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/include/crypto/acompress.h b/include/crypto/acompress.h index 080e134df35c..f383a4008854 100644 --- a/include/crypto/acompress.h +++ b/include/crypto/acompress.h @@ -38,6 +38,12 @@ /* Set this bit if destination is a folio. */ #define CRYPTO_ACOMP_REQ_DST_FOLIO 0x00000040 +/* Private flags that should not be touched by the user. */ +#define CRYPTO_ACOMP_REQ_PRIVATE \ + (CRYPTO_ACOMP_REQ_SRC_VIRT | CRYPTO_ACOMP_REQ_SRC_NONDMA | \ + CRYPTO_ACOMP_REQ_DST_VIRT | CRYPTO_ACOMP_REQ_DST_NONDMA | \ + CRYPTO_ACOMP_REQ_SRC_FOLIO | CRYPTO_ACOMP_REQ_DST_FOLIO) + #define CRYPTO_ACOMP_DST_MAX 131072 #define MAX_SYNC_COMP_REQSIZE 0 @@ -201,7 +207,7 @@ static inline unsigned int crypto_acomp_reqsize(struct crypto_acomp *tfm) static inline void acomp_request_set_tfm(struct acomp_req *req, struct crypto_acomp *tfm) { - req->base.tfm = crypto_acomp_tfm(tfm); + crypto_request_set_tfm(&req->base, crypto_acomp_tfm(tfm)); } static inline bool acomp_is_async(struct crypto_acomp *tfm) @@ -298,6 +304,11 @@ static inline void *acomp_request_extra(struct acomp_req *req) return (void *)((char *)req + len); } +static inline bool acomp_req_on_stack(struct acomp_req *req) +{ + return crypto_req_on_stack(&req->base); +} + /** * acomp_request_free() -- zeroize and free asynchronous (de)compression * request as well as the output buffer if allocated @@ -307,7 +318,7 @@ static inline void *acomp_request_extra(struct acomp_req *req) */ static inline void acomp_request_free(struct acomp_req *req) { - if (!req || (req->base.flags & CRYPTO_TFM_REQ_ON_STACK)) + if (!req || acomp_req_on_stack(req)) return; kfree_sensitive(req); } @@ -328,15 +339,9 @@ static inline void acomp_request_set_callback(struct acomp_req *req, crypto_completion_t cmpl, void *data) { - u32 keep = CRYPTO_ACOMP_REQ_SRC_VIRT | CRYPTO_ACOMP_REQ_SRC_NONDMA | - CRYPTO_ACOMP_REQ_DST_VIRT | CRYPTO_ACOMP_REQ_DST_NONDMA | - CRYPTO_ACOMP_REQ_SRC_FOLIO | CRYPTO_ACOMP_REQ_DST_FOLIO | - CRYPTO_TFM_REQ_ON_STACK; - - req->base.complete = cmpl; - req->base.data = data; - req->base.flags &= keep; - req->base.flags |= flgs & ~keep; + flgs &= ~CRYPTO_ACOMP_REQ_PRIVATE; + flgs |= req->base.flags & CRYPTO_ACOMP_REQ_PRIVATE; + crypto_request_set_callback(&req->base, flgs, cmpl, data); } /** diff --git a/include/crypto/internal/acompress.h b/include/crypto/internal/acompress.h index 960cdd1f3a57..5483ca5b46ad 100644 --- a/include/crypto/internal/acompress.h +++ b/include/crypto/internal/acompress.h @@ -229,4 +229,10 @@ static inline bool acomp_walk_more_src(const struct acomp_walk *walk, int cur) { return walk->slen != cur; } + +static inline u32 acomp_request_flags(struct acomp_req *req) +{ + return crypto_request_flags(&req->base) & ~CRYPTO_ACOMP_REQ_PRIVATE; +} + #endif From 05fa2c6e87da31eab150cdaca6697cd1de122ec7 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Mon, 7 Apr 2025 18:02:55 +0800 Subject: [PATCH 082/382] crypto: acomp - Add ACOMP_FBREQ_ON_STACK Add a helper to create an on-stack fallback request from a given request. Use this helper in acomp_do_nondma. Signed-off-by: Herbert Xu --- crypto/acompress.c | 14 +------------- include/crypto/internal/acompress.h | 26 ++++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 13 deletions(-) diff --git a/crypto/acompress.c b/crypto/acompress.c index b682a88781f0..f343b1a4b1d1 100644 --- a/crypto/acompress.c +++ b/crypto/acompress.c @@ -253,21 +253,9 @@ static void acomp_virt_to_sg(struct acomp_req *req) static int acomp_do_nondma(struct acomp_req *req, bool comp) { - u32 keep = CRYPTO_ACOMP_REQ_SRC_VIRT | - CRYPTO_ACOMP_REQ_SRC_NONDMA | - CRYPTO_ACOMP_REQ_DST_VIRT | - CRYPTO_ACOMP_REQ_DST_NONDMA; - ACOMP_REQUEST_ON_STACK(fbreq, crypto_acomp_reqtfm(req)); + ACOMP_FBREQ_ON_STACK(fbreq, req); int err; - acomp_request_set_callback(fbreq, req->base.flags, NULL, NULL); - fbreq->base.flags &= ~keep; - fbreq->base.flags |= req->base.flags & keep; - fbreq->src = req->src; - fbreq->dst = req->dst; - fbreq->slen = req->slen; - fbreq->dlen = req->dlen; - if (comp) err = crypto_acomp_compress(fbreq); else diff --git a/include/crypto/internal/acompress.h b/include/crypto/internal/acompress.h index 5483ca5b46ad..8840fd2c1db5 100644 --- a/include/crypto/internal/acompress.h +++ b/include/crypto/internal/acompress.h @@ -23,6 +23,12 @@ struct acomp_req *name = acomp_request_on_stack_init( \ __##name##_req, (tfm), 0, true) +#define ACOMP_FBREQ_ON_STACK(name, req) \ + char __##name##_req[sizeof(struct acomp_req) + \ + MAX_SYNC_COMP_REQSIZE] CRYPTO_MINALIGN_ATTR; \ + struct acomp_req *name = acomp_fbreq_on_stack_init( \ + __##name##_req, (req)) + /** * struct acomp_alg - asynchronous compression algorithm * @@ -235,4 +241,24 @@ static inline u32 acomp_request_flags(struct acomp_req *req) return crypto_request_flags(&req->base) & ~CRYPTO_ACOMP_REQ_PRIVATE; } +static inline struct acomp_req *acomp_fbreq_on_stack_init( + char *buf, struct acomp_req *old) +{ + struct crypto_acomp *tfm = crypto_acomp_reqtfm(old); + struct acomp_req *req; + + req = acomp_request_on_stack_init(buf, tfm, 0, true); + acomp_request_set_callback(req, acomp_request_flags(old), NULL, NULL); + req->base.flags &= ~CRYPTO_ACOMP_REQ_PRIVATE; + req->base.flags |= old->base.flags & CRYPTO_ACOMP_REQ_PRIVATE; + req->src = old->src; + req->dst = old->dst; + req->slen = old->slen; + req->dlen = old->dlen; + req->soff = old->soff; + req->doff = old->doff; + + return req; +} + #endif From d0a5c9d079decad95a77638c19bc5b3a6a0ffe21 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Mon, 7 Apr 2025 18:02:58 +0800 Subject: [PATCH 083/382] crypto: iaa - Switch to ACOMP_FBREQ_ON_STACK Rather than copying the request by hand, use the ACOMP_FBREQ_ON_STACK helper to do it. Signed-off-by: Herbert Xu --- drivers/crypto/intel/iaa/iaa_crypto_main.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/drivers/crypto/intel/iaa/iaa_crypto_main.c b/drivers/crypto/intel/iaa/iaa_crypto_main.c index aa63df16f20b..b4f15e738cee 100644 --- a/drivers/crypto/intel/iaa/iaa_crypto_main.c +++ b/drivers/crypto/intel/iaa/iaa_crypto_main.c @@ -999,12 +999,9 @@ out: static int deflate_generic_decompress(struct acomp_req *req) { - ACOMP_REQUEST_ON_STACK(fbreq, crypto_acomp_reqtfm(req)); + ACOMP_FBREQ_ON_STACK(fbreq, req); int ret; - acomp_request_set_callback(fbreq, 0, NULL, NULL); - acomp_request_set_params(fbreq, req->src, req->dst, req->slen, - req->dlen); ret = crypto_acomp_decompress(fbreq); req->dlen = fbreq->dlen; From 097c432caaa6d91f87732fe991cb08139e31101a Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Mon, 7 Apr 2025 18:03:00 +0800 Subject: [PATCH 084/382] crypto: acomp - Add ACOMP_REQUEST_CLONE Add a new helper ACOMP_REQUEST_CLONE that will transform a stack request into a dynamically allocated one if possible, and otherwise switch it over to the sycnrhonous fallback transform. The intended usage is: ACOMP_STACK_ON_REQUEST(req, tfm); ... err = crypto_acomp_compress(req); /* The request cannot complete synchronously. */ if (err == -EAGAIN) { /* This will not fail. */ req = ACOMP_REQUEST_CLONE(req, gfp); /* Redo operation. */ err = crypto_acomp_compress(req); } Signed-off-by: Herbert Xu --- crypto/acompress.c | 23 ++++++++++++++++++++++ include/crypto/acompress.h | 30 +++++++++++++++++++++++++---- include/crypto/internal/acompress.h | 11 +++-------- 3 files changed, 52 insertions(+), 12 deletions(-) diff --git a/crypto/acompress.c b/crypto/acompress.c index f343b1a4b1d1..530b9bfd03a5 100644 --- a/crypto/acompress.c +++ b/crypto/acompress.c @@ -316,6 +316,8 @@ int crypto_acomp_compress(struct acomp_req *req) { struct crypto_acomp *tfm = crypto_acomp_reqtfm(req); + if (acomp_req_on_stack(req) && acomp_is_async(tfm)) + return -EAGAIN; if (crypto_acomp_req_chain(tfm) || acomp_request_issg(req)) crypto_acomp_reqtfm(req)->compress(req); return acomp_do_req_chain(req, true); @@ -326,6 +328,8 @@ int crypto_acomp_decompress(struct acomp_req *req) { struct crypto_acomp *tfm = crypto_acomp_reqtfm(req); + if (acomp_req_on_stack(req) && acomp_is_async(tfm)) + return -EAGAIN; if (crypto_acomp_req_chain(tfm) || acomp_request_issg(req)) crypto_acomp_reqtfm(req)->decompress(req); return acomp_do_req_chain(req, false); @@ -603,5 +607,24 @@ int acomp_walk_virt(struct acomp_walk *__restrict walk, } EXPORT_SYMBOL_GPL(acomp_walk_virt); +struct acomp_req *acomp_request_clone(struct acomp_req *req, + size_t total, gfp_t gfp) +{ + struct crypto_acomp *tfm = crypto_acomp_reqtfm(req); + struct acomp_req *nreq; + + nreq = kmalloc(total, gfp); + if (!nreq) { + acomp_request_set_tfm(req, tfm->fb); + req->base.flags = CRYPTO_TFM_REQ_ON_STACK; + return req; + } + + memcpy(nreq, req, total); + acomp_request_set_tfm(req, tfm); + return req; +} +EXPORT_SYMBOL_GPL(acomp_request_clone); + MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Asynchronous compression type"); diff --git a/include/crypto/acompress.h b/include/crypto/acompress.h index f383a4008854..93cee67c27c0 100644 --- a/include/crypto/acompress.h +++ b/include/crypto/acompress.h @@ -49,10 +49,19 @@ #define MAX_SYNC_COMP_REQSIZE 0 #define ACOMP_REQUEST_ALLOC(name, tfm, gfp) \ + char __##name##_req[sizeof(struct acomp_req) + \ + MAX_SYNC_COMP_REQSIZE] CRYPTO_MINALIGN_ATTR; \ + struct acomp_req *name = acomp_request_alloc_init( \ + __##name##_req, (tfm), (gfp)) + +#define ACOMP_REQUEST_ON_STACK(name, tfm) \ char __##name##_req[sizeof(struct acomp_req) + \ MAX_SYNC_COMP_REQSIZE] CRYPTO_MINALIGN_ATTR; \ struct acomp_req *name = acomp_request_on_stack_init( \ - __##name##_req, (tfm), (gfp), false) + __##name##_req, (tfm)) + +#define ACOMP_REQUEST_CLONE(name, gfp) \ + acomp_request_clone(name, sizeof(__##name##_req), gfp) struct acomp_req; struct folio; @@ -571,12 +580,12 @@ int crypto_acomp_compress(struct acomp_req *req); */ int crypto_acomp_decompress(struct acomp_req *req); -static inline struct acomp_req *acomp_request_on_stack_init( - char *buf, struct crypto_acomp *tfm, gfp_t gfp, bool stackonly) +static inline struct acomp_req *acomp_request_alloc_init( + char *buf, struct crypto_acomp *tfm, gfp_t gfp) { struct acomp_req *req; - if (!stackonly && (req = acomp_request_alloc(tfm, gfp))) + if ((req = acomp_request_alloc(tfm, gfp))) return req; req = (void *)buf; @@ -586,4 +595,17 @@ static inline struct acomp_req *acomp_request_on_stack_init( return req; } +static inline struct acomp_req *acomp_request_on_stack_init( + char *buf, struct crypto_acomp *tfm) +{ + struct acomp_req *req = (void *)buf; + + acomp_request_set_tfm(req, tfm); + req->base.flags = CRYPTO_TFM_REQ_ON_STACK; + return req; +} + +struct acomp_req *acomp_request_clone(struct acomp_req *req, + size_t total, gfp_t gfp); + #endif diff --git a/include/crypto/internal/acompress.h b/include/crypto/internal/acompress.h index 8840fd2c1db5..b51d66633935 100644 --- a/include/crypto/internal/acompress.h +++ b/include/crypto/internal/acompress.h @@ -17,12 +17,6 @@ #include #include -#define ACOMP_REQUEST_ON_STACK(name, tfm) \ - char __##name##_req[sizeof(struct acomp_req) + \ - MAX_SYNC_COMP_REQSIZE] CRYPTO_MINALIGN_ATTR; \ - struct acomp_req *name = acomp_request_on_stack_init( \ - __##name##_req, (tfm), 0, true) - #define ACOMP_FBREQ_ON_STACK(name, req) \ char __##name##_req[sizeof(struct acomp_req) + \ MAX_SYNC_COMP_REQSIZE] CRYPTO_MINALIGN_ATTR; \ @@ -245,9 +239,10 @@ static inline struct acomp_req *acomp_fbreq_on_stack_init( char *buf, struct acomp_req *old) { struct crypto_acomp *tfm = crypto_acomp_reqtfm(old); - struct acomp_req *req; + struct acomp_req *req = (void *)buf; - req = acomp_request_on_stack_init(buf, tfm, 0, true); + acomp_request_set_tfm(req, tfm->fb); + req->base.flags = CRYPTO_TFM_REQ_ON_STACK; acomp_request_set_callback(req, acomp_request_flags(old), NULL, NULL); req->base.flags &= ~CRYPTO_ACOMP_REQ_PRIVATE; req->base.flags |= old->base.flags & CRYPTO_ACOMP_REQ_PRIVATE; From e87e95d8dde623f199afbb364f89e6b20ffa61ec Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Mon, 7 Apr 2025 18:03:02 +0800 Subject: [PATCH 085/382] ubifs: Use ACOMP_REQUEST_CLONE Switch to the new acomp API where stacks requests are used by default and a dynamic request is only allocted when necessary. Signed-off-by: Herbert Xu --- fs/ubifs/compress.c | 251 ++++++++++++++++++++------------------------ 1 file changed, 116 insertions(+), 135 deletions(-) diff --git a/fs/ubifs/compress.c b/fs/ubifs/compress.c index ea6f06adcd43..059a02691edd 100644 --- a/fs/ubifs/compress.c +++ b/fs/ubifs/compress.c @@ -19,6 +19,11 @@ #include #include "ubifs.h" +union ubifs_in_ptr { + const void *buf; + struct folio *folio; +}; + /* Fake description object for the "none" compressor */ static struct ubifs_compressor none_compr = { .compr_type = UBIFS_COMPR_NONE, @@ -68,28 +73,61 @@ static struct ubifs_compressor zstd_compr = { /* All UBIFS compressors */ struct ubifs_compressor *ubifs_compressors[UBIFS_COMPR_TYPES_CNT]; -static int ubifs_compress_req(const struct ubifs_info *c, - struct acomp_req *req, - void *out_buf, int *out_len, - const char *compr_name) +static void ubifs_compress_common(int *compr_type, union ubifs_in_ptr in_ptr, + size_t in_offset, int in_len, bool in_folio, + void *out_buf, int *out_len) { - struct crypto_wait wait; - int in_len = req->slen; + struct ubifs_compressor *compr = ubifs_compressors[*compr_type]; int dlen = *out_len; int err; + if (*compr_type == UBIFS_COMPR_NONE) + goto no_compr; + + /* If the input data is small, do not even try to compress it */ + if (in_len < UBIFS_MIN_COMPR_LEN) + goto no_compr; + dlen = min(dlen, in_len - UBIFS_MIN_COMPRESS_DIFF); - crypto_init_wait(&wait); - acomp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, - crypto_req_done, &wait); - acomp_request_set_dst_dma(req, out_buf, dlen); - err = crypto_acomp_compress(req); - err = crypto_wait_req(err, &wait); - *out_len = req->dlen; - acomp_request_free(req); + do { + ACOMP_REQUEST_ON_STACK(req, compr->cc); + DECLARE_CRYPTO_WAIT(wait); - return err; + acomp_request_set_callback(req, 0, NULL, NULL); + if (in_folio) + acomp_request_set_src_folio(req, in_ptr.folio, + in_offset, in_len); + else + acomp_request_set_src_dma(req, in_ptr.buf, in_len); + acomp_request_set_dst_dma(req, out_buf, dlen); + err = crypto_acomp_compress(req); + dlen = req->dlen; + if (err != -EAGAIN) + break; + + req = ACOMP_REQUEST_CLONE(req, GFP_NOFS | __GFP_NOWARN); + acomp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, + crypto_req_done, &wait); + err = crypto_acomp_compress(req); + err = crypto_wait_req(err, &wait); + dlen = req->dlen; + acomp_request_free(req); + } while (0); + + *out_len = dlen; + if (err) + goto no_compr; + + return; + +no_compr: + if (in_folio) + memcpy_from_folio(out_buf, in_ptr.folio, in_offset, in_len); + else + memcpy(out_buf, in_ptr.buf, in_len); + *out_len = in_len; + *compr_type = UBIFS_COMPR_NONE; } /** @@ -114,32 +152,10 @@ static int ubifs_compress_req(const struct ubifs_info *c, void ubifs_compress(const struct ubifs_info *c, const void *in_buf, int in_len, void *out_buf, int *out_len, int *compr_type) { - int err; - struct ubifs_compressor *compr = ubifs_compressors[*compr_type]; + union ubifs_in_ptr in_ptr = { .buf = in_buf }; - if (*compr_type == UBIFS_COMPR_NONE) - goto no_compr; - - /* If the input data is small, do not even try to compress it */ - if (in_len < UBIFS_MIN_COMPR_LEN) - goto no_compr; - - { - ACOMP_REQUEST_ALLOC(req, compr->cc, GFP_NOFS | __GFP_NOWARN); - - acomp_request_set_src_dma(req, in_buf, in_len); - err = ubifs_compress_req(c, req, out_buf, out_len, compr->name); - } - - if (err) - goto no_compr; - - return; - -no_compr: - memcpy(out_buf, in_buf, in_len); - *out_len = in_len; - *compr_type = UBIFS_COMPR_NONE; + ubifs_compress_common(compr_type, in_ptr, 0, in_len, false, + out_buf, out_len); } /** @@ -166,55 +182,71 @@ void ubifs_compress_folio(const struct ubifs_info *c, struct folio *in_folio, size_t in_offset, int in_len, void *out_buf, int *out_len, int *compr_type) { - int err; - struct ubifs_compressor *compr = ubifs_compressors[*compr_type]; + union ubifs_in_ptr in_ptr = { .folio = in_folio }; - if (*compr_type == UBIFS_COMPR_NONE) - goto no_compr; - - /* If the input data is small, do not even try to compress it */ - if (in_len < UBIFS_MIN_COMPR_LEN) - goto no_compr; - - { - ACOMP_REQUEST_ALLOC(req, compr->cc, GFP_NOFS | __GFP_NOWARN); - - acomp_request_set_src_folio(req, in_folio, in_offset, in_len); - err = ubifs_compress_req(c, req, out_buf, out_len, compr->name); - } - - if (err) - goto no_compr; - - return; - -no_compr: - memcpy_from_folio(out_buf, in_folio, in_offset, in_len); - *out_len = in_len; - *compr_type = UBIFS_COMPR_NONE; + ubifs_compress_common(compr_type, in_ptr, in_offset, in_len, true, + out_buf, out_len); } -static int ubifs_decompress_req(const struct ubifs_info *c, - struct acomp_req *req, - const void *in_buf, int in_len, int *out_len, - const char *compr_name) +static int ubifs_decompress_common(const struct ubifs_info *c, + const void *in_buf, int in_len, + void *out_ptr, size_t out_offset, + int *out_len, bool out_folio, + int compr_type) { - struct crypto_wait wait; + struct ubifs_compressor *compr; + int dlen = *out_len; int err; - crypto_init_wait(&wait); - acomp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, - crypto_req_done, &wait); - acomp_request_set_src_dma(req, in_buf, in_len); - err = crypto_acomp_decompress(req); - err = crypto_wait_req(err, &wait); - *out_len = req->dlen; + if (unlikely(compr_type < 0 || compr_type >= UBIFS_COMPR_TYPES_CNT)) { + ubifs_err(c, "invalid compression type %d", compr_type); + return -EINVAL; + } + compr = ubifs_compressors[compr_type]; + + if (unlikely(!compr->capi_name)) { + ubifs_err(c, "%s compression is not compiled in", compr->name); + return -EINVAL; + } + + if (compr_type == UBIFS_COMPR_NONE) { + if (out_folio) + memcpy_to_folio(out_ptr, out_offset, in_buf, in_len); + else + memcpy(out_ptr, in_buf, in_len); + *out_len = in_len; + return 0; + } + + do { + ACOMP_REQUEST_ON_STACK(req, compr->cc); + DECLARE_CRYPTO_WAIT(wait); + + acomp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, + crypto_req_done, &wait); + acomp_request_set_src_dma(req, in_buf, in_len); + if (out_folio) + acomp_request_set_dst_folio(req, out_ptr, out_offset, + dlen); + else + acomp_request_set_dst_dma(req, out_ptr, dlen); + err = crypto_acomp_decompress(req); + dlen = req->dlen; + if (err != -EAGAIN) + break; + + req = ACOMP_REQUEST_CLONE(req, GFP_NOFS | __GFP_NOWARN); + err = crypto_acomp_decompress(req); + err = crypto_wait_req(err, &wait); + dlen = req->dlen; + acomp_request_free(req); + } while (0); + + *out_len = dlen; if (err) ubifs_err(c, "cannot decompress %d bytes, compressor %s, error %d", - in_len, compr_name, err); - - acomp_request_free(req); + in_len, compr->name, err); return err; } @@ -235,33 +267,8 @@ static int ubifs_decompress_req(const struct ubifs_info *c, int ubifs_decompress(const struct ubifs_info *c, const void *in_buf, int in_len, void *out_buf, int *out_len, int compr_type) { - struct ubifs_compressor *compr; - - if (unlikely(compr_type < 0 || compr_type >= UBIFS_COMPR_TYPES_CNT)) { - ubifs_err(c, "invalid compression type %d", compr_type); - return -EINVAL; - } - - compr = ubifs_compressors[compr_type]; - - if (unlikely(!compr->capi_name)) { - ubifs_err(c, "%s compression is not compiled in", compr->name); - return -EINVAL; - } - - if (compr_type == UBIFS_COMPR_NONE) { - memcpy(out_buf, in_buf, in_len); - *out_len = in_len; - return 0; - } - - { - ACOMP_REQUEST_ALLOC(req, compr->cc, GFP_NOFS | __GFP_NOWARN); - - acomp_request_set_dst_dma(req, out_buf, *out_len); - return ubifs_decompress_req(c, req, in_buf, in_len, out_len, - compr->name); - } + return ubifs_decompress_common(c, in_buf, in_len, out_buf, 0, out_len, + false, compr_type); } /** @@ -283,34 +290,8 @@ int ubifs_decompress_folio(const struct ubifs_info *c, const void *in_buf, int in_len, struct folio *out_folio, size_t out_offset, int *out_len, int compr_type) { - struct ubifs_compressor *compr; - - if (unlikely(compr_type < 0 || compr_type >= UBIFS_COMPR_TYPES_CNT)) { - ubifs_err(c, "invalid compression type %d", compr_type); - return -EINVAL; - } - - compr = ubifs_compressors[compr_type]; - - if (unlikely(!compr->capi_name)) { - ubifs_err(c, "%s compression is not compiled in", compr->name); - return -EINVAL; - } - - if (compr_type == UBIFS_COMPR_NONE) { - memcpy_to_folio(out_folio, out_offset, in_buf, in_len); - *out_len = in_len; - return 0; - } - - { - ACOMP_REQUEST_ALLOC(req, compr->cc, GFP_NOFS | __GFP_NOWARN); - - acomp_request_set_dst_folio(req, out_folio, out_offset, - *out_len); - return ubifs_decompress_req(c, req, in_buf, in_len, out_len, - compr->name); - } + return ubifs_decompress_common(c, in_buf, in_len, out_folio, + out_offset, out_len, true, compr_type); } /** From 018cba2ecc3bb97d3cb24470d2e1245cd90d98c3 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Mon, 7 Apr 2025 18:03:05 +0800 Subject: [PATCH 086/382] crypto: acomp - Remove ACOMP_REQUEST_ALLOC Remove ACOMP_REQUEST_ALLOC in favour of ACOMP_REQUEST_ON_STACK with ACOMP_REQUEST_CLONE. Signed-off-by: Herbert Xu --- include/crypto/acompress.h | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/include/crypto/acompress.h b/include/crypto/acompress.h index 93cee67c27c0..96ec0090a855 100644 --- a/include/crypto/acompress.h +++ b/include/crypto/acompress.h @@ -48,12 +48,6 @@ #define MAX_SYNC_COMP_REQSIZE 0 -#define ACOMP_REQUEST_ALLOC(name, tfm, gfp) \ - char __##name##_req[sizeof(struct acomp_req) + \ - MAX_SYNC_COMP_REQSIZE] CRYPTO_MINALIGN_ATTR; \ - struct acomp_req *name = acomp_request_alloc_init( \ - __##name##_req, (tfm), (gfp)) - #define ACOMP_REQUEST_ON_STACK(name, tfm) \ char __##name##_req[sizeof(struct acomp_req) + \ MAX_SYNC_COMP_REQSIZE] CRYPTO_MINALIGN_ATTR; \ @@ -580,21 +574,6 @@ int crypto_acomp_compress(struct acomp_req *req); */ int crypto_acomp_decompress(struct acomp_req *req); -static inline struct acomp_req *acomp_request_alloc_init( - char *buf, struct crypto_acomp *tfm, gfp_t gfp) -{ - struct acomp_req *req; - - if ((req = acomp_request_alloc(tfm, gfp))) - return req; - - req = (void *)buf; - acomp_request_set_tfm(req, tfm->fb); - req->base.flags = CRYPTO_TFM_REQ_ON_STACK; - - return req; -} - static inline struct acomp_req *acomp_request_on_stack_init( char *buf, struct crypto_acomp *tfm) { From 5f3437e9c89eec7bbf0ee5f582894d41f57528bc Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Mon, 7 Apr 2025 18:05:27 +0800 Subject: [PATCH 087/382] crypto: acomp - Simplify folio handling Rather than storing the folio as is and handling it later, convert it to a scatterlist right away. Signed-off-by: Herbert Xu --- crypto/acompress.c | 42 ++------------------------- crypto/scompress.c | 10 ++----- include/crypto/acompress.h | 45 +++++------------------------ include/crypto/internal/acompress.h | 16 +--------- 4 files changed, 12 insertions(+), 101 deletions(-) diff --git a/crypto/acompress.c b/crypto/acompress.c index 530b9bfd03a5..674325ecefbc 100644 --- a/crypto/acompress.c +++ b/crypto/acompress.c @@ -33,9 +33,7 @@ struct crypto_scomp; enum { ACOMP_WALK_SLEEP = 1 << 0, ACOMP_WALK_SRC_LINEAR = 1 << 1, - ACOMP_WALK_SRC_FOLIO = 1 << 2, - ACOMP_WALK_DST_LINEAR = 1 << 3, - ACOMP_WALK_DST_FOLIO = 1 << 4, + ACOMP_WALK_DST_LINEAR = 1 << 2, }; static const struct crypto_type crypto_acomp_type; @@ -195,12 +193,8 @@ static void acomp_reqchain_virt(struct acomp_req *req) if (state->flags & CRYPTO_ACOMP_REQ_SRC_VIRT) acomp_request_set_src_dma(req, state->src, slen); - else if (state->flags & CRYPTO_ACOMP_REQ_SRC_FOLIO) - acomp_request_set_src_folio(req, state->sfolio, req->soff, slen); if (state->flags & CRYPTO_ACOMP_REQ_DST_VIRT) acomp_request_set_dst_dma(req, state->dst, dlen); - else if (state->flags & CRYPTO_ACOMP_REQ_DST_FOLIO) - acomp_request_set_dst_folio(req, state->dfolio, req->doff, dlen); } static void acomp_virt_to_sg(struct acomp_req *req) @@ -208,9 +202,7 @@ static void acomp_virt_to_sg(struct acomp_req *req) struct acomp_req_chain *state = &req->chain; state->flags = req->base.flags & (CRYPTO_ACOMP_REQ_SRC_VIRT | - CRYPTO_ACOMP_REQ_DST_VIRT | - CRYPTO_ACOMP_REQ_SRC_FOLIO | - CRYPTO_ACOMP_REQ_DST_FOLIO); + CRYPTO_ACOMP_REQ_DST_VIRT); if (acomp_request_src_isvirt(req)) { unsigned int slen = req->slen; @@ -219,16 +211,6 @@ static void acomp_virt_to_sg(struct acomp_req *req) state->src = svirt; sg_init_one(&state->ssg, svirt, slen); acomp_request_set_src_sg(req, &state->ssg, slen); - } else if (acomp_request_src_isfolio(req)) { - struct folio *folio = req->sfolio; - unsigned int slen = req->slen; - size_t off = req->soff; - - state->sfolio = folio; - sg_init_table(&state->ssg, 1); - sg_set_page(&state->ssg, folio_page(folio, off / PAGE_SIZE), - slen, off % PAGE_SIZE); - acomp_request_set_src_sg(req, &state->ssg, slen); } if (acomp_request_dst_isvirt(req)) { @@ -238,16 +220,6 @@ static void acomp_virt_to_sg(struct acomp_req *req) state->dst = dvirt; sg_init_one(&state->dsg, dvirt, dlen); acomp_request_set_dst_sg(req, &state->dsg, dlen); - } else if (acomp_request_dst_isfolio(req)) { - struct folio *folio = req->dfolio; - unsigned int dlen = req->dlen; - size_t off = req->doff; - - state->dfolio = folio; - sg_init_table(&state->dsg, 1); - sg_set_page(&state->dsg, folio_page(folio, off / PAGE_SIZE), - dlen, off % PAGE_SIZE); - acomp_request_set_src_sg(req, &state->dsg, dlen); } } @@ -579,18 +551,8 @@ int acomp_walk_virt(struct acomp_walk *__restrict walk, walk->flags |= ACOMP_WALK_SLEEP; if ((req->base.flags & CRYPTO_ACOMP_REQ_SRC_VIRT)) walk->flags |= ACOMP_WALK_SRC_LINEAR; - else if ((req->base.flags & CRYPTO_ACOMP_REQ_SRC_FOLIO)) { - src = &req->chain.ssg; - sg_init_table(src, 1); - sg_set_folio(src, req->sfolio, walk->slen, req->soff); - } if ((req->base.flags & CRYPTO_ACOMP_REQ_DST_VIRT)) walk->flags |= ACOMP_WALK_DST_LINEAR; - else if ((req->base.flags & CRYPTO_ACOMP_REQ_DST_FOLIO)) { - dst = &req->chain.dsg; - sg_init_table(dst, 1); - sg_set_folio(dst, req->dfolio, walk->dlen, req->doff); - } if ((walk->flags & ACOMP_WALK_SRC_LINEAR)) { walk->in.sg = (void *)req->svirt; diff --git a/crypto/scompress.c b/crypto/scompress.c index 7ade3f2fee7e..c330b81bc5a6 100644 --- a/crypto/scompress.c +++ b/crypto/scompress.c @@ -193,10 +193,7 @@ static int scomp_acomp_comp_decomp(struct acomp_req *req, int dir) if (dst_isvirt) dst = req->dvirt; else { - if (acomp_request_dst_isfolio(req)) { - dpage = folio_page(req->dfolio, 0); - doff = req->doff; - } else if (dlen <= req->dst->length) { + if (dlen <= req->dst->length) { dpage = sg_page(req->dst); doff = req->dst->offset; } else @@ -218,10 +215,7 @@ static int scomp_acomp_comp_decomp(struct acomp_req *req, int dir) else { src = NULL; do { - if (acomp_request_src_isfolio(req)) { - spage = folio_page(req->sfolio, 0); - soff = req->soff; - } else if (slen <= req->src->length) { + if (slen <= req->src->length) { spage = sg_page(req->src); soff = req->src->offset; } else diff --git a/include/crypto/acompress.h b/include/crypto/acompress.h index 96ec0090a855..1b30290d6380 100644 --- a/include/crypto/acompress.h +++ b/include/crypto/acompress.h @@ -32,17 +32,10 @@ /* Set this bit for if virtual address destination cannot be used for DMA. */ #define CRYPTO_ACOMP_REQ_DST_NONDMA 0x00000010 -/* Set this bit if source is a folio. */ -#define CRYPTO_ACOMP_REQ_SRC_FOLIO 0x00000020 - -/* Set this bit if destination is a folio. */ -#define CRYPTO_ACOMP_REQ_DST_FOLIO 0x00000040 - /* Private flags that should not be touched by the user. */ #define CRYPTO_ACOMP_REQ_PRIVATE \ (CRYPTO_ACOMP_REQ_SRC_VIRT | CRYPTO_ACOMP_REQ_SRC_NONDMA | \ - CRYPTO_ACOMP_REQ_DST_VIRT | CRYPTO_ACOMP_REQ_DST_NONDMA | \ - CRYPTO_ACOMP_REQ_SRC_FOLIO | CRYPTO_ACOMP_REQ_DST_FOLIO) + CRYPTO_ACOMP_REQ_DST_VIRT | CRYPTO_ACOMP_REQ_DST_NONDMA) #define CRYPTO_ACOMP_DST_MAX 131072 @@ -84,10 +77,6 @@ struct acomp_req_chain { * @dst: Destination scatterlist * @svirt: Source virtual address * @dvirt: Destination virtual address - * @sfolio: Source folio - * @soff: Source folio offset - * @dfolio: Destination folio - * @doff: Destination folio offset * @slen: Size of the input buffer * @dlen: Size of the output buffer and number of bytes produced * @chain: Private API code data, do not use @@ -98,15 +87,11 @@ struct acomp_req { union { struct scatterlist *src; const u8 *svirt; - struct folio *sfolio; }; union { struct scatterlist *dst; u8 *dvirt; - struct folio *dfolio; }; - size_t soff; - size_t doff; unsigned int slen; unsigned int dlen; @@ -373,8 +358,6 @@ static inline void acomp_request_set_params(struct acomp_req *req, req->base.flags &= ~(CRYPTO_ACOMP_REQ_SRC_VIRT | CRYPTO_ACOMP_REQ_SRC_NONDMA | - CRYPTO_ACOMP_REQ_SRC_FOLIO | - CRYPTO_ACOMP_REQ_DST_FOLIO | CRYPTO_ACOMP_REQ_DST_VIRT | CRYPTO_ACOMP_REQ_DST_NONDMA); } @@ -397,7 +380,6 @@ static inline void acomp_request_set_src_sg(struct acomp_req *req, req->base.flags &= ~CRYPTO_ACOMP_REQ_SRC_NONDMA; req->base.flags &= ~CRYPTO_ACOMP_REQ_SRC_VIRT; - req->base.flags &= ~CRYPTO_ACOMP_REQ_SRC_FOLIO; } /** @@ -417,7 +399,6 @@ static inline void acomp_request_set_src_dma(struct acomp_req *req, req->slen = slen; req->base.flags &= ~CRYPTO_ACOMP_REQ_SRC_NONDMA; - req->base.flags &= ~CRYPTO_ACOMP_REQ_SRC_FOLIO; req->base.flags |= CRYPTO_ACOMP_REQ_SRC_VIRT; } @@ -438,7 +419,6 @@ static inline void acomp_request_set_src_nondma(struct acomp_req *req, req->svirt = src; req->slen = slen; - req->base.flags &= ~CRYPTO_ACOMP_REQ_SRC_FOLIO; req->base.flags |= CRYPTO_ACOMP_REQ_SRC_NONDMA; req->base.flags |= CRYPTO_ACOMP_REQ_SRC_VIRT; } @@ -457,13 +437,9 @@ static inline void acomp_request_set_src_folio(struct acomp_req *req, struct folio *folio, size_t off, unsigned int len) { - req->sfolio = folio; - req->soff = off; - req->slen = len; - - req->base.flags &= ~CRYPTO_ACOMP_REQ_SRC_NONDMA; - req->base.flags &= ~CRYPTO_ACOMP_REQ_SRC_VIRT; - req->base.flags |= CRYPTO_ACOMP_REQ_SRC_FOLIO; + sg_init_table(&req->chain.ssg, 1); + sg_set_folio(&req->chain.ssg, folio, len, off); + acomp_request_set_src_sg(req, &req->chain.ssg, len); } /** @@ -484,7 +460,6 @@ static inline void acomp_request_set_dst_sg(struct acomp_req *req, req->base.flags &= ~CRYPTO_ACOMP_REQ_DST_NONDMA; req->base.flags &= ~CRYPTO_ACOMP_REQ_DST_VIRT; - req->base.flags &= ~CRYPTO_ACOMP_REQ_DST_FOLIO; } /** @@ -504,7 +479,6 @@ static inline void acomp_request_set_dst_dma(struct acomp_req *req, req->dlen = dlen; req->base.flags &= ~CRYPTO_ACOMP_REQ_DST_NONDMA; - req->base.flags &= ~CRYPTO_ACOMP_REQ_DST_FOLIO; req->base.flags |= CRYPTO_ACOMP_REQ_DST_VIRT; } @@ -524,7 +498,6 @@ static inline void acomp_request_set_dst_nondma(struct acomp_req *req, req->dvirt = dst; req->dlen = dlen; - req->base.flags &= ~CRYPTO_ACOMP_REQ_DST_FOLIO; req->base.flags |= CRYPTO_ACOMP_REQ_DST_NONDMA; req->base.flags |= CRYPTO_ACOMP_REQ_DST_VIRT; } @@ -543,13 +516,9 @@ static inline void acomp_request_set_dst_folio(struct acomp_req *req, struct folio *folio, size_t off, unsigned int len) { - req->dfolio = folio; - req->doff = off; - req->dlen = len; - - req->base.flags &= ~CRYPTO_ACOMP_REQ_DST_NONDMA; - req->base.flags &= ~CRYPTO_ACOMP_REQ_DST_VIRT; - req->base.flags |= CRYPTO_ACOMP_REQ_DST_FOLIO; + sg_init_table(&req->chain.dsg, 1); + sg_set_folio(&req->chain.dsg, folio, len, off); + acomp_request_set_dst_sg(req, &req->chain.dsg, len); } /** diff --git a/include/crypto/internal/acompress.h b/include/crypto/internal/acompress.h index b51d66633935..d6d53c7696fd 100644 --- a/include/crypto/internal/acompress.h +++ b/include/crypto/internal/acompress.h @@ -154,9 +154,7 @@ void crypto_unregister_acomps(struct acomp_alg *algs, int count); static inline bool acomp_request_issg(struct acomp_req *req) { return !(req->base.flags & (CRYPTO_ACOMP_REQ_SRC_VIRT | - CRYPTO_ACOMP_REQ_DST_VIRT | - CRYPTO_ACOMP_REQ_SRC_FOLIO | - CRYPTO_ACOMP_REQ_DST_FOLIO)); + CRYPTO_ACOMP_REQ_DST_VIRT)); } static inline bool acomp_request_src_isvirt(struct acomp_req *req) @@ -191,16 +189,6 @@ static inline bool acomp_request_isnondma(struct acomp_req *req) CRYPTO_ACOMP_REQ_DST_NONDMA); } -static inline bool acomp_request_src_isfolio(struct acomp_req *req) -{ - return req->base.flags & CRYPTO_ACOMP_REQ_SRC_FOLIO; -} - -static inline bool acomp_request_dst_isfolio(struct acomp_req *req) -{ - return req->base.flags & CRYPTO_ACOMP_REQ_DST_FOLIO; -} - static inline bool crypto_acomp_req_chain(struct crypto_acomp *tfm) { return crypto_tfm_req_chain(&tfm->base); @@ -250,8 +238,6 @@ static inline struct acomp_req *acomp_fbreq_on_stack_init( req->dst = old->dst; req->slen = old->slen; req->dlen = old->dlen; - req->soff = old->soff; - req->doff = old->doff; return req; } From ab0f46cefde5d5382e6ae039ac59df34bdd502d2 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Mon, 7 Apr 2025 18:11:26 +0800 Subject: [PATCH 088/382] crypto: nx - Add missing header inclusions The gutting of crypto/ctr.h uncovered missing header inclusions. Add them. Signed-off-by: Herbert Xu --- drivers/crypto/nx/nx-aes-cbc.c | 8 +++++--- drivers/crypto/nx/nx-aes-ctr.c | 8 +++++--- drivers/crypto/nx/nx-aes-ecb.c | 8 +++++--- drivers/crypto/nx/nx.c | 4 ++-- drivers/crypto/nx/nx.h | 5 ++++- 5 files changed, 21 insertions(+), 12 deletions(-) diff --git a/drivers/crypto/nx/nx-aes-cbc.c b/drivers/crypto/nx/nx-aes-cbc.c index 0e440f704a8f..35fa5bad1d9f 100644 --- a/drivers/crypto/nx/nx-aes-cbc.c +++ b/drivers/crypto/nx/nx-aes-cbc.c @@ -8,10 +8,12 @@ */ #include -#include +#include +#include +#include #include -#include -#include +#include +#include #include #include "nx_csbcpb.h" diff --git a/drivers/crypto/nx/nx-aes-ctr.c b/drivers/crypto/nx/nx-aes-ctr.c index dfa3ad1a12f2..709b3ee74657 100644 --- a/drivers/crypto/nx/nx-aes-ctr.c +++ b/drivers/crypto/nx/nx-aes-ctr.c @@ -9,10 +9,12 @@ #include #include -#include +#include +#include +#include #include -#include -#include +#include +#include #include #include "nx_csbcpb.h" diff --git a/drivers/crypto/nx/nx-aes-ecb.c b/drivers/crypto/nx/nx-aes-ecb.c index 502a565074e9..4039cf3b22d4 100644 --- a/drivers/crypto/nx/nx-aes-ecb.c +++ b/drivers/crypto/nx/nx-aes-ecb.c @@ -8,10 +8,12 @@ */ #include -#include +#include +#include +#include #include -#include -#include +#include +#include #include #include "nx_csbcpb.h" diff --git a/drivers/crypto/nx/nx.c b/drivers/crypto/nx/nx.c index a3b979193d9b..4e4a371ba390 100644 --- a/drivers/crypto/nx/nx.c +++ b/drivers/crypto/nx/nx.c @@ -7,11 +7,11 @@ * Author: Kent Yoder */ +#include #include #include -#include +#include #include -#include #include #include #include diff --git a/drivers/crypto/nx/nx.h b/drivers/crypto/nx/nx.h index e1b4b6927bec..b1f6634a1644 100644 --- a/drivers/crypto/nx/nx.h +++ b/drivers/crypto/nx/nx.h @@ -4,6 +4,9 @@ #define __NX_H__ #include +#include +#include +#include #define NX_NAME "nx-crypto" #define NX_STRING "IBM Power7+ Nest Accelerator Crypto Driver" @@ -139,7 +142,7 @@ struct nx_crypto_ctx { } priv; }; -struct crypto_aead; +struct scatterlist; /* prototypes */ int nx_crypto_ctx_aes_ccm_init(struct crypto_aead *tfm); From 6611dcd503e464c84e2eb19a4878a5bc0bb10e21 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Mon, 7 Apr 2025 18:11:28 +0800 Subject: [PATCH 089/382] crypto: ccp - Add missing header inclusions The gutting of crypto/ctr.h uncovered missing header inclusions. Add them. Signed-off-by: Herbert Xu --- drivers/crypto/ccp/ccp-crypto-aes.c | 15 ++++++++------- drivers/crypto/ccp/ccp-crypto-des3.c | 15 ++++++++------- drivers/crypto/ccp/ccp-crypto-main.c | 13 ++++++++----- 3 files changed, 24 insertions(+), 19 deletions(-) diff --git a/drivers/crypto/ccp/ccp-crypto-aes.c b/drivers/crypto/ccp/ccp-crypto-aes.c index d11daaf47f06..685d42ec7ade 100644 --- a/drivers/crypto/ccp/ccp-crypto-aes.c +++ b/drivers/crypto/ccp/ccp-crypto-aes.c @@ -7,15 +7,16 @@ * Author: Tom Lendacky */ -#include -#include -#include -#include -#include -#include #include #include -#include +#include +#include +#include +#include +#include +#include +#include +#include #include "ccp-crypto.h" diff --git a/drivers/crypto/ccp/ccp-crypto-des3.c b/drivers/crypto/ccp/ccp-crypto-des3.c index afae30adb703..91b1189c47de 100644 --- a/drivers/crypto/ccp/ccp-crypto-des3.c +++ b/drivers/crypto/ccp/ccp-crypto-des3.c @@ -7,14 +7,15 @@ * Author: Gary R Hook */ -#include -#include -#include -#include -#include -#include -#include #include +#include +#include +#include +#include +#include +#include +#include +#include #include "ccp-crypto.h" diff --git a/drivers/crypto/ccp/ccp-crypto-main.c b/drivers/crypto/ccp/ccp-crypto-main.c index ecd58b38c46e..bc90aba5162a 100644 --- a/drivers/crypto/ccp/ccp-crypto-main.c +++ b/drivers/crypto/ccp/ccp-crypto-main.c @@ -7,14 +7,17 @@ * Author: Tom Lendacky */ -#include -#include +#include +#include +#include +#include +#include #include #include -#include +#include #include -#include -#include +#include +#include #include "ccp-crypto.h" From 60323cb47c842b0d2f0b5ff192e4a4fb4adfa363 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Mon, 7 Apr 2025 18:11:31 +0800 Subject: [PATCH 090/382] crypto: s5p-sss - Add missing header inclusions The gutting of crypto/ctr.h uncovered missing header inclusions. Add them. Signed-off-by: Herbert Xu --- drivers/crypto/s5p-sss.c | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/drivers/crypto/s5p-sss.c b/drivers/crypto/s5p-sss.c index b4c3c14dafd5..b829c84f60f2 100644 --- a/drivers/crypto/s5p-sss.c +++ b/drivers/crypto/s5p-sss.c @@ -9,11 +9,17 @@ // // Hash part based on omap-sham.c driver. +#include +#include +#include +#include +#include +#include +#include +#include #include -#include #include #include -#include #include #include #include @@ -22,17 +28,9 @@ #include #include #include - -#include -#include -#include -#include - -#include -#include -#include -#include -#include +#include +#include +#include #define _SBF(s, v) ((v) << (s)) From fcfbdddc6f0260174f76e185c9768a950f0872be Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Mon, 7 Apr 2025 18:11:33 +0800 Subject: [PATCH 091/382] crypto: ctr - Remove unnecessary header inclusions Now that the broken drivers have been fixed, remove the unnecessary inclusions from crypto/ctr.h. Signed-off-by: Herbert Xu --- include/crypto/ctr.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/include/crypto/ctr.h b/include/crypto/ctr.h index c41685874f00..06984a26c8cf 100644 --- a/include/crypto/ctr.h +++ b/include/crypto/ctr.h @@ -8,9 +8,6 @@ #ifndef _CRYPTO_CTR_H #define _CRYPTO_CTR_H -#include -#include - #define CTR_RFC3686_NONCE_SIZE 4 #define CTR_RFC3686_IV_SIZE 8 #define CTR_RFC3686_BLOCK_SIZE 16 From bebe54b8be12a7b7e8ecb4946fb95aea8374efa2 Mon Sep 17 00:00:00 2001 From: Zixun LI Date: Mon, 7 Apr 2025 12:20:49 +0200 Subject: [PATCH 092/382] crypto: atmel - add CRYPTO_ALG_KERN_DRIVER_ONLY flag This patch introduces the CRYPTO_ALG_KERN_DRIVER_ONLY flag to the atmel-aes, atmel-sha, and atmel-tdes drivers. This flag is set for hardware accelerated ciphers accessible through a kernel driver only, which is the case of these drivers. Signed-off-by: Zixun LI Signed-off-by: Herbert Xu --- drivers/crypto/atmel-aes.c | 5 +++-- drivers/crypto/atmel-sha.c | 6 ++++-- drivers/crypto/atmel-tdes.c | 2 +- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/drivers/crypto/atmel-aes.c b/drivers/crypto/atmel-aes.c index 14bf86957d31..27c5d000b4b2 100644 --- a/drivers/crypto/atmel-aes.c +++ b/drivers/crypto/atmel-aes.c @@ -1743,7 +1743,8 @@ static struct skcipher_alg aes_xts_alg = { .base.cra_driver_name = "atmel-xts-aes", .base.cra_blocksize = AES_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct atmel_aes_xts_ctx), - .base.cra_flags = CRYPTO_ALG_NEED_FALLBACK, + .base.cra_flags = CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_ALG_KERN_DRIVER_ONLY, .min_keysize = 2 * AES_MIN_KEY_SIZE, .max_keysize = 2 * AES_MAX_KEY_SIZE, @@ -2220,7 +2221,7 @@ static void atmel_aes_unregister_algs(struct atmel_aes_dev *dd) static void atmel_aes_crypto_alg_init(struct crypto_alg *alg) { - alg->cra_flags |= CRYPTO_ALG_ASYNC; + alg->cra_flags |= CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY; alg->cra_alignmask = 0xf; alg->cra_priority = ATMEL_AES_PRIORITY; alg->cra_module = THIS_MODULE; diff --git a/drivers/crypto/atmel-sha.c b/drivers/crypto/atmel-sha.c index 67a170608566..2cc36da163e8 100644 --- a/drivers/crypto/atmel-sha.c +++ b/drivers/crypto/atmel-sha.c @@ -1254,7 +1254,8 @@ static int atmel_sha_cra_init(struct crypto_tfm *tfm) static void atmel_sha_alg_init(struct ahash_alg *alg) { alg->halg.base.cra_priority = ATMEL_SHA_PRIORITY; - alg->halg.base.cra_flags = CRYPTO_ALG_ASYNC; + alg->halg.base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY; alg->halg.base.cra_ctxsize = sizeof(struct atmel_sha_ctx); alg->halg.base.cra_module = THIS_MODULE; alg->halg.base.cra_init = atmel_sha_cra_init; @@ -2041,7 +2042,8 @@ static void atmel_sha_hmac_cra_exit(struct crypto_tfm *tfm) static void atmel_sha_hmac_alg_init(struct ahash_alg *alg) { alg->halg.base.cra_priority = ATMEL_SHA_PRIORITY; - alg->halg.base.cra_flags = CRYPTO_ALG_ASYNC; + alg->halg.base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY; alg->halg.base.cra_ctxsize = sizeof(struct atmel_sha_hmac_ctx); alg->halg.base.cra_module = THIS_MODULE; alg->halg.base.cra_init = atmel_sha_hmac_cra_init; diff --git a/drivers/crypto/atmel-tdes.c b/drivers/crypto/atmel-tdes.c index de9717e221e4..098f5532f389 100644 --- a/drivers/crypto/atmel-tdes.c +++ b/drivers/crypto/atmel-tdes.c @@ -785,7 +785,7 @@ static int atmel_tdes_init_tfm(struct crypto_skcipher *tfm) static void atmel_tdes_skcipher_alg_init(struct skcipher_alg *alg) { alg->base.cra_priority = ATMEL_TDES_PRIORITY; - alg->base.cra_flags = CRYPTO_ALG_ASYNC; + alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY; alg->base.cra_ctxsize = sizeof(struct atmel_tdes_ctx); alg->base.cra_module = THIS_MODULE; From 6eed1e3552fc076d2617f6793cb148d485696ab6 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Mon, 7 Apr 2025 18:20:57 +0800 Subject: [PATCH 093/382] crypto: api - Mark cra_init/cra_exit as deprecated These functions have been obsoleted by the type-specific init/exit functions. Signed-off-by: Herbert Xu --- include/linux/crypto.h | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/include/linux/crypto.h b/include/linux/crypto.h index a387f1547ea0..56cf229e2530 100644 --- a/include/linux/crypto.h +++ b/include/linux/crypto.h @@ -300,17 +300,8 @@ struct cipher_alg { * by @cra_type and @cra_flags above, the associated structure must be * filled with callbacks. This field might be empty. This is the case * for ahash, shash. - * @cra_init: Initialize the cryptographic transformation object. This function - * is used to initialize the cryptographic transformation object. - * This function is called only once at the instantiation time, right - * after the transformation context was allocated. In case the - * cryptographic hardware has some special requirements which need to - * be handled by software, this function shall check for the precise - * requirement of the transformation and put any software fallbacks - * in place. - * @cra_exit: Deinitialize the cryptographic transformation object. This is a - * counterpart to @cra_init, used to remove various changes set in - * @cra_init. + * @cra_init: Deprecated, do not use. + * @cra_exit: Deprecated, do not use. * @cra_u.cipher: Union member which contains a single-block symmetric cipher * definition. See @struct @cipher_alg. * @cra_module: Owner of this transformation implementation. Set to THIS_MODULE From afddce13ce81d52a13898fa0700917835c71acd6 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Mon, 7 Apr 2025 18:20:59 +0800 Subject: [PATCH 094/382] crypto: api - Add reqsize to crypto_alg Add a reqsize field to crypto_alg with the intention of replacing the type-specific reqsize field currently used by ahash and acomp. Signed-off-by: Herbert Xu --- include/linux/crypto.h | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/include/linux/crypto.h b/include/linux/crypto.h index 56cf229e2530..15476b085ce3 100644 --- a/include/linux/crypto.h +++ b/include/linux/crypto.h @@ -276,6 +276,7 @@ struct cipher_alg { * to the alignmask of the algorithm being used, in order to * avoid the API having to realign them. Note: the alignmask is * not supported for hash algorithms and is always 0 for them. + * @cra_reqsize: Size of the request context for this algorithm. * @cra_priority: Priority of this transformation implementation. In case * multiple transformations with same @cra_name are available to * the Crypto API, the kernel will use the one with highest @@ -322,6 +323,7 @@ struct crypto_alg { unsigned int cra_blocksize; unsigned int cra_ctxsize; unsigned int cra_alignmask; + unsigned int cra_reqsize; int cra_priority; refcount_t cra_refcnt; @@ -441,6 +443,11 @@ static inline unsigned int crypto_tfm_alg_alignmask(struct crypto_tfm *tfm) return tfm->__crt_alg->cra_alignmask; } +static inline unsigned int crypto_tfm_alg_reqsize(struct crypto_tfm *tfm) +{ + return tfm->__crt_alg->cra_reqsize; +} + static inline u32 crypto_tfm_get_flags(struct crypto_tfm *tfm) { return tfm->crt_flags; From dbad301d9f8565f0e14004c0dbbe169e09756cb9 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Mon, 7 Apr 2025 18:21:02 +0800 Subject: [PATCH 095/382] crypto: acomp - Use cra_reqsize Use the common reqsize if present. Signed-off-by: Herbert Xu --- crypto/acompress.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crypto/acompress.c b/crypto/acompress.c index 674325ecefbc..36b52a72a43b 100644 --- a/crypto/acompress.c +++ b/crypto/acompress.c @@ -109,7 +109,7 @@ static int crypto_acomp_init_tfm(struct crypto_tfm *tfm) acomp->compress = alg->compress; acomp->decompress = alg->decompress; - acomp->reqsize = alg->reqsize; + acomp->reqsize = alg->base.cra_reqsize ?: alg->reqsize; acomp->base.exit = crypto_acomp_exit_tfm; From 1d10f790125d8a696c51fd7585caf3dc7672058e Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Mon, 7 Apr 2025 18:21:04 +0800 Subject: [PATCH 096/382] crypto: qat - Use cra_reqsize for acomp Use the common reqsize field for acomp algorithms. Signed-off-by: Herbert Xu --- drivers/crypto/intel/qat/qat_common/qat_comp_algs.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/intel/qat/qat_common/qat_comp_algs.c b/drivers/crypto/intel/qat/qat_common/qat_comp_algs.c index a6e02405d402..a0a29b97a749 100644 --- a/drivers/crypto/intel/qat/qat_common/qat_comp_algs.c +++ b/drivers/crypto/intel/qat/qat_common/qat_comp_algs.c @@ -241,13 +241,13 @@ static struct acomp_alg qat_acomp[] = { { .cra_priority = 4001, .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, .cra_ctxsize = sizeof(struct qat_compression_ctx), + .cra_reqsize = sizeof(struct qat_compression_req), .cra_module = THIS_MODULE, }, .init = qat_comp_alg_init_tfm, .exit = qat_comp_alg_exit_tfm, .compress = qat_comp_alg_compress, .decompress = qat_comp_alg_decompress, - .reqsize = sizeof(struct qat_compression_req), }}; int qat_comp_algs_register(void) From 47b5b6f9eb736b1868b0f9c1a1575b5922451cc6 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Mon, 7 Apr 2025 18:21:06 +0800 Subject: [PATCH 097/382] crypto: iaa - Use cra_reqsize for acomp Use the common reqsize field for acomp algorithms. Signed-off-by: Herbert Xu --- drivers/crypto/intel/iaa/iaa_crypto_main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/intel/iaa/iaa_crypto_main.c b/drivers/crypto/intel/iaa/iaa_crypto_main.c index b4f15e738cee..be3b899c6977 100644 --- a/drivers/crypto/intel/iaa/iaa_crypto_main.c +++ b/drivers/crypto/intel/iaa/iaa_crypto_main.c @@ -1685,12 +1685,12 @@ static struct acomp_alg iaa_acomp_fixed_deflate = { .init = iaa_comp_init_fixed, .compress = iaa_comp_acompress, .decompress = iaa_comp_adecompress, - .reqsize = sizeof(u32), .base = { .cra_name = "deflate", .cra_driver_name = "deflate-iaa", .cra_flags = CRYPTO_ALG_ASYNC, .cra_ctxsize = sizeof(struct iaa_compression_ctx), + .cra_reqsize = sizeof(u32), .cra_module = THIS_MODULE, .cra_priority = IAA_ALG_PRIORITY, } From 300e6d6e9ebf2dcc0ca3eab68ed65207e947926c Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Mon, 7 Apr 2025 18:21:09 +0800 Subject: [PATCH 098/382] crypto: acomp - Remove reqsize field Remove the type-specific reqsize field in favour of the common one. Signed-off-by: Herbert Xu --- crypto/acompress.c | 2 +- include/crypto/internal/acompress.h | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/crypto/acompress.c b/crypto/acompress.c index 36b52a72a43b..d5605b0ad266 100644 --- a/crypto/acompress.c +++ b/crypto/acompress.c @@ -109,7 +109,7 @@ static int crypto_acomp_init_tfm(struct crypto_tfm *tfm) acomp->compress = alg->compress; acomp->decompress = alg->decompress; - acomp->reqsize = alg->base.cra_reqsize ?: alg->reqsize; + acomp->reqsize = alg->base.cra_reqsize; acomp->base.exit = crypto_acomp_exit_tfm; diff --git a/include/crypto/internal/acompress.h b/include/crypto/internal/acompress.h index d6d53c7696fd..0f3ad65be2d9 100644 --- a/include/crypto/internal/acompress.h +++ b/include/crypto/internal/acompress.h @@ -40,7 +40,6 @@ * counterpart to @init, used to remove various changes set in * @init. * - * @reqsize: Context size for (de)compression requests * @base: Common crypto API algorithm data structure * @calg: Cmonn algorithm data structure shared with scomp */ @@ -50,8 +49,6 @@ struct acomp_alg { int (*init)(struct crypto_acomp *tfm); void (*exit)(struct crypto_acomp *tfm); - unsigned int reqsize; - union { struct COMP_ALG_COMMON; struct comp_alg_common calg; From c360df01c6c19959d2218cbad56d5b7c2ac7a476 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Mon, 7 Apr 2025 18:21:11 +0800 Subject: [PATCH 099/382] crypto: ahash - Use cra_reqsize Use the common reqsize field and remove reqsize from ahash_alg. Signed-off-by: Herbert Xu --- crypto/ahash.c | 4 ++-- include/crypto/hash.h | 3 --- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/crypto/ahash.c b/crypto/ahash.c index fc59897c234c..ba0a639144af 100644 --- a/crypto/ahash.c +++ b/crypto/ahash.c @@ -723,7 +723,7 @@ static int crypto_ahash_init_tfm(struct crypto_tfm *tfm) struct ahash_alg *alg = crypto_ahash_alg(hash); crypto_ahash_set_statesize(hash, alg->halg.statesize); - crypto_ahash_set_reqsize(hash, alg->reqsize); + crypto_ahash_set_reqsize(hash, crypto_tfm_alg_reqsize(tfm)); if (tfm->__crt_alg->cra_type == &crypto_shash_type) return crypto_init_ahash_using_shash(tfm); @@ -889,7 +889,7 @@ static int ahash_prepare_alg(struct ahash_alg *alg) if (alg->halg.statesize == 0) return -EINVAL; - if (alg->reqsize && alg->reqsize < alg->halg.statesize) + if (base->cra_reqsize && base->cra_reqsize < alg->halg.statesize) return -EINVAL; err = hash_prepare_alg(&alg->halg); diff --git a/include/crypto/hash.h b/include/crypto/hash.h index 113a5835e586..eceb2ed04f26 100644 --- a/include/crypto/hash.h +++ b/include/crypto/hash.h @@ -136,7 +136,6 @@ struct ahash_request { * This is a counterpart to @init_tfm, used to remove * various changes set in @init_tfm. * @clone_tfm: Copy transform into new object, may allocate memory. - * @reqsize: Size of the request context. * @halg: see struct hash_alg_common */ struct ahash_alg { @@ -153,8 +152,6 @@ struct ahash_alg { void (*exit_tfm)(struct crypto_ahash *tfm); int (*clone_tfm)(struct crypto_ahash *dst, struct crypto_ahash *src); - unsigned int reqsize; - struct hash_alg_common halg; }; From 2f0c856af6258b7d0110872b72a53267bfe72715 Mon Sep 17 00:00:00 2001 From: Ovidiu Panait Date: Mon, 7 Apr 2025 15:36:01 +0300 Subject: [PATCH 100/382] crypto: img-hash - use API helpers to setup fallback request Rather than setting up the fallback request by hand, use ahash_request_set_callback() and ahash_request_set_crypt() API helpers to properly setup the new request. This also ensures that the completion callback is properly passed down to the fallback algorithm, which avoids a crash with async fallbacks. Signed-off-by: Ovidiu Panait Signed-off-by: Herbert Xu --- drivers/crypto/img-hash.c | 41 ++++++++++++++++++++++----------------- 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/drivers/crypto/img-hash.c b/drivers/crypto/img-hash.c index 1dc2378aa88b..e050f5ff5efb 100644 --- a/drivers/crypto/img-hash.c +++ b/drivers/crypto/img-hash.c @@ -491,8 +491,9 @@ static int img_hash_init(struct ahash_request *req) struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback); - rctx->fallback_req.base.flags = req->base.flags - & CRYPTO_TFM_REQ_MAY_SLEEP; + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); return crypto_ahash_init(&rctx->fallback_req); } @@ -555,10 +556,10 @@ static int img_hash_update(struct ahash_request *req) struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback); - rctx->fallback_req.base.flags = req->base.flags - & CRYPTO_TFM_REQ_MAY_SLEEP; - rctx->fallback_req.nbytes = req->nbytes; - rctx->fallback_req.src = req->src; + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); + ahash_request_set_crypt(&rctx->fallback_req, req->src, NULL, req->nbytes); return crypto_ahash_update(&rctx->fallback_req); } @@ -570,9 +571,10 @@ static int img_hash_final(struct ahash_request *req) struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback); - rctx->fallback_req.base.flags = req->base.flags - & CRYPTO_TFM_REQ_MAY_SLEEP; - rctx->fallback_req.result = req->result; + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); + ahash_request_set_crypt(&rctx->fallback_req, NULL, req->result, 0); return crypto_ahash_final(&rctx->fallback_req); } @@ -584,11 +586,12 @@ static int img_hash_finup(struct ahash_request *req) struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback); - rctx->fallback_req.base.flags = req->base.flags - & CRYPTO_TFM_REQ_MAY_SLEEP; - rctx->fallback_req.nbytes = req->nbytes; - rctx->fallback_req.src = req->src; - rctx->fallback_req.result = req->result; + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); + ahash_request_set_crypt(&rctx->fallback_req, req->src, req->result, + req->nbytes); + return crypto_ahash_finup(&rctx->fallback_req); } @@ -600,8 +603,9 @@ static int img_hash_import(struct ahash_request *req, const void *in) struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback); - rctx->fallback_req.base.flags = req->base.flags - & CRYPTO_TFM_REQ_MAY_SLEEP; + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); return crypto_ahash_import(&rctx->fallback_req, in); } @@ -613,8 +617,9 @@ static int img_hash_export(struct ahash_request *req, void *out) struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback); - rctx->fallback_req.base.flags = req->base.flags - & CRYPTO_TFM_REQ_MAY_SLEEP; + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); return crypto_ahash_export(&rctx->fallback_req, out); } From cbc9f5bcfa72d7e58cbace18c97e4df7e3a73058 Mon Sep 17 00:00:00 2001 From: Ovidiu Panait Date: Mon, 7 Apr 2025 15:36:02 +0300 Subject: [PATCH 101/382] crypto: rk3288 - use API helpers to setup fallback request Rather than setting up the fallback request by hand, use ahash_request_set_callback() and ahash_request_set_crypt() API helpers to properly setup the new request. This also ensures that the completion callback is properly passed down to the fallback algorithm, which avoids a crash with async fallbacks. Signed-off-by: Ovidiu Panait Signed-off-by: Herbert Xu --- drivers/crypto/rockchip/rk3288_crypto_ahash.c | 52 ++++++++++--------- 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/drivers/crypto/rockchip/rk3288_crypto_ahash.c b/drivers/crypto/rockchip/rk3288_crypto_ahash.c index 69d6019d8abc..d6928ebe9526 100644 --- a/drivers/crypto/rockchip/rk3288_crypto_ahash.c +++ b/drivers/crypto/rockchip/rk3288_crypto_ahash.c @@ -52,12 +52,11 @@ static int rk_ahash_digest_fb(struct ahash_request *areq) algt->stat_fb++; ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); - rctx->fallback_req.base.flags = areq->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; - - rctx->fallback_req.nbytes = areq->nbytes; - rctx->fallback_req.src = areq->src; - rctx->fallback_req.result = areq->result; + ahash_request_set_callback(&rctx->fallback_req, + areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + areq->base.complete, areq->base.data); + ahash_request_set_crypt(&rctx->fallback_req, areq->src, areq->result, + areq->nbytes); return crypto_ahash_digest(&rctx->fallback_req); } @@ -124,8 +123,9 @@ static int rk_ahash_init(struct ahash_request *req) struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); - rctx->fallback_req.base.flags = req->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); return crypto_ahash_init(&rctx->fallback_req); } @@ -137,10 +137,10 @@ static int rk_ahash_update(struct ahash_request *req) struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); - rctx->fallback_req.base.flags = req->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; - rctx->fallback_req.nbytes = req->nbytes; - rctx->fallback_req.src = req->src; + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); + ahash_request_set_crypt(&rctx->fallback_req, req->src, NULL, req->nbytes); return crypto_ahash_update(&rctx->fallback_req); } @@ -152,9 +152,10 @@ static int rk_ahash_final(struct ahash_request *req) struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); - rctx->fallback_req.base.flags = req->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; - rctx->fallback_req.result = req->result; + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); + ahash_request_set_crypt(&rctx->fallback_req, NULL, req->result, 0); return crypto_ahash_final(&rctx->fallback_req); } @@ -166,12 +167,11 @@ static int rk_ahash_finup(struct ahash_request *req) struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); - rctx->fallback_req.base.flags = req->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; - - rctx->fallback_req.nbytes = req->nbytes; - rctx->fallback_req.src = req->src; - rctx->fallback_req.result = req->result; + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); + ahash_request_set_crypt(&rctx->fallback_req, req->src, req->result, + req->nbytes); return crypto_ahash_finup(&rctx->fallback_req); } @@ -183,8 +183,9 @@ static int rk_ahash_import(struct ahash_request *req, const void *in) struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); - rctx->fallback_req.base.flags = req->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); return crypto_ahash_import(&rctx->fallback_req, in); } @@ -196,8 +197,9 @@ static int rk_ahash_export(struct ahash_request *req, void *out) struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); - rctx->fallback_req.base.flags = req->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); return crypto_ahash_export(&rctx->fallback_req, out); } From c1024581ff2c34db4fdfe857b29f606e5117eb12 Mon Sep 17 00:00:00 2001 From: Ovidiu Panait Date: Mon, 7 Apr 2025 15:36:04 +0300 Subject: [PATCH 102/382] crypto: tegra - use API helpers to setup fallback request Rather than setting up the fallback request by hand, use ahash_request_set_callback() and ahash_request_set_crypt() API helpers to properly setup the new request. Signed-off-by: Ovidiu Panait Signed-off-by: Herbert Xu --- drivers/crypto/tegra/tegra-se-hash.c | 52 +++++++++++++++------------- 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/drivers/crypto/tegra/tegra-se-hash.c b/drivers/crypto/tegra/tegra-se-hash.c index 42d007b7af45..d09b4aaeecef 100644 --- a/drivers/crypto/tegra/tegra-se-hash.c +++ b/drivers/crypto/tegra/tegra-se-hash.c @@ -117,8 +117,9 @@ static int tegra_sha_fallback_init(struct ahash_request *req) struct tegra_sha_ctx *ctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); - rctx->fallback_req.base.flags = req->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); return crypto_ahash_init(&rctx->fallback_req); } @@ -130,10 +131,10 @@ static int tegra_sha_fallback_update(struct ahash_request *req) struct tegra_sha_ctx *ctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); - rctx->fallback_req.base.flags = req->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; - rctx->fallback_req.nbytes = req->nbytes; - rctx->fallback_req.src = req->src; + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); + ahash_request_set_crypt(&rctx->fallback_req, req->src, NULL, req->nbytes); return crypto_ahash_update(&rctx->fallback_req); } @@ -145,9 +146,10 @@ static int tegra_sha_fallback_final(struct ahash_request *req) struct tegra_sha_ctx *ctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); - rctx->fallback_req.base.flags = req->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; - rctx->fallback_req.result = req->result; + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); + ahash_request_set_crypt(&rctx->fallback_req, NULL, req->result, 0); return crypto_ahash_final(&rctx->fallback_req); } @@ -159,12 +161,11 @@ static int tegra_sha_fallback_finup(struct ahash_request *req) struct tegra_sha_ctx *ctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); - rctx->fallback_req.base.flags = req->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; - - rctx->fallback_req.nbytes = req->nbytes; - rctx->fallback_req.src = req->src; - rctx->fallback_req.result = req->result; + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); + ahash_request_set_crypt(&rctx->fallback_req, req->src, req->result, + req->nbytes); return crypto_ahash_finup(&rctx->fallback_req); } @@ -176,12 +177,11 @@ static int tegra_sha_fallback_digest(struct ahash_request *req) struct tegra_sha_ctx *ctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); - rctx->fallback_req.base.flags = req->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; - - rctx->fallback_req.nbytes = req->nbytes; - rctx->fallback_req.src = req->src; - rctx->fallback_req.result = req->result; + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); + ahash_request_set_crypt(&rctx->fallback_req, req->src, req->result, + req->nbytes); return crypto_ahash_digest(&rctx->fallback_req); } @@ -193,8 +193,9 @@ static int tegra_sha_fallback_import(struct ahash_request *req, const void *in) struct tegra_sha_ctx *ctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); - rctx->fallback_req.base.flags = req->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); return crypto_ahash_import(&rctx->fallback_req, in); } @@ -206,8 +207,9 @@ static int tegra_sha_fallback_export(struct ahash_request *req, void *out) struct tegra_sha_ctx *ctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); - rctx->fallback_req.base.flags = req->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); return crypto_ahash_export(&rctx->fallback_req, out); } From 61bb8db6f682bf3fed9feac8948daefb37146265 Mon Sep 17 00:00:00 2001 From: Thomas Richard Date: Mon, 7 Apr 2025 19:33:14 +0200 Subject: [PATCH 103/382] crypto: caam - Add support for i.MX8QM On i.MX8QM, caam clocks are turned on automatically and Linux does not have access to the caam controller's register page, so skip clocks initialization. Signed-off-by: Thomas Richard Reviewed-by: Gaurav Jain Signed-off-by: Herbert Xu --- drivers/crypto/caam/ctrl.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/crypto/caam/ctrl.c b/drivers/crypto/caam/ctrl.c index d4b39184dbdb..38ff931059b4 100644 --- a/drivers/crypto/caam/ctrl.c +++ b/drivers/crypto/caam/ctrl.c @@ -573,6 +573,7 @@ static const struct soc_device_attribute caam_imx_soc_table[] = { { .soc_id = "i.MX7*", .data = &caam_imx7_data }, { .soc_id = "i.MX8M*", .data = &caam_imx7_data }, { .soc_id = "i.MX8ULP", .data = &caam_imx8ulp_data }, + { .soc_id = "i.MX8QM", .data = &caam_imx8ulp_data }, { .soc_id = "VF*", .data = &caam_vf610_data }, { .family = "Freescale i.MX" }, { /* sentinel */ } From 3828485e1c7b111290122ab6e083c2a37132b5c2 Mon Sep 17 00:00:00 2001 From: Lukas Wunner Date: Mon, 7 Apr 2025 21:32:41 +0200 Subject: [PATCH 104/382] crypto: ecdsa - Fix enc/dec size reported by KEYCTL_PKEY_QUERY KEYCTL_PKEY_QUERY system calls for ecdsa keys return the key size as max_enc_size and max_dec_size, even though such keys cannot be used for encryption/decryption. They're exclusively for signature generation or verification. Only rsa keys with pkcs1 encoding can also be used for encryption or decryption. Return 0 instead for ecdsa keys (as well as ecrdsa keys). Signed-off-by: Lukas Wunner Reviewed-by: Stefan Berger Reviewed-by: Ignat Korchagin Signed-off-by: Herbert Xu --- crypto/asymmetric_keys/public_key.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/crypto/asymmetric_keys/public_key.c b/crypto/asymmetric_keys/public_key.c index bf165d321440..dd44a966947f 100644 --- a/crypto/asymmetric_keys/public_key.c +++ b/crypto/asymmetric_keys/public_key.c @@ -188,6 +188,8 @@ static int software_key_query(const struct kernel_pkey_params *params, ptr = pkey_pack_u32(ptr, pkey->paramlen); memcpy(ptr, pkey->params, pkey->paramlen); + memset(info, 0, sizeof(*info)); + if (issig) { sig = crypto_alloc_sig(alg_name, 0, 0); if (IS_ERR(sig)) { @@ -211,6 +213,9 @@ static int software_key_query(const struct kernel_pkey_params *params, info->supported_ops |= KEYCTL_SUPPORTS_SIGN; if (strcmp(params->encoding, "pkcs1") == 0) { + info->max_enc_size = len; + info->max_dec_size = len; + info->supported_ops |= KEYCTL_SUPPORTS_ENCRYPT; if (pkey->key_is_private) info->supported_ops |= KEYCTL_SUPPORTS_DECRYPT; @@ -232,6 +237,8 @@ static int software_key_query(const struct kernel_pkey_params *params, len = crypto_akcipher_maxsize(tfm); info->max_sig_size = len; info->max_data_size = len; + info->max_enc_size = len; + info->max_dec_size = len; info->supported_ops = KEYCTL_SUPPORTS_ENCRYPT; if (pkey->key_is_private) @@ -239,8 +246,6 @@ static int software_key_query(const struct kernel_pkey_params *params, } info->key_size = len * 8; - info->max_enc_size = len; - info->max_dec_size = len; ret = 0; From 6b7f9397c98c72902f9364056413c73fe6dee1d8 Mon Sep 17 00:00:00 2001 From: Lukas Wunner Date: Mon, 7 Apr 2025 21:32:42 +0200 Subject: [PATCH 105/382] crypto: ecdsa - Fix NIST P521 key size reported by KEYCTL_PKEY_QUERY When user space issues a KEYCTL_PKEY_QUERY system call for a NIST P521 key, the key_size is incorrectly reported as 528 bits instead of 521. That's because the key size obtained through crypto_sig_keysize() is in bytes and software_key_query() multiplies by 8 to yield the size in bits. The underlying assumption is that the key size is always a multiple of 8. With the recent addition of NIST P521, that's no longer the case. Fix by returning the key_size in bits from crypto_sig_keysize() and adjusting the calculations in software_key_query(). The ->key_size() callbacks of sig_alg algorithms now return the size in bits, whereas the ->digest_size() and ->max_size() callbacks return the size in bytes. This matches with the units in struct keyctl_pkey_query. Fixes: a7d45ba77d3d ("crypto: ecdsa - Register NIST P521 and extend test suite") Signed-off-by: Lukas Wunner Reviewed-by: Stefan Berger Reviewed-by: Ignat Korchagin Signed-off-by: Herbert Xu --- crypto/asymmetric_keys/public_key.c | 8 ++++---- crypto/ecdsa-p1363.c | 6 ++++-- crypto/ecdsa-x962.c | 5 +++-- crypto/ecdsa.c | 2 +- crypto/ecrdsa.c | 2 +- crypto/rsassa-pkcs1.c | 2 +- crypto/sig.c | 9 +++++++-- include/crypto/sig.h | 2 +- 8 files changed, 22 insertions(+), 14 deletions(-) diff --git a/crypto/asymmetric_keys/public_key.c b/crypto/asymmetric_keys/public_key.c index dd44a966947f..89dc887d2c5c 100644 --- a/crypto/asymmetric_keys/public_key.c +++ b/crypto/asymmetric_keys/public_key.c @@ -205,6 +205,7 @@ static int software_key_query(const struct kernel_pkey_params *params, goto error_free_tfm; len = crypto_sig_keysize(sig); + info->key_size = len; info->max_sig_size = crypto_sig_maxsize(sig); info->max_data_size = crypto_sig_digestsize(sig); @@ -213,8 +214,8 @@ static int software_key_query(const struct kernel_pkey_params *params, info->supported_ops |= KEYCTL_SUPPORTS_SIGN; if (strcmp(params->encoding, "pkcs1") == 0) { - info->max_enc_size = len; - info->max_dec_size = len; + info->max_enc_size = len / BITS_PER_BYTE; + info->max_dec_size = len / BITS_PER_BYTE; info->supported_ops |= KEYCTL_SUPPORTS_ENCRYPT; if (pkey->key_is_private) @@ -235,6 +236,7 @@ static int software_key_query(const struct kernel_pkey_params *params, goto error_free_tfm; len = crypto_akcipher_maxsize(tfm); + info->key_size = len * BITS_PER_BYTE; info->max_sig_size = len; info->max_data_size = len; info->max_enc_size = len; @@ -245,8 +247,6 @@ static int software_key_query(const struct kernel_pkey_params *params, info->supported_ops |= KEYCTL_SUPPORTS_DECRYPT; } - info->key_size = len * 8; - ret = 0; error_free_tfm: diff --git a/crypto/ecdsa-p1363.c b/crypto/ecdsa-p1363.c index 4454f1f8f33f..e0c55c64711c 100644 --- a/crypto/ecdsa-p1363.c +++ b/crypto/ecdsa-p1363.c @@ -21,7 +21,8 @@ static int ecdsa_p1363_verify(struct crypto_sig *tfm, const void *digest, unsigned int dlen) { struct ecdsa_p1363_ctx *ctx = crypto_sig_ctx(tfm); - unsigned int keylen = crypto_sig_keysize(ctx->child); + unsigned int keylen = DIV_ROUND_UP_POW2(crypto_sig_keysize(ctx->child), + BITS_PER_BYTE); unsigned int ndigits = DIV_ROUND_UP_POW2(keylen, sizeof(u64)); struct ecdsa_raw_sig sig; @@ -45,7 +46,8 @@ static unsigned int ecdsa_p1363_max_size(struct crypto_sig *tfm) { struct ecdsa_p1363_ctx *ctx = crypto_sig_ctx(tfm); - return 2 * crypto_sig_keysize(ctx->child); + return 2 * DIV_ROUND_UP_POW2(crypto_sig_keysize(ctx->child), + BITS_PER_BYTE); } static unsigned int ecdsa_p1363_digest_size(struct crypto_sig *tfm) diff --git a/crypto/ecdsa-x962.c b/crypto/ecdsa-x962.c index 90a04f4b9a2f..ee71594d10a0 100644 --- a/crypto/ecdsa-x962.c +++ b/crypto/ecdsa-x962.c @@ -82,7 +82,7 @@ static int ecdsa_x962_verify(struct crypto_sig *tfm, int err; sig_ctx.ndigits = DIV_ROUND_UP_POW2(crypto_sig_keysize(ctx->child), - sizeof(u64)); + sizeof(u64) * BITS_PER_BYTE); err = asn1_ber_decoder(&ecdsasignature_decoder, &sig_ctx, src, slen); if (err < 0) @@ -103,7 +103,8 @@ static unsigned int ecdsa_x962_max_size(struct crypto_sig *tfm) { struct ecdsa_x962_ctx *ctx = crypto_sig_ctx(tfm); struct sig_alg *alg = crypto_sig_alg(ctx->child); - int slen = crypto_sig_keysize(ctx->child); + int slen = DIV_ROUND_UP_POW2(crypto_sig_keysize(ctx->child), + BITS_PER_BYTE); /* * Verify takes ECDSA-Sig-Value (described in RFC 5480) as input, diff --git a/crypto/ecdsa.c b/crypto/ecdsa.c index 117526d15dde..a70b60a90a3c 100644 --- a/crypto/ecdsa.c +++ b/crypto/ecdsa.c @@ -167,7 +167,7 @@ static unsigned int ecdsa_key_size(struct crypto_sig *tfm) { struct ecc_ctx *ctx = crypto_sig_ctx(tfm); - return DIV_ROUND_UP(ctx->curve->nbits, 8); + return ctx->curve->nbits; } static unsigned int ecdsa_digest_size(struct crypto_sig *tfm) diff --git a/crypto/ecrdsa.c b/crypto/ecrdsa.c index b3dd8a3ddeb7..2c0602f0cd40 100644 --- a/crypto/ecrdsa.c +++ b/crypto/ecrdsa.c @@ -249,7 +249,7 @@ static unsigned int ecrdsa_key_size(struct crypto_sig *tfm) * Verify doesn't need any output, so it's just informational * for keyctl to determine the key bit size. */ - return ctx->pub_key.ndigits * sizeof(u64); + return ctx->pub_key.ndigits * sizeof(u64) * BITS_PER_BYTE; } static unsigned int ecrdsa_max_size(struct crypto_sig *tfm) diff --git a/crypto/rsassa-pkcs1.c b/crypto/rsassa-pkcs1.c index d01ac75635e0..94fa5e9600e7 100644 --- a/crypto/rsassa-pkcs1.c +++ b/crypto/rsassa-pkcs1.c @@ -301,7 +301,7 @@ static unsigned int rsassa_pkcs1_key_size(struct crypto_sig *tfm) { struct rsassa_pkcs1_ctx *ctx = crypto_sig_ctx(tfm); - return ctx->key_size; + return ctx->key_size * BITS_PER_BYTE; } static int rsassa_pkcs1_set_pub_key(struct crypto_sig *tfm, diff --git a/crypto/sig.c b/crypto/sig.c index dfc7cae90802..53a3dd6fbe3f 100644 --- a/crypto/sig.c +++ b/crypto/sig.c @@ -102,6 +102,11 @@ static int sig_default_set_key(struct crypto_sig *tfm, return -ENOSYS; } +static unsigned int sig_default_size(struct crypto_sig *tfm) +{ + return DIV_ROUND_UP_POW2(crypto_sig_keysize(tfm), BITS_PER_BYTE); +} + static int sig_prepare_alg(struct sig_alg *alg) { struct crypto_alg *base = &alg->base; @@ -117,9 +122,9 @@ static int sig_prepare_alg(struct sig_alg *alg) if (!alg->key_size) return -EINVAL; if (!alg->max_size) - alg->max_size = alg->key_size; + alg->max_size = sig_default_size; if (!alg->digest_size) - alg->digest_size = alg->key_size; + alg->digest_size = sig_default_size; base->cra_type = &crypto_sig_type; base->cra_flags &= ~CRYPTO_ALG_TYPE_MASK; diff --git a/include/crypto/sig.h b/include/crypto/sig.h index 11024708c069..fa6dafafab3f 100644 --- a/include/crypto/sig.h +++ b/include/crypto/sig.h @@ -128,7 +128,7 @@ static inline void crypto_free_sig(struct crypto_sig *tfm) /** * crypto_sig_keysize() - Get key size * - * Function returns the key size in bytes. + * Function returns the key size in bits. * Function assumes that the key is already set in the transformation. If this * function is called without a setkey or with a failed setkey, you may end up * in a NULL dereference. From c80d6598ffef3154698fb991c15211762cbf550e Mon Sep 17 00:00:00 2001 From: Thorsten Blum Date: Tue, 8 Apr 2025 13:22:59 +0200 Subject: [PATCH 106/382] crypto: essiv - Remove unnecessary strscpy() size argument If the destination buffer has a fixed length, strscpy() automatically determines its size using sizeof() when the argument is omitted. This makes the explicit size argument unnecessary - remove it. No functional changes intended. Signed-off-by: Thorsten Blum Signed-off-by: Herbert Xu --- crypto/essiv.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crypto/essiv.c b/crypto/essiv.c index ec0ec8992c2d..cfce8ef7ab1f 100644 --- a/crypto/essiv.c +++ b/crypto/essiv.c @@ -548,8 +548,7 @@ static int essiv_create(struct crypto_template *tmpl, struct rtattr **tb) } /* record the driver name so we can instantiate this exact algo later */ - strscpy(ictx->shash_driver_name, hash_alg->base.cra_driver_name, - CRYPTO_MAX_ALG_NAME); + strscpy(ictx->shash_driver_name, hash_alg->base.cra_driver_name); /* Instance fields */ From d701722aa343e59dd7c18fc62894edf4497251e2 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Wed, 9 Apr 2025 11:29:03 +0800 Subject: [PATCH 107/382] crypto: api - Allow delayed algorithm destruction The current algorithm unregistration mechanism originated from software crypto. The code relies on module reference counts to stop in-use algorithms from being unregistered. Therefore if the unregistration function is reached, it is assumed that the module reference count has hit zero and thus the algorithm reference count should be exactly 1. This is completely broken for hardware devices, which can be unplugged at random. Fix this by allowing algorithms to be destroyed later if a destroy callback is provided. Reported-by: Sean Anderson Signed-off-by: Herbert Xu --- crypto/algapi.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crypto/algapi.c b/crypto/algapi.c index 5b8a4c787387..f368c0dc0d6d 100644 --- a/crypto/algapi.c +++ b/crypto/algapi.c @@ -481,10 +481,10 @@ void crypto_unregister_alg(struct crypto_alg *alg) if (WARN(ret, "Algorithm %s is not registered", alg->cra_driver_name)) return; - if (WARN_ON(refcount_read(&alg->cra_refcnt) != 1)) - return; - - if (alg->cra_type && alg->cra_type->destroy) + if (alg->cra_destroy) + crypto_alg_put(alg); + else if (!WARN_ON(refcount_read(&alg->cra_refcnt) != 1) && + alg->cra_type && alg->cra_type->destroy) alg->cra_type->destroy(alg); crypto_remove_final(&list); From 9af6339a65426b0b4850b07ff3b6959497bfcac5 Mon Sep 17 00:00:00 2001 From: Ashish Kalra Date: Wed, 9 Apr 2025 19:34:29 +0000 Subject: [PATCH 108/382] crypto: ccp - Fix __sev_snp_shutdown_locked Fix smatch warning: drivers/crypto/ccp/sev-dev.c:1755 __sev_snp_shutdown_locked() error: uninitialized symbol 'dfflush_error'. Fixes: 9770b428b1a2 ("crypto: ccp - Move dev_info/err messages for SEV/SNP init and shutdown") Reported-by: Dan Carpenter Closes: https://lore.kernel.org/linux-crypto/d9c2e79c-e26e-47b7-8243-ff6e7b101ec3@stanley.mountain/ Signed-off-by: Ashish Kalra Acked-by: Tom Lendacky Signed-off-by: Herbert Xu --- drivers/crypto/ccp/sev-dev.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/ccp/sev-dev.c b/drivers/crypto/ccp/sev-dev.c index 19fb51558a7d..1ccff5e3f4bc 100644 --- a/drivers/crypto/ccp/sev-dev.c +++ b/drivers/crypto/ccp/sev-dev.c @@ -1744,7 +1744,7 @@ static int __sev_snp_shutdown_locked(int *error, bool panic) ret = __sev_do_cmd_locked(SEV_CMD_SNP_SHUTDOWN_EX, &data, error); /* SHUTDOWN may require DF_FLUSH */ if (*error == SEV_RET_DFFLUSH_REQUIRED) { - int dfflush_error; + int dfflush_error = SEV_RET_NO_FW_CALL; ret = __sev_do_cmd_locked(SEV_CMD_SNP_DF_FLUSH, NULL, &dfflush_error); if (ret) { From 695912e619ee3945075af2de248e1e0fcef94a57 Mon Sep 17 00:00:00 2001 From: Sakari Ailus Date: Thu, 10 Apr 2025 18:22:37 +0300 Subject: [PATCH 109/382] hwrng: atmel - Add struct device pointer to device context struct Add a struct device pointer field to the device's context struct. This makes using the unsigned long priv pointer in struct hwrng unnecessary, so remove that one as well. Signed-off-by: Sakari Ailus Signed-off-by: Herbert Xu --- drivers/char/hw_random/atmel-rng.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/drivers/char/hw_random/atmel-rng.c b/drivers/char/hw_random/atmel-rng.c index 143406bc6939..d2b00458761e 100644 --- a/drivers/char/hw_random/atmel-rng.c +++ b/drivers/char/hw_random/atmel-rng.c @@ -37,6 +37,7 @@ struct atmel_trng { struct clk *clk; void __iomem *base; struct hwrng rng; + struct device *dev; bool has_half_rate; }; @@ -59,9 +60,9 @@ static int atmel_trng_read(struct hwrng *rng, void *buf, size_t max, u32 *data = buf; int ret; - ret = pm_runtime_get_sync((struct device *)trng->rng.priv); + ret = pm_runtime_get_sync(trng->dev); if (ret < 0) { - pm_runtime_put_sync((struct device *)trng->rng.priv); + pm_runtime_put_sync(trng->dev); return ret; } @@ -79,8 +80,8 @@ static int atmel_trng_read(struct hwrng *rng, void *buf, size_t max, ret = 4; out: - pm_runtime_mark_last_busy((struct device *)trng->rng.priv); - pm_runtime_put_sync_autosuspend((struct device *)trng->rng.priv); + pm_runtime_mark_last_busy(trng->dev); + pm_runtime_put_sync_autosuspend(trng->dev); return ret; } @@ -134,9 +135,9 @@ static int atmel_trng_probe(struct platform_device *pdev) return -ENODEV; trng->has_half_rate = data->has_half_rate; + trng->dev = &pdev->dev; trng->rng.name = pdev->name; trng->rng.read = atmel_trng_read; - trng->rng.priv = (unsigned long)&pdev->dev; platform_set_drvdata(pdev, trng); #ifndef CONFIG_PM From c2c460c30af5240d1894692306aa3a2377964f9e Mon Sep 17 00:00:00 2001 From: Sakari Ailus Date: Thu, 10 Apr 2025 18:22:38 +0300 Subject: [PATCH 110/382] hwrng: mtk - Add struct device pointer to device context struct Add a struct device pointer field to the device's context struct. This makes using the unsigned long priv pointer in struct hwrng unnecessary, so remove that one as well. Signed-off-by: Sakari Ailus Signed-off-by: Herbert Xu --- drivers/char/hw_random/mtk-rng.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/drivers/char/hw_random/mtk-rng.c b/drivers/char/hw_random/mtk-rng.c index 1e3048f2bb38..b7fa1bc1122b 100644 --- a/drivers/char/hw_random/mtk-rng.c +++ b/drivers/char/hw_random/mtk-rng.c @@ -36,6 +36,7 @@ struct mtk_rng { void __iomem *base; struct clk *clk; struct hwrng rng; + struct device *dev; }; static int mtk_rng_init(struct hwrng *rng) @@ -85,7 +86,7 @@ static int mtk_rng_read(struct hwrng *rng, void *buf, size_t max, bool wait) struct mtk_rng *priv = to_mtk_rng(rng); int retval = 0; - pm_runtime_get_sync((struct device *)priv->rng.priv); + pm_runtime_get_sync(priv->dev); while (max >= sizeof(u32)) { if (!mtk_rng_wait_ready(rng, wait)) @@ -97,8 +98,8 @@ static int mtk_rng_read(struct hwrng *rng, void *buf, size_t max, bool wait) max -= sizeof(u32); } - pm_runtime_mark_last_busy((struct device *)priv->rng.priv); - pm_runtime_put_sync_autosuspend((struct device *)priv->rng.priv); + pm_runtime_mark_last_busy(priv->dev); + pm_runtime_put_sync_autosuspend(priv->dev); return retval || !wait ? retval : -EIO; } @@ -112,13 +113,13 @@ static int mtk_rng_probe(struct platform_device *pdev) if (!priv) return -ENOMEM; + priv->dev = &pdev->dev; priv->rng.name = pdev->name; #ifndef CONFIG_PM priv->rng.init = mtk_rng_init; priv->rng.cleanup = mtk_rng_cleanup; #endif priv->rng.read = mtk_rng_read; - priv->rng.priv = (unsigned long)&pdev->dev; priv->rng.quality = 900; priv->clk = devm_clk_get(&pdev->dev, "rng"); From a6b3e1f9a5b694f8b06952e3ede89a715efccf0c Mon Sep 17 00:00:00 2001 From: Sakari Ailus Date: Thu, 10 Apr 2025 18:22:39 +0300 Subject: [PATCH 111/382] hwrng: npcm - Add struct device pointer to device context struct Add a struct device pointer field to the device's context struct. This makes using the unsigned long priv pointer in struct hwrng unnecessary, so remove that one as well. Signed-off-by: Sakari Ailus Signed-off-by: Herbert Xu --- drivers/char/hw_random/npcm-rng.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/drivers/char/hw_random/npcm-rng.c b/drivers/char/hw_random/npcm-rng.c index 9ff00f096f38..3e308c890bd2 100644 --- a/drivers/char/hw_random/npcm-rng.c +++ b/drivers/char/hw_random/npcm-rng.c @@ -32,6 +32,7 @@ struct npcm_rng { void __iomem *base; struct hwrng rng; + struct device *dev; u32 clkp; }; @@ -57,7 +58,7 @@ static int npcm_rng_read(struct hwrng *rng, void *buf, size_t max, bool wait) int retval = 0; int ready; - pm_runtime_get_sync((struct device *)priv->rng.priv); + pm_runtime_get_sync(priv->dev); while (max) { if (wait) { @@ -79,8 +80,8 @@ static int npcm_rng_read(struct hwrng *rng, void *buf, size_t max, bool wait) max--; } - pm_runtime_mark_last_busy((struct device *)priv->rng.priv); - pm_runtime_put_sync_autosuspend((struct device *)priv->rng.priv); + pm_runtime_mark_last_busy(priv->dev); + pm_runtime_put_sync_autosuspend(priv->dev); return retval || !wait ? retval : -EIO; } @@ -109,7 +110,7 @@ static int npcm_rng_probe(struct platform_device *pdev) #endif priv->rng.name = pdev->name; priv->rng.read = npcm_rng_read; - priv->rng.priv = (unsigned long)&pdev->dev; + priv->dev = &pdev->dev; priv->clkp = (u32)(uintptr_t)of_device_get_match_data(&pdev->dev); writel(NPCM_RNG_M1ROSEL, priv->base + NPCM_RNGMODE_REG); From 9b27a1b2001a70e1a47d824475a19839194f74dd Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Wed, 16 Apr 2025 15:34:59 +0800 Subject: [PATCH 112/382] crypto: powerpc - Include uaccess.h and others The powerpc aes/ghash code was relying on pagefault_disable from being pulled in by random header files. Fix this by explicitly including uaccess.h. Also add other missing header files to prevent similar problems in future. Signed-off-by: Herbert Xu --- arch/powerpc/crypto/aes.c | 8 ++++---- arch/powerpc/crypto/aes_cbc.c | 4 ++++ arch/powerpc/crypto/aes_ctr.c | 4 ++++ arch/powerpc/crypto/aes_xts.c | 4 ++++ arch/powerpc/crypto/ghash.c | 14 +++++++------- 5 files changed, 23 insertions(+), 11 deletions(-) diff --git a/arch/powerpc/crypto/aes.c b/arch/powerpc/crypto/aes.c index ec06189fbf99..3f1e5e894902 100644 --- a/arch/powerpc/crypto/aes.c +++ b/arch/powerpc/crypto/aes.c @@ -7,15 +7,15 @@ * Author: Marcelo Henrique Cerri */ -#include -#include -#include -#include #include #include #include #include #include +#include +#include +#include +#include #include "aesp8-ppc.h" diff --git a/arch/powerpc/crypto/aes_cbc.c b/arch/powerpc/crypto/aes_cbc.c index ed0debc7acb5..5f2a4f375eef 100644 --- a/arch/powerpc/crypto/aes_cbc.c +++ b/arch/powerpc/crypto/aes_cbc.c @@ -12,6 +12,10 @@ #include #include #include +#include +#include +#include +#include #include "aesp8-ppc.h" diff --git a/arch/powerpc/crypto/aes_ctr.c b/arch/powerpc/crypto/aes_ctr.c index 3da75f42529a..e27c4036e711 100644 --- a/arch/powerpc/crypto/aes_ctr.c +++ b/arch/powerpc/crypto/aes_ctr.c @@ -12,6 +12,10 @@ #include #include #include +#include +#include +#include +#include #include "aesp8-ppc.h" diff --git a/arch/powerpc/crypto/aes_xts.c b/arch/powerpc/crypto/aes_xts.c index dabbccb41550..9440e771cede 100644 --- a/arch/powerpc/crypto/aes_xts.c +++ b/arch/powerpc/crypto/aes_xts.c @@ -13,6 +13,10 @@ #include #include #include +#include +#include +#include +#include #include "aesp8-ppc.h" diff --git a/arch/powerpc/crypto/ghash.c b/arch/powerpc/crypto/ghash.c index 77eca20bc7ac..9bb61a843fd3 100644 --- a/arch/powerpc/crypto/ghash.c +++ b/arch/powerpc/crypto/ghash.c @@ -11,19 +11,19 @@ * Copyright (C) 2014 - 2018 Linaro Ltd. */ -#include -#include -#include -#include +#include "aesp8-ppc.h" #include #include #include +#include #include -#include #include #include -#include -#include "aesp8-ppc.h" +#include +#include +#include +#include +#include void gcm_init_p8(u128 htable[16], const u64 Xi[2]); void gcm_gmult_p8(u64 Xi[2], const u128 htable[16]); From 7ba8df47810f0731da1c9edd9aa8c1836279a93d Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 11 Apr 2025 15:38:43 +0800 Subject: [PATCH 113/382] asm-generic: Make simd.h more resilient Add missing header inclusions and protect against double inclusion. Signed-off-by: Herbert Xu --- include/asm-generic/simd.h | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/include/asm-generic/simd.h b/include/asm-generic/simd.h index d0343d58a74a..ac29a22eb7cf 100644 --- a/include/asm-generic/simd.h +++ b/include/asm-generic/simd.h @@ -1,6 +1,10 @@ /* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_GENERIC_SIMD_H +#define _ASM_GENERIC_SIMD_H -#include +#include +#include +#include /* * may_use_simd - whether it is allowable at this time to issue SIMD @@ -13,3 +17,5 @@ static __must_check inline bool may_use_simd(void) { return !in_interrupt(); } + +#endif /* _ASM_GENERIC_SIMD_H */ From d01dd2700b2d3690e4371466eb6f8a9b24c03e71 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 11 Apr 2025 15:38:45 +0800 Subject: [PATCH 114/382] arm: Make simd.h more resilient Add missing header inclusions and protect against double inclusion. Signed-off-by: Herbert Xu --- arch/arm/include/asm/simd.h | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/arch/arm/include/asm/simd.h b/arch/arm/include/asm/simd.h index 82191dbd7e78..d37559762180 100644 --- a/arch/arm/include/asm/simd.h +++ b/arch/arm/include/asm/simd.h @@ -1,8 +1,14 @@ /* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SIMD_H +#define _ASM_SIMD_H -#include +#include +#include +#include static __must_check inline bool may_use_simd(void) { return IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && !in_hardirq(); } + +#endif /* _ASM_SIMD_H */ From f1c09a0b6add651e6a0be22302680ac444dee29b Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 11 Apr 2025 15:38:47 +0800 Subject: [PATCH 115/382] x86: Make simd.h more resilient Add missing header inclusions and protect against double inclusion. Signed-off-by: Herbert Xu --- arch/x86/include/asm/simd.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/arch/x86/include/asm/simd.h b/arch/x86/include/asm/simd.h index a341c878e977..b8027b63cd7a 100644 --- a/arch/x86/include/asm/simd.h +++ b/arch/x86/include/asm/simd.h @@ -1,6 +1,10 @@ /* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SIMD_H +#define _ASM_SIMD_H #include +#include +#include /* * may_use_simd - whether it is allowable at this time to issue SIMD @@ -10,3 +14,5 @@ static __must_check inline bool may_use_simd(void) { return irq_fpu_usable(); } + +#endif /* _ASM_SIMD_H */ From 34f170a590710737f9611231cd77b37ea41f707a Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 11 Apr 2025 15:38:50 +0800 Subject: [PATCH 116/382] crypto: simd - Include asm/simd.h in internal/simd.h Now that the asm/simd.h files have been made safe against double inclusion, include it directly in internal/simd.h. Signed-off-by: Herbert Xu --- include/crypto/internal/simd.h | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/include/crypto/internal/simd.h b/include/crypto/internal/simd.h index be97b97a75dd..f56049bd1660 100644 --- a/include/crypto/internal/simd.h +++ b/include/crypto/internal/simd.h @@ -6,6 +6,7 @@ #ifndef _CRYPTO_INTERNAL_SIMD_H #define _CRYPTO_INTERNAL_SIMD_H +#include #include #include @@ -46,9 +47,6 @@ void simd_unregister_aeads(struct aead_alg *algs, int count, * self-tests, in order to test the no-SIMD fallback code. This override is * currently limited to configurations where the extra self-tests are enabled, * because it might be a bit too invasive to be part of the regular self-tests. - * - * This is a macro so that , which some architectures don't have, - * doesn't have to be included directly here. */ #ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS DECLARE_PER_CPU(bool, crypto_simd_disabled_for_test); From 66fecd9d94034b6193636fe3bc01f089439f5413 Mon Sep 17 00:00:00 2001 From: Thorsten Blum Date: Fri, 11 Apr 2025 21:20:51 +0200 Subject: [PATCH 117/382] crypto: skcipher - Realign struct skcipher_walk to save 8 bytes Reduce skcipher_walk's struct size by 8 bytes by realigning its members. pahole output before: /* size: 120, cachelines: 2, members: 13 */ /* sum members: 108, holes: 2, sum holes: 8 */ /* padding: 4 */ /* last cacheline: 56 bytes */ and after: /* size: 112, cachelines: 2, members: 13 */ /* padding: 4 */ /* last cacheline: 48 bytes */ No functional changes intended. Signed-off-by: Thorsten Blum Signed-off-by: Herbert Xu --- include/crypto/internal/skcipher.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/include/crypto/internal/skcipher.h b/include/crypto/internal/skcipher.h index a958ab0636ad..0cad8e7364c8 100644 --- a/include/crypto/internal/skcipher.h +++ b/include/crypto/internal/skcipher.h @@ -67,8 +67,6 @@ struct skcipher_walk { struct scatter_walk in; }; - unsigned int nbytes; - union { /* Virtual address of the destination. */ struct { @@ -81,6 +79,7 @@ struct skcipher_walk { struct scatter_walk out; }; + unsigned int nbytes; unsigned int total; u8 *page; From f1440a90465bea1993f937ac7add592ce1e4ff44 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sat, 12 Apr 2025 13:16:43 +0800 Subject: [PATCH 118/382] crypto: api - Add support for duplicating algorithms before registration If the bit CRYPTO_ALG_DUP_FIRST is set, an algorithm will be duplicated by kmemdup before registration. This is inteded for hardware-based algorithms that may be unplugged at will. Do not use this if the algorithm data structure is embedded in a bigger data structure. Perform the duplication in the driver instead. Signed-off-by: Herbert Xu --- crypto/acompress.c | 1 + crypto/aead.c | 1 + crypto/ahash.c | 1 + crypto/akcipher.c | 1 + crypto/algapi.c | 41 ++++++++++++++++++++++++++++------------- crypto/api.c | 9 +++++++++ crypto/internal.h | 5 ++++- crypto/kpp.c | 1 + crypto/lskcipher.c | 1 + crypto/rng.c | 1 + crypto/scompress.c | 1 + crypto/shash.c | 1 + crypto/sig.c | 1 + crypto/skcipher.c | 1 + include/linux/crypto.h | 9 +++++++++ 15 files changed, 61 insertions(+), 14 deletions(-) diff --git a/crypto/acompress.c b/crypto/acompress.c index d5605b0ad266..606d09a7fbfd 100644 --- a/crypto/acompress.c +++ b/crypto/acompress.c @@ -150,6 +150,7 @@ static const struct crypto_type crypto_acomp_type = { .maskset = CRYPTO_ALG_TYPE_ACOMPRESS_MASK, .type = CRYPTO_ALG_TYPE_ACOMPRESS, .tfmsize = offsetof(struct crypto_acomp, base), + .algsize = offsetof(struct acomp_alg, base), }; struct crypto_acomp *crypto_alloc_acomp(const char *alg_name, u32 type, diff --git a/crypto/aead.c b/crypto/aead.c index 12f5b42171af..5d14b775036e 100644 --- a/crypto/aead.c +++ b/crypto/aead.c @@ -186,6 +186,7 @@ static const struct crypto_type crypto_aead_type = { .maskset = CRYPTO_ALG_TYPE_MASK, .type = CRYPTO_ALG_TYPE_AEAD, .tfmsize = offsetof(struct crypto_aead, base), + .algsize = offsetof(struct aead_alg, base), }; int crypto_grab_aead(struct crypto_aead_spawn *spawn, diff --git a/crypto/ahash.c b/crypto/ahash.c index ba0a639144af..7c9c0931197f 100644 --- a/crypto/ahash.c +++ b/crypto/ahash.c @@ -792,6 +792,7 @@ static const struct crypto_type crypto_ahash_type = { .maskset = CRYPTO_ALG_TYPE_AHASH_MASK, .type = CRYPTO_ALG_TYPE_AHASH, .tfmsize = offsetof(struct crypto_ahash, base), + .algsize = offsetof(struct ahash_alg, halg.base), }; int crypto_grab_ahash(struct crypto_ahash_spawn *spawn, diff --git a/crypto/akcipher.c b/crypto/akcipher.c index 72c82d9aa077..a36f50c83827 100644 --- a/crypto/akcipher.c +++ b/crypto/akcipher.c @@ -97,6 +97,7 @@ static const struct crypto_type crypto_akcipher_type = { .maskset = CRYPTO_ALG_TYPE_AHASH_MASK, .type = CRYPTO_ALG_TYPE_AKCIPHER, .tfmsize = offsetof(struct crypto_akcipher, base), + .algsize = offsetof(struct akcipher_alg, base), }; int crypto_grab_akcipher(struct crypto_akcipher_spawn *spawn, diff --git a/crypto/algapi.c b/crypto/algapi.c index f368c0dc0d6d..532d3efc3c7d 100644 --- a/crypto/algapi.c +++ b/crypto/algapi.c @@ -66,13 +66,7 @@ static int crypto_check_alg(struct crypto_alg *alg) static void crypto_free_instance(struct crypto_instance *inst) { - struct crypto_alg *alg = &inst->alg; - const struct crypto_type *type; - - type = alg->cra_type; - if (type->destroy) - type->destroy(alg); - type->free(inst); + inst->alg.cra_type->free(inst); } static void crypto_destroy_instance_workfn(struct work_struct *w) @@ -424,6 +418,15 @@ void crypto_remove_final(struct list_head *list) } EXPORT_SYMBOL_GPL(crypto_remove_final); +static void crypto_free_alg(struct crypto_alg *alg) +{ + unsigned int algsize = alg->cra_type->algsize; + u8 *p = (u8 *)alg - algsize; + + crypto_destroy_alg(alg); + kfree(p); +} + int crypto_register_alg(struct crypto_alg *alg) { struct crypto_larval *larval; @@ -436,6 +439,19 @@ int crypto_register_alg(struct crypto_alg *alg) if (err) return err; + if (alg->cra_flags & CRYPTO_ALG_DUP_FIRST && + !WARN_ON_ONCE(alg->cra_destroy)) { + unsigned int algsize = alg->cra_type->algsize; + u8 *p = (u8 *)alg - algsize; + + p = kmemdup(p, algsize + sizeof(*alg), GFP_KERNEL); + if (!p) + return -ENOMEM; + + alg = (void *)(p + algsize); + alg->cra_destroy = crypto_free_alg; + } + down_write(&crypto_alg_sem); larval = __crypto_register_alg(alg, &algs_to_put); if (!IS_ERR_OR_NULL(larval)) { @@ -444,8 +460,10 @@ int crypto_register_alg(struct crypto_alg *alg) } up_write(&crypto_alg_sem); - if (IS_ERR(larval)) + if (IS_ERR(larval)) { + crypto_alg_put(alg); return PTR_ERR(larval); + } if (test_started) crypto_schedule_test(larval); @@ -481,12 +499,9 @@ void crypto_unregister_alg(struct crypto_alg *alg) if (WARN(ret, "Algorithm %s is not registered", alg->cra_driver_name)) return; - if (alg->cra_destroy) - crypto_alg_put(alg); - else if (!WARN_ON(refcount_read(&alg->cra_refcnt) != 1) && - alg->cra_type && alg->cra_type->destroy) - alg->cra_type->destroy(alg); + WARN_ON(!alg->cra_destroy && refcount_read(&alg->cra_refcnt) != 1); + list_add(&alg->cra_list, &list); crypto_remove_final(&list); } EXPORT_SYMBOL_GPL(crypto_unregister_alg); diff --git a/crypto/api.c b/crypto/api.c index 2880aa04bb99..e427cc5662b5 100644 --- a/crypto/api.c +++ b/crypto/api.c @@ -703,5 +703,14 @@ void crypto_req_done(void *data, int err) } EXPORT_SYMBOL_GPL(crypto_req_done); +void crypto_destroy_alg(struct crypto_alg *alg) +{ + if (alg->cra_type && alg->cra_type->destroy) + alg->cra_type->destroy(alg); + if (alg->cra_destroy) + alg->cra_destroy(alg); +} +EXPORT_SYMBOL_GPL(crypto_destroy_alg); + MODULE_DESCRIPTION("Cryptographic core API"); MODULE_LICENSE("GPL"); diff --git a/crypto/internal.h b/crypto/internal.h index 2edefb546ad4..2ed79bf208ca 100644 --- a/crypto/internal.h +++ b/crypto/internal.h @@ -46,6 +46,7 @@ struct crypto_type { unsigned int maskclear; unsigned int maskset; unsigned int tfmsize; + unsigned int algsize; }; enum { @@ -162,10 +163,12 @@ static inline struct crypto_alg *crypto_alg_get(struct crypto_alg *alg) return alg; } +void crypto_destroy_alg(struct crypto_alg *alg); + static inline void crypto_alg_put(struct crypto_alg *alg) { if (refcount_dec_and_test(&alg->cra_refcnt)) - alg->cra_destroy(alg); + crypto_destroy_alg(alg); } static inline int crypto_tmpl_get(struct crypto_template *tmpl) diff --git a/crypto/kpp.c b/crypto/kpp.c index ecc63a1a948d..2e0cefe7a25f 100644 --- a/crypto/kpp.c +++ b/crypto/kpp.c @@ -80,6 +80,7 @@ static const struct crypto_type crypto_kpp_type = { .maskset = CRYPTO_ALG_TYPE_MASK, .type = CRYPTO_ALG_TYPE_KPP, .tfmsize = offsetof(struct crypto_kpp, base), + .algsize = offsetof(struct kpp_alg, base), }; struct crypto_kpp *crypto_alloc_kpp(const char *alg_name, u32 type, u32 mask) diff --git a/crypto/lskcipher.c b/crypto/lskcipher.c index cdb4897c63e6..c2e2c38b5aa8 100644 --- a/crypto/lskcipher.c +++ b/crypto/lskcipher.c @@ -294,6 +294,7 @@ static const struct crypto_type crypto_lskcipher_type = { .maskset = CRYPTO_ALG_TYPE_MASK, .type = CRYPTO_ALG_TYPE_LSKCIPHER, .tfmsize = offsetof(struct crypto_lskcipher, base), + .algsize = offsetof(struct lskcipher_alg, co.base), }; static void crypto_lskcipher_exit_tfm_sg(struct crypto_tfm *tfm) diff --git a/crypto/rng.c b/crypto/rng.c index 9d8804e46422..b8ae6ebc091d 100644 --- a/crypto/rng.c +++ b/crypto/rng.c @@ -98,6 +98,7 @@ static const struct crypto_type crypto_rng_type = { .maskset = CRYPTO_ALG_TYPE_MASK, .type = CRYPTO_ALG_TYPE_RNG, .tfmsize = offsetof(struct crypto_rng, base), + .algsize = offsetof(struct rng_alg, base), }; struct crypto_rng *crypto_alloc_rng(const char *alg_name, u32 type, u32 mask) diff --git a/crypto/scompress.c b/crypto/scompress.c index c330b81bc5a6..f7ce29b4cdb8 100644 --- a/crypto/scompress.c +++ b/crypto/scompress.c @@ -347,6 +347,7 @@ static const struct crypto_type crypto_scomp_type = { .maskset = CRYPTO_ALG_TYPE_MASK, .type = CRYPTO_ALG_TYPE_SCOMPRESS, .tfmsize = offsetof(struct crypto_scomp, base), + .algsize = offsetof(struct scomp_alg, base), }; static void scomp_prepare_alg(struct scomp_alg *alg) diff --git a/crypto/shash.c b/crypto/shash.c index 301ab42bf849..a2a7d6609172 100644 --- a/crypto/shash.c +++ b/crypto/shash.c @@ -227,6 +227,7 @@ const struct crypto_type crypto_shash_type = { .maskset = CRYPTO_ALG_TYPE_MASK, .type = CRYPTO_ALG_TYPE_SHASH, .tfmsize = offsetof(struct crypto_shash, base), + .algsize = offsetof(struct shash_alg, base), }; int crypto_grab_shash(struct crypto_shash_spawn *spawn, diff --git a/crypto/sig.c b/crypto/sig.c index 53a3dd6fbe3f..beba745b6405 100644 --- a/crypto/sig.c +++ b/crypto/sig.c @@ -74,6 +74,7 @@ static const struct crypto_type crypto_sig_type = { .maskset = CRYPTO_ALG_TYPE_MASK, .type = CRYPTO_ALG_TYPE_SIG, .tfmsize = offsetof(struct crypto_sig, base), + .algsize = offsetof(struct sig_alg, base), }; struct crypto_sig *crypto_alloc_sig(const char *alg_name, u32 type, u32 mask) diff --git a/crypto/skcipher.c b/crypto/skcipher.c index 132075a905d9..319215cfded5 100644 --- a/crypto/skcipher.c +++ b/crypto/skcipher.c @@ -620,6 +620,7 @@ static const struct crypto_type crypto_skcipher_type = { .maskset = CRYPTO_ALG_TYPE_SKCIPHER_MASK, .type = CRYPTO_ALG_TYPE_SKCIPHER, .tfmsize = offsetof(struct crypto_skcipher, base), + .algsize = offsetof(struct skcipher_alg, base), }; int crypto_grab_skcipher(struct crypto_skcipher_spawn *spawn, diff --git a/include/linux/crypto.h b/include/linux/crypto.h index 15476b085ce3..b89b1b348095 100644 --- a/include/linux/crypto.h +++ b/include/linux/crypto.h @@ -49,6 +49,15 @@ */ #define CRYPTO_ALG_NEED_FALLBACK 0x00000100 +/* + * Set if the algorithm data structure should be duplicated into + * kmalloc memory before registration. This is useful for hardware + * that can be disconnected at will. Do not use this if the data + * structure is embedded into a bigger one. Duplicate the overall + * data structure in the driver in that case. + */ +#define CRYPTO_ALG_DUP_FIRST 0x00000200 + /* * Set if the algorithm has passed automated run-time testing. Note that * if there is no run-time testing for a given algorithm it is considered From 34f442a3a08aec3bd993dda89c0fdc8447ce4fee Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sat, 12 Apr 2025 18:47:40 +0800 Subject: [PATCH 119/382] crypto: padlock-sha - Make descsize an algorithm attribute Rather than setting descsize in init_tfm, set it statically and double-check it in init_tfm. Signed-off-by: Herbert Xu --- drivers/crypto/padlock-sha.c | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/drivers/crypto/padlock-sha.c b/drivers/crypto/padlock-sha.c index db9e84c0c9fb..466493907d48 100644 --- a/drivers/crypto/padlock-sha.c +++ b/drivers/crypto/padlock-sha.c @@ -206,8 +206,14 @@ static int padlock_init_tfm(struct crypto_shash *hash) return PTR_ERR(fallback_tfm); } + if (crypto_shash_descsize(hash) < sizeof(struct padlock_sha_desc) + + crypto_shash_descsize(fallback_tfm)) { + crypto_free_shash(fallback_tfm); + return -EINVAL; + } + ctx->fallback = fallback_tfm; - hash->descsize += crypto_shash_descsize(fallback_tfm); + return 0; } @@ -228,7 +234,8 @@ static struct shash_alg sha1_alg = { .import = padlock_sha_import, .init_tfm = padlock_init_tfm, .exit_tfm = padlock_exit_tfm, - .descsize = sizeof(struct padlock_sha_desc), + .descsize = sizeof(struct padlock_sha_desc) + + sizeof(struct sha1_state), .statesize = sizeof(struct sha1_state), .base = { .cra_name = "sha1", @@ -251,7 +258,8 @@ static struct shash_alg sha256_alg = { .import = padlock_sha_import, .init_tfm = padlock_init_tfm, .exit_tfm = padlock_exit_tfm, - .descsize = sizeof(struct padlock_sha_desc), + .descsize = sizeof(struct padlock_sha_desc) + + sizeof(struct sha256_state), .statesize = sizeof(struct sha256_state), .base = { .cra_name = "sha256", From c1dd353d18e53b39dffe5d8fce948f959ac73bbc Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sat, 12 Apr 2025 18:47:42 +0800 Subject: [PATCH 120/382] crypto: zynqmp-sha - Make descsize an algorithm attribute Rather than setting descsize in init_tfm, set it statically and double-check it in init_tfm. Signed-off-by: Herbert Xu --- drivers/crypto/xilinx/zynqmp-sha.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/xilinx/zynqmp-sha.c b/drivers/crypto/xilinx/zynqmp-sha.c index 580649f9bff8..f66e0a4c1169 100644 --- a/drivers/crypto/xilinx/zynqmp-sha.c +++ b/drivers/crypto/xilinx/zynqmp-sha.c @@ -60,8 +60,14 @@ static int zynqmp_sha_init_tfm(struct crypto_shash *hash) if (IS_ERR(fallback_tfm)) return PTR_ERR(fallback_tfm); + if (crypto_shash_descsize(hash) < + sizeof(struct zynqmp_sha_desc_ctx) + + crypto_shash_descsize(tfm_ctx->fbk_tfm)) { + crypto_free_shash(fallback_tfm); + return -EINVAL; + } + tfm_ctx->fbk_tfm = fallback_tfm; - hash->descsize += crypto_shash_descsize(tfm_ctx->fbk_tfm); return 0; } @@ -170,7 +176,8 @@ static struct zynqmp_sha_drv_ctx sha3_drv_ctx = { .import = zynqmp_sha_import, .init_tfm = zynqmp_sha_init_tfm, .exit_tfm = zynqmp_sha_exit_tfm, - .descsize = sizeof(struct zynqmp_sha_desc_ctx), + .descsize = sizeof(struct zynqmp_sha_desc_ctx) + + sizeof(struct sha3_state), .statesize = sizeof(struct sha3_state), .digestsize = SHA3_384_DIGEST_SIZE, .base = { From aeffd9093820c16e10f61d1c894c16d435b82975 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sat, 12 Apr 2025 18:47:45 +0800 Subject: [PATCH 121/382] crypto: hmac - Make descsize an algorithm attribute Rather than setting descsize in init_tfm, make it an algorithm attribute and set it during instance construction. Signed-off-by: Herbert Xu --- crypto/hmac.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/crypto/hmac.c b/crypto/hmac.c index 7cec25ff9889..dfb153511865 100644 --- a/crypto/hmac.c +++ b/crypto/hmac.c @@ -146,9 +146,6 @@ static int hmac_init_tfm(struct crypto_shash *parent) if (IS_ERR(hash)) return PTR_ERR(hash); - parent->descsize = sizeof(struct shash_desc) + - crypto_shash_descsize(hash); - tctx->hash = hash; return 0; } @@ -222,6 +219,7 @@ static int hmac_create(struct crypto_template *tmpl, struct rtattr **tb) inst->alg.digestsize = ds; inst->alg.statesize = ss; + inst->alg.descsize = sizeof(struct shash_desc) + salg->descsize; inst->alg.init = hmac_init; inst->alg.update = hmac_update; inst->alg.final = hmac_final; From 90916934fd093edf62bc0c5c9a940a8efa7db2f8 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sat, 12 Apr 2025 18:47:47 +0800 Subject: [PATCH 122/382] crypto: shash - Remove dynamic descsize As all users of the dynamic descsize have been converted to use a static one instead, remove support for dynamic descsize. Signed-off-by: Herbert Xu --- crypto/shash.c | 18 +----------------- include/crypto/hash.h | 3 +-- 2 files changed, 2 insertions(+), 19 deletions(-) diff --git a/crypto/shash.c b/crypto/shash.c index a2a7d6609172..f23bd9cb1873 100644 --- a/crypto/shash.c +++ b/crypto/shash.c @@ -153,9 +153,6 @@ static int crypto_shash_init_tfm(struct crypto_tfm *tfm) { struct crypto_shash *hash = __crypto_shash_cast(tfm); struct shash_alg *alg = crypto_shash_alg(hash); - int err; - - hash->descsize = alg->descsize; shash_set_needkey(hash, alg); @@ -165,18 +162,7 @@ static int crypto_shash_init_tfm(struct crypto_tfm *tfm) if (!alg->init_tfm) return 0; - err = alg->init_tfm(hash); - if (err) - return err; - - /* ->init_tfm() may have increased the descsize. */ - if (WARN_ON_ONCE(hash->descsize > HASH_MAX_DESCSIZE)) { - if (alg->exit_tfm) - alg->exit_tfm(hash); - return -EINVAL; - } - - return 0; + return alg->init_tfm(hash); } static void crypto_shash_free_instance(struct crypto_instance *inst) @@ -274,8 +260,6 @@ struct crypto_shash *crypto_clone_shash(struct crypto_shash *hash) if (IS_ERR(nhash)) return nhash; - nhash->descsize = hash->descsize; - if (alg->clone_tfm) { err = alg->clone_tfm(nhash, hash); if (err) { diff --git a/include/crypto/hash.h b/include/crypto/hash.h index eceb2ed04f26..87518cf3b2d8 100644 --- a/include/crypto/hash.h +++ b/include/crypto/hash.h @@ -235,7 +235,6 @@ struct crypto_ahash { }; struct crypto_shash { - unsigned int descsize; struct crypto_tfm base; }; @@ -810,7 +809,7 @@ static inline void crypto_shash_clear_flags(struct crypto_shash *tfm, u32 flags) */ static inline unsigned int crypto_shash_descsize(struct crypto_shash *tfm) { - return tfm->descsize; + return crypto_shash_alg(tfm)->descsize; } static inline void *shash_desc_ctx(struct shash_desc *desc) From 04bfa4c7d5119ca38f8133bfdae7957a60c8b221 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sat, 12 Apr 2025 18:57:19 +0800 Subject: [PATCH 123/382] crypto: hash - Add HASH_REQUEST_ON_STACK Allow any ahash to be used with a stack request, with optional dynamic allocation when async is needed. The intended usage is: HASH_REQUEST_ON_STACK(req, tfm); ... err = crypto_ahash_digest(req); /* The request cannot complete synchronously. */ if (err == -EAGAIN) { /* This will not fail. */ req = HASH_REQUEST_CLONE(req, gfp); /* Redo operation. */ err = crypto_ahash_digest(req); } Signed-off-by: Herbert Xu --- crypto/ahash.c | 106 +++++++++++++++++++++++++++++++-- include/crypto/hash.h | 54 +++++++++++++---- include/crypto/internal/hash.h | 26 +++++++- 3 files changed, 171 insertions(+), 15 deletions(-) diff --git a/crypto/ahash.c b/crypto/ahash.c index 7c9c0931197f..7a74092323b9 100644 --- a/crypto/ahash.c +++ b/crypto/ahash.c @@ -300,6 +300,8 @@ int crypto_ahash_setkey(struct crypto_ahash *tfm, const u8 *key, int err; err = alg->setkey(tfm, key, keylen); + if (!err && ahash_is_async(tfm)) + err = crypto_ahash_setkey(tfm->fb, key, keylen); if (unlikely(err)) { ahash_set_needkey(tfm, alg); return err; @@ -473,6 +475,8 @@ int crypto_ahash_init(struct ahash_request *req) return crypto_shash_init(prepare_shash_desc(req, tfm)); if (crypto_ahash_get_flags(tfm) & CRYPTO_TFM_NEED_KEY) return -ENOKEY; + if (ahash_req_on_stack(req) && ahash_is_async(tfm)) + return -EAGAIN; return ahash_do_req_chain(req, crypto_ahash_alg(tfm)->init); } EXPORT_SYMBOL_GPL(crypto_ahash_init); @@ -520,6 +524,8 @@ int crypto_ahash_update(struct ahash_request *req) if (likely(tfm->using_shash)) return shash_ahash_update(req, ahash_request_ctx(req)); + if (ahash_req_on_stack(req) && ahash_is_async(tfm)) + return -EAGAIN; return ahash_do_req_chain(req, crypto_ahash_alg(tfm)->update); } EXPORT_SYMBOL_GPL(crypto_ahash_update); @@ -530,6 +536,8 @@ int crypto_ahash_final(struct ahash_request *req) if (likely(tfm->using_shash)) return crypto_shash_final(ahash_request_ctx(req), req->result); + if (ahash_req_on_stack(req) && ahash_is_async(tfm)) + return -EAGAIN; return ahash_do_req_chain(req, crypto_ahash_alg(tfm)->final); } EXPORT_SYMBOL_GPL(crypto_ahash_final); @@ -540,6 +548,8 @@ int crypto_ahash_finup(struct ahash_request *req) if (likely(tfm->using_shash)) return shash_ahash_finup(req, ahash_request_ctx(req)); + if (ahash_req_on_stack(req) && ahash_is_async(tfm)) + return -EAGAIN; if (!crypto_ahash_alg(tfm)->finup || (!crypto_ahash_req_chain(tfm) && ahash_request_isvirt(req))) return ahash_def_finup(req); @@ -611,6 +621,8 @@ int crypto_ahash_digest(struct ahash_request *req) if (likely(tfm->using_shash)) return shash_ahash_digest(req, prepare_shash_desc(req, tfm)); + if (ahash_req_on_stack(req) && ahash_is_async(tfm)) + return -EAGAIN; if (!crypto_ahash_req_chain(tfm) && ahash_request_isvirt(req)) return ahash_def_digest(req); if (crypto_ahash_get_flags(tfm) & CRYPTO_TFM_NEED_KEY) @@ -714,26 +726,63 @@ static void crypto_ahash_exit_tfm(struct crypto_tfm *tfm) struct crypto_ahash *hash = __crypto_ahash_cast(tfm); struct ahash_alg *alg = crypto_ahash_alg(hash); - alg->exit_tfm(hash); + if (alg->exit_tfm) + alg->exit_tfm(hash); + else if (tfm->__crt_alg->cra_exit) + tfm->__crt_alg->cra_exit(tfm); + + if (ahash_is_async(hash)) + crypto_free_ahash(hash->fb); } static int crypto_ahash_init_tfm(struct crypto_tfm *tfm) { struct crypto_ahash *hash = __crypto_ahash_cast(tfm); struct ahash_alg *alg = crypto_ahash_alg(hash); + struct crypto_ahash *fb = NULL; + int err; crypto_ahash_set_statesize(hash, alg->halg.statesize); crypto_ahash_set_reqsize(hash, crypto_tfm_alg_reqsize(tfm)); + hash->fb = hash; + if (tfm->__crt_alg->cra_type == &crypto_shash_type) return crypto_init_ahash_using_shash(tfm); + if (ahash_is_async(hash)) { + fb = crypto_alloc_ahash(crypto_ahash_alg_name(hash), + 0, CRYPTO_ALG_ASYNC); + if (IS_ERR(fb)) + return PTR_ERR(fb); + + hash->fb = fb; + } + ahash_set_needkey(hash, alg); - if (alg->exit_tfm) - tfm->exit = crypto_ahash_exit_tfm; + tfm->exit = crypto_ahash_exit_tfm; - return alg->init_tfm ? alg->init_tfm(hash) : 0; + if (!alg->init_tfm) { + if (!tfm->__crt_alg->cra_init) + return 0; + + err = tfm->__crt_alg->cra_init(tfm); + if (err) + goto out_free_sync_hash; + + return 0; + } + + err = alg->init_tfm(hash); + if (err) + goto out_free_sync_hash; + + return 0; + +out_free_sync_hash: + crypto_free_ahash(fb); + return err; } static unsigned int crypto_ahash_extsize(struct crypto_alg *alg) @@ -970,5 +1019,54 @@ int ahash_register_instance(struct crypto_template *tmpl, } EXPORT_SYMBOL_GPL(ahash_register_instance); +void ahash_request_free(struct ahash_request *req) +{ + if (unlikely(!req)) + return; + + if (!ahash_req_on_stack(req)) { + kfree(req); + return; + } + + ahash_request_zero(req); +} +EXPORT_SYMBOL_GPL(ahash_request_free); + +int crypto_hash_digest(struct crypto_ahash *tfm, const u8 *data, + unsigned int len, u8 *out) +{ + HASH_REQUEST_ON_STACK(req, tfm->fb); + int err; + + ahash_request_set_callback(req, 0, NULL, NULL); + ahash_request_set_virt(req, data, out, len); + err = crypto_ahash_digest(req); + + ahash_request_zero(req); + + return err; +} +EXPORT_SYMBOL_GPL(crypto_hash_digest); + +struct ahash_request *ahash_request_clone(struct ahash_request *req, + size_t total, gfp_t gfp) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct ahash_request *nreq; + + nreq = kmalloc(total, gfp); + if (!nreq) { + ahash_request_set_tfm(req, tfm->fb); + req->base.flags = CRYPTO_TFM_REQ_ON_STACK; + return req; + } + + memcpy(nreq, req, total); + ahash_request_set_tfm(req, tfm); + return req; +} +EXPORT_SYMBOL_GPL(ahash_request_clone); + MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Asynchronous cryptographic hash type"); diff --git a/include/crypto/hash.h b/include/crypto/hash.h index 87518cf3b2d8..b00ea8407f9c 100644 --- a/include/crypto/hash.h +++ b/include/crypto/hash.h @@ -16,6 +16,9 @@ /* Set this bit for virtual address instead of SG list. */ #define CRYPTO_AHASH_REQ_VIRT 0x00000001 +#define CRYPTO_AHASH_REQ_PRIVATE \ + CRYPTO_AHASH_REQ_VIRT + struct crypto_ahash; /** @@ -167,12 +170,22 @@ struct shash_desc { * containing a 'struct sha3_state'. */ #define HASH_MAX_DESCSIZE (sizeof(struct shash_desc) + 360) +#define MAX_SYNC_HASH_REQSIZE HASH_MAX_DESCSIZE #define SHASH_DESC_ON_STACK(shash, ctx) \ char __##shash##_desc[sizeof(struct shash_desc) + HASH_MAX_DESCSIZE] \ __aligned(__alignof__(struct shash_desc)); \ struct shash_desc *shash = (struct shash_desc *)__##shash##_desc +#define HASH_REQUEST_ON_STACK(name, _tfm) \ + char __##name##_req[sizeof(struct ahash_request) + \ + MAX_SYNC_HASH_REQSIZE] CRYPTO_MINALIGN_ATTR; \ + struct ahash_request *name = \ + ahash_request_on_stack_init(__##name##_req, (_tfm)) + +#define HASH_REQUEST_CLONE(name, gfp) \ + hash_request_clone(name, sizeof(__##name##_req), gfp) + /** * struct shash_alg - synchronous message digest definition * @init: see struct ahash_alg @@ -231,6 +244,7 @@ struct crypto_ahash { bool using_shash; /* Underlying algorithm is shash, not ahash */ unsigned int statesize; unsigned int reqsize; + struct crypto_ahash *fb; struct crypto_tfm base; }; @@ -248,6 +262,11 @@ struct crypto_shash { * CRYPTO_ALG_TYPE_SKCIPHER API applies here as well. */ +static inline bool ahash_req_on_stack(struct ahash_request *req) +{ + return crypto_req_on_stack(&req->base); +} + static inline struct crypto_ahash *__crypto_ahash_cast(struct crypto_tfm *tfm) { return container_of(tfm, struct crypto_ahash, base); @@ -544,7 +563,7 @@ int crypto_ahash_update(struct ahash_request *req); static inline void ahash_request_set_tfm(struct ahash_request *req, struct crypto_ahash *tfm) { - req->base.tfm = crypto_ahash_tfm(tfm); + crypto_request_set_tfm(&req->base, crypto_ahash_tfm(tfm)); } /** @@ -578,9 +597,12 @@ static inline struct ahash_request *ahash_request_alloc_noprof( * ahash_request_free() - zeroize and free the request data structure * @req: request data structure cipher handle to be freed */ -static inline void ahash_request_free(struct ahash_request *req) +void ahash_request_free(struct ahash_request *req); + +static inline void ahash_request_zero(struct ahash_request *req) { - kfree_sensitive(req); + memzero_explicit(req, sizeof(*req) + + crypto_ahash_reqsize(crypto_ahash_reqtfm(req))); } static inline struct ahash_request *ahash_request_cast( @@ -619,13 +641,9 @@ static inline void ahash_request_set_callback(struct ahash_request *req, crypto_completion_t compl, void *data) { - u32 keep = CRYPTO_AHASH_REQ_VIRT; - - req->base.complete = compl; - req->base.data = data; - flags &= ~keep; - req->base.flags &= keep; - req->base.flags |= flags; + flags &= ~CRYPTO_AHASH_REQ_PRIVATE; + flags |= req->base.flags & CRYPTO_AHASH_REQ_PRIVATE; + crypto_request_set_callback(&req->base, flags, compl, data); } /** @@ -870,6 +888,9 @@ int crypto_shash_digest(struct shash_desc *desc, const u8 *data, int crypto_shash_tfm_digest(struct crypto_shash *tfm, const u8 *data, unsigned int len, u8 *out); +int crypto_hash_digest(struct crypto_ahash *tfm, const u8 *data, + unsigned int len, u8 *out); + /** * crypto_shash_export() - extract operational state for message digest * @desc: reference to the operational state handle whose state is exported @@ -980,4 +1001,17 @@ static inline bool ahash_is_async(struct crypto_ahash *tfm) return crypto_tfm_is_async(&tfm->base); } +static inline struct ahash_request *ahash_request_on_stack_init( + char *buf, struct crypto_ahash *tfm) +{ + struct ahash_request *req = (void *)buf; + + ahash_request_set_tfm(req, tfm); + req->base.flags = CRYPTO_TFM_REQ_ON_STACK; + return req; +} + +struct ahash_request *ahash_request_clone(struct ahash_request *req, + size_t total, gfp_t gfp); + #endif /* _CRYPTO_HASH_H */ diff --git a/include/crypto/internal/hash.h b/include/crypto/internal/hash.h index e2a1fac38610..45c728ac2621 100644 --- a/include/crypto/internal/hash.h +++ b/include/crypto/internal/hash.h @@ -11,6 +11,12 @@ #include #include +#define HASH_FBREQ_ON_STACK(name, req) \ + char __##name##_req[sizeof(struct ahash_request) + \ + MAX_SYNC_HASH_REQSIZE] CRYPTO_MINALIGN_ATTR; \ + struct ahash_request *name = ahash_fbreq_on_stack_init( \ + __##name##_req, (req)) + struct ahash_request; struct ahash_instance { @@ -187,7 +193,7 @@ static inline void ahash_request_complete(struct ahash_request *req, int err) static inline u32 ahash_request_flags(struct ahash_request *req) { - return req->base.flags; + return crypto_request_flags(&req->base) & ~CRYPTO_AHASH_REQ_PRIVATE; } static inline struct crypto_ahash *crypto_spawn_ahash( @@ -257,5 +263,23 @@ static inline bool crypto_ahash_req_chain(struct crypto_ahash *tfm) return crypto_tfm_req_chain(&tfm->base); } +static inline struct ahash_request *ahash_fbreq_on_stack_init( + char *buf, struct ahash_request *old) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(old); + struct ahash_request *req = (void *)buf; + + ahash_request_set_tfm(req, tfm->fb); + req->base.flags = CRYPTO_TFM_REQ_ON_STACK; + ahash_request_set_callback(req, ahash_request_flags(old), NULL, NULL); + req->base.flags &= ~CRYPTO_AHASH_REQ_PRIVATE; + req->base.flags |= old->base.flags & CRYPTO_AHASH_REQ_PRIVATE; + req->src = old->src; + req->result = old->result; + req->nbytes = old->nbytes; + + return req; +} + #endif /* _CRYPTO_INTERNAL_HASH_H */ From c7e68043620e0d5f89a37e573c667beab72d2937 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sat, 12 Apr 2025 18:57:22 +0800 Subject: [PATCH 124/382] crypto: zynqmp-sha - Add locking The hardwrae is only capable of one hash at a time, so add a lock to make sure that it isn't used concurrently. Fixes: 7ecc3e34474b ("crypto: xilinx - Add Xilinx SHA3 driver") Signed-off-by: Herbert Xu --- drivers/crypto/xilinx/zynqmp-sha.c | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/drivers/crypto/xilinx/zynqmp-sha.c b/drivers/crypto/xilinx/zynqmp-sha.c index f66e0a4c1169..4db8c717906f 100644 --- a/drivers/crypto/xilinx/zynqmp-sha.c +++ b/drivers/crypto/xilinx/zynqmp-sha.c @@ -3,18 +3,19 @@ * Xilinx ZynqMP SHA Driver. * Copyright (c) 2022 Xilinx Inc. */ -#include #include #include #include -#include +#include +#include #include #include +#include #include -#include #include #include #include +#include #include #define ZYNQMP_DMA_BIT_MASK 32U @@ -43,6 +44,8 @@ struct zynqmp_sha_desc_ctx { static dma_addr_t update_dma_addr, final_dma_addr; static char *ubuf, *fbuf; +static DEFINE_SPINLOCK(zynqmp_sha_lock); + static int zynqmp_sha_init_tfm(struct crypto_shash *hash) { const char *fallback_driver_name = crypto_shash_alg_name(hash); @@ -130,7 +133,8 @@ static int zynqmp_sha_export(struct shash_desc *desc, void *out) return crypto_shash_export(&dctx->fbk_req, out); } -static int zynqmp_sha_digest(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) +static int __zynqmp_sha_digest(struct shash_desc *desc, const u8 *data, + unsigned int len, u8 *out) { unsigned int remaining_len = len; int update_size; @@ -165,6 +169,12 @@ static int zynqmp_sha_digest(struct shash_desc *desc, const u8 *data, unsigned i return ret; } +static int zynqmp_sha_digest(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) +{ + scoped_guard(spinlock_bh, &zynqmp_sha_lock) + return __zynqmp_sha_digest(desc, data, len, out); +} + static struct zynqmp_sha_drv_ctx sha3_drv_ctx = { .sha3_384 = { .init = zynqmp_sha_init, From 16aeed07c0debede447ceb9cae6ad047ab654ccf Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sat, 12 Apr 2025 18:57:24 +0800 Subject: [PATCH 125/382] crypto: arm64/sha512 - Fix header inclusions Instead of relying on linux/module.h being included through the header file sha512_base.h, include it directly. Signed-off-by: Herbert Xu --- arch/arm64/crypto/sha512-glue.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/arch/arm64/crypto/sha512-glue.c b/arch/arm64/crypto/sha512-glue.c index 62f129dea83d..f789deabefc0 100644 --- a/arch/arm64/crypto/sha512-glue.c +++ b/arch/arm64/crypto/sha512-glue.c @@ -6,11 +6,10 @@ */ #include -#include -#include +#include +#include #include #include -#include MODULE_DESCRIPTION("SHA-384/SHA-512 secure hash for arm64"); MODULE_AUTHOR("Andy Polyakov "); From a58d1c0be43b9f21da16471f9786d5d1fc3a2aa7 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sat, 12 Apr 2025 18:57:26 +0800 Subject: [PATCH 126/382] crypto: hash - Update HASH_MAX_DESCSIZE comment The biggest context is not sha3_generic (356), but sha-s390 (360). Signed-off-by: Herbert Xu --- include/crypto/hash.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/crypto/hash.h b/include/crypto/hash.h index b00ea8407f9c..58ac1423dc38 100644 --- a/include/crypto/hash.h +++ b/include/crypto/hash.h @@ -166,8 +166,8 @@ struct shash_desc { #define HASH_MAX_DIGESTSIZE 64 /* - * Worst case is hmac(sha3-224-generic). Its context is a nested 'shash_desc' - * containing a 'struct sha3_state'. + * Worst case is hmac(sha-224-s390). Its context is a nested 'shash_desc' + * containing a 'struct s390_sha_ctx'. */ #define HASH_MAX_DESCSIZE (sizeof(struct shash_desc) + 360) #define MAX_SYNC_HASH_REQSIZE HASH_MAX_DESCSIZE From f4065b2f63cfc220a852c2c2e760753e5cae0d54 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sat, 12 Apr 2025 18:57:29 +0800 Subject: [PATCH 127/382] crypto: lib/sm3 - Move sm3 library into lib/crypto Move the sm3 library code into lib/crypto. Signed-off-by: Herbert Xu --- arch/arm64/crypto/Kconfig | 4 ++-- arch/riscv/crypto/Kconfig | 2 +- arch/x86/crypto/Kconfig | 2 +- crypto/Kconfig | 5 +---- crypto/Makefile | 1 - lib/crypto/Kconfig | 3 +++ lib/crypto/Makefile | 3 +++ {crypto => lib/crypto}/sm3.c | 0 8 files changed, 11 insertions(+), 9 deletions(-) rename {crypto => lib/crypto}/sm3.c (100%) diff --git a/arch/arm64/crypto/Kconfig b/arch/arm64/crypto/Kconfig index ce655da0fbee..9dd1ae7a68a4 100644 --- a/arch/arm64/crypto/Kconfig +++ b/arch/arm64/crypto/Kconfig @@ -101,7 +101,7 @@ config CRYPTO_SM3_NEON tristate "Hash functions: SM3 (NEON)" depends on KERNEL_MODE_NEON select CRYPTO_HASH - select CRYPTO_SM3 + select CRYPTO_LIB_SM3 help SM3 (ShangMi 3) secure hash function (OSCCA GM/T 0004-2012) @@ -112,7 +112,7 @@ config CRYPTO_SM3_ARM64_CE tristate "Hash functions: SM3 (ARMv8.2 Crypto Extensions)" depends on KERNEL_MODE_NEON select CRYPTO_HASH - select CRYPTO_SM3 + select CRYPTO_LIB_SM3 help SM3 (ShangMi 3) secure hash function (OSCCA GM/T 0004-2012) diff --git a/arch/riscv/crypto/Kconfig b/arch/riscv/crypto/Kconfig index 6392e1e11bc9..27a1f26d41bd 100644 --- a/arch/riscv/crypto/Kconfig +++ b/arch/riscv/crypto/Kconfig @@ -61,7 +61,7 @@ config CRYPTO_SM3_RISCV64 tristate "Hash functions: SM3 (ShangMi 3)" depends on 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO select CRYPTO_HASH - select CRYPTO_SM3 + select CRYPTO_LIB_SM3 help SM3 (ShangMi 3) secure hash function (OSCCA GM/T 0004-2012) diff --git a/arch/x86/crypto/Kconfig b/arch/x86/crypto/Kconfig index 0cec75926380..4e0bd1258751 100644 --- a/arch/x86/crypto/Kconfig +++ b/arch/x86/crypto/Kconfig @@ -455,7 +455,7 @@ config CRYPTO_SM3_AVX_X86_64 tristate "Hash functions: SM3 (AVX)" depends on X86 && 64BIT select CRYPTO_HASH - select CRYPTO_SM3 + select CRYPTO_LIB_SM3 help SM3 secure hash function as defined by OSCCA GM/T 0004-2012 SM3 diff --git a/crypto/Kconfig b/crypto/Kconfig index dbf97c4e7c59..9322e42e562d 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -1012,13 +1012,10 @@ config CRYPTO_SHA3 help SHA-3 secure hash algorithms (FIPS 202, ISO/IEC 10118-3) -config CRYPTO_SM3 - tristate - config CRYPTO_SM3_GENERIC tristate "SM3 (ShangMi 3)" select CRYPTO_HASH - select CRYPTO_SM3 + select CRYPTO_LIB_SM3 help SM3 (ShangMi 3) secure hash function (OSCCA GM/T 0004-2012, ISO/IEC 10118-3) diff --git a/crypto/Makefile b/crypto/Makefile index 98510a2aa0b1..baf5040ca661 100644 --- a/crypto/Makefile +++ b/crypto/Makefile @@ -79,7 +79,6 @@ obj-$(CONFIG_CRYPTO_SHA1) += sha1_generic.o obj-$(CONFIG_CRYPTO_SHA256) += sha256_generic.o obj-$(CONFIG_CRYPTO_SHA512) += sha512_generic.o obj-$(CONFIG_CRYPTO_SHA3) += sha3_generic.o -obj-$(CONFIG_CRYPTO_SM3) += sm3.o obj-$(CONFIG_CRYPTO_SM3_GENERIC) += sm3_generic.o obj-$(CONFIG_CRYPTO_STREEBOG) += streebog_generic.o obj-$(CONFIG_CRYPTO_WP512) += wp512.o diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index 798972b29b68..2c6ab80e0cdc 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -152,4 +152,7 @@ config CRYPTO_LIB_SHA1 config CRYPTO_LIB_SHA256 tristate +config CRYPTO_LIB_SM3 + tristate + endmenu diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index 01fac1cd05a1..4dd62bc5bee3 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -60,3 +60,6 @@ endif obj-$(CONFIG_MPILIB) += mpi/ obj-$(CONFIG_CRYPTO_MANAGER_EXTRA_TESTS) += simd.o + +obj-$(CONFIG_CRYPTO_LIB_SM3) += libsm3.o +libsm3-y := sm3.o diff --git a/crypto/sm3.c b/lib/crypto/sm3.c similarity index 100% rename from crypto/sm3.c rename to lib/crypto/sm3.c From cb16ba46958e41f8d805cbdb027a601b3a0c65b4 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sat, 12 Apr 2025 18:57:31 +0800 Subject: [PATCH 128/382] crypto: lib/sm3 - Export generic block function Export the generic block function so that it can be used by the Crypto API. Signed-off-by: Herbert Xu --- include/crypto/sm3.h | 1 + lib/crypto/sm3.c | 25 ++++++++++++------------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/include/crypto/sm3.h b/include/crypto/sm3.h index 1f021ad0533f..d49211ba9a20 100644 --- a/include/crypto/sm3.h +++ b/include/crypto/sm3.h @@ -58,6 +58,7 @@ static inline void sm3_init(struct sm3_state *sctx) sctx->count = 0; } +void sm3_block_generic(struct sm3_state *sctx, u8 const *data, int blocks); void sm3_update(struct sm3_state *sctx, const u8 *data, unsigned int len); void sm3_final(struct sm3_state *sctx, u8 *out); diff --git a/lib/crypto/sm3.c b/lib/crypto/sm3.c index 18c2fb73ba16..de64aa913280 100644 --- a/lib/crypto/sm3.c +++ b/lib/crypto/sm3.c @@ -166,19 +166,22 @@ static void sm3_transform(struct sm3_state *sctx, u8 const *data, u32 W[16]) #undef W1 #undef W2 -static inline void sm3_block(struct sm3_state *sctx, - u8 const *data, int blocks, u32 W[16]) +void sm3_block_generic(struct sm3_state *sctx, u8 const *data, int blocks) { - while (blocks--) { + u32 W[16]; + + do { sm3_transform(sctx, data, W); data += SM3_BLOCK_SIZE; - } + } while (--blocks); + + memzero_explicit(W, sizeof(W)); } +EXPORT_SYMBOL_GPL(sm3_block_generic); void sm3_update(struct sm3_state *sctx, const u8 *data, unsigned int len) { unsigned int partial = sctx->count % SM3_BLOCK_SIZE; - u32 W[16]; sctx->count += len; @@ -192,19 +195,17 @@ void sm3_update(struct sm3_state *sctx, const u8 *data, unsigned int len) data += p; len -= p; - sm3_block(sctx, sctx->buffer, 1, W); + sm3_block_generic(sctx, sctx->buffer, 1); } blocks = len / SM3_BLOCK_SIZE; len %= SM3_BLOCK_SIZE; if (blocks) { - sm3_block(sctx, data, blocks, W); + sm3_block_generic(sctx, data, blocks); data += blocks * SM3_BLOCK_SIZE; } - memzero_explicit(W, sizeof(W)); - partial = 0; } if (len) @@ -218,7 +219,6 @@ void sm3_final(struct sm3_state *sctx, u8 *out) __be64 *bits = (__be64 *)(sctx->buffer + bit_offset); __be32 *digest = (__be32 *)out; unsigned int partial = sctx->count % SM3_BLOCK_SIZE; - u32 W[16]; int i; sctx->buffer[partial++] = 0x80; @@ -226,18 +226,17 @@ void sm3_final(struct sm3_state *sctx, u8 *out) memset(sctx->buffer + partial, 0, SM3_BLOCK_SIZE - partial); partial = 0; - sm3_block(sctx, sctx->buffer, 1, W); + sm3_block_generic(sctx, sctx->buffer, 1); } memset(sctx->buffer + partial, 0, bit_offset - partial); *bits = cpu_to_be64(sctx->count << 3); - sm3_block(sctx, sctx->buffer, 1, W); + sm3_block_generic(sctx, sctx->buffer, 1); for (i = 0; i < 8; i++) put_unaligned_be32(sctx->state[i], digest++); /* Zeroize sensitive information. */ - memzero_explicit(W, sizeof(W)); memzero_explicit(sctx, sizeof(*sctx)); } EXPORT_SYMBOL_GPL(sm3_final); From e13b67e9d02829620c1c57d1b008e9c7ef954d19 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sat, 12 Apr 2025 18:57:33 +0800 Subject: [PATCH 129/382] crypto: sm3-base - Use sm3_init Remove the duplicate init code and simply call sm3_init. Signed-off-by: Herbert Xu --- include/crypto/sm3_base.h | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/include/crypto/sm3_base.h b/include/crypto/sm3_base.h index b33ed39c2bce..835896228e8e 100644 --- a/include/crypto/sm3_base.h +++ b/include/crypto/sm3_base.h @@ -20,18 +20,7 @@ typedef void (sm3_block_fn)(struct sm3_state *sst, u8 const *src, int blocks); static inline int sm3_base_init(struct shash_desc *desc) { - struct sm3_state *sctx = shash_desc_ctx(desc); - - sctx->state[0] = SM3_IVA; - sctx->state[1] = SM3_IVB; - sctx->state[2] = SM3_IVC; - sctx->state[3] = SM3_IVD; - sctx->state[4] = SM3_IVE; - sctx->state[5] = SM3_IVF; - sctx->state[6] = SM3_IVG; - sctx->state[7] = SM3_IVH; - sctx->count = 0; - + sm3_init(shash_desc_ctx(desc)); return 0; } From 0a1376744caa553eaf073b9f71d3afd1733c0b12 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sat, 12 Apr 2025 18:57:36 +0800 Subject: [PATCH 130/382] crypto: cbcmac - Set block size properly The block size of a hash algorithm is meant to be the number of bytes its block function can handle. For cbcmac that should be the block size of the underlying block cipher instead of one. Set the block size of all cbcmac implementations accordingly. Signed-off-by: Herbert Xu --- arch/arm64/crypto/aes-glue.c | 2 +- arch/arm64/crypto/sm4-ce-glue.c | 2 +- crypto/ccm.c | 2 +- drivers/crypto/inside-secure/safexcel_hash.c | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/arch/arm64/crypto/aes-glue.c b/arch/arm64/crypto/aes-glue.c index b0150999743f..5ca3b5661749 100644 --- a/arch/arm64/crypto/aes-glue.c +++ b/arch/arm64/crypto/aes-glue.c @@ -1009,7 +1009,7 @@ static struct shash_alg mac_algs[] = { { .base.cra_name = "cbcmac(aes)", .base.cra_driver_name = "cbcmac-aes-" MODE, .base.cra_priority = PRIO, - .base.cra_blocksize = 1, + .base.cra_blocksize = AES_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct mac_tfm_ctx), .base.cra_module = THIS_MODULE, diff --git a/arch/arm64/crypto/sm4-ce-glue.c b/arch/arm64/crypto/sm4-ce-glue.c index 43741bed874e..f11cf26e5a20 100644 --- a/arch/arm64/crypto/sm4-ce-glue.c +++ b/arch/arm64/crypto/sm4-ce-glue.c @@ -723,7 +723,7 @@ static struct shash_alg sm4_mac_algs[] = { .cra_name = "cbcmac(sm4)", .cra_driver_name = "cbcmac-sm4-ce", .cra_priority = 400, - .cra_blocksize = 1, + .cra_blocksize = SM4_BLOCK_SIZE, .cra_ctxsize = sizeof(struct sm4_mac_tfm_ctx), .cra_module = THIS_MODULE, }, diff --git a/crypto/ccm.c b/crypto/ccm.c index 06476b53b491..a0610ff6ce02 100644 --- a/crypto/ccm.c +++ b/crypto/ccm.c @@ -883,7 +883,7 @@ static int cbcmac_create(struct crypto_template *tmpl, struct rtattr **tb) goto err_free_inst; inst->alg.base.cra_priority = alg->cra_priority; - inst->alg.base.cra_blocksize = 1; + inst->alg.base.cra_blocksize = alg->cra_blocksize; inst->alg.digestsize = alg->cra_blocksize; inst->alg.descsize = sizeof(struct cbcmac_desc_ctx) + diff --git a/drivers/crypto/inside-secure/safexcel_hash.c b/drivers/crypto/inside-secure/safexcel_hash.c index f44c08f5f5ec..d2b632193beb 100644 --- a/drivers/crypto/inside-secure/safexcel_hash.c +++ b/drivers/crypto/inside-secure/safexcel_hash.c @@ -2043,7 +2043,7 @@ struct safexcel_alg_template safexcel_alg_cbcmac = { .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY | CRYPTO_ALG_KERN_DRIVER_ONLY, - .cra_blocksize = 1, + .cra_blocksize = AES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), .cra_init = safexcel_ahash_cra_init, .cra_exit = safexcel_ahash_cra_exit, From b03892c2f836f3daa3a7a3c2e36f82fb76210d40 Mon Sep 17 00:00:00 2001 From: "Dr. David Alan Gilbert" Date: Sun, 13 Apr 2025 00:32:41 +0100 Subject: [PATCH 131/382] crypto: deadcode structs from 'comp' removal Ard's recent series of patches removing 'comp' implementations left behind a bunch of trivial structs, remove them. These are: crypto842_ctx - commit 2d985ff0072f ("crypto: 842 - drop obsolete 'comp' implementation") lz4_ctx - commit 33335afe33c9 ("crypto: lz4 - drop obsolete 'comp' implementation") lz4hc_ctx - commit dbae96559eef ("crypto: lz4hc - drop obsolete 'comp' implementation") lzo_ctx - commit a3e43a25bad0 ("crypto: lzo - drop obsolete 'comp' implementation") lzorle_ctx - commit d32da55c5b0c ("crypto: lzo-rle - drop obsolete 'comp' implementation") Signed-off-by: Dr. David Alan Gilbert Signed-off-by: Herbert Xu --- crypto/842.c | 4 ---- crypto/lz4.c | 4 ---- crypto/lz4hc.c | 4 ---- crypto/lzo-rle.c | 4 ---- crypto/lzo.c | 4 ---- 5 files changed, 20 deletions(-) diff --git a/crypto/842.c b/crypto/842.c index 5fb37a925989..881945d44328 100644 --- a/crypto/842.c +++ b/crypto/842.c @@ -23,10 +23,6 @@ #include #include -struct crypto842_ctx { - void *wmem; /* working memory for compress */ -}; - static void *crypto842_alloc_ctx(void) { void *ctx; diff --git a/crypto/lz4.c b/crypto/lz4.c index 82588607fb2e..9661ed01692f 100644 --- a/crypto/lz4.c +++ b/crypto/lz4.c @@ -12,10 +12,6 @@ #include #include -struct lz4_ctx { - void *lz4_comp_mem; -}; - static void *lz4_alloc_ctx(void) { void *ctx; diff --git a/crypto/lz4hc.c b/crypto/lz4hc.c index 997e76c0183a..a637fddc1ccd 100644 --- a/crypto/lz4hc.c +++ b/crypto/lz4hc.c @@ -10,10 +10,6 @@ #include #include -struct lz4hc_ctx { - void *lz4hc_comp_mem; -}; - static void *lz4hc_alloc_ctx(void) { void *ctx; diff --git a/crypto/lzo-rle.c b/crypto/lzo-rle.c index b1350ae278b8..e7efcf107179 100644 --- a/crypto/lzo-rle.c +++ b/crypto/lzo-rle.c @@ -9,10 +9,6 @@ #include #include -struct lzorle_ctx { - void *lzorle_comp_mem; -}; - static void *lzorle_alloc_ctx(void) { void *ctx; diff --git a/crypto/lzo.c b/crypto/lzo.c index dfe5a07ca35f..f1b36a1ca6f6 100644 --- a/crypto/lzo.c +++ b/crypto/lzo.c @@ -9,10 +9,6 @@ #include #include -struct lzo_ctx { - void *lzo_comp_mem; -}; - static void *lzo_alloc_ctx(void) { void *ctx; From 378a337ab40f88d63ba71d68ff578ead7f5ac8f1 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Sat, 12 Apr 2025 21:54:13 -0700 Subject: [PATCH 132/382] crypto: powerpc/poly1305 - implement library instead of shash Currently the Power10 optimized Poly1305 is only wired up to the crypto_shash API, which makes it unavailable to users of the library API. The crypto_shash API for Poly1305 is going to change to be implemented on top of the library API, so the library API needs to be supported. And of course it's needed anyway to serve the library users. Therefore, change the Power10 optimized Poly1305 code to implement the library API instead of the crypto_shash API. Cc: Danny Tsen Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/powerpc/crypto/Kconfig | 11 +- arch/powerpc/crypto/poly1305-p10-glue.c | 130 +++++++----------------- 2 files changed, 38 insertions(+), 103 deletions(-) diff --git a/arch/powerpc/crypto/Kconfig b/arch/powerpc/crypto/Kconfig index 47dccdd49637..cbcf76953d83 100644 --- a/arch/powerpc/crypto/Kconfig +++ b/arch/powerpc/crypto/Kconfig @@ -100,16 +100,11 @@ config CRYPTO_CHACHA20_P10 default CRYPTO_LIB_CHACHA_INTERNAL config CRYPTO_POLY1305_P10 - tristate "Hash functions: Poly1305 (P10 or later)" + tristate depends on PPC64 && CPU_LITTLE_ENDIAN && VSX - select CRYPTO_HASH + select CRYPTO_ARCH_HAVE_LIB_POLY1305 select CRYPTO_LIB_POLY1305_GENERIC - help - Poly1305 authenticator algorithm (RFC7539) - - Architecture: PowerPC64 - - Power10 or later - - Little-endian + default CRYPTO_LIB_POLY1305_INTERNAL config CRYPTO_DEV_VMX bool "Support for VMX cryptographic acceleration instructions" diff --git a/arch/powerpc/crypto/poly1305-p10-glue.c b/arch/powerpc/crypto/poly1305-p10-glue.c index 369686e9370b..87bd5eba4497 100644 --- a/arch/powerpc/crypto/poly1305-p10-glue.c +++ b/arch/powerpc/crypto/poly1305-p10-glue.c @@ -4,15 +4,11 @@ * * Copyright 2023- IBM Corp. All rights reserved. */ - -#include -#include #include #include #include -#include -#include #include +#include #include #include #include @@ -22,6 +18,8 @@ asmlinkage void poly1305_p10le_4blocks(void *h, const u8 *m, u32 mlen); asmlinkage void poly1305_64s(void *h, const u8 *m, u32 mlen, int highbit); asmlinkage void poly1305_emit_64(void *h, void *s, u8 *dst); +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_p10); + static void vsx_begin(void) { preempt_disable(); @@ -34,51 +32,29 @@ static void vsx_end(void) preempt_enable(); } -static int crypto_poly1305_p10_init(struct shash_desc *desc) +void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 key[POLY1305_KEY_SIZE]) { - struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc); + if (!static_key_enabled(&have_p10)) + return poly1305_init_generic(dctx, key); - poly1305_core_init(&dctx->h); + dctx->h = (struct poly1305_state){}; + dctx->core_r.key.r64[0] = get_unaligned_le64(key + 0); + dctx->core_r.key.r64[1] = get_unaligned_le64(key + 8); + dctx->s[0] = get_unaligned_le32(key + 16); + dctx->s[1] = get_unaligned_le32(key + 20); + dctx->s[2] = get_unaligned_le32(key + 24); + dctx->s[3] = get_unaligned_le32(key + 28); dctx->buflen = 0; - dctx->rset = 0; - dctx->sset = false; - - return 0; } +EXPORT_SYMBOL(poly1305_init_arch); -static unsigned int crypto_poly1305_setdctxkey(struct poly1305_desc_ctx *dctx, - const u8 *inp, unsigned int len) +void poly1305_update_arch(struct poly1305_desc_ctx *dctx, + const u8 *src, unsigned int srclen) { - unsigned int acc = 0; + unsigned int bytes; - if (unlikely(!dctx->sset)) { - if (!dctx->rset && len >= POLY1305_BLOCK_SIZE) { - struct poly1305_core_key *key = &dctx->core_r; - - key->key.r64[0] = get_unaligned_le64(&inp[0]); - key->key.r64[1] = get_unaligned_le64(&inp[8]); - inp += POLY1305_BLOCK_SIZE; - len -= POLY1305_BLOCK_SIZE; - acc += POLY1305_BLOCK_SIZE; - dctx->rset = 1; - } - if (len >= POLY1305_BLOCK_SIZE) { - dctx->s[0] = get_unaligned_le32(&inp[0]); - dctx->s[1] = get_unaligned_le32(&inp[4]); - dctx->s[2] = get_unaligned_le32(&inp[8]); - dctx->s[3] = get_unaligned_le32(&inp[12]); - acc += POLY1305_BLOCK_SIZE; - dctx->sset = true; - } - } - return acc; -} - -static int crypto_poly1305_p10_update(struct shash_desc *desc, - const u8 *src, unsigned int srclen) -{ - struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc); - unsigned int bytes, used; + if (!static_key_enabled(&have_p10)) + return poly1305_update_generic(dctx, src, srclen); if (unlikely(dctx->buflen)) { bytes = min(srclen, POLY1305_BLOCK_SIZE - dctx->buflen); @@ -86,26 +62,16 @@ static int crypto_poly1305_p10_update(struct shash_desc *desc, src += bytes; srclen -= bytes; dctx->buflen += bytes; - - if (dctx->buflen == POLY1305_BLOCK_SIZE) { - if (likely(!crypto_poly1305_setdctxkey(dctx, dctx->buf, - POLY1305_BLOCK_SIZE))) { - vsx_begin(); - poly1305_64s(&dctx->h, dctx->buf, - POLY1305_BLOCK_SIZE, 1); - vsx_end(); - } - dctx->buflen = 0; - } + if (dctx->buflen < POLY1305_BLOCK_SIZE) + return; + vsx_begin(); + poly1305_64s(&dctx->h, dctx->buf, POLY1305_BLOCK_SIZE, 1); + vsx_end(); + dctx->buflen = 0; } if (likely(srclen >= POLY1305_BLOCK_SIZE)) { bytes = round_down(srclen, POLY1305_BLOCK_SIZE); - used = crypto_poly1305_setdctxkey(dctx, src, bytes); - if (likely(used)) { - srclen -= used; - src += used; - } if (crypto_simd_usable() && (srclen >= POLY1305_BLOCK_SIZE*4)) { vsx_begin(); poly1305_p10le_4blocks(&dctx->h, src, srclen); @@ -126,61 +92,35 @@ static int crypto_poly1305_p10_update(struct shash_desc *desc, dctx->buflen = srclen; memcpy(dctx->buf, src, srclen); } - - return 0; } +EXPORT_SYMBOL(poly1305_update_arch); -static int crypto_poly1305_p10_final(struct shash_desc *desc, u8 *dst) +void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst) { - struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc); + if (!static_key_enabled(&have_p10)) + return poly1305_final_generic(dctx, dst); - if (unlikely(!dctx->sset)) - return -ENOKEY; - - if ((dctx->buflen)) { + if (dctx->buflen) { dctx->buf[dctx->buflen++] = 1; memset(dctx->buf + dctx->buflen, 0, POLY1305_BLOCK_SIZE - dctx->buflen); vsx_begin(); poly1305_64s(&dctx->h, dctx->buf, POLY1305_BLOCK_SIZE, 0); vsx_end(); - dctx->buflen = 0; } poly1305_emit_64(&dctx->h, &dctx->s, dst); - return 0; } - -static struct shash_alg poly1305_alg = { - .digestsize = POLY1305_DIGEST_SIZE, - .init = crypto_poly1305_p10_init, - .update = crypto_poly1305_p10_update, - .final = crypto_poly1305_p10_final, - .descsize = sizeof(struct poly1305_desc_ctx), - .base = { - .cra_name = "poly1305", - .cra_driver_name = "poly1305-p10", - .cra_priority = 300, - .cra_blocksize = POLY1305_BLOCK_SIZE, - .cra_module = THIS_MODULE, - }, -}; +EXPORT_SYMBOL(poly1305_final_arch); static int __init poly1305_p10_init(void) { - return crypto_register_shash(&poly1305_alg); + if (cpu_has_feature(CPU_FTR_ARCH_31)) + static_branch_enable(&have_p10); + return 0; } - -static void __exit poly1305_p10_exit(void) -{ - crypto_unregister_shash(&poly1305_alg); -} - -module_cpu_feature_match(PPC_MODULE_FEATURE_P10, poly1305_p10_init); -module_exit(poly1305_p10_exit); +arch_initcall(poly1305_p10_init); MODULE_LICENSE("GPL"); MODULE_AUTHOR("Danny Tsen "); MODULE_DESCRIPTION("Optimized Poly1305 for P10"); -MODULE_ALIAS_CRYPTO("poly1305"); -MODULE_ALIAS_CRYPTO("poly1305-p10"); From ecaa4be1280a4faf72dc4b6b4f6d867332d5762e Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Sat, 12 Apr 2025 21:54:14 -0700 Subject: [PATCH 133/382] crypto: poly1305 - centralize the shash wrappers for arch code Following the example of the crc32, crc32c, and chacha code, make the crypto subsystem register both generic and architecture-optimized poly1305 shash algorithms, both implemented on top of the appropriate library functions. This eliminates the need for every architecture to implement the same shash glue code. Note that the poly1305 shash requires that the key be prepended to the data, which differs from the library functions where the key is simply a parameter to poly1305_init(). Previously this was handled at a fairly low level, polluting the library code with shash-specific code. Reorganize things so that the shash code handles this quirk itself. Also, to register the architecture-optimized shashes only when architecture-optimized code is actually being used, add a function poly1305_is_arch_optimized() and make each arch implement it. Change each architecture's Poly1305 module_init function to arch_initcall so that the CPU feature detection is guaranteed to run before poly1305_is_arch_optimized() gets called by crypto/poly1305.c. (In cases where poly1305_is_arch_optimized() just returns true unconditionally, using arch_initcall is not strictly needed, but it's still good to be consistent across architectures.) Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/arm/crypto/poly1305-glue.c | 11 +- arch/arm64/crypto/poly1305-glue.c | 9 +- arch/mips/crypto/poly1305-glue.c | 10 +- arch/powerpc/crypto/poly1305-p10-glue.c | 6 + arch/x86/crypto/poly1305_glue.c | 8 +- crypto/Makefile | 3 +- crypto/poly1305.c | 152 ++++++++++++++++++++++++ crypto/poly1305_generic.c | 149 ----------------------- include/crypto/poly1305.h | 9 ++ 9 files changed, 201 insertions(+), 156 deletions(-) create mode 100644 crypto/poly1305.c delete mode 100644 crypto/poly1305_generic.c diff --git a/arch/arm/crypto/poly1305-glue.c b/arch/arm/crypto/poly1305-glue.c index 4464ffbf8fd1..238dbf50d05d 100644 --- a/arch/arm/crypto/poly1305-glue.c +++ b/arch/arm/crypto/poly1305-glue.c @@ -218,7 +218,7 @@ static struct shash_alg arm_poly1305_algs[] = {{ .descsize = sizeof(struct poly1305_desc_ctx), .base.cra_name = "poly1305", - .base.cra_driver_name = "poly1305-arm", + .base.cra_driver_name = "poly1305-arm-old", .base.cra_priority = 150, .base.cra_blocksize = POLY1305_BLOCK_SIZE, .base.cra_module = THIS_MODULE, @@ -238,6 +238,13 @@ static struct shash_alg arm_poly1305_algs[] = {{ #endif }}; +bool poly1305_is_arch_optimized(void) +{ + /* We always can use at least the ARM scalar implementation. */ + return true; +} +EXPORT_SYMBOL(poly1305_is_arch_optimized); + static int __init arm_poly1305_mod_init(void) { if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && @@ -264,7 +271,7 @@ static void __exit arm_poly1305_mod_exit(void) ARRAY_SIZE(arm_poly1305_algs)); } -module_init(arm_poly1305_mod_init); +arch_initcall(arm_poly1305_mod_init); module_exit(arm_poly1305_mod_exit); MODULE_DESCRIPTION("Accelerated Poly1305 transform for ARM"); diff --git a/arch/arm64/crypto/poly1305-glue.c b/arch/arm64/crypto/poly1305-glue.c index 18883ea438f3..89415731d9be 100644 --- a/arch/arm64/crypto/poly1305-glue.c +++ b/arch/arm64/crypto/poly1305-glue.c @@ -206,6 +206,13 @@ static struct shash_alg neon_poly1305_alg = { .base.cra_module = THIS_MODULE, }; +bool poly1305_is_arch_optimized(void) +{ + /* We always can use at least the ARM64 scalar implementation. */ + return true; +} +EXPORT_SYMBOL(poly1305_is_arch_optimized); + static int __init neon_poly1305_mod_init(void) { if (!cpu_have_named_feature(ASIMD)) @@ -223,7 +230,7 @@ static void __exit neon_poly1305_mod_exit(void) crypto_unregister_shash(&neon_poly1305_alg); } -module_init(neon_poly1305_mod_init); +arch_initcall(neon_poly1305_mod_init); module_exit(neon_poly1305_mod_exit); MODULE_DESCRIPTION("Poly1305 transform using NEON instructions"); diff --git a/arch/mips/crypto/poly1305-glue.c b/arch/mips/crypto/poly1305-glue.c index c03ad0bbe69c..9eee71dc0b99 100644 --- a/arch/mips/crypto/poly1305-glue.c +++ b/arch/mips/crypto/poly1305-glue.c @@ -165,12 +165,18 @@ static struct shash_alg mips_poly1305_alg = { .descsize = sizeof(struct poly1305_desc_ctx), .base.cra_name = "poly1305", - .base.cra_driver_name = "poly1305-mips", + .base.cra_driver_name = "poly1305-mips-old", .base.cra_priority = 200, .base.cra_blocksize = POLY1305_BLOCK_SIZE, .base.cra_module = THIS_MODULE, }; +bool poly1305_is_arch_optimized(void) +{ + return true; +} +EXPORT_SYMBOL(poly1305_is_arch_optimized); + static int __init mips_poly1305_mod_init(void) { return IS_REACHABLE(CONFIG_CRYPTO_HASH) ? @@ -183,7 +189,7 @@ static void __exit mips_poly1305_mod_exit(void) crypto_unregister_shash(&mips_poly1305_alg); } -module_init(mips_poly1305_mod_init); +arch_initcall(mips_poly1305_mod_init); module_exit(mips_poly1305_mod_exit); MODULE_DESCRIPTION("Poly1305 transform (MIPS accelerated"); diff --git a/arch/powerpc/crypto/poly1305-p10-glue.c b/arch/powerpc/crypto/poly1305-p10-glue.c index 87bd5eba4497..40d296d52c23 100644 --- a/arch/powerpc/crypto/poly1305-p10-glue.c +++ b/arch/powerpc/crypto/poly1305-p10-glue.c @@ -113,6 +113,12 @@ void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst) } EXPORT_SYMBOL(poly1305_final_arch); +bool poly1305_is_arch_optimized(void) +{ + return static_key_enabled(&have_p10); +} +EXPORT_SYMBOL(poly1305_is_arch_optimized); + static int __init poly1305_p10_init(void) { if (cpu_has_feature(CPU_FTR_ARCH_31)) diff --git a/arch/x86/crypto/poly1305_glue.c b/arch/x86/crypto/poly1305_glue.c index 8b5593c46da7..40d03a9a4fb5 100644 --- a/arch/x86/crypto/poly1305_glue.c +++ b/arch/x86/crypto/poly1305_glue.c @@ -257,6 +257,12 @@ static struct shash_alg alg = { }, }; +bool poly1305_is_arch_optimized(void) +{ + return static_key_enabled(&poly1305_use_avx); +} +EXPORT_SYMBOL(poly1305_is_arch_optimized); + static int __init poly1305_simd_mod_init(void) { if (boot_cpu_has(X86_FEATURE_AVX) && @@ -280,7 +286,7 @@ static void __exit poly1305_simd_mod_exit(void) crypto_unregister_shash(&alg); } -module_init(poly1305_simd_mod_init); +arch_initcall(poly1305_simd_mod_init); module_exit(poly1305_simd_mod_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/Makefile b/crypto/Makefile index baf5040ca661..5d2f2a28d8a0 100644 --- a/crypto/Makefile +++ b/crypto/Makefile @@ -149,7 +149,8 @@ obj-$(CONFIG_CRYPTO_SEED) += seed.o obj-$(CONFIG_CRYPTO_ARIA) += aria_generic.o obj-$(CONFIG_CRYPTO_CHACHA20) += chacha.o CFLAGS_chacha.o += -DARCH=$(ARCH) -obj-$(CONFIG_CRYPTO_POLY1305) += poly1305_generic.o +obj-$(CONFIG_CRYPTO_POLY1305) += poly1305.o +CFLAGS_poly1305.o += -DARCH=$(ARCH) obj-$(CONFIG_CRYPTO_DEFLATE) += deflate.o obj-$(CONFIG_CRYPTO_MICHAEL_MIC) += michael_mic.o obj-$(CONFIG_CRYPTO_CRC32C) += crc32c_generic.o diff --git a/crypto/poly1305.c b/crypto/poly1305.c new file mode 100644 index 000000000000..e0436bdc462b --- /dev/null +++ b/crypto/poly1305.c @@ -0,0 +1,152 @@ +/* + * Crypto API wrapper for the Poly1305 library functions + * + * Copyright (C) 2015 Martin Willi + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + */ + +#include +#include +#include +#include +#include +#include + +struct crypto_poly1305_desc_ctx { + struct poly1305_desc_ctx base; + u8 key[POLY1305_KEY_SIZE]; + unsigned int keysize; +}; + +static int crypto_poly1305_init(struct shash_desc *desc) +{ + struct crypto_poly1305_desc_ctx *dctx = shash_desc_ctx(desc); + + dctx->keysize = 0; + return 0; +} + +static int crypto_poly1305_update(struct shash_desc *desc, + const u8 *src, unsigned int srclen, bool arch) +{ + struct crypto_poly1305_desc_ctx *dctx = shash_desc_ctx(desc); + unsigned int bytes; + + /* + * The key is passed as the first 32 "data" bytes. The actual + * poly1305_init() can be called only once the full key is available. + */ + if (dctx->keysize < POLY1305_KEY_SIZE) { + bytes = min(srclen, POLY1305_KEY_SIZE - dctx->keysize); + memcpy(&dctx->key[dctx->keysize], src, bytes); + dctx->keysize += bytes; + if (dctx->keysize < POLY1305_KEY_SIZE) + return 0; + if (arch) + poly1305_init(&dctx->base, dctx->key); + else + poly1305_init_generic(&dctx->base, dctx->key); + src += bytes; + srclen -= bytes; + } + + if (arch) + poly1305_update(&dctx->base, src, srclen); + else + poly1305_update_generic(&dctx->base, src, srclen); + + return 0; +} + +static int crypto_poly1305_update_generic(struct shash_desc *desc, + const u8 *src, unsigned int srclen) +{ + return crypto_poly1305_update(desc, src, srclen, false); +} + +static int crypto_poly1305_update_arch(struct shash_desc *desc, + const u8 *src, unsigned int srclen) +{ + return crypto_poly1305_update(desc, src, srclen, true); +} + +static int crypto_poly1305_final(struct shash_desc *desc, u8 *dst, bool arch) +{ + struct crypto_poly1305_desc_ctx *dctx = shash_desc_ctx(desc); + + if (unlikely(dctx->keysize != POLY1305_KEY_SIZE)) + return -ENOKEY; + + if (arch) + poly1305_final(&dctx->base, dst); + else + poly1305_final_generic(&dctx->base, dst); + memzero_explicit(&dctx->key, sizeof(dctx->key)); + return 0; +} + +static int crypto_poly1305_final_generic(struct shash_desc *desc, u8 *dst) +{ + return crypto_poly1305_final(desc, dst, false); +} + +static int crypto_poly1305_final_arch(struct shash_desc *desc, u8 *dst) +{ + return crypto_poly1305_final(desc, dst, true); +} + +static struct shash_alg poly1305_algs[] = { + { + .base.cra_name = "poly1305", + .base.cra_driver_name = "poly1305-generic", + .base.cra_priority = 100, + .base.cra_blocksize = POLY1305_BLOCK_SIZE, + .base.cra_module = THIS_MODULE, + .digestsize = POLY1305_DIGEST_SIZE, + .init = crypto_poly1305_init, + .update = crypto_poly1305_update_generic, + .final = crypto_poly1305_final_generic, + .descsize = sizeof(struct crypto_poly1305_desc_ctx), + }, + { + .base.cra_name = "poly1305", + .base.cra_driver_name = "poly1305-" __stringify(ARCH), + .base.cra_priority = 300, + .base.cra_blocksize = POLY1305_BLOCK_SIZE, + .base.cra_module = THIS_MODULE, + .digestsize = POLY1305_DIGEST_SIZE, + .init = crypto_poly1305_init, + .update = crypto_poly1305_update_arch, + .final = crypto_poly1305_final_arch, + .descsize = sizeof(struct crypto_poly1305_desc_ctx), + }, +}; + +static int num_algs; + +static int __init poly1305_mod_init(void) +{ + /* register the arch flavours only if they differ from generic */ + num_algs = poly1305_is_arch_optimized() ? 2 : 1; + + return crypto_register_shashes(poly1305_algs, num_algs); +} + +static void __exit poly1305_mod_exit(void) +{ + crypto_unregister_shashes(poly1305_algs, num_algs); +} + +subsys_initcall(poly1305_mod_init); +module_exit(poly1305_mod_exit); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Martin Willi "); +MODULE_DESCRIPTION("Crypto API wrapper for the Poly1305 library functions"); +MODULE_ALIAS_CRYPTO("poly1305"); +MODULE_ALIAS_CRYPTO("poly1305-generic"); +MODULE_ALIAS_CRYPTO("poly1305-" __stringify(ARCH)); diff --git a/crypto/poly1305_generic.c b/crypto/poly1305_generic.c deleted file mode 100644 index e6f29a98725a..000000000000 --- a/crypto/poly1305_generic.c +++ /dev/null @@ -1,149 +0,0 @@ -/* - * Poly1305 authenticator algorithm, RFC7539 - * - * Copyright (C) 2015 Martin Willi - * - * Based on public domain code by Andrew Moon and Daniel J. Bernstein. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - */ - -#include -#include -#include -#include -#include -#include -#include - -static int crypto_poly1305_init(struct shash_desc *desc) -{ - struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc); - - poly1305_core_init(&dctx->h); - dctx->buflen = 0; - dctx->rset = 0; - dctx->sset = false; - - return 0; -} - -static unsigned int crypto_poly1305_setdesckey(struct poly1305_desc_ctx *dctx, - const u8 *src, unsigned int srclen) -{ - if (!dctx->sset) { - if (!dctx->rset && srclen >= POLY1305_BLOCK_SIZE) { - poly1305_core_setkey(&dctx->core_r, src); - src += POLY1305_BLOCK_SIZE; - srclen -= POLY1305_BLOCK_SIZE; - dctx->rset = 2; - } - if (srclen >= POLY1305_BLOCK_SIZE) { - dctx->s[0] = get_unaligned_le32(src + 0); - dctx->s[1] = get_unaligned_le32(src + 4); - dctx->s[2] = get_unaligned_le32(src + 8); - dctx->s[3] = get_unaligned_le32(src + 12); - src += POLY1305_BLOCK_SIZE; - srclen -= POLY1305_BLOCK_SIZE; - dctx->sset = true; - } - } - return srclen; -} - -static void poly1305_blocks(struct poly1305_desc_ctx *dctx, const u8 *src, - unsigned int srclen) -{ - unsigned int datalen; - - if (unlikely(!dctx->sset)) { - datalen = crypto_poly1305_setdesckey(dctx, src, srclen); - src += srclen - datalen; - srclen = datalen; - } - - poly1305_core_blocks(&dctx->h, &dctx->core_r, src, - srclen / POLY1305_BLOCK_SIZE, 1); -} - -static int crypto_poly1305_update(struct shash_desc *desc, - const u8 *src, unsigned int srclen) -{ - struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc); - unsigned int bytes; - - if (unlikely(dctx->buflen)) { - bytes = min(srclen, POLY1305_BLOCK_SIZE - dctx->buflen); - memcpy(dctx->buf + dctx->buflen, src, bytes); - src += bytes; - srclen -= bytes; - dctx->buflen += bytes; - - if (dctx->buflen == POLY1305_BLOCK_SIZE) { - poly1305_blocks(dctx, dctx->buf, - POLY1305_BLOCK_SIZE); - dctx->buflen = 0; - } - } - - if (likely(srclen >= POLY1305_BLOCK_SIZE)) { - poly1305_blocks(dctx, src, srclen); - src += srclen - (srclen % POLY1305_BLOCK_SIZE); - srclen %= POLY1305_BLOCK_SIZE; - } - - if (unlikely(srclen)) { - dctx->buflen = srclen; - memcpy(dctx->buf, src, srclen); - } - - return 0; -} - -static int crypto_poly1305_final(struct shash_desc *desc, u8 *dst) -{ - struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc); - - if (unlikely(!dctx->sset)) - return -ENOKEY; - - poly1305_final_generic(dctx, dst); - return 0; -} - -static struct shash_alg poly1305_alg = { - .digestsize = POLY1305_DIGEST_SIZE, - .init = crypto_poly1305_init, - .update = crypto_poly1305_update, - .final = crypto_poly1305_final, - .descsize = sizeof(struct poly1305_desc_ctx), - .base = { - .cra_name = "poly1305", - .cra_driver_name = "poly1305-generic", - .cra_priority = 100, - .cra_blocksize = POLY1305_BLOCK_SIZE, - .cra_module = THIS_MODULE, - }, -}; - -static int __init poly1305_mod_init(void) -{ - return crypto_register_shash(&poly1305_alg); -} - -static void __exit poly1305_mod_exit(void) -{ - crypto_unregister_shash(&poly1305_alg); -} - -subsys_initcall(poly1305_mod_init); -module_exit(poly1305_mod_exit); - -MODULE_LICENSE("GPL"); -MODULE_AUTHOR("Martin Willi "); -MODULE_DESCRIPTION("Poly1305 authenticator"); -MODULE_ALIAS_CRYPTO("poly1305"); -MODULE_ALIAS_CRYPTO("poly1305-generic"); diff --git a/include/crypto/poly1305.h b/include/crypto/poly1305.h index 090692ec3bc7..91444965772a 100644 --- a/include/crypto/poly1305.h +++ b/include/crypto/poly1305.h @@ -96,4 +96,13 @@ static inline void poly1305_final(struct poly1305_desc_ctx *desc, u8 *digest) poly1305_final_generic(desc, digest); } +#if IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_POLY1305) +bool poly1305_is_arch_optimized(void); +#else +static inline bool poly1305_is_arch_optimized(void) +{ + return false; +} +#endif + #endif From 1f81c58279c73c813a6967460b26df3a959457e3 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Sat, 12 Apr 2025 21:54:15 -0700 Subject: [PATCH 134/382] crypto: arm/poly1305 - remove redundant shash algorithm Since crypto/poly1305.c now registers a poly1305-$(ARCH) shash algorithm that uses the architecture's Poly1305 library functions, individual architectures no longer need to do the same. Therefore, remove the redundant shash algorithm from the arch-specific code and leave just the library functions there. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/arm/crypto/Kconfig | 6 -- arch/arm/crypto/poly1305-glue.c | 167 +------------------------------- 2 files changed, 3 insertions(+), 170 deletions(-) diff --git a/arch/arm/crypto/Kconfig b/arch/arm/crypto/Kconfig index 332f2430beec..3530e7c80793 100644 --- a/arch/arm/crypto/Kconfig +++ b/arch/arm/crypto/Kconfig @@ -48,14 +48,8 @@ config CRYPTO_NHPOLY1305_NEON config CRYPTO_POLY1305_ARM tristate - select CRYPTO_HASH select CRYPTO_ARCH_HAVE_LIB_POLY1305 default CRYPTO_LIB_POLY1305_INTERNAL - help - Poly1305 authenticator algorithm (RFC7539) - - Architecture: arm optionally using - - NEON (Advanced SIMD) extensions config CRYPTO_BLAKE2S_ARM bool "Hash functions: BLAKE2s" diff --git a/arch/arm/crypto/poly1305-glue.c b/arch/arm/crypto/poly1305-glue.c index 238dbf50d05d..6d6998b3ec7e 100644 --- a/arch/arm/crypto/poly1305-glue.c +++ b/arch/arm/crypto/poly1305-glue.c @@ -8,15 +8,12 @@ #include #include #include -#include -#include -#include -#include +#include #include #include -#include #include #include +#include void poly1305_init_arm(void *state, const u8 *key); void poly1305_blocks_arm(void *state, const u8 *src, u32 len, u32 hibit); @@ -40,102 +37,6 @@ void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 key[POLY1305_KE } EXPORT_SYMBOL(poly1305_init_arch); -static int arm_poly1305_init(struct shash_desc *desc) -{ - struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc); - - dctx->buflen = 0; - dctx->rset = 0; - dctx->sset = false; - - return 0; -} - -static void arm_poly1305_blocks(struct poly1305_desc_ctx *dctx, const u8 *src, - u32 len, u32 hibit, bool do_neon) -{ - if (unlikely(!dctx->sset)) { - if (!dctx->rset) { - poly1305_init_arm(&dctx->h, src); - src += POLY1305_BLOCK_SIZE; - len -= POLY1305_BLOCK_SIZE; - dctx->rset = 1; - } - if (len >= POLY1305_BLOCK_SIZE) { - dctx->s[0] = get_unaligned_le32(src + 0); - dctx->s[1] = get_unaligned_le32(src + 4); - dctx->s[2] = get_unaligned_le32(src + 8); - dctx->s[3] = get_unaligned_le32(src + 12); - src += POLY1305_BLOCK_SIZE; - len -= POLY1305_BLOCK_SIZE; - dctx->sset = true; - } - if (len < POLY1305_BLOCK_SIZE) - return; - } - - len &= ~(POLY1305_BLOCK_SIZE - 1); - - if (static_branch_likely(&have_neon) && likely(do_neon)) - poly1305_blocks_neon(&dctx->h, src, len, hibit); - else - poly1305_blocks_arm(&dctx->h, src, len, hibit); -} - -static void arm_poly1305_do_update(struct poly1305_desc_ctx *dctx, - const u8 *src, u32 len, bool do_neon) -{ - if (unlikely(dctx->buflen)) { - u32 bytes = min(len, POLY1305_BLOCK_SIZE - dctx->buflen); - - memcpy(dctx->buf + dctx->buflen, src, bytes); - src += bytes; - len -= bytes; - dctx->buflen += bytes; - - if (dctx->buflen == POLY1305_BLOCK_SIZE) { - arm_poly1305_blocks(dctx, dctx->buf, - POLY1305_BLOCK_SIZE, 1, false); - dctx->buflen = 0; - } - } - - if (likely(len >= POLY1305_BLOCK_SIZE)) { - arm_poly1305_blocks(dctx, src, len, 1, do_neon); - src += round_down(len, POLY1305_BLOCK_SIZE); - len %= POLY1305_BLOCK_SIZE; - } - - if (unlikely(len)) { - dctx->buflen = len; - memcpy(dctx->buf, src, len); - } -} - -static int arm_poly1305_update(struct shash_desc *desc, - const u8 *src, unsigned int srclen) -{ - struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc); - - arm_poly1305_do_update(dctx, src, srclen, false); - return 0; -} - -static int __maybe_unused arm_poly1305_update_neon(struct shash_desc *desc, - const u8 *src, - unsigned int srclen) -{ - struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc); - bool do_neon = crypto_simd_usable() && srclen > 128; - - if (static_branch_likely(&have_neon) && do_neon) - kernel_neon_begin(); - arm_poly1305_do_update(dctx, src, srclen, do_neon); - if (static_branch_likely(&have_neon) && do_neon) - kernel_neon_end(); - return 0; -} - void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src, unsigned int nbytes) { @@ -199,45 +100,6 @@ void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst) } EXPORT_SYMBOL(poly1305_final_arch); -static int arm_poly1305_final(struct shash_desc *desc, u8 *dst) -{ - struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc); - - if (unlikely(!dctx->sset)) - return -ENOKEY; - - poly1305_final_arch(dctx, dst); - return 0; -} - -static struct shash_alg arm_poly1305_algs[] = {{ - .init = arm_poly1305_init, - .update = arm_poly1305_update, - .final = arm_poly1305_final, - .digestsize = POLY1305_DIGEST_SIZE, - .descsize = sizeof(struct poly1305_desc_ctx), - - .base.cra_name = "poly1305", - .base.cra_driver_name = "poly1305-arm-old", - .base.cra_priority = 150, - .base.cra_blocksize = POLY1305_BLOCK_SIZE, - .base.cra_module = THIS_MODULE, -#ifdef CONFIG_KERNEL_MODE_NEON -}, { - .init = arm_poly1305_init, - .update = arm_poly1305_update_neon, - .final = arm_poly1305_final, - .digestsize = POLY1305_DIGEST_SIZE, - .descsize = sizeof(struct poly1305_desc_ctx), - - .base.cra_name = "poly1305", - .base.cra_driver_name = "poly1305-neon", - .base.cra_priority = 200, - .base.cra_blocksize = POLY1305_BLOCK_SIZE, - .base.cra_module = THIS_MODULE, -#endif -}}; - bool poly1305_is_arch_optimized(void) { /* We always can use at least the ARM scalar implementation. */ @@ -250,32 +112,9 @@ static int __init arm_poly1305_mod_init(void) if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && (elf_hwcap & HWCAP_NEON)) static_branch_enable(&have_neon); - else if (IS_REACHABLE(CONFIG_CRYPTO_HASH)) - /* register only the first entry */ - return crypto_register_shash(&arm_poly1305_algs[0]); - - return IS_REACHABLE(CONFIG_CRYPTO_HASH) ? - crypto_register_shashes(arm_poly1305_algs, - ARRAY_SIZE(arm_poly1305_algs)) : 0; + return 0; } - -static void __exit arm_poly1305_mod_exit(void) -{ - if (!IS_REACHABLE(CONFIG_CRYPTO_HASH)) - return; - if (!static_branch_likely(&have_neon)) { - crypto_unregister_shash(&arm_poly1305_algs[0]); - return; - } - crypto_unregister_shashes(arm_poly1305_algs, - ARRAY_SIZE(arm_poly1305_algs)); -} - arch_initcall(arm_poly1305_mod_init); -module_exit(arm_poly1305_mod_exit); MODULE_DESCRIPTION("Accelerated Poly1305 transform for ARM"); MODULE_LICENSE("GPL v2"); -MODULE_ALIAS_CRYPTO("poly1305"); -MODULE_ALIAS_CRYPTO("poly1305-arm"); -MODULE_ALIAS_CRYPTO("poly1305-neon"); From f4b1a73aec5c32ea533cc897b4b150b3a002a023 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Sat, 12 Apr 2025 21:54:16 -0700 Subject: [PATCH 135/382] crypto: arm64/poly1305 - remove redundant shash algorithm Since crypto/poly1305.c now registers a poly1305-$(ARCH) shash algorithm that uses the architecture's Poly1305 library functions, individual architectures no longer need to do the same. Therefore, remove the redundant shash algorithm from the arch-specific code and leave just the library functions there. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/arm64/crypto/Kconfig | 6 -- arch/arm64/crypto/poly1305-glue.c | 140 ++---------------------------- 2 files changed, 6 insertions(+), 140 deletions(-) diff --git a/arch/arm64/crypto/Kconfig b/arch/arm64/crypto/Kconfig index 9dd1ae7a68a4..7c2f63f2e307 100644 --- a/arch/arm64/crypto/Kconfig +++ b/arch/arm64/crypto/Kconfig @@ -28,14 +28,8 @@ config CRYPTO_NHPOLY1305_NEON config CRYPTO_POLY1305_NEON tristate depends on KERNEL_MODE_NEON - select CRYPTO_HASH select CRYPTO_ARCH_HAVE_LIB_POLY1305 default CRYPTO_LIB_POLY1305_INTERNAL - help - Poly1305 authenticator algorithm (RFC7539) - - Architecture: arm64 using: - - NEON (Advanced SIMD) extensions config CRYPTO_SHA1_ARM64_CE tristate "Hash functions: SHA-1 (ARMv8 Crypto Extensions)" diff --git a/arch/arm64/crypto/poly1305-glue.c b/arch/arm64/crypto/poly1305-glue.c index 89415731d9be..cb152ceac14a 100644 --- a/arch/arm64/crypto/poly1305-glue.c +++ b/arch/arm64/crypto/poly1305-glue.c @@ -8,15 +8,12 @@ #include #include #include -#include -#include -#include -#include +#include #include #include -#include #include #include +#include asmlinkage void poly1305_init_arm64(void *state, const u8 *key); asmlinkage void poly1305_blocks(void *state, const u8 *src, u32 len, u32 hibit); @@ -36,92 +33,6 @@ void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 key[POLY1305_KE } EXPORT_SYMBOL(poly1305_init_arch); -static int neon_poly1305_init(struct shash_desc *desc) -{ - struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc); - - dctx->buflen = 0; - dctx->rset = 0; - dctx->sset = false; - - return 0; -} - -static void neon_poly1305_blocks(struct poly1305_desc_ctx *dctx, const u8 *src, - u32 len, u32 hibit, bool do_neon) -{ - if (unlikely(!dctx->sset)) { - if (!dctx->rset) { - poly1305_init_arm64(&dctx->h, src); - src += POLY1305_BLOCK_SIZE; - len -= POLY1305_BLOCK_SIZE; - dctx->rset = 1; - } - if (len >= POLY1305_BLOCK_SIZE) { - dctx->s[0] = get_unaligned_le32(src + 0); - dctx->s[1] = get_unaligned_le32(src + 4); - dctx->s[2] = get_unaligned_le32(src + 8); - dctx->s[3] = get_unaligned_le32(src + 12); - src += POLY1305_BLOCK_SIZE; - len -= POLY1305_BLOCK_SIZE; - dctx->sset = true; - } - if (len < POLY1305_BLOCK_SIZE) - return; - } - - len &= ~(POLY1305_BLOCK_SIZE - 1); - - if (static_branch_likely(&have_neon) && likely(do_neon)) - poly1305_blocks_neon(&dctx->h, src, len, hibit); - else - poly1305_blocks(&dctx->h, src, len, hibit); -} - -static void neon_poly1305_do_update(struct poly1305_desc_ctx *dctx, - const u8 *src, u32 len, bool do_neon) -{ - if (unlikely(dctx->buflen)) { - u32 bytes = min(len, POLY1305_BLOCK_SIZE - dctx->buflen); - - memcpy(dctx->buf + dctx->buflen, src, bytes); - src += bytes; - len -= bytes; - dctx->buflen += bytes; - - if (dctx->buflen == POLY1305_BLOCK_SIZE) { - neon_poly1305_blocks(dctx, dctx->buf, - POLY1305_BLOCK_SIZE, 1, false); - dctx->buflen = 0; - } - } - - if (likely(len >= POLY1305_BLOCK_SIZE)) { - neon_poly1305_blocks(dctx, src, len, 1, do_neon); - src += round_down(len, POLY1305_BLOCK_SIZE); - len %= POLY1305_BLOCK_SIZE; - } - - if (unlikely(len)) { - dctx->buflen = len; - memcpy(dctx->buf, src, len); - } -} - -static int neon_poly1305_update(struct shash_desc *desc, - const u8 *src, unsigned int srclen) -{ - bool do_neon = crypto_simd_usable() && srclen > 128; - struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc); - - if (static_branch_likely(&have_neon) && do_neon) - kernel_neon_begin(); - neon_poly1305_do_update(dctx, src, srclen, do_neon); - if (static_branch_likely(&have_neon) && do_neon) - kernel_neon_end(); - return 0; -} - void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src, unsigned int nbytes) { @@ -181,31 +92,6 @@ void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst) } EXPORT_SYMBOL(poly1305_final_arch); -static int neon_poly1305_final(struct shash_desc *desc, u8 *dst) -{ - struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc); - - if (unlikely(!dctx->sset)) - return -ENOKEY; - - poly1305_final_arch(dctx, dst); - return 0; -} - -static struct shash_alg neon_poly1305_alg = { - .init = neon_poly1305_init, - .update = neon_poly1305_update, - .final = neon_poly1305_final, - .digestsize = POLY1305_DIGEST_SIZE, - .descsize = sizeof(struct poly1305_desc_ctx), - - .base.cra_name = "poly1305", - .base.cra_driver_name = "poly1305-neon", - .base.cra_priority = 200, - .base.cra_blocksize = POLY1305_BLOCK_SIZE, - .base.cra_module = THIS_MODULE, -}; - bool poly1305_is_arch_optimized(void) { /* We always can use at least the ARM64 scalar implementation. */ @@ -215,25 +101,11 @@ EXPORT_SYMBOL(poly1305_is_arch_optimized); static int __init neon_poly1305_mod_init(void) { - if (!cpu_have_named_feature(ASIMD)) - return 0; - - static_branch_enable(&have_neon); - - return IS_REACHABLE(CONFIG_CRYPTO_HASH) ? - crypto_register_shash(&neon_poly1305_alg) : 0; + if (cpu_have_named_feature(ASIMD)) + static_branch_enable(&have_neon); + return 0; } - -static void __exit neon_poly1305_mod_exit(void) -{ - if (IS_REACHABLE(CONFIG_CRYPTO_HASH) && cpu_have_named_feature(ASIMD)) - crypto_unregister_shash(&neon_poly1305_alg); -} - arch_initcall(neon_poly1305_mod_init); -module_exit(neon_poly1305_mod_exit); -MODULE_DESCRIPTION("Poly1305 transform using NEON instructions"); +MODULE_DESCRIPTION("Poly1305 authenticator (ARM64 optimized)"); MODULE_LICENSE("GPL v2"); -MODULE_ALIAS_CRYPTO("poly1305"); -MODULE_ALIAS_CRYPTO("poly1305-neon"); From 25221872660c9971c3684b66c8cefe0b95849aa6 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Sat, 12 Apr 2025 21:54:17 -0700 Subject: [PATCH 136/382] crypto: mips/poly1305 - drop redundant dependency on CONFIG_MIPS arch/mips/crypto/Kconfig is sourced only when CONFIG_MIPS is enabled, so there is no need for options defined in that file to depend on it. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/mips/crypto/Kconfig | 1 - 1 file changed, 1 deletion(-) diff --git a/arch/mips/crypto/Kconfig b/arch/mips/crypto/Kconfig index 0189686de3a1..9892391f74ac 100644 --- a/arch/mips/crypto/Kconfig +++ b/arch/mips/crypto/Kconfig @@ -4,7 +4,6 @@ menu "Accelerated Cryptographic Algorithms for CPU (mips)" config CRYPTO_POLY1305_MIPS tristate - depends on MIPS select CRYPTO_HASH select CRYPTO_ARCH_HAVE_LIB_POLY1305 default CRYPTO_LIB_POLY1305_INTERNAL From 632dcef5bb44c1548de16e2c72bb5037a895d00f Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Sat, 12 Apr 2025 21:54:18 -0700 Subject: [PATCH 137/382] crypto: mips/poly1305 - remove redundant shash algorithm Since crypto/poly1305.c now registers a poly1305-$(ARCH) shash algorithm that uses the architecture's Poly1305 library functions, individual architectures no longer need to do the same. Therefore, remove the redundant shash algorithm from the arch-specific code and leave just the library functions there. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/mips/crypto/Kconfig | 5 -- arch/mips/crypto/poly1305-glue.c | 120 +------------------------------ 2 files changed, 2 insertions(+), 123 deletions(-) diff --git a/arch/mips/crypto/Kconfig b/arch/mips/crypto/Kconfig index 9892391f74ac..8283664a1f24 100644 --- a/arch/mips/crypto/Kconfig +++ b/arch/mips/crypto/Kconfig @@ -4,13 +4,8 @@ menu "Accelerated Cryptographic Algorithms for CPU (mips)" config CRYPTO_POLY1305_MIPS tristate - select CRYPTO_HASH select CRYPTO_ARCH_HAVE_LIB_POLY1305 default CRYPTO_LIB_POLY1305_INTERNAL - help - Poly1305 authenticator algorithm (RFC7539) - - Architecture: mips config CRYPTO_MD5_OCTEON tristate "Digests: MD5 (OCTEON)" diff --git a/arch/mips/crypto/poly1305-glue.c b/arch/mips/crypto/poly1305-glue.c index 9eee71dc0b99..576e7a58e0b1 100644 --- a/arch/mips/crypto/poly1305-glue.c +++ b/arch/mips/crypto/poly1305-glue.c @@ -5,13 +5,10 @@ * Copyright (C) 2019 Linaro Ltd. */ -#include -#include -#include -#include +#include #include -#include #include +#include asmlinkage void poly1305_init_mips(void *state, const u8 *key); asmlinkage void poly1305_blocks_mips(void *state, const u8 *src, u32 len, u32 hibit); @@ -28,77 +25,6 @@ void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 key[POLY1305_KE } EXPORT_SYMBOL(poly1305_init_arch); -static int mips_poly1305_init(struct shash_desc *desc) -{ - struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc); - - dctx->buflen = 0; - dctx->rset = 0; - dctx->sset = false; - - return 0; -} - -static void mips_poly1305_blocks(struct poly1305_desc_ctx *dctx, const u8 *src, - u32 len, u32 hibit) -{ - if (unlikely(!dctx->sset)) { - if (!dctx->rset) { - poly1305_init_mips(&dctx->h, src); - src += POLY1305_BLOCK_SIZE; - len -= POLY1305_BLOCK_SIZE; - dctx->rset = 1; - } - if (len >= POLY1305_BLOCK_SIZE) { - dctx->s[0] = get_unaligned_le32(src + 0); - dctx->s[1] = get_unaligned_le32(src + 4); - dctx->s[2] = get_unaligned_le32(src + 8); - dctx->s[3] = get_unaligned_le32(src + 12); - src += POLY1305_BLOCK_SIZE; - len -= POLY1305_BLOCK_SIZE; - dctx->sset = true; - } - if (len < POLY1305_BLOCK_SIZE) - return; - } - - len &= ~(POLY1305_BLOCK_SIZE - 1); - - poly1305_blocks_mips(&dctx->h, src, len, hibit); -} - -static int mips_poly1305_update(struct shash_desc *desc, const u8 *src, - unsigned int len) -{ - struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc); - - if (unlikely(dctx->buflen)) { - u32 bytes = min(len, POLY1305_BLOCK_SIZE - dctx->buflen); - - memcpy(dctx->buf + dctx->buflen, src, bytes); - src += bytes; - len -= bytes; - dctx->buflen += bytes; - - if (dctx->buflen == POLY1305_BLOCK_SIZE) { - mips_poly1305_blocks(dctx, dctx->buf, POLY1305_BLOCK_SIZE, 1); - dctx->buflen = 0; - } - } - - if (likely(len >= POLY1305_BLOCK_SIZE)) { - mips_poly1305_blocks(dctx, src, len, 1); - src += round_down(len, POLY1305_BLOCK_SIZE); - len %= POLY1305_BLOCK_SIZE; - } - - if (unlikely(len)) { - dctx->buflen = len; - memcpy(dctx->buf, src, len); - } - return 0; -} - void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src, unsigned int nbytes) { @@ -146,53 +72,11 @@ void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst) } EXPORT_SYMBOL(poly1305_final_arch); -static int mips_poly1305_final(struct shash_desc *desc, u8 *dst) -{ - struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc); - - if (unlikely(!dctx->sset)) - return -ENOKEY; - - poly1305_final_arch(dctx, dst); - return 0; -} - -static struct shash_alg mips_poly1305_alg = { - .init = mips_poly1305_init, - .update = mips_poly1305_update, - .final = mips_poly1305_final, - .digestsize = POLY1305_DIGEST_SIZE, - .descsize = sizeof(struct poly1305_desc_ctx), - - .base.cra_name = "poly1305", - .base.cra_driver_name = "poly1305-mips-old", - .base.cra_priority = 200, - .base.cra_blocksize = POLY1305_BLOCK_SIZE, - .base.cra_module = THIS_MODULE, -}; - bool poly1305_is_arch_optimized(void) { return true; } EXPORT_SYMBOL(poly1305_is_arch_optimized); -static int __init mips_poly1305_mod_init(void) -{ - return IS_REACHABLE(CONFIG_CRYPTO_HASH) ? - crypto_register_shash(&mips_poly1305_alg) : 0; -} - -static void __exit mips_poly1305_mod_exit(void) -{ - if (IS_REACHABLE(CONFIG_CRYPTO_HASH)) - crypto_unregister_shash(&mips_poly1305_alg); -} - -arch_initcall(mips_poly1305_mod_init); -module_exit(mips_poly1305_mod_exit); - MODULE_DESCRIPTION("Poly1305 transform (MIPS accelerated"); MODULE_LICENSE("GPL v2"); -MODULE_ALIAS_CRYPTO("poly1305"); -MODULE_ALIAS_CRYPTO("poly1305-mips"); From 21969da642a263dde2f7b483ad676b901ae129a3 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Sat, 12 Apr 2025 21:54:19 -0700 Subject: [PATCH 138/382] crypto: x86/poly1305 - remove redundant shash algorithm Since crypto/poly1305.c now registers a poly1305-$(ARCH) shash algorithm that uses the architecture's Poly1305 library functions, individual architectures no longer need to do the same. Therefore, remove the redundant shash algorithm from the arch-specific code and leave just the library functions there. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/x86/crypto/Kconfig | 7 --- arch/x86/crypto/poly1305_glue.c | 97 +++------------------------------ 2 files changed, 7 insertions(+), 97 deletions(-) diff --git a/arch/x86/crypto/Kconfig b/arch/x86/crypto/Kconfig index 4e0bd1258751..b4a4dc722ba8 100644 --- a/arch/x86/crypto/Kconfig +++ b/arch/x86/crypto/Kconfig @@ -399,16 +399,9 @@ config CRYPTO_POLYVAL_CLMUL_NI config CRYPTO_POLY1305_X86_64 tristate depends on X86 && 64BIT - select CRYPTO_HASH select CRYPTO_LIB_POLY1305_GENERIC select CRYPTO_ARCH_HAVE_LIB_POLY1305 default CRYPTO_LIB_POLY1305_INTERNAL - help - Poly1305 authenticator algorithm (RFC7539) - - Architecture: x86_64 using: - - SSE2 (Streaming SIMD Extensions 2) - - AVX2 (Advanced Vector Extensions 2) config CRYPTO_SHA1_SSSE3 tristate "Hash functions: SHA-1 (SSSE3/AVX/AVX2/SHA-NI)" diff --git a/arch/x86/crypto/poly1305_glue.c b/arch/x86/crypto/poly1305_glue.c index 40d03a9a4fb5..a0fc543a0d68 100644 --- a/arch/x86/crypto/poly1305_glue.c +++ b/arch/x86/crypto/poly1305_glue.c @@ -3,15 +3,13 @@ * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. */ -#include -#include -#include #include -#include +#include #include #include #include #include +#include #include #include @@ -137,38 +135,13 @@ void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 key[POLY1305_KE dctx->s[2] = get_unaligned_le32(&key[24]); dctx->s[3] = get_unaligned_le32(&key[28]); dctx->buflen = 0; - dctx->sset = true; } EXPORT_SYMBOL(poly1305_init_arch); -static unsigned int crypto_poly1305_setdctxkey(struct poly1305_desc_ctx *dctx, - const u8 *inp, unsigned int len) -{ - unsigned int acc = 0; - if (unlikely(!dctx->sset)) { - if (!dctx->rset && len >= POLY1305_BLOCK_SIZE) { - poly1305_simd_init(&dctx->h, inp); - inp += POLY1305_BLOCK_SIZE; - len -= POLY1305_BLOCK_SIZE; - acc += POLY1305_BLOCK_SIZE; - dctx->rset = 1; - } - if (len >= POLY1305_BLOCK_SIZE) { - dctx->s[0] = get_unaligned_le32(&inp[0]); - dctx->s[1] = get_unaligned_le32(&inp[4]); - dctx->s[2] = get_unaligned_le32(&inp[8]); - dctx->s[3] = get_unaligned_le32(&inp[12]); - acc += POLY1305_BLOCK_SIZE; - dctx->sset = true; - } - } - return acc; -} - void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src, unsigned int srclen) { - unsigned int bytes, used; + unsigned int bytes; if (unlikely(dctx->buflen)) { bytes = min(srclen, POLY1305_BLOCK_SIZE - dctx->buflen); @@ -178,19 +151,16 @@ void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src, dctx->buflen += bytes; if (dctx->buflen == POLY1305_BLOCK_SIZE) { - if (likely(!crypto_poly1305_setdctxkey(dctx, dctx->buf, POLY1305_BLOCK_SIZE))) - poly1305_simd_blocks(&dctx->h, dctx->buf, POLY1305_BLOCK_SIZE, 1); + poly1305_simd_blocks(&dctx->h, dctx->buf, POLY1305_BLOCK_SIZE, 1); dctx->buflen = 0; } } if (likely(srclen >= POLY1305_BLOCK_SIZE)) { bytes = round_down(srclen, POLY1305_BLOCK_SIZE); - srclen -= bytes; - used = crypto_poly1305_setdctxkey(dctx, src, bytes); - if (likely(bytes - used)) - poly1305_simd_blocks(&dctx->h, src + used, bytes - used, 1); + poly1305_simd_blocks(&dctx->h, src, bytes, 1); src += bytes; + srclen -= bytes; } if (unlikely(srclen)) { @@ -214,49 +184,6 @@ void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst) } EXPORT_SYMBOL(poly1305_final_arch); -static int crypto_poly1305_init(struct shash_desc *desc) -{ - struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc); - - *dctx = (struct poly1305_desc_ctx){}; - return 0; -} - -static int crypto_poly1305_update(struct shash_desc *desc, - const u8 *src, unsigned int srclen) -{ - struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc); - - poly1305_update_arch(dctx, src, srclen); - return 0; -} - -static int crypto_poly1305_final(struct shash_desc *desc, u8 *dst) -{ - struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc); - - if (unlikely(!dctx->sset)) - return -ENOKEY; - - poly1305_final_arch(dctx, dst); - return 0; -} - -static struct shash_alg alg = { - .digestsize = POLY1305_DIGEST_SIZE, - .init = crypto_poly1305_init, - .update = crypto_poly1305_update, - .final = crypto_poly1305_final, - .descsize = sizeof(struct poly1305_desc_ctx), - .base = { - .cra_name = "poly1305", - .cra_driver_name = "poly1305-simd", - .cra_priority = 300, - .cra_blocksize = POLY1305_BLOCK_SIZE, - .cra_module = THIS_MODULE, - }, -}; - bool poly1305_is_arch_optimized(void) { return static_key_enabled(&poly1305_use_avx); @@ -277,20 +204,10 @@ static int __init poly1305_simd_mod_init(void) /* Skylake downclocks unacceptably much when using zmm, but later generations are fast. */ boot_cpu_data.x86_vfm != INTEL_SKYLAKE_X) static_branch_enable(&poly1305_use_avx512); - return IS_REACHABLE(CONFIG_CRYPTO_HASH) ? crypto_register_shash(&alg) : 0; + return 0; } - -static void __exit poly1305_simd_mod_exit(void) -{ - if (IS_REACHABLE(CONFIG_CRYPTO_HASH)) - crypto_unregister_shash(&alg); -} - arch_initcall(poly1305_simd_mod_init); -module_exit(poly1305_simd_mod_exit); MODULE_LICENSE("GPL"); MODULE_AUTHOR("Jason A. Donenfeld "); MODULE_DESCRIPTION("Poly1305 authenticator"); -MODULE_ALIAS_CRYPTO("poly1305"); -MODULE_ALIAS_CRYPTO("poly1305-simd"); From 34374f76af9b65f7cf89c2fe62489593d6ecc443 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Sat, 12 Apr 2025 21:54:20 -0700 Subject: [PATCH 139/382] crypto: x86/poly1305 - don't select CRYPTO_LIB_POLY1305_GENERIC The x86 Poly1305 code never falls back to the generic code, so selecting CRYPTO_LIB_POLY1305_GENERIC is unnecessary. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/x86/crypto/Kconfig | 1 - 1 file changed, 1 deletion(-) diff --git a/arch/x86/crypto/Kconfig b/arch/x86/crypto/Kconfig index b4a4dc722ba8..9a3cade49f63 100644 --- a/arch/x86/crypto/Kconfig +++ b/arch/x86/crypto/Kconfig @@ -399,7 +399,6 @@ config CRYPTO_POLYVAL_CLMUL_NI config CRYPTO_POLY1305_X86_64 tristate depends on X86 && 64BIT - select CRYPTO_LIB_POLY1305_GENERIC select CRYPTO_ARCH_HAVE_LIB_POLY1305 default CRYPTO_LIB_POLY1305_INTERNAL From 5f7325fbb3d416ba66f163a9272e17d70f1d9bf4 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Sat, 12 Apr 2025 21:54:21 -0700 Subject: [PATCH 140/382] crypto: poly1305 - remove rset and sset fields of poly1305_desc_ctx These fields are no longer needed, so remove them. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- include/crypto/poly1305.h | 4 ---- lib/crypto/poly1305.c | 2 -- 2 files changed, 6 deletions(-) diff --git a/include/crypto/poly1305.h b/include/crypto/poly1305.h index 91444965772a..6e21ec2d1dc2 100644 --- a/include/crypto/poly1305.h +++ b/include/crypto/poly1305.h @@ -43,10 +43,6 @@ struct poly1305_desc_ctx { u8 buf[POLY1305_BLOCK_SIZE]; /* bytes used in partial buffer */ unsigned int buflen; - /* how many keys have been set in r[] */ - unsigned short rset; - /* whether s[] has been set */ - bool sset; /* finalize key */ u32 s[4]; /* accumulator */ diff --git a/lib/crypto/poly1305.c b/lib/crypto/poly1305.c index 6e80214ebad8..b633b043f0f6 100644 --- a/lib/crypto/poly1305.c +++ b/lib/crypto/poly1305.c @@ -22,8 +22,6 @@ void poly1305_init_generic(struct poly1305_desc_ctx *desc, desc->s[3] = get_unaligned_le32(key + 28); poly1305_core_init(&desc->h); desc->buflen = 0; - desc->sset = true; - desc->rset = 2; } EXPORT_SYMBOL_GPL(poly1305_init_generic); From 2f5f3cd003fa303574da1a8fcdec9f602b7fc221 Mon Sep 17 00:00:00 2001 From: Ovidiu Panait Date: Mon, 7 Apr 2025 15:36:03 +0300 Subject: [PATCH 141/382] crypto: sun8i-ss - use API helpers to setup fallback request Rather than setting up the fallback request by hand, use ahash_request_set_callback() and ahash_request_set_crypt() API helpers to properly setup the new request. This also ensures that the completion callback is properly passed down to the fallback algorithm, which avoids a crash with async fallbacks. Signed-off-by: Ovidiu Panait Tested-by: Corentin Labbe Acked-by: Corentin Labbe Signed-off-by: Herbert Xu --- .../crypto/allwinner/sun8i-ss/sun8i-ss-hash.c | 49 ++++++++++--------- 1 file changed, 27 insertions(+), 22 deletions(-) diff --git a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c index 753f67a36dc5..8bc08089f044 100644 --- a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c +++ b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c @@ -150,7 +150,9 @@ int sun8i_ss_hash_init(struct ahash_request *areq) memset(rctx, 0, sizeof(struct sun8i_ss_hash_reqctx)); ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); - rctx->fallback_req.base.flags = areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; + ahash_request_set_callback(&rctx->fallback_req, + areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + areq->base.complete, areq->base.data); return crypto_ahash_init(&rctx->fallback_req); } @@ -162,7 +164,9 @@ int sun8i_ss_hash_export(struct ahash_request *areq, void *out) struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); - rctx->fallback_req.base.flags = areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; + ahash_request_set_callback(&rctx->fallback_req, + areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + areq->base.complete, areq->base.data); return crypto_ahash_export(&rctx->fallback_req, out); } @@ -174,7 +178,9 @@ int sun8i_ss_hash_import(struct ahash_request *areq, const void *in) struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); - rctx->fallback_req.base.flags = areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; + ahash_request_set_callback(&rctx->fallback_req, + areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + areq->base.complete, areq->base.data); return crypto_ahash_import(&rctx->fallback_req, in); } @@ -186,9 +192,10 @@ int sun8i_ss_hash_final(struct ahash_request *areq) struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); - rctx->fallback_req.base.flags = areq->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; - rctx->fallback_req.result = areq->result; + ahash_request_set_callback(&rctx->fallback_req, + areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + areq->base.complete, areq->base.data); + ahash_request_set_crypt(&rctx->fallback_req, NULL, areq->result, 0); if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG)) { struct ahash_alg *alg = crypto_ahash_alg(tfm); @@ -212,10 +219,10 @@ int sun8i_ss_hash_update(struct ahash_request *areq) struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); - rctx->fallback_req.base.flags = areq->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; - rctx->fallback_req.nbytes = areq->nbytes; - rctx->fallback_req.src = areq->src; + ahash_request_set_callback(&rctx->fallback_req, + areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + areq->base.complete, areq->base.data); + ahash_request_set_crypt(&rctx->fallback_req, areq->src, NULL, areq->nbytes); return crypto_ahash_update(&rctx->fallback_req); } @@ -227,12 +234,11 @@ int sun8i_ss_hash_finup(struct ahash_request *areq) struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); - rctx->fallback_req.base.flags = areq->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; - - rctx->fallback_req.nbytes = areq->nbytes; - rctx->fallback_req.src = areq->src; - rctx->fallback_req.result = areq->result; + ahash_request_set_callback(&rctx->fallback_req, + areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + areq->base.complete, areq->base.data); + ahash_request_set_crypt(&rctx->fallback_req, areq->src, areq->result, + areq->nbytes); if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG)) { struct ahash_alg *alg = crypto_ahash_alg(tfm); @@ -256,12 +262,11 @@ static int sun8i_ss_hash_digest_fb(struct ahash_request *areq) struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); - rctx->fallback_req.base.flags = areq->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; - - rctx->fallback_req.nbytes = areq->nbytes; - rctx->fallback_req.src = areq->src; - rctx->fallback_req.result = areq->result; + ahash_request_set_callback(&rctx->fallback_req, + areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + areq->base.complete, areq->base.data); + ahash_request_set_crypt(&rctx->fallback_req, areq->src, areq->result, + areq->nbytes); if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG)) { struct ahash_alg *alg = crypto_ahash_alg(tfm); From ddd0855fa3c3f1de020ab3aeddce15fe38e116f7 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Tue, 15 Apr 2025 17:23:19 +0800 Subject: [PATCH 142/382] crypto: deflate - Make the acomp walk atomic Add an atomic flag to the acomp walk and use that in deflate. Due to the use of a per-cpu context, it is impossible to sleep during the walk in deflate. Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-lkp/202504151654.4c3b6393-lkp@intel.com Fixes: 08cabc7d3c86 ("crypto: deflate - Convert to acomp") Signed-off-by: Herbert Xu --- crypto/acompress.c | 4 ++-- crypto/deflate.c | 4 ++-- include/crypto/internal/acompress.h | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/crypto/acompress.c b/crypto/acompress.c index 606d09a7fbfd..b0f9192f6b2e 100644 --- a/crypto/acompress.c +++ b/crypto/acompress.c @@ -536,7 +536,7 @@ int acomp_walk_next_dst(struct acomp_walk *walk) EXPORT_SYMBOL_GPL(acomp_walk_next_dst); int acomp_walk_virt(struct acomp_walk *__restrict walk, - struct acomp_req *__restrict req) + struct acomp_req *__restrict req, bool atomic) { struct scatterlist *src = req->src; struct scatterlist *dst = req->dst; @@ -548,7 +548,7 @@ int acomp_walk_virt(struct acomp_walk *__restrict walk, return -EINVAL; walk->flags = 0; - if ((req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP)) + if ((req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) && !atomic) walk->flags |= ACOMP_WALK_SLEEP; if ((req->base.flags & CRYPTO_ACOMP_REQ_SRC_VIRT)) walk->flags |= ACOMP_WALK_SRC_LINEAR; diff --git a/crypto/deflate.c b/crypto/deflate.c index 57d7af4dfdfb..0d2b64d96d6e 100644 --- a/crypto/deflate.c +++ b/crypto/deflate.c @@ -60,7 +60,7 @@ static int deflate_compress_one(struct acomp_req *req, struct acomp_walk walk; int ret; - ret = acomp_walk_virt(&walk, req); + ret = acomp_walk_virt(&walk, req, true); if (ret) return ret; @@ -140,7 +140,7 @@ static int deflate_decompress_one(struct acomp_req *req, struct acomp_walk walk; int ret; - ret = acomp_walk_virt(&walk, req); + ret = acomp_walk_virt(&walk, req, true); if (ret) return ret; diff --git a/include/crypto/internal/acompress.h b/include/crypto/internal/acompress.h index 0f3ad65be2d9..7eda32619024 100644 --- a/include/crypto/internal/acompress.h +++ b/include/crypto/internal/acompress.h @@ -208,7 +208,7 @@ void acomp_walk_done_dst(struct acomp_walk *walk, int used); int acomp_walk_next_src(struct acomp_walk *walk); int acomp_walk_next_dst(struct acomp_walk *walk); int acomp_walk_virt(struct acomp_walk *__restrict walk, - struct acomp_req *__restrict req); + struct acomp_req *__restrict req, bool atomic); static inline bool acomp_walk_more_src(const struct acomp_walk *walk, int cur) { From 02db42856e430a915b8f9d7074a51910d2184134 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Wed, 16 Apr 2025 15:48:26 +0800 Subject: [PATCH 143/382] crypto: public_key - Make sig/tfm local to if clause in software_key_query The recent code changes in this function triggered a false-positive maybe-uninitialized warning in software_key_query. Rearrange the code by moving the sig/tfm variables into the if clause where they are actually used. Signed-off-by: Herbert Xu --- crypto/asymmetric_keys/public_key.c | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/crypto/asymmetric_keys/public_key.c b/crypto/asymmetric_keys/public_key.c index 89dc887d2c5c..e5b177c8e842 100644 --- a/crypto/asymmetric_keys/public_key.c +++ b/crypto/asymmetric_keys/public_key.c @@ -163,10 +163,8 @@ static u8 *pkey_pack_u32(u8 *dst, u32 val) static int software_key_query(const struct kernel_pkey_params *params, struct kernel_pkey_query *info) { - struct crypto_akcipher *tfm; struct public_key *pkey = params->key->payload.data[asym_crypto]; char alg_name[CRYPTO_MAX_ALG_NAME]; - struct crypto_sig *sig; u8 *key, *ptr; int ret, len; bool issig; @@ -191,6 +189,8 @@ static int software_key_query(const struct kernel_pkey_params *params, memset(info, 0, sizeof(*info)); if (issig) { + struct crypto_sig *sig; + sig = crypto_alloc_sig(alg_name, 0, 0); if (IS_ERR(sig)) { ret = PTR_ERR(sig); @@ -202,7 +202,7 @@ static int software_key_query(const struct kernel_pkey_params *params, else ret = crypto_sig_set_pubkey(sig, key, pkey->keylen); if (ret < 0) - goto error_free_tfm; + goto error_free_sig; len = crypto_sig_keysize(sig); info->key_size = len; @@ -221,7 +221,12 @@ static int software_key_query(const struct kernel_pkey_params *params, if (pkey->key_is_private) info->supported_ops |= KEYCTL_SUPPORTS_DECRYPT; } + +error_free_sig: + crypto_free_sig(sig); } else { + struct crypto_akcipher *tfm; + tfm = crypto_alloc_akcipher(alg_name, 0, 0); if (IS_ERR(tfm)) { ret = PTR_ERR(tfm); @@ -233,7 +238,7 @@ static int software_key_query(const struct kernel_pkey_params *params, else ret = crypto_akcipher_set_pub_key(tfm, key, pkey->keylen); if (ret < 0) - goto error_free_tfm; + goto error_free_akcipher; len = crypto_akcipher_maxsize(tfm); info->key_size = len * BITS_PER_BYTE; @@ -245,15 +250,11 @@ static int software_key_query(const struct kernel_pkey_params *params, info->supported_ops = KEYCTL_SUPPORTS_ENCRYPT; if (pkey->key_is_private) info->supported_ops |= KEYCTL_SUPPORTS_DECRYPT; + +error_free_akcipher: + crypto_free_akcipher(tfm); } - ret = 0; - -error_free_tfm: - if (issig) - crypto_free_sig(sig); - else - crypto_free_akcipher(tfm); error_free_key: kfree_sensitive(key); pr_devel("<==%s() = %d\n", __func__, ret); From da4cb617bc7d827946cbb368034940b379a1de90 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Thu, 17 Apr 2025 10:26:28 +0800 Subject: [PATCH 144/382] powerpc/crc: Include uaccess.h and others The powerpc crc code was relying on pagefault_disable from being pulled in by random header files. Fix this by explicitly including uaccess.h. Also add other missing header files to prevent similar problems in future. Reported-by: Eric Biggers Reported-by: Stephen Rothwell Fixes: 7ba8df47810f ("asm-generic: Make simd.h more resilient") Signed-off-by: Herbert Xu --- arch/powerpc/lib/crc-t10dif-glue.c | 16 ++++++++-------- arch/powerpc/lib/crc32-glue.c | 15 ++++++++------- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/arch/powerpc/lib/crc-t10dif-glue.c b/arch/powerpc/lib/crc-t10dif-glue.c index f411b0120cc5..fa988e249f01 100644 --- a/arch/powerpc/lib/crc-t10dif-glue.c +++ b/arch/powerpc/lib/crc-t10dif-glue.c @@ -6,15 +6,15 @@ * [based on crc32c-vpmsum_glue.c] */ -#include -#include -#include -#include -#include -#include -#include -#include #include +#include +#include +#include +#include +#include +#include +#include +#include #define VMX_ALIGN 16 #define VMX_ALIGN_MASK (VMX_ALIGN-1) diff --git a/arch/powerpc/lib/crc32-glue.c b/arch/powerpc/lib/crc32-glue.c index dbd10f339183..28450fe04e86 100644 --- a/arch/powerpc/lib/crc32-glue.c +++ b/arch/powerpc/lib/crc32-glue.c @@ -1,12 +1,13 @@ // SPDX-License-Identifier: GPL-2.0-only -#include -#include -#include -#include -#include -#include -#include #include +#include +#include +#include +#include +#include +#include +#include +#include #define VMX_ALIGN 16 #define VMX_ALIGN_MASK (VMX_ALIGN-1) From 31b20bc22f6897a881059c0c85635966e0fd23a5 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 18 Apr 2025 10:52:34 +0800 Subject: [PATCH 145/382] crypto: acomp - Add missing return statements in compress/decompress The return statements were missing which causes REQ_CHAIN algorithms to execute twice for every request. Reported-by: Eric Biggers Fixes: 64929fe8c0a4 ("crypto: acomp - Remove request chaining") Signed-off-by: Herbert Xu --- crypto/acompress.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crypto/acompress.c b/crypto/acompress.c index b0f9192f6b2e..4c665c6fb5d6 100644 --- a/crypto/acompress.c +++ b/crypto/acompress.c @@ -292,7 +292,7 @@ int crypto_acomp_compress(struct acomp_req *req) if (acomp_req_on_stack(req) && acomp_is_async(tfm)) return -EAGAIN; if (crypto_acomp_req_chain(tfm) || acomp_request_issg(req)) - crypto_acomp_reqtfm(req)->compress(req); + return crypto_acomp_reqtfm(req)->compress(req); return acomp_do_req_chain(req, true); } EXPORT_SYMBOL_GPL(crypto_acomp_compress); @@ -304,7 +304,7 @@ int crypto_acomp_decompress(struct acomp_req *req) if (acomp_req_on_stack(req) && acomp_is_async(tfm)) return -EAGAIN; if (crypto_acomp_req_chain(tfm) || acomp_request_issg(req)) - crypto_acomp_reqtfm(req)->decompress(req); + return crypto_acomp_reqtfm(req)->decompress(req); return acomp_do_req_chain(req, false); } EXPORT_SYMBOL_GPL(crypto_acomp_decompress); From 8821d26926ea6c1115974a989ec5a467da402f4f Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Thu, 17 Apr 2025 20:59:09 -0700 Subject: [PATCH 146/382] crypto: lib/chacha - restore ability to remove modules Though the module_exit functions are now no-ops, they should still be defined, since otherwise the modules become unremovable. Fixes: 08820553f33a ("crypto: arm/chacha - remove the redundant skcipher algorithms") Fixes: 8c28abede16c ("crypto: arm64/chacha - remove the skcipher algorithms") Fixes: f7915484c020 ("crypto: powerpc/chacha - remove the skcipher algorithms") Fixes: ceba0eda8313 ("crypto: riscv/chacha - implement library instead of skcipher") Fixes: 632ab0978f08 ("crypto: x86/chacha - remove the skcipher algorithms") Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/arm/crypto/chacha-glue.c | 5 +++++ arch/arm64/crypto/chacha-neon-glue.c | 5 +++++ arch/powerpc/crypto/chacha-p10-glue.c | 5 +++++ arch/riscv/crypto/chacha-riscv64-glue.c | 5 +++++ arch/x86/crypto/chacha_glue.c | 5 +++++ 5 files changed, 25 insertions(+) diff --git a/arch/arm/crypto/chacha-glue.c b/arch/arm/crypto/chacha-glue.c index 3a5c75c95d43..12afb40cf1ff 100644 --- a/arch/arm/crypto/chacha-glue.c +++ b/arch/arm/crypto/chacha-glue.c @@ -124,6 +124,11 @@ static int __init chacha_arm_mod_init(void) } arch_initcall(chacha_arm_mod_init); +static void __exit chacha_arm_mod_exit(void) +{ +} +module_exit(chacha_arm_mod_exit); + MODULE_DESCRIPTION("ChaCha and HChaCha functions (ARM optimized)"); MODULE_AUTHOR("Ard Biesheuvel "); MODULE_LICENSE("GPL v2"); diff --git a/arch/arm64/crypto/chacha-neon-glue.c b/arch/arm64/crypto/chacha-neon-glue.c index a0c336b28402..14a2836eff61 100644 --- a/arch/arm64/crypto/chacha-neon-glue.c +++ b/arch/arm64/crypto/chacha-neon-glue.c @@ -106,6 +106,11 @@ static int __init chacha_simd_mod_init(void) } arch_initcall(chacha_simd_mod_init); +static void __exit chacha_simd_mod_exit(void) +{ +} +module_exit(chacha_simd_mod_exit); + MODULE_DESCRIPTION("ChaCha and HChaCha functions (ARM64 optimized)"); MODULE_AUTHOR("Ard Biesheuvel "); MODULE_LICENSE("GPL v2"); diff --git a/arch/powerpc/crypto/chacha-p10-glue.c b/arch/powerpc/crypto/chacha-p10-glue.c index 9982929573ad..351ed409f9b2 100644 --- a/arch/powerpc/crypto/chacha-p10-glue.c +++ b/arch/powerpc/crypto/chacha-p10-glue.c @@ -89,6 +89,11 @@ static int __init chacha_p10_init(void) } arch_initcall(chacha_p10_init); +static void __exit chacha_p10_exit(void) +{ +} +module_exit(chacha_p10_exit); + MODULE_DESCRIPTION("ChaCha stream cipher (P10 accelerated)"); MODULE_AUTHOR("Danny Tsen "); MODULE_LICENSE("GPL v2"); diff --git a/arch/riscv/crypto/chacha-riscv64-glue.c b/arch/riscv/crypto/chacha-riscv64-glue.c index ccaab0dea383..afc4e3be3cac 100644 --- a/arch/riscv/crypto/chacha-riscv64-glue.c +++ b/arch/riscv/crypto/chacha-riscv64-glue.c @@ -64,6 +64,11 @@ static int __init riscv64_chacha_mod_init(void) } arch_initcall(riscv64_chacha_mod_init); +static void __exit riscv64_chacha_mod_exit(void) +{ +} +module_exit(riscv64_chacha_mod_exit); + MODULE_DESCRIPTION("ChaCha stream cipher (RISC-V optimized)"); MODULE_AUTHOR("Jerry Shih "); MODULE_LICENSE("GPL"); diff --git a/arch/x86/crypto/chacha_glue.c b/arch/x86/crypto/chacha_glue.c index fcc14c006bde..59bf63c00072 100644 --- a/arch/x86/crypto/chacha_glue.c +++ b/arch/x86/crypto/chacha_glue.c @@ -176,6 +176,11 @@ static int __init chacha_simd_mod_init(void) } arch_initcall(chacha_simd_mod_init); +static void __exit chacha_simd_mod_exit(void) +{ +} +module_exit(chacha_simd_mod_exit); + MODULE_LICENSE("GPL"); MODULE_AUTHOR("Martin Willi "); MODULE_DESCRIPTION("ChaCha and HChaCha functions (x86_64 optimized)"); From bb9c648b334be581a791c7669abaa594e4b5ebb7 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Thu, 17 Apr 2025 21:00:17 -0700 Subject: [PATCH 147/382] crypto: lib/poly1305 - restore ability to remove modules Though the module_exit functions are now no-ops, they should still be defined, since otherwise the modules become unremovable. Fixes: 1f81c58279c7 ("crypto: arm/poly1305 - remove redundant shash algorithm") Fixes: f4b1a73aec5c ("crypto: arm64/poly1305 - remove redundant shash algorithm") Fixes: 378a337ab40f ("crypto: powerpc/poly1305 - implement library instead of shash") Fixes: 21969da642a2 ("crypto: x86/poly1305 - remove redundant shash algorithm") Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/arm/crypto/poly1305-glue.c | 5 +++++ arch/arm64/crypto/poly1305-glue.c | 5 +++++ arch/powerpc/crypto/poly1305-p10-glue.c | 5 +++++ arch/x86/crypto/poly1305_glue.c | 5 +++++ 4 files changed, 20 insertions(+) diff --git a/arch/arm/crypto/poly1305-glue.c b/arch/arm/crypto/poly1305-glue.c index 6d6998b3ec7e..42d0ebde1ae1 100644 --- a/arch/arm/crypto/poly1305-glue.c +++ b/arch/arm/crypto/poly1305-glue.c @@ -116,5 +116,10 @@ static int __init arm_poly1305_mod_init(void) } arch_initcall(arm_poly1305_mod_init); +static void __exit arm_poly1305_mod_exit(void) +{ +} +module_exit(arm_poly1305_mod_exit); + MODULE_DESCRIPTION("Accelerated Poly1305 transform for ARM"); MODULE_LICENSE("GPL v2"); diff --git a/arch/arm64/crypto/poly1305-glue.c b/arch/arm64/crypto/poly1305-glue.c index cb152ceac14a..906970dd5373 100644 --- a/arch/arm64/crypto/poly1305-glue.c +++ b/arch/arm64/crypto/poly1305-glue.c @@ -107,5 +107,10 @@ static int __init neon_poly1305_mod_init(void) } arch_initcall(neon_poly1305_mod_init); +static void __exit neon_poly1305_mod_exit(void) +{ +} +module_exit(neon_poly1305_mod_exit); + MODULE_DESCRIPTION("Poly1305 authenticator (ARM64 optimized)"); MODULE_LICENSE("GPL v2"); diff --git a/arch/powerpc/crypto/poly1305-p10-glue.c b/arch/powerpc/crypto/poly1305-p10-glue.c index 40d296d52c23..00617f4c58e6 100644 --- a/arch/powerpc/crypto/poly1305-p10-glue.c +++ b/arch/powerpc/crypto/poly1305-p10-glue.c @@ -127,6 +127,11 @@ static int __init poly1305_p10_init(void) } arch_initcall(poly1305_p10_init); +static void __exit poly1305_p10_exit(void) +{ +} +module_exit(poly1305_p10_exit); + MODULE_LICENSE("GPL"); MODULE_AUTHOR("Danny Tsen "); MODULE_DESCRIPTION("Optimized Poly1305 for P10"); diff --git a/arch/x86/crypto/poly1305_glue.c b/arch/x86/crypto/poly1305_glue.c index a0fc543a0d68..cff35ca5822a 100644 --- a/arch/x86/crypto/poly1305_glue.c +++ b/arch/x86/crypto/poly1305_glue.c @@ -208,6 +208,11 @@ static int __init poly1305_simd_mod_init(void) } arch_initcall(poly1305_simd_mod_init); +static void __exit poly1305_simd_mod_exit(void) +{ +} +module_exit(poly1305_simd_mod_exit); + MODULE_LICENSE("GPL"); MODULE_AUTHOR("Jason A. Donenfeld "); MODULE_DESCRIPTION("Poly1305 authenticator"); From 2ee97c190b2b6769d7c1688e80b84c951177a22b Mon Sep 17 00:00:00 2001 From: Thorsten Blum Date: Fri, 18 Apr 2025 00:36:49 +0200 Subject: [PATCH 148/382] crypto: engine - Realign struct crypto_engine to save 8 bytes Realign struct crypto_engine to reduce its size by 8 bytes. Total size is now 192 bytes, allowing it to fit within 3 cachelines instead of 4. pahole output before: /* size: 200, cachelines: 4, members: 17 */ /* sum members: 183, holes: 3, sum holes: 17 */ /* paddings: 1, sum paddings: 4 */ /* last cacheline: 8 bytes */ and after: /* size: 192, cachelines: 3, members: 17 */ /* sum members: 183, holes: 2, sum holes: 9 */ /* paddings: 1, sum paddings: 4 */ No functional changes intended. Signed-off-by: Thorsten Blum Signed-off-by: Herbert Xu --- include/crypto/internal/engine.h | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/include/crypto/internal/engine.h b/include/crypto/internal/engine.h index fbf4be56cf12..b6a4ea2240fc 100644 --- a/include/crypto/internal/engine.h +++ b/include/crypto/internal/engine.h @@ -27,10 +27,10 @@ struct device; * @retry_support: indication that the hardware allows re-execution * of a failed backlog request * crypto-engine, in head position to keep order + * @rt: whether this queue is set to run as a realtime task * @list: link with the global crypto engine list * @queue_lock: spinlock to synchronise access to request queue * @queue: the crypto queue of the engine - * @rt: whether this queue is set to run as a realtime task * @prepare_crypt_hardware: a request will soon arrive from the queue * so the subsystem requests the driver to prepare the hardware * by issuing this call @@ -51,14 +51,13 @@ struct crypto_engine { bool running; bool retry_support; + bool rt; struct list_head list; spinlock_t queue_lock; struct crypto_queue queue; struct device *dev; - bool rt; - int (*prepare_crypt_hardware)(struct crypto_engine *engine); int (*unprepare_crypt_hardware)(struct crypto_engine *engine); int (*do_batch_requests)(struct crypto_engine *engine); From 7650f826f7b2d84782f9147c51687ff0364125e9 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 18 Apr 2025 10:58:41 +0800 Subject: [PATCH 149/382] crypto: shash - Handle partial blocks in API Provide an option to handle the partial blocks in the shash API. Almost every hash algorithm has a block size and are only able to hash partial blocks on finalisation. Rather than duplicating the partial block handling many times, add this functionality to the shash API. It is optional (e.g., hmac would never need this by relying on the partial block handling of the underlying hash), and to enable it set the bit CRYPTO_AHASH_ALG_BLOCK_ONLY. The export format is always that of the underlying hash export, plus the partial block buffer, followed by a single-byte for the partial block length. Set the bit CRYPTO_AHASH_ALG_FINAL_NONZERO to withhold an extra byte in the partial block. This will come in handy when this is extended to ahash where hardware often can't deal with a zero-length final. It will also be used for algorithms requiring an extra block for finalisation (e.g., cmac). As an optimisation, set the bit CRYPTO_AHASH_ALG_FINUP_MAX if the algorithm wishes to get as much data as possible instead of just the last partial block. The descriptor will be zeroed after finalisation. Signed-off-by: Herbert Xu --- crypto/shash.c | 229 +++++++++++++++++++++++++++------ include/crypto/hash.h | 104 +++++++++------ include/crypto/internal/hash.h | 15 +++ include/linux/crypto.h | 2 + 4 files changed, 271 insertions(+), 79 deletions(-) diff --git a/crypto/shash.c b/crypto/shash.c index f23bd9cb1873..b6c79a4a044a 100644 --- a/crypto/shash.c +++ b/crypto/shash.c @@ -16,6 +16,24 @@ #include "hash.h" +static inline bool crypto_shash_block_only(struct crypto_shash *tfm) +{ + return crypto_shash_alg(tfm)->base.cra_flags & + CRYPTO_AHASH_ALG_BLOCK_ONLY; +} + +static inline bool crypto_shash_final_nonzero(struct crypto_shash *tfm) +{ + return crypto_shash_alg(tfm)->base.cra_flags & + CRYPTO_AHASH_ALG_FINAL_NONZERO; +} + +static inline bool crypto_shash_finup_max(struct crypto_shash *tfm) +{ + return crypto_shash_alg(tfm)->base.cra_flags & + CRYPTO_AHASH_ALG_FINUP_MAX; +} + int shash_no_setkey(struct crypto_shash *tfm, const u8 *key, unsigned int keylen) { @@ -46,18 +64,27 @@ int crypto_shash_setkey(struct crypto_shash *tfm, const u8 *key, } EXPORT_SYMBOL_GPL(crypto_shash_setkey); -int crypto_shash_update(struct shash_desc *desc, const u8 *data, - unsigned int len) +static int __crypto_shash_init(struct shash_desc *desc) { - return crypto_shash_alg(desc->tfm)->update(desc, data, len); -} -EXPORT_SYMBOL_GPL(crypto_shash_update); + struct crypto_shash *tfm = desc->tfm; -int crypto_shash_final(struct shash_desc *desc, u8 *out) -{ - return crypto_shash_alg(desc->tfm)->final(desc, out); + if (crypto_shash_block_only(tfm)) { + u8 *buf = shash_desc_ctx(desc); + + buf += crypto_shash_descsize(tfm) - 1; + *buf = 0; + } + + return crypto_shash_alg(tfm)->init(desc); } -EXPORT_SYMBOL_GPL(crypto_shash_final); + +int crypto_shash_init(struct shash_desc *desc) +{ + if (crypto_shash_get_flags(desc->tfm) & CRYPTO_TFM_NEED_KEY) + return -ENOKEY; + return __crypto_shash_init(desc); +} +EXPORT_SYMBOL_GPL(crypto_shash_init); static int shash_default_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) @@ -68,20 +95,89 @@ static int shash_default_finup(struct shash_desc *desc, const u8 *data, shash->final(desc, out); } -int crypto_shash_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) +static int crypto_shash_op_and_zero( + int (*op)(struct shash_desc *desc, const u8 *data, + unsigned int len, u8 *out), + struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) { - return crypto_shash_alg(desc->tfm)->finup(desc, data, len, out); + int err; + + err = op(desc, data, len, out); + memset(shash_desc_ctx(desc), 0, crypto_shash_descsize(desc->tfm)); + return err; +} + +int crypto_shash_finup(struct shash_desc *restrict desc, const u8 *data, + unsigned int len, u8 *restrict out) +{ + struct crypto_shash *tfm = desc->tfm; + u8 *blenp = shash_desc_ctx(desc); + bool finup_max, nonzero; + unsigned int bs; + int err; + u8 *buf; + + if (!crypto_shash_block_only(tfm)) { + if (out) + goto finup; + return crypto_shash_alg(tfm)->update(desc, data, len); + } + + finup_max = out && crypto_shash_finup_max(tfm); + + /* Retain extra block for final nonzero algorithms. */ + nonzero = crypto_shash_final_nonzero(tfm); + + /* + * The partial block buffer follows the algorithm desc context. + * The byte following that contains the length. + */ + blenp += crypto_shash_descsize(tfm) - 1; + bs = crypto_shash_blocksize(tfm); + buf = blenp - bs; + + if (likely(!*blenp && finup_max)) + goto finup; + + while ((*blenp + len) >= bs + nonzero) { + unsigned int nbytes = len - nonzero; + const u8 *src = data; + + if (*blenp) { + memcpy(buf + *blenp, data, bs - *blenp); + nbytes = bs; + src = buf; + } + + err = crypto_shash_alg(tfm)->update(desc, src, nbytes); + if (err < 0) + return err; + + data += nbytes - err - *blenp; + len -= nbytes - err - *blenp; + *blenp = 0; + } + + if (*blenp || !out) { + memcpy(buf + *blenp, data, len); + *blenp += len; + if (!out) + return 0; + data = buf; + len = *blenp; + } + +finup: + return crypto_shash_op_and_zero(crypto_shash_alg(tfm)->finup, desc, + data, len, out); } EXPORT_SYMBOL_GPL(crypto_shash_finup); static int shash_default_digest(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) { - struct shash_alg *shash = crypto_shash_alg(desc->tfm); - - return shash->init(desc) ?: - shash->finup(desc, data, len, out); + return __crypto_shash_init(desc) ?: + crypto_shash_finup(desc, data, len, out); } int crypto_shash_digest(struct shash_desc *desc, const u8 *data, @@ -92,7 +188,8 @@ int crypto_shash_digest(struct shash_desc *desc, const u8 *data, if (crypto_shash_get_flags(tfm) & CRYPTO_TFM_NEED_KEY) return -ENOKEY; - return crypto_shash_alg(tfm)->digest(desc, data, len, out); + return crypto_shash_op_and_zero(crypto_shash_alg(tfm)->digest, desc, + data, len, out); } EXPORT_SYMBOL_GPL(crypto_shash_digest); @@ -100,44 +197,92 @@ int crypto_shash_tfm_digest(struct crypto_shash *tfm, const u8 *data, unsigned int len, u8 *out) { SHASH_DESC_ON_STACK(desc, tfm); - int err; desc->tfm = tfm; - - err = crypto_shash_digest(desc, data, len, out); - - shash_desc_zero(desc); - - return err; + return crypto_shash_digest(desc, data, len, out); } EXPORT_SYMBOL_GPL(crypto_shash_tfm_digest); +int crypto_shash_export_core(struct shash_desc *desc, void *out) +{ + int (*export)(struct shash_desc *desc, void *out); + struct crypto_shash *tfm = desc->tfm; + u8 *buf = shash_desc_ctx(desc); + unsigned int plen, ss; + + plen = crypto_shash_blocksize(tfm) + 1; + ss = crypto_shash_statesize(tfm); + if (crypto_shash_block_only(tfm)) + ss -= plen; + export = crypto_shash_alg(tfm)->export; + if (!export) { + memcpy(out, buf, ss); + return 0; + } + + return export(desc, out); +} +EXPORT_SYMBOL_GPL(crypto_shash_export_core); + int crypto_shash_export(struct shash_desc *desc, void *out) { struct crypto_shash *tfm = desc->tfm; - struct shash_alg *shash = crypto_shash_alg(tfm); - if (shash->export) - return shash->export(desc, out); + if (crypto_shash_block_only(tfm)) { + unsigned int plen = crypto_shash_blocksize(tfm) + 1; + unsigned int descsize = crypto_shash_descsize(tfm); + unsigned int ss = crypto_shash_statesize(tfm); + u8 *buf = shash_desc_ctx(desc); - memcpy(out, shash_desc_ctx(desc), crypto_shash_descsize(tfm)); - return 0; + memcpy(out + ss - plen, buf + descsize - plen, plen); + } + return crypto_shash_export_core(desc, out); } EXPORT_SYMBOL_GPL(crypto_shash_export); -int crypto_shash_import(struct shash_desc *desc, const void *in) +int crypto_shash_import_core(struct shash_desc *desc, const void *in) { + int (*import)(struct shash_desc *desc, const void *in); struct crypto_shash *tfm = desc->tfm; - struct shash_alg *shash = crypto_shash_alg(tfm); + unsigned int descsize, plen, ss; + u8 *buf = shash_desc_ctx(desc); if (crypto_shash_get_flags(tfm) & CRYPTO_TFM_NEED_KEY) return -ENOKEY; - if (shash->import) - return shash->import(desc, in); + plen = crypto_shash_blocksize(tfm) + 1; + descsize = crypto_shash_descsize(tfm); + ss = crypto_shash_statesize(tfm); + buf[descsize - 1] = 0; + if (crypto_shash_block_only(tfm)) + ss -= plen; + import = crypto_shash_alg(tfm)->import; + if (!import) { + memcpy(buf, in, ss); + return 0; + } - memcpy(shash_desc_ctx(desc), in, crypto_shash_descsize(tfm)); - return 0; + return import(desc, in); +} +EXPORT_SYMBOL_GPL(crypto_shash_import_core); + +int crypto_shash_import(struct shash_desc *desc, const void *in) +{ + struct crypto_shash *tfm = desc->tfm; + int err; + + err = crypto_shash_import_core(desc, in); + if (crypto_shash_block_only(tfm)) { + unsigned int plen = crypto_shash_blocksize(tfm) + 1; + unsigned int descsize = crypto_shash_descsize(tfm); + unsigned int ss = crypto_shash_statesize(tfm); + u8 *buf = shash_desc_ctx(desc); + + memcpy(buf + descsize - plen, in + ss - plen, plen); + if (buf[descsize - 1] >= plen) + err = -EOVERFLOW; + } + return err; } EXPORT_SYMBOL_GPL(crypto_shash_import); @@ -293,9 +438,6 @@ static int shash_prepare_alg(struct shash_alg *alg) struct crypto_alg *base = &alg->halg.base; int err; - if (alg->descsize > HASH_MAX_DESCSIZE) - return -EINVAL; - if ((alg->export && !alg->import) || (alg->import && !alg->export)) return -EINVAL; @@ -321,11 +463,20 @@ static int shash_prepare_alg(struct shash_alg *alg) alg->finup = shash_default_finup; if (!alg->digest) alg->digest = shash_default_digest; - if (!alg->export) + if (!alg->export && !alg->halg.statesize) alg->halg.statesize = alg->descsize; if (!alg->setkey) alg->setkey = shash_no_setkey; + if (base->cra_flags & CRYPTO_AHASH_ALG_BLOCK_ONLY) { + BUILD_BUG_ON(MAX_ALGAPI_BLOCKSIZE >= 256); + alg->descsize += base->cra_blocksize + 1; + alg->statesize += base->cra_blocksize + 1; + } + + if (alg->descsize > HASH_MAX_DESCSIZE) + return -EINVAL; + return 0; } diff --git a/include/crypto/hash.h b/include/crypto/hash.h index 58ac1423dc38..5f87d1040a7c 100644 --- a/include/crypto/hash.h +++ b/include/crypto/hash.h @@ -85,6 +85,8 @@ struct ahash_request { * transformation object. Data processing can happen synchronously * [SHASH] or asynchronously [AHASH] at this point. Driver must not use * req->result. + * For block-only algorithms, @update must return the number + * of bytes to store in the API partial block buffer. * @final: **[mandatory]** Retrieve result from the driver. This function finalizes the * transformation and retrieves the resulting hash from the driver and * pushes it back to upper layers. No data processing happens at this @@ -905,6 +907,18 @@ int crypto_hash_digest(struct crypto_ahash *tfm, const u8 *data, */ int crypto_shash_export(struct shash_desc *desc, void *out); +/** + * crypto_shash_export_core() - extract core state for message digest + * @desc: reference to the operational state handle whose state is exported + * @out: output buffer of sufficient size that can hold the hash state + * + * Export the hash state without the partial block buffer. + * + * Context: Softirq or process context. + * Return: 0 if the export creation was successful; < 0 if an error occurred + */ +int crypto_shash_export_core(struct shash_desc *desc, void *out); + /** * crypto_shash_import() - import operational state * @desc: reference to the operational state handle the state imported into @@ -919,6 +933,18 @@ int crypto_shash_export(struct shash_desc *desc, void *out); */ int crypto_shash_import(struct shash_desc *desc, const void *in); +/** + * crypto_shash_import_core() - import core state + * @desc: reference to the operational state handle the state imported into + * @in: buffer holding the state + * + * Import the hash state without the partial block buffer. + * + * Context: Softirq or process context. + * Return: 0 if the import was successful; < 0 if an error occurred + */ +int crypto_shash_import_core(struct shash_desc *desc, const void *in); + /** * crypto_shash_init() - (re)initialize message digest * @desc: operational state handle that is already filled @@ -931,46 +957,7 @@ int crypto_shash_import(struct shash_desc *desc, const void *in); * Return: 0 if the message digest initialization was successful; < 0 if an * error occurred */ -static inline int crypto_shash_init(struct shash_desc *desc) -{ - struct crypto_shash *tfm = desc->tfm; - - if (crypto_shash_get_flags(tfm) & CRYPTO_TFM_NEED_KEY) - return -ENOKEY; - - return crypto_shash_alg(tfm)->init(desc); -} - -/** - * crypto_shash_update() - add data to message digest for processing - * @desc: operational state handle that is already initialized - * @data: input data to be added to the message digest - * @len: length of the input data - * - * Updates the message digest state of the operational state handle. - * - * Context: Softirq or process context. - * Return: 0 if the message digest update was successful; < 0 if an error - * occurred - */ -int crypto_shash_update(struct shash_desc *desc, const u8 *data, - unsigned int len); - -/** - * crypto_shash_final() - calculate message digest - * @desc: operational state handle that is already filled with data - * @out: output buffer filled with the message digest - * - * Finalize the message digest operation and create the message digest - * based on all data added to the cipher handle. The message digest is placed - * into the output buffer. The caller must ensure that the output buffer is - * large enough by using crypto_shash_digestsize. - * - * Context: Softirq or process context. - * Return: 0 if the message digest creation was successful; < 0 if an error - * occurred - */ -int crypto_shash_final(struct shash_desc *desc, u8 *out); +int crypto_shash_init(struct shash_desc *desc); /** * crypto_shash_finup() - calculate message digest of buffer @@ -990,6 +977,43 @@ int crypto_shash_final(struct shash_desc *desc, u8 *out); int crypto_shash_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out); +/** + * crypto_shash_update() - add data to message digest for processing + * @desc: operational state handle that is already initialized + * @data: input data to be added to the message digest + * @len: length of the input data + * + * Updates the message digest state of the operational state handle. + * + * Context: Softirq or process context. + * Return: 0 if the message digest update was successful; < 0 if an error + * occurred + */ +static inline int crypto_shash_update(struct shash_desc *desc, const u8 *data, + unsigned int len) +{ + return crypto_shash_finup(desc, data, len, NULL); +} + +/** + * crypto_shash_final() - calculate message digest + * @desc: operational state handle that is already filled with data + * @out: output buffer filled with the message digest + * + * Finalize the message digest operation and create the message digest + * based on all data added to the cipher handle. The message digest is placed + * into the output buffer. The caller must ensure that the output buffer is + * large enough by using crypto_shash_digestsize. + * + * Context: Softirq or process context. + * Return: 0 if the message digest creation was successful; < 0 if an error + * occurred + */ +static inline int crypto_shash_final(struct shash_desc *desc, u8 *out) +{ + return crypto_shash_finup(desc, NULL, 0, out); +} + static inline void shash_desc_zero(struct shash_desc *desc) { memzero_explicit(desc, diff --git a/include/crypto/internal/hash.h b/include/crypto/internal/hash.h index 45c728ac2621..1e80dd084a23 100644 --- a/include/crypto/internal/hash.h +++ b/include/crypto/internal/hash.h @@ -11,6 +11,15 @@ #include #include +/* Set this bit to handle partial blocks in the API. */ +#define CRYPTO_AHASH_ALG_BLOCK_ONLY 0x01000000 + +/* Set this bit if final requires at least one byte. */ +#define CRYPTO_AHASH_ALG_FINAL_NONZERO 0x02000000 + +/* Set this bit if finup can deal with multiple blocks. */ +#define CRYPTO_AHASH_ALG_FINUP_MAX 0x04000000 + #define HASH_FBREQ_ON_STACK(name, req) \ char __##name##_req[sizeof(struct ahash_request) + \ MAX_SYNC_HASH_REQSIZE] CRYPTO_MINALIGN_ATTR; \ @@ -281,5 +290,11 @@ static inline struct ahash_request *ahash_fbreq_on_stack_init( return req; } +/* Return the state size without partial block for block-only algorithms. */ +static inline unsigned int crypto_shash_coresize(struct crypto_shash *tfm) +{ + return crypto_shash_statesize(tfm) - crypto_shash_blocksize(tfm) - 1; +} + #endif /* _CRYPTO_INTERNAL_HASH_H */ diff --git a/include/linux/crypto.h b/include/linux/crypto.h index b89b1b348095..f691ce01745e 100644 --- a/include/linux/crypto.h +++ b/include/linux/crypto.h @@ -136,6 +136,8 @@ /* Set if the algorithm supports request chains and virtual addresses. */ #define CRYPTO_ALG_REQ_CHAIN 0x00040000 +/* The high bits 0xff000000 are reserved for type-specific flags. */ + /* * Transform masks and values (for crt_flags). */ From aa54e17020852858d6e1639f1c7ef516fd217e08 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 18 Apr 2025 10:58:43 +0800 Subject: [PATCH 150/382] crypto: blake2b-generic - Use API partial block handling Use the Crypto API partial block handling. Signed-off-by: Herbert Xu --- crypto/blake2b_generic.c | 31 ++++++++------ include/crypto/blake2b.h | 14 ++----- include/crypto/internal/blake2b.h | 70 ++++++++++++++++++++++++++----- 3 files changed, 80 insertions(+), 35 deletions(-) diff --git a/crypto/blake2b_generic.c b/crypto/blake2b_generic.c index 04a712ddfb43..6fa38965a493 100644 --- a/crypto/blake2b_generic.c +++ b/crypto/blake2b_generic.c @@ -15,12 +15,12 @@ * More information about BLAKE2 can be found at https://blake2.net. */ -#include -#include -#include -#include #include #include +#include +#include +#include +#include static const u8 blake2b_sigma[12][16] = { { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 }, @@ -111,8 +111,8 @@ static void blake2b_compress_one_generic(struct blake2b_state *S, #undef G #undef ROUND -void blake2b_compress_generic(struct blake2b_state *state, - const u8 *block, size_t nblocks, u32 inc) +static void blake2b_compress_generic(struct blake2b_state *state, + const u8 *block, size_t nblocks, u32 inc) { do { blake2b_increment_counter(state, inc); @@ -120,17 +120,19 @@ void blake2b_compress_generic(struct blake2b_state *state, block += BLAKE2B_BLOCK_SIZE; } while (--nblocks); } -EXPORT_SYMBOL(blake2b_compress_generic); static int crypto_blake2b_update_generic(struct shash_desc *desc, const u8 *in, unsigned int inlen) { - return crypto_blake2b_update(desc, in, inlen, blake2b_compress_generic); + return crypto_blake2b_update_bo(desc, in, inlen, + blake2b_compress_generic); } -static int crypto_blake2b_final_generic(struct shash_desc *desc, u8 *out) +static int crypto_blake2b_finup_generic(struct shash_desc *desc, const u8 *in, + unsigned int inlen, u8 *out) { - return crypto_blake2b_final(desc, out, blake2b_compress_generic); + return crypto_blake2b_finup(desc, in, inlen, out, + blake2b_compress_generic); } #define BLAKE2B_ALG(name, driver_name, digest_size) \ @@ -138,7 +140,9 @@ static int crypto_blake2b_final_generic(struct shash_desc *desc, u8 *out) .base.cra_name = name, \ .base.cra_driver_name = driver_name, \ .base.cra_priority = 100, \ - .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY, \ + .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY | \ + CRYPTO_AHASH_ALG_BLOCK_ONLY | \ + CRYPTO_AHASH_ALG_FINAL_NONZERO, \ .base.cra_blocksize = BLAKE2B_BLOCK_SIZE, \ .base.cra_ctxsize = sizeof(struct blake2b_tfm_ctx), \ .base.cra_module = THIS_MODULE, \ @@ -146,8 +150,9 @@ static int crypto_blake2b_final_generic(struct shash_desc *desc, u8 *out) .setkey = crypto_blake2b_setkey, \ .init = crypto_blake2b_init, \ .update = crypto_blake2b_update_generic, \ - .final = crypto_blake2b_final_generic, \ - .descsize = sizeof(struct blake2b_state), \ + .finup = crypto_blake2b_finup_generic, \ + .descsize = BLAKE2B_DESC_SIZE, \ + .statesize = BLAKE2B_STATE_SIZE, \ } static struct shash_alg blake2b_algs[] = { diff --git a/include/crypto/blake2b.h b/include/crypto/blake2b.h index 0c0176285349..68da368dc182 100644 --- a/include/crypto/blake2b.h +++ b/include/crypto/blake2b.h @@ -11,6 +11,8 @@ enum blake2b_lengths { BLAKE2B_BLOCK_SIZE = 128, BLAKE2B_HASH_SIZE = 64, BLAKE2B_KEY_SIZE = 64, + BLAKE2B_STATE_SIZE = 80, + BLAKE2B_DESC_SIZE = 96, BLAKE2B_160_HASH_SIZE = 20, BLAKE2B_256_HASH_SIZE = 32, @@ -25,7 +27,6 @@ struct blake2b_state { u64 f[2]; u8 buf[BLAKE2B_BLOCK_SIZE]; unsigned int buflen; - unsigned int outlen; }; enum blake2b_iv { @@ -40,7 +41,7 @@ enum blake2b_iv { }; static inline void __blake2b_init(struct blake2b_state *state, size_t outlen, - const void *key, size_t keylen) + size_t keylen) { state->h[0] = BLAKE2B_IV0 ^ (0x01010000 | keylen << 8 | outlen); state->h[1] = BLAKE2B_IV1; @@ -52,15 +53,6 @@ static inline void __blake2b_init(struct blake2b_state *state, size_t outlen, state->h[7] = BLAKE2B_IV7; state->t[0] = 0; state->t[1] = 0; - state->f[0] = 0; - state->f[1] = 0; - state->buflen = 0; - state->outlen = outlen; - if (keylen) { - memcpy(state->buf, key, keylen); - memset(&state->buf[keylen], 0, BLAKE2B_BLOCK_SIZE - keylen); - state->buflen = BLAKE2B_BLOCK_SIZE; - } } #endif /* _CRYPTO_BLAKE2B_H */ diff --git a/include/crypto/internal/blake2b.h b/include/crypto/internal/blake2b.h index 982fe5e8471c..48dc9830400d 100644 --- a/include/crypto/internal/blake2b.h +++ b/include/crypto/internal/blake2b.h @@ -7,16 +7,27 @@ #ifndef _CRYPTO_INTERNAL_BLAKE2B_H #define _CRYPTO_INTERNAL_BLAKE2B_H +#include #include #include +#include +#include +#include +#include +#include #include - -void blake2b_compress_generic(struct blake2b_state *state, - const u8 *block, size_t nblocks, u32 inc); +#include static inline void blake2b_set_lastblock(struct blake2b_state *state) { state->f[0] = -1; + state->f[1] = 0; +} + +static inline void blake2b_set_nonlast(struct blake2b_state *state) +{ + state->f[0] = 0; + state->f[1] = 0; } typedef void (*blake2b_compress_t)(struct blake2b_state *state, @@ -30,6 +41,7 @@ static inline void __blake2b_update(struct blake2b_state *state, if (unlikely(!inlen)) return; + blake2b_set_nonlast(state); if (inlen > fill) { memcpy(state->buf + state->buflen, in, fill); (*compress)(state, state->buf, 1, BLAKE2B_BLOCK_SIZE); @@ -49,6 +61,7 @@ static inline void __blake2b_update(struct blake2b_state *state, } static inline void __blake2b_final(struct blake2b_state *state, u8 *out, + unsigned int outlen, blake2b_compress_t compress) { int i; @@ -59,13 +72,13 @@ static inline void __blake2b_final(struct blake2b_state *state, u8 *out, (*compress)(state, state->buf, 1, state->buflen); for (i = 0; i < ARRAY_SIZE(state->h); i++) __cpu_to_le64s(&state->h[i]); - memcpy(out, state->h, state->outlen); + memcpy(out, state->h, outlen); } /* Helper functions for shash implementations of BLAKE2b */ struct blake2b_tfm_ctx { - u8 key[BLAKE2B_KEY_SIZE]; + u8 key[BLAKE2B_BLOCK_SIZE]; unsigned int keylen; }; @@ -74,10 +87,13 @@ static inline int crypto_blake2b_setkey(struct crypto_shash *tfm, { struct blake2b_tfm_ctx *tctx = crypto_shash_ctx(tfm); - if (keylen == 0 || keylen > BLAKE2B_KEY_SIZE) + if (keylen > BLAKE2B_KEY_SIZE) return -EINVAL; + BUILD_BUG_ON(BLAKE2B_KEY_SIZE > BLAKE2B_BLOCK_SIZE); + memcpy(tctx->key, key, keylen); + memset(tctx->key + keylen, 0, BLAKE2B_BLOCK_SIZE - keylen); tctx->keylen = keylen; return 0; @@ -89,8 +105,9 @@ static inline int crypto_blake2b_init(struct shash_desc *desc) struct blake2b_state *state = shash_desc_ctx(desc); unsigned int outlen = crypto_shash_digestsize(desc->tfm); - __blake2b_init(state, outlen, tctx->key, tctx->keylen); - return 0; + __blake2b_init(state, outlen, tctx->keylen); + return tctx->keylen ? + crypto_shash_update(desc, tctx->key, BLAKE2B_BLOCK_SIZE) : 0; } static inline int crypto_blake2b_update(struct shash_desc *desc, @@ -103,12 +120,43 @@ static inline int crypto_blake2b_update(struct shash_desc *desc, return 0; } -static inline int crypto_blake2b_final(struct shash_desc *desc, u8 *out, - blake2b_compress_t compress) +static inline int crypto_blake2b_update_bo(struct shash_desc *desc, + const u8 *in, unsigned int inlen, + blake2b_compress_t compress) { struct blake2b_state *state = shash_desc_ctx(desc); - __blake2b_final(state, out, compress); + blake2b_set_nonlast(state); + compress(state, in, inlen / BLAKE2B_BLOCK_SIZE, BLAKE2B_BLOCK_SIZE); + return inlen - round_down(inlen, BLAKE2B_BLOCK_SIZE); +} + +static inline int crypto_blake2b_final(struct shash_desc *desc, u8 *out, + blake2b_compress_t compress) +{ + unsigned int outlen = crypto_shash_digestsize(desc->tfm); + struct blake2b_state *state = shash_desc_ctx(desc); + + __blake2b_final(state, out, outlen, compress); + return 0; +} + +static inline int crypto_blake2b_finup(struct shash_desc *desc, const u8 *in, + unsigned int inlen, u8 *out, + blake2b_compress_t compress) +{ + struct blake2b_state *state = shash_desc_ctx(desc); + u8 buf[BLAKE2B_BLOCK_SIZE]; + int i; + + memcpy(buf, in, inlen); + memset(buf + inlen, 0, BLAKE2B_BLOCK_SIZE - inlen); + blake2b_set_lastblock(state); + compress(state, buf, 1, inlen); + for (i = 0; i < ARRAY_SIZE(state->h); i++) + __cpu_to_le64s(&state->h[i]); + memcpy(out, state->h, crypto_shash_digestsize(desc->tfm)); + memzero_explicit(buf, sizeof(buf)); return 0; } From cc28260ab4fbe110981bcffff5da50befcb5119e Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 18 Apr 2025 10:58:45 +0800 Subject: [PATCH 151/382] crypto: arm/blake2b - Use API partial block handling Use the Crypto API partial block handling. Also remove the unnecessary SIMD fallback path. Signed-off-by: Herbert Xu --- arch/arm/crypto/blake2b-neon-glue.c | 20 +++++----- include/crypto/blake2b.h | 21 +++++----- include/crypto/internal/blake2b.h | 62 ----------------------------- 3 files changed, 19 insertions(+), 84 deletions(-) diff --git a/arch/arm/crypto/blake2b-neon-glue.c b/arch/arm/crypto/blake2b-neon-glue.c index 4b59d027ba4a..7ae4ba0afe06 100644 --- a/arch/arm/crypto/blake2b-neon-glue.c +++ b/arch/arm/crypto/blake2b-neon-glue.c @@ -7,7 +7,6 @@ #include #include -#include #include #include @@ -21,11 +20,6 @@ asmlinkage void blake2b_compress_neon(struct blake2b_state *state, static void blake2b_compress_arch(struct blake2b_state *state, const u8 *block, size_t nblocks, u32 inc) { - if (!crypto_simd_usable()) { - blake2b_compress_generic(state, block, nblocks, inc); - return; - } - do { const size_t blocks = min_t(size_t, nblocks, SZ_4K / BLAKE2B_BLOCK_SIZE); @@ -42,12 +36,14 @@ static void blake2b_compress_arch(struct blake2b_state *state, static int crypto_blake2b_update_neon(struct shash_desc *desc, const u8 *in, unsigned int inlen) { - return crypto_blake2b_update(desc, in, inlen, blake2b_compress_arch); + return crypto_blake2b_update_bo(desc, in, inlen, blake2b_compress_arch); } -static int crypto_blake2b_final_neon(struct shash_desc *desc, u8 *out) +static int crypto_blake2b_finup_neon(struct shash_desc *desc, const u8 *in, + unsigned int inlen, u8 *out) { - return crypto_blake2b_final(desc, out, blake2b_compress_arch); + return crypto_blake2b_finup(desc, in, inlen, out, + blake2b_compress_arch); } #define BLAKE2B_ALG(name, driver_name, digest_size) \ @@ -55,7 +51,8 @@ static int crypto_blake2b_final_neon(struct shash_desc *desc, u8 *out) .base.cra_name = name, \ .base.cra_driver_name = driver_name, \ .base.cra_priority = 200, \ - .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY, \ + .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY | \ + CRYPTO_AHASH_ALG_BLOCK_ONLY, \ .base.cra_blocksize = BLAKE2B_BLOCK_SIZE, \ .base.cra_ctxsize = sizeof(struct blake2b_tfm_ctx), \ .base.cra_module = THIS_MODULE, \ @@ -63,8 +60,9 @@ static int crypto_blake2b_final_neon(struct shash_desc *desc, u8 *out) .setkey = crypto_blake2b_setkey, \ .init = crypto_blake2b_init, \ .update = crypto_blake2b_update_neon, \ - .final = crypto_blake2b_final_neon, \ + .finup = crypto_blake2b_finup_neon, \ .descsize = sizeof(struct blake2b_state), \ + .statesize = BLAKE2B_STATE_SIZE, \ } static struct shash_alg blake2b_neon_algs[] = { diff --git a/include/crypto/blake2b.h b/include/crypto/blake2b.h index 68da368dc182..dd7694477e50 100644 --- a/include/crypto/blake2b.h +++ b/include/crypto/blake2b.h @@ -7,12 +7,20 @@ #include #include +struct blake2b_state { + /* 'h', 't', and 'f' are used in assembly code, so keep them as-is. */ + u64 h[8]; + u64 t[2]; + /* The true state ends here. The rest is temporary storage. */ + u64 f[2]; +}; + enum blake2b_lengths { BLAKE2B_BLOCK_SIZE = 128, BLAKE2B_HASH_SIZE = 64, BLAKE2B_KEY_SIZE = 64, - BLAKE2B_STATE_SIZE = 80, - BLAKE2B_DESC_SIZE = 96, + BLAKE2B_STATE_SIZE = offsetof(struct blake2b_state, f), + BLAKE2B_DESC_SIZE = sizeof(struct blake2b_state), BLAKE2B_160_HASH_SIZE = 20, BLAKE2B_256_HASH_SIZE = 32, @@ -20,15 +28,6 @@ enum blake2b_lengths { BLAKE2B_512_HASH_SIZE = 64, }; -struct blake2b_state { - /* 'h', 't', and 'f' are used in assembly code, so keep them as-is. */ - u64 h[8]; - u64 t[2]; - u64 f[2]; - u8 buf[BLAKE2B_BLOCK_SIZE]; - unsigned int buflen; -}; - enum blake2b_iv { BLAKE2B_IV0 = 0x6A09E667F3BCC908ULL, BLAKE2B_IV1 = 0xBB67AE8584CAA73BULL, diff --git a/include/crypto/internal/blake2b.h b/include/crypto/internal/blake2b.h index 48dc9830400d..3e09e2485306 100644 --- a/include/crypto/internal/blake2b.h +++ b/include/crypto/internal/blake2b.h @@ -33,48 +33,6 @@ static inline void blake2b_set_nonlast(struct blake2b_state *state) typedef void (*blake2b_compress_t)(struct blake2b_state *state, const u8 *block, size_t nblocks, u32 inc); -static inline void __blake2b_update(struct blake2b_state *state, - const u8 *in, size_t inlen, - blake2b_compress_t compress) -{ - const size_t fill = BLAKE2B_BLOCK_SIZE - state->buflen; - - if (unlikely(!inlen)) - return; - blake2b_set_nonlast(state); - if (inlen > fill) { - memcpy(state->buf + state->buflen, in, fill); - (*compress)(state, state->buf, 1, BLAKE2B_BLOCK_SIZE); - state->buflen = 0; - in += fill; - inlen -= fill; - } - if (inlen > BLAKE2B_BLOCK_SIZE) { - const size_t nblocks = DIV_ROUND_UP(inlen, BLAKE2B_BLOCK_SIZE); - /* Hash one less (full) block than strictly possible */ - (*compress)(state, in, nblocks - 1, BLAKE2B_BLOCK_SIZE); - in += BLAKE2B_BLOCK_SIZE * (nblocks - 1); - inlen -= BLAKE2B_BLOCK_SIZE * (nblocks - 1); - } - memcpy(state->buf + state->buflen, in, inlen); - state->buflen += inlen; -} - -static inline void __blake2b_final(struct blake2b_state *state, u8 *out, - unsigned int outlen, - blake2b_compress_t compress) -{ - int i; - - blake2b_set_lastblock(state); - memset(state->buf + state->buflen, 0, - BLAKE2B_BLOCK_SIZE - state->buflen); /* Padding */ - (*compress)(state, state->buf, 1, state->buflen); - for (i = 0; i < ARRAY_SIZE(state->h); i++) - __cpu_to_le64s(&state->h[i]); - memcpy(out, state->h, outlen); -} - /* Helper functions for shash implementations of BLAKE2b */ struct blake2b_tfm_ctx { @@ -110,16 +68,6 @@ static inline int crypto_blake2b_init(struct shash_desc *desc) crypto_shash_update(desc, tctx->key, BLAKE2B_BLOCK_SIZE) : 0; } -static inline int crypto_blake2b_update(struct shash_desc *desc, - const u8 *in, unsigned int inlen, - blake2b_compress_t compress) -{ - struct blake2b_state *state = shash_desc_ctx(desc); - - __blake2b_update(state, in, inlen, compress); - return 0; -} - static inline int crypto_blake2b_update_bo(struct shash_desc *desc, const u8 *in, unsigned int inlen, blake2b_compress_t compress) @@ -131,16 +79,6 @@ static inline int crypto_blake2b_update_bo(struct shash_desc *desc, return inlen - round_down(inlen, BLAKE2B_BLOCK_SIZE); } -static inline int crypto_blake2b_final(struct shash_desc *desc, u8 *out, - blake2b_compress_t compress) -{ - unsigned int outlen = crypto_shash_digestsize(desc->tfm); - struct blake2b_state *state = shash_desc_ctx(desc); - - __blake2b_final(state, out, outlen, compress); - return 0; -} - static inline int crypto_blake2b_finup(struct shash_desc *desc, const u8 *in, unsigned int inlen, u8 *out, blake2b_compress_t compress) From ef11957b0ad0a2e1c159c172865d6923f1731b57 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 18 Apr 2025 10:58:47 +0800 Subject: [PATCH 152/382] crypto: ghash-generic - Use API partial block handling Use the Crypto API partial block handling. Signed-off-by: Herbert Xu --- crypto/ghash-generic.c | 56 +++++++++++++----------------------------- include/crypto/ghash.h | 3 ++- 2 files changed, 19 insertions(+), 40 deletions(-) diff --git a/crypto/ghash-generic.c b/crypto/ghash-generic.c index c70d163c1ac9..b5fc20a0dafc 100644 --- a/crypto/ghash-generic.c +++ b/crypto/ghash-generic.c @@ -34,14 +34,14 @@ * (https://csrc.nist.gov/publications/detail/sp/800-38d/final) */ -#include #include #include #include -#include -#include +#include +#include #include #include +#include static int ghash_init(struct shash_desc *desc) { @@ -82,59 +82,36 @@ static int ghash_update(struct shash_desc *desc, struct ghash_ctx *ctx = crypto_shash_ctx(desc->tfm); u8 *dst = dctx->buffer; - if (dctx->bytes) { - int n = min(srclen, dctx->bytes); - u8 *pos = dst + (GHASH_BLOCK_SIZE - dctx->bytes); - - dctx->bytes -= n; - srclen -= n; - - while (n--) - *pos++ ^= *src++; - - if (!dctx->bytes) - gf128mul_4k_lle((be128 *)dst, ctx->gf128); - } - - while (srclen >= GHASH_BLOCK_SIZE) { + do { crypto_xor(dst, src, GHASH_BLOCK_SIZE); gf128mul_4k_lle((be128 *)dst, ctx->gf128); src += GHASH_BLOCK_SIZE; srclen -= GHASH_BLOCK_SIZE; - } + } while (srclen >= GHASH_BLOCK_SIZE); - if (srclen) { - dctx->bytes = GHASH_BLOCK_SIZE - srclen; - while (srclen--) - *dst++ ^= *src++; - } - - return 0; + return srclen; } -static void ghash_flush(struct ghash_ctx *ctx, struct ghash_desc_ctx *dctx) +static void ghash_flush(struct shash_desc *desc, const u8 *src, + unsigned int len) { + struct ghash_ctx *ctx = crypto_shash_ctx(desc->tfm); + struct ghash_desc_ctx *dctx = shash_desc_ctx(desc); u8 *dst = dctx->buffer; - if (dctx->bytes) { - u8 *tmp = dst + (GHASH_BLOCK_SIZE - dctx->bytes); - - while (dctx->bytes--) - *tmp++ ^= 0; - + if (len) { + crypto_xor(dst, src, len); gf128mul_4k_lle((be128 *)dst, ctx->gf128); } - - dctx->bytes = 0; } -static int ghash_final(struct shash_desc *desc, u8 *dst) +static int ghash_finup(struct shash_desc *desc, const u8 *src, + unsigned int len, u8 *dst) { struct ghash_desc_ctx *dctx = shash_desc_ctx(desc); - struct ghash_ctx *ctx = crypto_shash_ctx(desc->tfm); u8 *buf = dctx->buffer; - ghash_flush(ctx, dctx); + ghash_flush(desc, src, len); memcpy(dst, buf, GHASH_BLOCK_SIZE); return 0; @@ -151,13 +128,14 @@ static struct shash_alg ghash_alg = { .digestsize = GHASH_DIGEST_SIZE, .init = ghash_init, .update = ghash_update, - .final = ghash_final, + .finup = ghash_finup, .setkey = ghash_setkey, .descsize = sizeof(struct ghash_desc_ctx), .base = { .cra_name = "ghash", .cra_driver_name = "ghash-generic", .cra_priority = 100, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_blocksize = GHASH_BLOCK_SIZE, .cra_ctxsize = sizeof(struct ghash_ctx), .cra_module = THIS_MODULE, diff --git a/include/crypto/ghash.h b/include/crypto/ghash.h index f832c9f2aca3..16904f2b5184 100644 --- a/include/crypto/ghash.h +++ b/include/crypto/ghash.h @@ -12,13 +12,14 @@ #define GHASH_BLOCK_SIZE 16 #define GHASH_DIGEST_SIZE 16 +struct gf128mul_4k; + struct ghash_ctx { struct gf128mul_4k *gf128; }; struct ghash_desc_ctx { u8 buffer[GHASH_BLOCK_SIZE]; - u32 bytes; }; #endif From f5bd7e84ac37d141696f81e2bb92f113cbbef9ed Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 18 Apr 2025 10:58:50 +0800 Subject: [PATCH 153/382] crypto: powerpc/ghash - Use API partial block handling Use the Crypto API partial block handling. Signed-off-by: Herbert Xu --- arch/powerpc/crypto/ghash.c | 81 +++++++++++++------------------------ 1 file changed, 28 insertions(+), 53 deletions(-) diff --git a/arch/powerpc/crypto/ghash.c b/arch/powerpc/crypto/ghash.c index 9bb61a843fd3..7308735bdb33 100644 --- a/arch/powerpc/crypto/ghash.c +++ b/arch/powerpc/crypto/ghash.c @@ -12,17 +12,16 @@ */ #include "aesp8-ppc.h" -#include #include #include -#include +#include #include #include #include -#include #include #include #include +#include #include void gcm_init_p8(u128 htable[16], const u64 Xi[2]); @@ -39,15 +38,12 @@ struct p8_ghash_ctx { struct p8_ghash_desc_ctx { u64 shash[2]; - u8 buffer[GHASH_DIGEST_SIZE]; - int bytes; }; static int p8_ghash_init(struct shash_desc *desc) { struct p8_ghash_desc_ctx *dctx = shash_desc_ctx(desc); - dctx->bytes = 0; memset(dctx->shash, 0, GHASH_DIGEST_SIZE); return 0; } @@ -74,27 +70,30 @@ static int p8_ghash_setkey(struct crypto_shash *tfm, const u8 *key, } static inline void __ghash_block(struct p8_ghash_ctx *ctx, - struct p8_ghash_desc_ctx *dctx) + struct p8_ghash_desc_ctx *dctx, + const u8 *src) { if (crypto_simd_usable()) { preempt_disable(); pagefault_disable(); enable_kernel_vsx(); - gcm_ghash_p8(dctx->shash, ctx->htable, - dctx->buffer, GHASH_DIGEST_SIZE); + gcm_ghash_p8(dctx->shash, ctx->htable, src, GHASH_BLOCK_SIZE); disable_kernel_vsx(); pagefault_enable(); preempt_enable(); } else { - crypto_xor((u8 *)dctx->shash, dctx->buffer, GHASH_BLOCK_SIZE); + crypto_xor((u8 *)dctx->shash, src, GHASH_BLOCK_SIZE); gf128mul_lle((be128 *)dctx->shash, &ctx->key); } } -static inline void __ghash_blocks(struct p8_ghash_ctx *ctx, - struct p8_ghash_desc_ctx *dctx, - const u8 *src, unsigned int srclen) +static inline int __ghash_blocks(struct p8_ghash_ctx *ctx, + struct p8_ghash_desc_ctx *dctx, + const u8 *src, unsigned int srclen) { + int remain = srclen - round_down(srclen, GHASH_BLOCK_SIZE); + + srclen -= remain; if (crypto_simd_usable()) { preempt_disable(); pagefault_disable(); @@ -105,62 +104,38 @@ static inline void __ghash_blocks(struct p8_ghash_ctx *ctx, pagefault_enable(); preempt_enable(); } else { - while (srclen >= GHASH_BLOCK_SIZE) { + do { crypto_xor((u8 *)dctx->shash, src, GHASH_BLOCK_SIZE); gf128mul_lle((be128 *)dctx->shash, &ctx->key); srclen -= GHASH_BLOCK_SIZE; src += GHASH_BLOCK_SIZE; - } + } while (srclen); } + + return remain; } static int p8_ghash_update(struct shash_desc *desc, const u8 *src, unsigned int srclen) { - unsigned int len; struct p8_ghash_ctx *ctx = crypto_tfm_ctx(crypto_shash_tfm(desc->tfm)); struct p8_ghash_desc_ctx *dctx = shash_desc_ctx(desc); - if (dctx->bytes) { - if (dctx->bytes + srclen < GHASH_DIGEST_SIZE) { - memcpy(dctx->buffer + dctx->bytes, src, - srclen); - dctx->bytes += srclen; - return 0; - } - memcpy(dctx->buffer + dctx->bytes, src, - GHASH_DIGEST_SIZE - dctx->bytes); - - __ghash_block(ctx, dctx); - - src += GHASH_DIGEST_SIZE - dctx->bytes; - srclen -= GHASH_DIGEST_SIZE - dctx->bytes; - dctx->bytes = 0; - } - len = srclen & ~(GHASH_DIGEST_SIZE - 1); - if (len) { - __ghash_blocks(ctx, dctx, src, len); - src += len; - srclen -= len; - } - if (srclen) { - memcpy(dctx->buffer, src, srclen); - dctx->bytes = srclen; - } - return 0; + return __ghash_blocks(ctx, dctx, src, srclen); } -static int p8_ghash_final(struct shash_desc *desc, u8 *out) +static int p8_ghash_finup(struct shash_desc *desc, const u8 *src, + unsigned int len, u8 *out) { - int i; struct p8_ghash_ctx *ctx = crypto_tfm_ctx(crypto_shash_tfm(desc->tfm)); struct p8_ghash_desc_ctx *dctx = shash_desc_ctx(desc); - if (dctx->bytes) { - for (i = dctx->bytes; i < GHASH_DIGEST_SIZE; i++) - dctx->buffer[i] = 0; - __ghash_block(ctx, dctx); - dctx->bytes = 0; + if (len) { + u8 buf[GHASH_BLOCK_SIZE] = {}; + + memcpy(buf, src, len); + __ghash_block(ctx, dctx, buf); + memzero_explicit(buf, sizeof(buf)); } memcpy(out, dctx->shash, GHASH_DIGEST_SIZE); return 0; @@ -170,14 +145,14 @@ struct shash_alg p8_ghash_alg = { .digestsize = GHASH_DIGEST_SIZE, .init = p8_ghash_init, .update = p8_ghash_update, - .final = p8_ghash_final, + .finup = p8_ghash_finup, .setkey = p8_ghash_setkey, - .descsize = sizeof(struct p8_ghash_desc_ctx) - + sizeof(struct ghash_desc_ctx), + .descsize = sizeof(struct p8_ghash_desc_ctx), .base = { .cra_name = "ghash", .cra_driver_name = "p8_ghash", .cra_priority = 1000, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_blocksize = GHASH_BLOCK_SIZE, .cra_ctxsize = sizeof(struct p8_ghash_ctx), .cra_module = THIS_MODULE, From 08bcb691352b4ab230889332694a181d06df72b6 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 18 Apr 2025 10:58:52 +0800 Subject: [PATCH 154/382] crypto: arm/ghash - Use API partial block handling Use the Crypto API partial block handling. Also switch to the generic export format. Finally remove a couple of stray may_use_simd() calls in gcm. Signed-off-by: Herbert Xu --- arch/arm/crypto/ghash-ce-glue.c | 110 +++++++++++++++----------------- 1 file changed, 50 insertions(+), 60 deletions(-) diff --git a/arch/arm/crypto/ghash-ce-glue.c b/arch/arm/crypto/ghash-ce-glue.c index aabfcf522a2c..a52dcc8c1e33 100644 --- a/arch/arm/crypto/ghash-ce-glue.c +++ b/arch/arm/crypto/ghash-ce-glue.c @@ -8,22 +8,22 @@ #include #include -#include -#include #include -#include #include -#include +#include +#include +#include #include #include -#include #include -#include #include #include -#include +#include #include +#include #include +#include +#include MODULE_DESCRIPTION("GHASH hash function using ARMv8 Crypto Extensions"); MODULE_AUTHOR("Ard Biesheuvel "); @@ -32,9 +32,6 @@ MODULE_ALIAS_CRYPTO("ghash"); MODULE_ALIAS_CRYPTO("gcm(aes)"); MODULE_ALIAS_CRYPTO("rfc4106(gcm(aes))"); -#define GHASH_BLOCK_SIZE 16 -#define GHASH_DIGEST_SIZE 16 - #define RFC4106_NONCE_SIZE 4 struct ghash_key { @@ -49,10 +46,8 @@ struct gcm_key { u8 nonce[]; // for RFC4106 nonce }; -struct ghash_desc_ctx { +struct arm_ghash_desc_ctx { u64 digest[GHASH_DIGEST_SIZE/sizeof(u64)]; - u8 buf[GHASH_BLOCK_SIZE]; - u32 count; }; asmlinkage void pmull_ghash_update_p64(int blocks, u64 dg[], const char *src, @@ -65,9 +60,9 @@ static __ro_after_init DEFINE_STATIC_KEY_FALSE(use_p64); static int ghash_init(struct shash_desc *desc) { - struct ghash_desc_ctx *ctx = shash_desc_ctx(desc); + struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc); - *ctx = (struct ghash_desc_ctx){}; + *ctx = (struct arm_ghash_desc_ctx){}; return 0; } @@ -85,54 +80,51 @@ static void ghash_do_update(int blocks, u64 dg[], const char *src, static int ghash_update(struct shash_desc *desc, const u8 *src, unsigned int len) { - struct ghash_desc_ctx *ctx = shash_desc_ctx(desc); - unsigned int partial = ctx->count % GHASH_BLOCK_SIZE; + struct ghash_key *key = crypto_shash_ctx(desc->tfm); + struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc); + int blocks; - ctx->count += len; + blocks = len / GHASH_BLOCK_SIZE; + ghash_do_update(blocks, ctx->digest, src, key, NULL); + return len - blocks * GHASH_BLOCK_SIZE; +} - if ((partial + len) >= GHASH_BLOCK_SIZE) { - struct ghash_key *key = crypto_shash_ctx(desc->tfm); - int blocks; +static int ghash_export(struct shash_desc *desc, void *out) +{ + struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc); + u8 *dst = out; - if (partial) { - int p = GHASH_BLOCK_SIZE - partial; - - memcpy(ctx->buf + partial, src, p); - src += p; - len -= p; - } - - blocks = len / GHASH_BLOCK_SIZE; - len %= GHASH_BLOCK_SIZE; - - ghash_do_update(blocks, ctx->digest, src, key, - partial ? ctx->buf : NULL); - src += blocks * GHASH_BLOCK_SIZE; - partial = 0; - } - if (len) - memcpy(ctx->buf + partial, src, len); + put_unaligned_be64(ctx->digest[1], dst); + put_unaligned_be64(ctx->digest[0], dst + 8); return 0; } -static int ghash_final(struct shash_desc *desc, u8 *dst) +static int ghash_import(struct shash_desc *desc, const void *in) { - struct ghash_desc_ctx *ctx = shash_desc_ctx(desc); - unsigned int partial = ctx->count % GHASH_BLOCK_SIZE; + struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc); + const u8 *src = in; - if (partial) { - struct ghash_key *key = crypto_shash_ctx(desc->tfm); - - memset(ctx->buf + partial, 0, GHASH_BLOCK_SIZE - partial); - ghash_do_update(1, ctx->digest, ctx->buf, key, NULL); - } - put_unaligned_be64(ctx->digest[1], dst); - put_unaligned_be64(ctx->digest[0], dst + 8); - - *ctx = (struct ghash_desc_ctx){}; + ctx->digest[1] = get_unaligned_be64(src); + ctx->digest[0] = get_unaligned_be64(src + 8); return 0; } +static int ghash_finup(struct shash_desc *desc, const u8 *src, + unsigned int len, u8 *dst) +{ + struct ghash_key *key = crypto_shash_ctx(desc->tfm); + struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc); + + if (len) { + u8 buf[GHASH_BLOCK_SIZE] = {}; + + memcpy(buf, src, len); + ghash_do_update(1, ctx->digest, buf, key, NULL); + memzero_explicit(buf, sizeof(buf)); + } + return ghash_export(desc, dst); +} + static void ghash_reflect(u64 h[], const be128 *k) { u64 carry = be64_to_cpu(k->a) >> 63; @@ -175,13 +167,17 @@ static struct shash_alg ghash_alg = { .digestsize = GHASH_DIGEST_SIZE, .init = ghash_init, .update = ghash_update, - .final = ghash_final, + .finup = ghash_finup, .setkey = ghash_setkey, - .descsize = sizeof(struct ghash_desc_ctx), + .export = ghash_export, + .import = ghash_import, + .descsize = sizeof(struct arm_ghash_desc_ctx), + .statesize = sizeof(struct ghash_desc_ctx), .base.cra_name = "ghash", .base.cra_driver_name = "ghash-ce", .base.cra_priority = 300, + .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .base.cra_blocksize = GHASH_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct ghash_key) + sizeof(u64[2]), .base.cra_module = THIS_MODULE, @@ -317,9 +313,6 @@ static int gcm_encrypt(struct aead_request *req, const u8 *iv, u32 assoclen) u8 *tag, *dst; int tail, err; - if (WARN_ON_ONCE(!may_use_simd())) - return -EBUSY; - err = skcipher_walk_aead_encrypt(&walk, req, false); kernel_neon_begin(); @@ -409,9 +402,6 @@ static int gcm_decrypt(struct aead_request *req, const u8 *iv, u32 assoclen) u8 *tag, *dst; int tail, err, ret; - if (WARN_ON_ONCE(!may_use_simd())) - return -EBUSY; - scatterwalk_map_and_copy(otag, req->src, req->assoclen + req->cryptlen - authsize, authsize, 0); From 9a7c987fb92b8aea61a94e28f2782d184e95d621 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 18 Apr 2025 10:58:54 +0800 Subject: [PATCH 155/382] crypto: arm64/ghash - Use API partial block handling Use the Crypto API partial block handling. Also remove the unnecessary SIMD fallback path. Signed-off-by: Herbert Xu --- arch/arm64/crypto/ghash-ce-glue.c | 151 ++++++++++++------------------ 1 file changed, 60 insertions(+), 91 deletions(-) diff --git a/arch/arm64/crypto/ghash-ce-glue.c b/arch/arm64/crypto/ghash-ce-glue.c index 071e122f9c37..4995b6e22335 100644 --- a/arch/arm64/crypto/ghash-ce-glue.c +++ b/arch/arm64/crypto/ghash-ce-glue.c @@ -6,30 +6,27 @@ */ #include -#include -#include #include -#include -#include #include +#include +#include #include #include #include -#include #include #include #include -#include +#include +#include #include +#include +#include MODULE_DESCRIPTION("GHASH and AES-GCM using ARMv8 Crypto Extensions"); MODULE_AUTHOR("Ard Biesheuvel "); MODULE_LICENSE("GPL v2"); MODULE_ALIAS_CRYPTO("ghash"); -#define GHASH_BLOCK_SIZE 16 -#define GHASH_DIGEST_SIZE 16 - #define RFC4106_NONCE_SIZE 4 struct ghash_key { @@ -37,10 +34,8 @@ struct ghash_key { u64 h[][2]; }; -struct ghash_desc_ctx { +struct arm_ghash_desc_ctx { u64 digest[GHASH_DIGEST_SIZE/sizeof(u64)]; - u8 buf[GHASH_BLOCK_SIZE]; - u32 count; }; struct gcm_aes_ctx { @@ -65,36 +60,12 @@ asmlinkage int pmull_gcm_decrypt(int bytes, u8 dst[], const u8 src[], static int ghash_init(struct shash_desc *desc) { - struct ghash_desc_ctx *ctx = shash_desc_ctx(desc); + struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc); - *ctx = (struct ghash_desc_ctx){}; + *ctx = (struct arm_ghash_desc_ctx){}; return 0; } -static void ghash_do_update(int blocks, u64 dg[], const char *src, - struct ghash_key *key, const char *head) -{ - be128 dst = { cpu_to_be64(dg[1]), cpu_to_be64(dg[0]) }; - - do { - const u8 *in = src; - - if (head) { - in = head; - blocks++; - head = NULL; - } else { - src += GHASH_BLOCK_SIZE; - } - - crypto_xor((u8 *)&dst, in, GHASH_BLOCK_SIZE); - gf128mul_lle(&dst, &key->k); - } while (--blocks); - - dg[0] = be64_to_cpu(dst.b); - dg[1] = be64_to_cpu(dst.a); -} - static __always_inline void ghash_do_simd_update(int blocks, u64 dg[], const char *src, struct ghash_key *key, const char *head, @@ -103,13 +74,9 @@ void ghash_do_simd_update(int blocks, u64 dg[], const char *src, u64 const h[][2], const char *head)) { - if (likely(crypto_simd_usable())) { - kernel_neon_begin(); - simd_update(blocks, dg, src, key->h, head); - kernel_neon_end(); - } else { - ghash_do_update(blocks, dg, src, key, head); - } + kernel_neon_begin(); + simd_update(blocks, dg, src, key->h, head); + kernel_neon_end(); } /* avoid hogging the CPU for too long */ @@ -118,63 +85,61 @@ void ghash_do_simd_update(int blocks, u64 dg[], const char *src, static int ghash_update(struct shash_desc *desc, const u8 *src, unsigned int len) { - struct ghash_desc_ctx *ctx = shash_desc_ctx(desc); - unsigned int partial = ctx->count % GHASH_BLOCK_SIZE; + struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc); + struct ghash_key *key = crypto_shash_ctx(desc->tfm); + int blocks; - ctx->count += len; + blocks = len / GHASH_BLOCK_SIZE; + len -= blocks * GHASH_BLOCK_SIZE; - if ((partial + len) >= GHASH_BLOCK_SIZE) { - struct ghash_key *key = crypto_shash_ctx(desc->tfm); - int blocks; + do { + int chunk = min(blocks, MAX_BLOCKS); - if (partial) { - int p = GHASH_BLOCK_SIZE - partial; + ghash_do_simd_update(chunk, ctx->digest, src, key, NULL, + pmull_ghash_update_p8); + blocks -= chunk; + src += chunk * GHASH_BLOCK_SIZE; + } while (unlikely(blocks > 0)); + return len; +} - memcpy(ctx->buf + partial, src, p); - src += p; - len -= p; - } +static int ghash_export(struct shash_desc *desc, void *out) +{ + struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc); + u8 *dst = out; - blocks = len / GHASH_BLOCK_SIZE; - len %= GHASH_BLOCK_SIZE; - - do { - int chunk = min(blocks, MAX_BLOCKS); - - ghash_do_simd_update(chunk, ctx->digest, src, key, - partial ? ctx->buf : NULL, - pmull_ghash_update_p8); - - blocks -= chunk; - src += chunk * GHASH_BLOCK_SIZE; - partial = 0; - } while (unlikely(blocks > 0)); - } - if (len) - memcpy(ctx->buf + partial, src, len); + put_unaligned_be64(ctx->digest[1], dst); + put_unaligned_be64(ctx->digest[0], dst + 8); return 0; } -static int ghash_final(struct shash_desc *desc, u8 *dst) +static int ghash_import(struct shash_desc *desc, const void *in) { - struct ghash_desc_ctx *ctx = shash_desc_ctx(desc); - unsigned int partial = ctx->count % GHASH_BLOCK_SIZE; + struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc); + const u8 *src = in; - if (partial) { - struct ghash_key *key = crypto_shash_ctx(desc->tfm); - - memset(ctx->buf + partial, 0, GHASH_BLOCK_SIZE - partial); - - ghash_do_simd_update(1, ctx->digest, ctx->buf, key, NULL, - pmull_ghash_update_p8); - } - put_unaligned_be64(ctx->digest[1], dst); - put_unaligned_be64(ctx->digest[0], dst + 8); - - memzero_explicit(ctx, sizeof(*ctx)); + ctx->digest[1] = get_unaligned_be64(src); + ctx->digest[0] = get_unaligned_be64(src + 8); return 0; } +static int ghash_finup(struct shash_desc *desc, const u8 *src, + unsigned int len, u8 *dst) +{ + struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc); + struct ghash_key *key = crypto_shash_ctx(desc->tfm); + + if (len) { + u8 buf[GHASH_BLOCK_SIZE] = {}; + + memcpy(buf, src, len); + ghash_do_simd_update(1, ctx->digest, src, key, NULL, + pmull_ghash_update_p8); + memzero_explicit(buf, sizeof(buf)); + } + return ghash_export(desc, dst); +} + static void ghash_reflect(u64 h[], const be128 *k) { u64 carry = be64_to_cpu(k->a) & BIT(63) ? 1 : 0; @@ -205,6 +170,7 @@ static struct shash_alg ghash_alg = { .base.cra_name = "ghash", .base.cra_driver_name = "ghash-neon", .base.cra_priority = 150, + .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .base.cra_blocksize = GHASH_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct ghash_key) + sizeof(u64[2]), .base.cra_module = THIS_MODULE, @@ -212,9 +178,12 @@ static struct shash_alg ghash_alg = { .digestsize = GHASH_DIGEST_SIZE, .init = ghash_init, .update = ghash_update, - .final = ghash_final, + .finup = ghash_finup, .setkey = ghash_setkey, - .descsize = sizeof(struct ghash_desc_ctx), + .export = ghash_export, + .import = ghash_import, + .descsize = sizeof(struct arm_ghash_desc_ctx), + .statesize = sizeof(struct ghash_desc_ctx), }; static int num_rounds(struct crypto_aes_ctx *ctx) From 867a2177c2c995d3a0c8b03f9be3b7f4fe84e0b1 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 18 Apr 2025 10:58:57 +0800 Subject: [PATCH 156/382] crypto: riscv/ghash - Use API partial block handling Use the Crypto API partial block handling. As this was the last user relying on crypto/ghash.h for gf128mul.h, remove the unnecessary inclusion of gf128mul.h from crypto/ghash.h. Signed-off-by: Herbert Xu --- arch/riscv/crypto/ghash-riscv64-glue.c | 58 ++++++++------------------ include/crypto/ghash.h | 1 - 2 files changed, 18 insertions(+), 41 deletions(-) diff --git a/arch/riscv/crypto/ghash-riscv64-glue.c b/arch/riscv/crypto/ghash-riscv64-glue.c index 312e7891fd0a..d86073d25387 100644 --- a/arch/riscv/crypto/ghash-riscv64-glue.c +++ b/arch/riscv/crypto/ghash-riscv64-glue.c @@ -11,11 +11,16 @@ #include #include +#include +#include #include #include #include -#include +#include +#include +#include #include +#include asmlinkage void ghash_zvkg(be128 *accumulator, const be128 *key, const u8 *data, size_t len); @@ -26,8 +31,6 @@ struct riscv64_ghash_tfm_ctx { struct riscv64_ghash_desc_ctx { be128 accumulator; - u8 buffer[GHASH_BLOCK_SIZE]; - u32 bytes; }; static int riscv64_ghash_setkey(struct crypto_shash *tfm, const u8 *key, @@ -78,50 +81,24 @@ static int riscv64_ghash_update(struct shash_desc *desc, const u8 *src, { const struct riscv64_ghash_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); struct riscv64_ghash_desc_ctx *dctx = shash_desc_ctx(desc); - unsigned int len; - if (dctx->bytes) { - if (dctx->bytes + srclen < GHASH_BLOCK_SIZE) { - memcpy(dctx->buffer + dctx->bytes, src, srclen); - dctx->bytes += srclen; - return 0; - } - memcpy(dctx->buffer + dctx->bytes, src, - GHASH_BLOCK_SIZE - dctx->bytes); - riscv64_ghash_blocks(tctx, dctx, dctx->buffer, - GHASH_BLOCK_SIZE); - src += GHASH_BLOCK_SIZE - dctx->bytes; - srclen -= GHASH_BLOCK_SIZE - dctx->bytes; - dctx->bytes = 0; - } - - len = round_down(srclen, GHASH_BLOCK_SIZE); - if (len) { - riscv64_ghash_blocks(tctx, dctx, src, len); - src += len; - srclen -= len; - } - - if (srclen) { - memcpy(dctx->buffer, src, srclen); - dctx->bytes = srclen; - } - - return 0; + riscv64_ghash_blocks(tctx, dctx, src, + round_down(srclen, GHASH_BLOCK_SIZE)); + return srclen - round_down(srclen, GHASH_BLOCK_SIZE); } -static int riscv64_ghash_final(struct shash_desc *desc, u8 *out) +static int riscv64_ghash_finup(struct shash_desc *desc, const u8 *src, + unsigned int len, u8 *out) { const struct riscv64_ghash_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); struct riscv64_ghash_desc_ctx *dctx = shash_desc_ctx(desc); - int i; - if (dctx->bytes) { - for (i = dctx->bytes; i < GHASH_BLOCK_SIZE; i++) - dctx->buffer[i] = 0; + if (len) { + u8 buf[GHASH_BLOCK_SIZE] = {}; - riscv64_ghash_blocks(tctx, dctx, dctx->buffer, - GHASH_BLOCK_SIZE); + memcpy(buf, src, len); + riscv64_ghash_blocks(tctx, dctx, buf, GHASH_BLOCK_SIZE); + memzero_explicit(buf, sizeof(buf)); } memcpy(out, &dctx->accumulator, GHASH_DIGEST_SIZE); @@ -131,7 +108,7 @@ static int riscv64_ghash_final(struct shash_desc *desc, u8 *out) static struct shash_alg riscv64_ghash_alg = { .init = riscv64_ghash_init, .update = riscv64_ghash_update, - .final = riscv64_ghash_final, + .finup = riscv64_ghash_finup, .setkey = riscv64_ghash_setkey, .descsize = sizeof(struct riscv64_ghash_desc_ctx), .digestsize = GHASH_DIGEST_SIZE, @@ -139,6 +116,7 @@ static struct shash_alg riscv64_ghash_alg = { .cra_blocksize = GHASH_BLOCK_SIZE, .cra_ctxsize = sizeof(struct riscv64_ghash_tfm_ctx), .cra_priority = 300, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_name = "ghash", .cra_driver_name = "ghash-riscv64-zvkg", .cra_module = THIS_MODULE, diff --git a/include/crypto/ghash.h b/include/crypto/ghash.h index 16904f2b5184..043d938e9a2c 100644 --- a/include/crypto/ghash.h +++ b/include/crypto/ghash.h @@ -7,7 +7,6 @@ #define __CRYPTO_GHASH_H__ #include -#include #define GHASH_BLOCK_SIZE 16 #define GHASH_DIGEST_SIZE 16 From dab2d7b66fb65a1d6e6cc26de261e64d1c89eaf4 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 18 Apr 2025 10:58:59 +0800 Subject: [PATCH 157/382] crypto: s390/ghash - Use API partial block handling Use the Crypto API partial block handling. Also switch to the generic export format. Signed-off-by: Herbert Xu --- arch/s390/crypto/ghash_s390.c | 110 ++++++++++++++++------------------ 1 file changed, 50 insertions(+), 60 deletions(-) diff --git a/arch/s390/crypto/ghash_s390.c b/arch/s390/crypto/ghash_s390.c index 0800a2a5799f..dcbcee37cb63 100644 --- a/arch/s390/crypto/ghash_s390.c +++ b/arch/s390/crypto/ghash_s390.c @@ -8,29 +8,28 @@ * Author(s): Gerald Schaefer */ -#include -#include -#include #include +#include +#include +#include +#include +#include +#include +#include -#define GHASH_BLOCK_SIZE 16 -#define GHASH_DIGEST_SIZE 16 - -struct ghash_ctx { +struct s390_ghash_ctx { u8 key[GHASH_BLOCK_SIZE]; }; -struct ghash_desc_ctx { +struct s390_ghash_desc_ctx { u8 icv[GHASH_BLOCK_SIZE]; u8 key[GHASH_BLOCK_SIZE]; - u8 buffer[GHASH_BLOCK_SIZE]; - u32 bytes; }; static int ghash_init(struct shash_desc *desc) { - struct ghash_desc_ctx *dctx = shash_desc_ctx(desc); - struct ghash_ctx *ctx = crypto_shash_ctx(desc->tfm); + struct s390_ghash_ctx *ctx = crypto_shash_ctx(desc->tfm); + struct s390_ghash_desc_ctx *dctx = shash_desc_ctx(desc); memset(dctx, 0, sizeof(*dctx)); memcpy(dctx->key, ctx->key, GHASH_BLOCK_SIZE); @@ -41,7 +40,7 @@ static int ghash_init(struct shash_desc *desc) static int ghash_setkey(struct crypto_shash *tfm, const u8 *key, unsigned int keylen) { - struct ghash_ctx *ctx = crypto_shash_ctx(tfm); + struct s390_ghash_ctx *ctx = crypto_shash_ctx(tfm); if (keylen != GHASH_BLOCK_SIZE) return -EINVAL; @@ -54,80 +53,71 @@ static int ghash_setkey(struct crypto_shash *tfm, static int ghash_update(struct shash_desc *desc, const u8 *src, unsigned int srclen) { - struct ghash_desc_ctx *dctx = shash_desc_ctx(desc); + struct s390_ghash_desc_ctx *dctx = shash_desc_ctx(desc); unsigned int n; - u8 *buf = dctx->buffer; - - if (dctx->bytes) { - u8 *pos = buf + (GHASH_BLOCK_SIZE - dctx->bytes); - - n = min(srclen, dctx->bytes); - dctx->bytes -= n; - srclen -= n; - - memcpy(pos, src, n); - src += n; - - if (!dctx->bytes) { - cpacf_kimd(CPACF_KIMD_GHASH, dctx, buf, - GHASH_BLOCK_SIZE); - } - } n = srclen & ~(GHASH_BLOCK_SIZE - 1); - if (n) { - cpacf_kimd(CPACF_KIMD_GHASH, dctx, src, n); - src += n; - srclen -= n; - } - - if (srclen) { - dctx->bytes = GHASH_BLOCK_SIZE - srclen; - memcpy(buf, src, srclen); - } - - return 0; + cpacf_kimd(CPACF_KIMD_GHASH, dctx, src, n); + return srclen - n; } -static int ghash_flush(struct ghash_desc_ctx *dctx) +static void ghash_flush(struct s390_ghash_desc_ctx *dctx, const u8 *src, + unsigned int len) { - u8 *buf = dctx->buffer; + if (len) { + u8 buf[GHASH_BLOCK_SIZE] = {}; - if (dctx->bytes) { - u8 *pos = buf + (GHASH_BLOCK_SIZE - dctx->bytes); - - memset(pos, 0, dctx->bytes); + memcpy(buf, src, len); cpacf_kimd(CPACF_KIMD_GHASH, dctx, buf, GHASH_BLOCK_SIZE); - dctx->bytes = 0; + memzero_explicit(buf, sizeof(buf)); } +} +static int ghash_finup(struct shash_desc *desc, const u8 *src, + unsigned int len, u8 *dst) +{ + struct s390_ghash_desc_ctx *dctx = shash_desc_ctx(desc); + + ghash_flush(dctx, src, len); + memcpy(dst, dctx->icv, GHASH_BLOCK_SIZE); return 0; } -static int ghash_final(struct shash_desc *desc, u8 *dst) +static int ghash_export(struct shash_desc *desc, void *out) { - struct ghash_desc_ctx *dctx = shash_desc_ctx(desc); - int ret; + struct s390_ghash_desc_ctx *dctx = shash_desc_ctx(desc); - ret = ghash_flush(dctx); - if (!ret) - memcpy(dst, dctx->icv, GHASH_BLOCK_SIZE); - return ret; + memcpy(out, dctx->icv, GHASH_DIGEST_SIZE); + return 0; +} + +static int ghash_import(struct shash_desc *desc, const void *in) +{ + struct s390_ghash_ctx *ctx = crypto_shash_ctx(desc->tfm); + struct s390_ghash_desc_ctx *dctx = shash_desc_ctx(desc); + + memcpy(dctx->icv, in, GHASH_DIGEST_SIZE); + memcpy(dctx->key, ctx->key, GHASH_BLOCK_SIZE); + return 0; } static struct shash_alg ghash_alg = { .digestsize = GHASH_DIGEST_SIZE, .init = ghash_init, .update = ghash_update, - .final = ghash_final, + .finup = ghash_finup, .setkey = ghash_setkey, - .descsize = sizeof(struct ghash_desc_ctx), + .export = ghash_export, + .import = ghash_import, + .statesize = sizeof(struct ghash_desc_ctx), + .descsize = sizeof(struct s390_ghash_desc_ctx), .base = { .cra_name = "ghash", .cra_driver_name = "ghash-s390", .cra_priority = 300, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_blocksize = GHASH_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct ghash_ctx), + .cra_ctxsize = sizeof(struct s390_ghash_ctx), .cra_module = THIS_MODULE, }, }; From 3942654223cce18a245d0ec134c7259941205744 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 18 Apr 2025 10:59:01 +0800 Subject: [PATCH 158/382] crypto: x86/ghash - Use API partial block handling Use the Crypto API partial block handling. Also remove the unnecessary SIMD fallback path. Signed-off-by: Herbert Xu --- arch/x86/crypto/ghash-clmulni-intel_asm.S | 5 +- arch/x86/crypto/ghash-clmulni-intel_glue.c | 301 +++------------------ 2 files changed, 43 insertions(+), 263 deletions(-) diff --git a/arch/x86/crypto/ghash-clmulni-intel_asm.S b/arch/x86/crypto/ghash-clmulni-intel_asm.S index 99cb983ded9e..c4fbaa82ed7a 100644 --- a/arch/x86/crypto/ghash-clmulni-intel_asm.S +++ b/arch/x86/crypto/ghash-clmulni-intel_asm.S @@ -103,8 +103,8 @@ SYM_FUNC_START(clmul_ghash_mul) SYM_FUNC_END(clmul_ghash_mul) /* - * void clmul_ghash_update(char *dst, const char *src, unsigned int srclen, - * const le128 *shash); + * int clmul_ghash_update(char *dst, const char *src, unsigned int srclen, + * const le128 *shash); */ SYM_FUNC_START(clmul_ghash_update) FRAME_BEGIN @@ -127,6 +127,7 @@ SYM_FUNC_START(clmul_ghash_update) pshufb BSWAP, DATA movups DATA, (%rdi) .Lupdate_just_ret: + mov %rdx, %rax FRAME_END RET SYM_FUNC_END(clmul_ghash_update) diff --git a/arch/x86/crypto/ghash-clmulni-intel_glue.c b/arch/x86/crypto/ghash-clmulni-intel_glue.c index c759ec808bf1..aea5d4d06be7 100644 --- a/arch/x86/crypto/ghash-clmulni-intel_glue.c +++ b/arch/x86/crypto/ghash-clmulni-intel_glue.c @@ -7,41 +7,27 @@ * Author: Huang Ying */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include #include #include +#include +#include +#include +#include +#include +#include +#include +#include #include -#define GHASH_BLOCK_SIZE 16 -#define GHASH_DIGEST_SIZE 16 +asmlinkage void clmul_ghash_mul(char *dst, const le128 *shash); -void clmul_ghash_mul(char *dst, const le128 *shash); +asmlinkage int clmul_ghash_update(char *dst, const char *src, + unsigned int srclen, const le128 *shash); -void clmul_ghash_update(char *dst, const char *src, unsigned int srclen, - const le128 *shash); - -struct ghash_async_ctx { - struct cryptd_ahash *cryptd_tfm; -}; - -struct ghash_ctx { +struct x86_ghash_ctx { le128 shash; }; -struct ghash_desc_ctx { - u8 buffer[GHASH_BLOCK_SIZE]; - u32 bytes; -}; - static int ghash_init(struct shash_desc *desc) { struct ghash_desc_ctx *dctx = shash_desc_ctx(desc); @@ -54,7 +40,7 @@ static int ghash_init(struct shash_desc *desc) static int ghash_setkey(struct crypto_shash *tfm, const u8 *key, unsigned int keylen) { - struct ghash_ctx *ctx = crypto_shash_ctx(tfm); + struct x86_ghash_ctx *ctx = crypto_shash_ctx(tfm); u64 a, b; if (keylen != GHASH_BLOCK_SIZE) @@ -95,64 +81,38 @@ static int ghash_setkey(struct crypto_shash *tfm, static int ghash_update(struct shash_desc *desc, const u8 *src, unsigned int srclen) { + struct x86_ghash_ctx *ctx = crypto_shash_ctx(desc->tfm); struct ghash_desc_ctx *dctx = shash_desc_ctx(desc); - struct ghash_ctx *ctx = crypto_shash_ctx(desc->tfm); + u8 *dst = dctx->buffer; + int remain; + + kernel_fpu_begin(); + remain = clmul_ghash_update(dst, src, srclen, &ctx->shash); + kernel_fpu_end(); + return remain; +} + +static void ghash_flush(struct x86_ghash_ctx *ctx, struct ghash_desc_ctx *dctx, + const u8 *src, unsigned int len) +{ u8 *dst = dctx->buffer; kernel_fpu_begin(); - if (dctx->bytes) { - int n = min(srclen, dctx->bytes); - u8 *pos = dst + (GHASH_BLOCK_SIZE - dctx->bytes); - - dctx->bytes -= n; - srclen -= n; - - while (n--) - *pos++ ^= *src++; - - if (!dctx->bytes) - clmul_ghash_mul(dst, &ctx->shash); - } - - clmul_ghash_update(dst, src, srclen, &ctx->shash); - kernel_fpu_end(); - - if (srclen & 0xf) { - src += srclen - (srclen & 0xf); - srclen &= 0xf; - dctx->bytes = GHASH_BLOCK_SIZE - srclen; - while (srclen--) - *dst++ ^= *src++; - } - - return 0; -} - -static void ghash_flush(struct ghash_ctx *ctx, struct ghash_desc_ctx *dctx) -{ - u8 *dst = dctx->buffer; - - if (dctx->bytes) { - u8 *tmp = dst + (GHASH_BLOCK_SIZE - dctx->bytes); - - while (dctx->bytes--) - *tmp++ ^= 0; - - kernel_fpu_begin(); + if (len) { + crypto_xor(dst, src, len); clmul_ghash_mul(dst, &ctx->shash); - kernel_fpu_end(); } - - dctx->bytes = 0; + kernel_fpu_end(); } -static int ghash_final(struct shash_desc *desc, u8 *dst) +static int ghash_finup(struct shash_desc *desc, const u8 *src, + unsigned int len, u8 *dst) { + struct x86_ghash_ctx *ctx = crypto_shash_ctx(desc->tfm); struct ghash_desc_ctx *dctx = shash_desc_ctx(desc); - struct ghash_ctx *ctx = crypto_shash_ctx(desc->tfm); u8 *buf = dctx->buffer; - ghash_flush(ctx, dctx); + ghash_flush(ctx, dctx, src, len); memcpy(dst, buf, GHASH_BLOCK_SIZE); return 0; @@ -162,186 +122,20 @@ static struct shash_alg ghash_alg = { .digestsize = GHASH_DIGEST_SIZE, .init = ghash_init, .update = ghash_update, - .final = ghash_final, + .finup = ghash_finup, .setkey = ghash_setkey, .descsize = sizeof(struct ghash_desc_ctx), .base = { - .cra_name = "__ghash", - .cra_driver_name = "__ghash-pclmulqdqni", - .cra_priority = 0, - .cra_flags = CRYPTO_ALG_INTERNAL, + .cra_name = "ghash", + .cra_driver_name = "ghash-pclmulqdqni", + .cra_priority = 400, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_blocksize = GHASH_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct ghash_ctx), + .cra_ctxsize = sizeof(struct x86_ghash_ctx), .cra_module = THIS_MODULE, }, }; -static int ghash_async_init(struct ahash_request *req) -{ - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm); - struct ahash_request *cryptd_req = ahash_request_ctx(req); - struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm; - struct shash_desc *desc = cryptd_shash_desc(cryptd_req); - struct crypto_shash *child = cryptd_ahash_child(cryptd_tfm); - - desc->tfm = child; - return crypto_shash_init(desc); -} - -static void ghash_init_cryptd_req(struct ahash_request *req) -{ - struct ahash_request *cryptd_req = ahash_request_ctx(req); - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm); - struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm; - - ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base); - ahash_request_set_callback(cryptd_req, req->base.flags, - req->base.complete, req->base.data); - ahash_request_set_crypt(cryptd_req, req->src, req->result, - req->nbytes); -} - -static int ghash_async_update(struct ahash_request *req) -{ - struct ahash_request *cryptd_req = ahash_request_ctx(req); - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm); - struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm; - - if (!crypto_simd_usable() || - (in_atomic() && cryptd_ahash_queued(cryptd_tfm))) { - ghash_init_cryptd_req(req); - return crypto_ahash_update(cryptd_req); - } else { - struct shash_desc *desc = cryptd_shash_desc(cryptd_req); - return shash_ahash_update(req, desc); - } -} - -static int ghash_async_final(struct ahash_request *req) -{ - struct ahash_request *cryptd_req = ahash_request_ctx(req); - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm); - struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm; - - if (!crypto_simd_usable() || - (in_atomic() && cryptd_ahash_queued(cryptd_tfm))) { - ghash_init_cryptd_req(req); - return crypto_ahash_final(cryptd_req); - } else { - struct shash_desc *desc = cryptd_shash_desc(cryptd_req); - return crypto_shash_final(desc, req->result); - } -} - -static int ghash_async_import(struct ahash_request *req, const void *in) -{ - struct ahash_request *cryptd_req = ahash_request_ctx(req); - struct shash_desc *desc = cryptd_shash_desc(cryptd_req); - struct ghash_desc_ctx *dctx = shash_desc_ctx(desc); - - ghash_async_init(req); - memcpy(dctx, in, sizeof(*dctx)); - return 0; - -} - -static int ghash_async_export(struct ahash_request *req, void *out) -{ - struct ahash_request *cryptd_req = ahash_request_ctx(req); - struct shash_desc *desc = cryptd_shash_desc(cryptd_req); - struct ghash_desc_ctx *dctx = shash_desc_ctx(desc); - - memcpy(out, dctx, sizeof(*dctx)); - return 0; - -} - -static int ghash_async_digest(struct ahash_request *req) -{ - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm); - struct ahash_request *cryptd_req = ahash_request_ctx(req); - struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm; - - if (!crypto_simd_usable() || - (in_atomic() && cryptd_ahash_queued(cryptd_tfm))) { - ghash_init_cryptd_req(req); - return crypto_ahash_digest(cryptd_req); - } else { - struct shash_desc *desc = cryptd_shash_desc(cryptd_req); - struct crypto_shash *child = cryptd_ahash_child(cryptd_tfm); - - desc->tfm = child; - return shash_ahash_digest(req, desc); - } -} - -static int ghash_async_setkey(struct crypto_ahash *tfm, const u8 *key, - unsigned int keylen) -{ - struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm); - struct crypto_ahash *child = &ctx->cryptd_tfm->base; - - crypto_ahash_clear_flags(child, CRYPTO_TFM_REQ_MASK); - crypto_ahash_set_flags(child, crypto_ahash_get_flags(tfm) - & CRYPTO_TFM_REQ_MASK); - return crypto_ahash_setkey(child, key, keylen); -} - -static int ghash_async_init_tfm(struct crypto_tfm *tfm) -{ - struct cryptd_ahash *cryptd_tfm; - struct ghash_async_ctx *ctx = crypto_tfm_ctx(tfm); - - cryptd_tfm = cryptd_alloc_ahash("__ghash-pclmulqdqni", - CRYPTO_ALG_INTERNAL, - CRYPTO_ALG_INTERNAL); - if (IS_ERR(cryptd_tfm)) - return PTR_ERR(cryptd_tfm); - ctx->cryptd_tfm = cryptd_tfm; - crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), - sizeof(struct ahash_request) + - crypto_ahash_reqsize(&cryptd_tfm->base)); - - return 0; -} - -static void ghash_async_exit_tfm(struct crypto_tfm *tfm) -{ - struct ghash_async_ctx *ctx = crypto_tfm_ctx(tfm); - - cryptd_free_ahash(ctx->cryptd_tfm); -} - -static struct ahash_alg ghash_async_alg = { - .init = ghash_async_init, - .update = ghash_async_update, - .final = ghash_async_final, - .setkey = ghash_async_setkey, - .digest = ghash_async_digest, - .export = ghash_async_export, - .import = ghash_async_import, - .halg = { - .digestsize = GHASH_DIGEST_SIZE, - .statesize = sizeof(struct ghash_desc_ctx), - .base = { - .cra_name = "ghash", - .cra_driver_name = "ghash-clmulni", - .cra_priority = 400, - .cra_ctxsize = sizeof(struct ghash_async_ctx), - .cra_flags = CRYPTO_ALG_ASYNC, - .cra_blocksize = GHASH_BLOCK_SIZE, - .cra_module = THIS_MODULE, - .cra_init = ghash_async_init_tfm, - .cra_exit = ghash_async_exit_tfm, - }, - }, -}; - static const struct x86_cpu_id pcmul_cpu_id[] = { X86_MATCH_FEATURE(X86_FEATURE_PCLMULQDQ, NULL), /* Pickle-Mickle-Duck */ {} @@ -350,29 +144,14 @@ MODULE_DEVICE_TABLE(x86cpu, pcmul_cpu_id); static int __init ghash_pclmulqdqni_mod_init(void) { - int err; - if (!x86_match_cpu(pcmul_cpu_id)) return -ENODEV; - err = crypto_register_shash(&ghash_alg); - if (err) - goto err_out; - err = crypto_register_ahash(&ghash_async_alg); - if (err) - goto err_shash; - - return 0; - -err_shash: - crypto_unregister_shash(&ghash_alg); -err_out: - return err; + return crypto_register_shash(&ghash_alg); } static void __exit ghash_pclmulqdqni_mod_exit(void) { - crypto_unregister_ahash(&ghash_async_alg); crypto_unregister_shash(&ghash_alg); } From efd62c85525e212705368b24eb90ef10778190f5 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 18 Apr 2025 10:59:04 +0800 Subject: [PATCH 159/382] crypto: md5-generic - Use API partial block handling Use the Crypto API partial block handling. Signed-off-by: Herbert Xu --- crypto/md5.c | 102 ++++++++++++++++--------------------------- include/crypto/md5.h | 3 +- 2 files changed, 40 insertions(+), 65 deletions(-) diff --git a/crypto/md5.c b/crypto/md5.c index 72c0c46fb5ee..994005cd977d 100644 --- a/crypto/md5.c +++ b/crypto/md5.c @@ -17,11 +17,9 @@ */ #include #include -#include +#include #include #include -#include -#include const u8 md5_zero_message_hash[MD5_DIGEST_SIZE] = { 0xd4, 0x1d, 0x8c, 0xd9, 0x8f, 0x00, 0xb2, 0x04, @@ -120,10 +118,11 @@ static void md5_transform(__u32 *hash, __u32 const *in) hash[3] += d; } -static inline void md5_transform_helper(struct md5_state *ctx) +static inline void md5_transform_helper(struct md5_state *ctx, + u32 block[MD5_BLOCK_WORDS]) { - le32_to_cpu_array(ctx->block, sizeof(ctx->block) / sizeof(u32)); - md5_transform(ctx->hash, ctx->block); + le32_to_cpu_array(block, MD5_BLOCK_WORDS); + md5_transform(ctx->hash, block); } static int md5_init(struct shash_desc *desc) @@ -142,91 +141,66 @@ static int md5_init(struct shash_desc *desc) static int md5_update(struct shash_desc *desc, const u8 *data, unsigned int len) { struct md5_state *mctx = shash_desc_ctx(desc); - const u32 avail = sizeof(mctx->block) - (mctx->byte_count & 0x3f); + u32 block[MD5_BLOCK_WORDS]; mctx->byte_count += len; - - if (avail > len) { - memcpy((char *)mctx->block + (sizeof(mctx->block) - avail), - data, len); - return 0; - } - - memcpy((char *)mctx->block + (sizeof(mctx->block) - avail), - data, avail); - - md5_transform_helper(mctx); - data += avail; - len -= avail; - - while (len >= sizeof(mctx->block)) { - memcpy(mctx->block, data, sizeof(mctx->block)); - md5_transform_helper(mctx); - data += sizeof(mctx->block); - len -= sizeof(mctx->block); - } - - memcpy(mctx->block, data, len); - - return 0; + do { + memcpy(block, data, sizeof(block)); + md5_transform_helper(mctx, block); + data += sizeof(block); + len -= sizeof(block); + } while (len >= sizeof(block)); + memzero_explicit(block, sizeof(block)); + mctx->byte_count -= len; + return len; } -static int md5_final(struct shash_desc *desc, u8 *out) +static int md5_finup(struct shash_desc *desc, const u8 *data, unsigned int len, + u8 *out) { struct md5_state *mctx = shash_desc_ctx(desc); - const unsigned int offset = mctx->byte_count & 0x3f; - char *p = (char *)mctx->block + offset; - int padding = 56 - (offset + 1); + u32 block[MD5_BLOCK_WORDS]; + unsigned int offset; + int padding; + char *p; + + memcpy(block, data, len); + + offset = len; + p = (char *)block + offset; + padding = 56 - (offset + 1); *p++ = 0x80; if (padding < 0) { memset(p, 0x00, padding + sizeof (u64)); - md5_transform_helper(mctx); - p = (char *)mctx->block; + md5_transform_helper(mctx, block); + p = (char *)block; padding = 56; } memset(p, 0, padding); - mctx->block[14] = mctx->byte_count << 3; - mctx->block[15] = mctx->byte_count >> 29; - le32_to_cpu_array(mctx->block, (sizeof(mctx->block) - - sizeof(u64)) / sizeof(u32)); - md5_transform(mctx->hash, mctx->block); + mctx->byte_count += len; + block[14] = mctx->byte_count << 3; + block[15] = mctx->byte_count >> 29; + le32_to_cpu_array(block, (sizeof(block) - sizeof(u64)) / sizeof(u32)); + md5_transform(mctx->hash, block); + memzero_explicit(block, sizeof(block)); cpu_to_le32_array(mctx->hash, sizeof(mctx->hash) / sizeof(u32)); memcpy(out, mctx->hash, sizeof(mctx->hash)); - memset(mctx, 0, sizeof(*mctx)); return 0; } -static int md5_export(struct shash_desc *desc, void *out) -{ - struct md5_state *ctx = shash_desc_ctx(desc); - - memcpy(out, ctx, sizeof(*ctx)); - return 0; -} - -static int md5_import(struct shash_desc *desc, const void *in) -{ - struct md5_state *ctx = shash_desc_ctx(desc); - - memcpy(ctx, in, sizeof(*ctx)); - return 0; -} - static struct shash_alg alg = { .digestsize = MD5_DIGEST_SIZE, .init = md5_init, .update = md5_update, - .final = md5_final, - .export = md5_export, - .import = md5_import, - .descsize = sizeof(struct md5_state), - .statesize = sizeof(struct md5_state), + .finup = md5_finup, + .descsize = MD5_STATE_SIZE, .base = { .cra_name = "md5", .cra_driver_name = "md5-generic", + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_blocksize = MD5_HMAC_BLOCK_SIZE, .cra_module = THIS_MODULE, } diff --git a/include/crypto/md5.h b/include/crypto/md5.h index cf9e9dec3d21..198b5d69b92f 100644 --- a/include/crypto/md5.h +++ b/include/crypto/md5.h @@ -8,6 +8,7 @@ #define MD5_HMAC_BLOCK_SIZE 64 #define MD5_BLOCK_WORDS 16 #define MD5_HASH_WORDS 4 +#define MD5_STATE_SIZE 24 #define MD5_H0 0x67452301UL #define MD5_H1 0xefcdab89UL @@ -18,8 +19,8 @@ extern const u8 md5_zero_message_hash[MD5_DIGEST_SIZE]; struct md5_state { u32 hash[MD5_HASH_WORDS]; - u32 block[MD5_BLOCK_WORDS]; u64 byte_count; + u32 block[MD5_BLOCK_WORDS]; }; #endif From f53fc0d8e9909a55c0121a0fc0c0a8c0e436a494 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 18 Apr 2025 10:59:06 +0800 Subject: [PATCH 160/382] crypto: mips/octeon-md5 - Use API partial block handling Use the Crypto API partial block handling. Also switch to the generic export format. Signed-off-by: Herbert Xu --- arch/mips/cavium-octeon/crypto/octeon-md5.c | 119 +++++++++++--------- 1 file changed, 63 insertions(+), 56 deletions(-) diff --git a/arch/mips/cavium-octeon/crypto/octeon-md5.c b/arch/mips/cavium-octeon/crypto/octeon-md5.c index 5ee4ade99b99..fbc84eb7fedf 100644 --- a/arch/mips/cavium-octeon/crypto/octeon-md5.c +++ b/arch/mips/cavium-octeon/crypto/octeon-md5.c @@ -19,22 +19,26 @@ * any later version. */ -#include -#include -#include -#include -#include -#include #include #include +#include +#include +#include +#include +#include #include "octeon-crypto.h" +struct octeon_md5_state { + __le32 hash[MD5_HASH_WORDS]; + u64 byte_count; +}; + /* * We pass everything as 64-bit. OCTEON can handle misaligned data. */ -static void octeon_md5_store_hash(struct md5_state *ctx) +static void octeon_md5_store_hash(struct octeon_md5_state *ctx) { u64 *hash = (u64 *)ctx->hash; @@ -42,7 +46,7 @@ static void octeon_md5_store_hash(struct md5_state *ctx) write_octeon_64bit_hash_dword(hash[1], 1); } -static void octeon_md5_read_hash(struct md5_state *ctx) +static void octeon_md5_read_hash(struct octeon_md5_state *ctx) { u64 *hash = (u64 *)ctx->hash; @@ -66,13 +70,12 @@ static void octeon_md5_transform(const void *_block) static int octeon_md5_init(struct shash_desc *desc) { - struct md5_state *mctx = shash_desc_ctx(desc); + struct octeon_md5_state *mctx = shash_desc_ctx(desc); - mctx->hash[0] = MD5_H0; - mctx->hash[1] = MD5_H1; - mctx->hash[2] = MD5_H2; - mctx->hash[3] = MD5_H3; - cpu_to_le32_array(mctx->hash, 4); + mctx->hash[0] = cpu_to_le32(MD5_H0); + mctx->hash[1] = cpu_to_le32(MD5_H1); + mctx->hash[2] = cpu_to_le32(MD5_H2); + mctx->hash[3] = cpu_to_le32(MD5_H3); mctx->byte_count = 0; return 0; @@ -81,52 +84,38 @@ static int octeon_md5_init(struct shash_desc *desc) static int octeon_md5_update(struct shash_desc *desc, const u8 *data, unsigned int len) { - struct md5_state *mctx = shash_desc_ctx(desc); - const u32 avail = sizeof(mctx->block) - (mctx->byte_count & 0x3f); + struct octeon_md5_state *mctx = shash_desc_ctx(desc); struct octeon_cop2_state state; unsigned long flags; mctx->byte_count += len; - - if (avail > len) { - memcpy((char *)mctx->block + (sizeof(mctx->block) - avail), - data, len); - return 0; - } - - memcpy((char *)mctx->block + (sizeof(mctx->block) - avail), data, - avail); - flags = octeon_crypto_enable(&state); octeon_md5_store_hash(mctx); - octeon_md5_transform(mctx->block); - data += avail; - len -= avail; - - while (len >= sizeof(mctx->block)) { + do { octeon_md5_transform(data); - data += sizeof(mctx->block); - len -= sizeof(mctx->block); - } + data += MD5_HMAC_BLOCK_SIZE; + len -= MD5_HMAC_BLOCK_SIZE; + } while (len >= MD5_HMAC_BLOCK_SIZE); octeon_md5_read_hash(mctx); octeon_crypto_disable(&state, flags); - - memcpy(mctx->block, data, len); - - return 0; + mctx->byte_count -= len; + return len; } -static int octeon_md5_final(struct shash_desc *desc, u8 *out) +static int octeon_md5_finup(struct shash_desc *desc, const u8 *src, + unsigned int offset, u8 *out) { - struct md5_state *mctx = shash_desc_ctx(desc); - const unsigned int offset = mctx->byte_count & 0x3f; - char *p = (char *)mctx->block + offset; + struct octeon_md5_state *mctx = shash_desc_ctx(desc); int padding = 56 - (offset + 1); struct octeon_cop2_state state; + u32 block[MD5_BLOCK_WORDS]; unsigned long flags; + char *p; + p = memcpy(block, src, offset); + p += offset; *p++ = 0x80; flags = octeon_crypto_enable(&state); @@ -134,39 +123,56 @@ static int octeon_md5_final(struct shash_desc *desc, u8 *out) if (padding < 0) { memset(p, 0x00, padding + sizeof(u64)); - octeon_md5_transform(mctx->block); - p = (char *)mctx->block; + octeon_md5_transform(block); + p = (char *)block; padding = 56; } memset(p, 0, padding); - mctx->block[14] = mctx->byte_count << 3; - mctx->block[15] = mctx->byte_count >> 29; - cpu_to_le32_array(mctx->block + 14, 2); - octeon_md5_transform(mctx->block); + mctx->byte_count += offset; + block[14] = mctx->byte_count << 3; + block[15] = mctx->byte_count >> 29; + cpu_to_le32_array(block + 14, 2); + octeon_md5_transform(block); octeon_md5_read_hash(mctx); octeon_crypto_disable(&state, flags); + memzero_explicit(block, sizeof(block)); memcpy(out, mctx->hash, sizeof(mctx->hash)); - memset(mctx, 0, sizeof(*mctx)); return 0; } static int octeon_md5_export(struct shash_desc *desc, void *out) { - struct md5_state *ctx = shash_desc_ctx(desc); + struct octeon_md5_state *ctx = shash_desc_ctx(desc); + union { + u8 *u8; + u32 *u32; + u64 *u64; + } p = { .u8 = out }; + int i; - memcpy(out, ctx, sizeof(*ctx)); + for (i = 0; i < MD5_HASH_WORDS; i++) + put_unaligned(le32_to_cpu(ctx->hash[i]), p.u32++); + put_unaligned(ctx->byte_count, p.u64); return 0; } static int octeon_md5_import(struct shash_desc *desc, const void *in) { - struct md5_state *ctx = shash_desc_ctx(desc); + struct octeon_md5_state *ctx = shash_desc_ctx(desc); + union { + const u8 *u8; + const u32 *u32; + const u64 *u64; + } p = { .u8 = in }; + int i; - memcpy(ctx, in, sizeof(*ctx)); + for (i = 0; i < MD5_HASH_WORDS; i++) + ctx->hash[i] = cpu_to_le32(get_unaligned(p.u32++)); + ctx->byte_count = get_unaligned(p.u64); return 0; } @@ -174,15 +180,16 @@ static struct shash_alg alg = { .digestsize = MD5_DIGEST_SIZE, .init = octeon_md5_init, .update = octeon_md5_update, - .final = octeon_md5_final, + .finup = octeon_md5_finup, .export = octeon_md5_export, .import = octeon_md5_import, - .descsize = sizeof(struct md5_state), - .statesize = sizeof(struct md5_state), + .statesize = MD5_STATE_SIZE, + .descsize = sizeof(struct octeon_md5_state), .base = { .cra_name = "md5", .cra_driver_name= "octeon-md5", .cra_priority = OCTEON_CR_OPCODE_PRIORITY, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_blocksize = MD5_HMAC_BLOCK_SIZE, .cra_module = THIS_MODULE, } From 60fded57098fab456dbdff06fd50f819236d4e65 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 18 Apr 2025 10:59:08 +0800 Subject: [PATCH 161/382] crypto: powerpc/md5 - Use API partial block handling Use the Crypto API partial block handling. Signed-off-by: Herbert Xu --- arch/powerpc/crypto/md5-glue.c | 99 +++++++--------------------------- 1 file changed, 20 insertions(+), 79 deletions(-) diff --git a/arch/powerpc/crypto/md5-glue.c b/arch/powerpc/crypto/md5-glue.c index c24f605033bd..204440a90cd8 100644 --- a/arch/powerpc/crypto/md5-glue.c +++ b/arch/powerpc/crypto/md5-glue.c @@ -8,25 +8,13 @@ */ #include -#include -#include -#include -#include #include -#include +#include +#include +#include extern void ppc_md5_transform(u32 *state, const u8 *src, u32 blocks); -static inline void ppc_md5_clear_context(struct md5_state *sctx) -{ - int count = sizeof(struct md5_state) >> 2; - u32 *ptr = (u32 *)sctx; - - /* make sure we can clear the fast way */ - BUILD_BUG_ON(sizeof(struct md5_state) % 4); - do { *ptr++ = 0; } while (--count); -} - static int ppc_md5_init(struct shash_desc *desc) { struct md5_state *sctx = shash_desc_ctx(desc); @@ -44,79 +32,34 @@ static int ppc_md5_update(struct shash_desc *desc, const u8 *data, unsigned int len) { struct md5_state *sctx = shash_desc_ctx(desc); - const unsigned int offset = sctx->byte_count & 0x3f; - unsigned int avail = 64 - offset; - const u8 *src = data; - sctx->byte_count += len; - - if (avail > len) { - memcpy((char *)sctx->block + offset, src, len); - return 0; - } - - if (offset) { - memcpy((char *)sctx->block + offset, src, avail); - ppc_md5_transform(sctx->hash, (const u8 *)sctx->block, 1); - len -= avail; - src += avail; - } - - if (len > 63) { - ppc_md5_transform(sctx->hash, src, len >> 6); - src += len & ~0x3f; - len &= 0x3f; - } - - memcpy((char *)sctx->block, src, len); - return 0; + sctx->byte_count += round_down(len, MD5_HMAC_BLOCK_SIZE); + ppc_md5_transform(sctx->hash, data, len >> 6); + return len - round_down(len, MD5_HMAC_BLOCK_SIZE); } -static int ppc_md5_final(struct shash_desc *desc, u8 *out) +static int ppc_md5_finup(struct shash_desc *desc, const u8 *src, + unsigned int offset, u8 *out) { struct md5_state *sctx = shash_desc_ctx(desc); - const unsigned int offset = sctx->byte_count & 0x3f; - const u8 *src = (const u8 *)sctx->block; - u8 *p = (u8 *)src + offset; - int padlen = 55 - offset; - __le64 *pbits = (__le64 *)((char *)sctx->block + 56); + __le64 block[MD5_BLOCK_WORDS] = {}; + u8 *p = memcpy(block, src, offset); __le32 *dst = (__le32 *)out; + __le64 *pbits; + src = p; + p += offset; *p++ = 0x80; - - if (padlen < 0) { - memset(p, 0x00, padlen + sizeof (u64)); - ppc_md5_transform(sctx->hash, src, 1); - p = (char *)sctx->block; - padlen = 56; - } - - memset(p, 0, padlen); + sctx->byte_count += offset; + pbits = &block[(MD5_BLOCK_WORDS / (offset > 55 ? 1 : 2)) - 1]; *pbits = cpu_to_le64(sctx->byte_count << 3); - ppc_md5_transform(sctx->hash, src, 1); + ppc_md5_transform(sctx->hash, src, (pbits - block + 1) / 8); + memzero_explicit(block, sizeof(block)); dst[0] = cpu_to_le32(sctx->hash[0]); dst[1] = cpu_to_le32(sctx->hash[1]); dst[2] = cpu_to_le32(sctx->hash[2]); dst[3] = cpu_to_le32(sctx->hash[3]); - - ppc_md5_clear_context(sctx); - return 0; -} - -static int ppc_md5_export(struct shash_desc *desc, void *out) -{ - struct md5_state *sctx = shash_desc_ctx(desc); - - memcpy(out, sctx, sizeof(*sctx)); - return 0; -} - -static int ppc_md5_import(struct shash_desc *desc, const void *in) -{ - struct md5_state *sctx = shash_desc_ctx(desc); - - memcpy(sctx, in, sizeof(*sctx)); return 0; } @@ -124,15 +67,13 @@ static struct shash_alg alg = { .digestsize = MD5_DIGEST_SIZE, .init = ppc_md5_init, .update = ppc_md5_update, - .final = ppc_md5_final, - .export = ppc_md5_export, - .import = ppc_md5_import, - .descsize = sizeof(struct md5_state), - .statesize = sizeof(struct md5_state), + .finup = ppc_md5_finup, + .descsize = MD5_STATE_SIZE, .base = { .cra_name = "md5", .cra_driver_name= "md5-ppc", .cra_priority = 200, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_blocksize = MD5_HMAC_BLOCK_SIZE, .cra_module = THIS_MODULE, } From cc1f5bbe428c915b2076a77d07ad0477c1ee6d10 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 18 Apr 2025 10:59:11 +0800 Subject: [PATCH 162/382] crypto: sparc/md5 - Use API partial block handling Use the Crypto API partial block handling. Also switch to the generic export format. Signed-off-by: Herbert Xu --- arch/sparc/crypto/md5_glue.c | 141 ++++++++++++++++------------------- 1 file changed, 63 insertions(+), 78 deletions(-) diff --git a/arch/sparc/crypto/md5_glue.c b/arch/sparc/crypto/md5_glue.c index 511db98d590a..5b018c6a376c 100644 --- a/arch/sparc/crypto/md5_glue.c +++ b/arch/sparc/crypto/md5_glue.c @@ -14,121 +14,105 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -#include -#include -#include -#include -#include -#include - -#include #include +#include +#include +#include +#include +#include +#include +#include +#include #include "opcodes.h" -asmlinkage void md5_sparc64_transform(u32 *digest, const char *data, +struct sparc_md5_state { + __le32 hash[MD5_HASH_WORDS]; + u64 byte_count; +}; + +asmlinkage void md5_sparc64_transform(__le32 *digest, const char *data, unsigned int rounds); static int md5_sparc64_init(struct shash_desc *desc) { - struct md5_state *mctx = shash_desc_ctx(desc); + struct sparc_md5_state *mctx = shash_desc_ctx(desc); - mctx->hash[0] = MD5_H0; - mctx->hash[1] = MD5_H1; - mctx->hash[2] = MD5_H2; - mctx->hash[3] = MD5_H3; - le32_to_cpu_array(mctx->hash, 4); + mctx->hash[0] = cpu_to_le32(MD5_H0); + mctx->hash[1] = cpu_to_le32(MD5_H1); + mctx->hash[2] = cpu_to_le32(MD5_H2); + mctx->hash[3] = cpu_to_le32(MD5_H3); mctx->byte_count = 0; return 0; } -static void __md5_sparc64_update(struct md5_state *sctx, const u8 *data, - unsigned int len, unsigned int partial) -{ - unsigned int done = 0; - - sctx->byte_count += len; - if (partial) { - done = MD5_HMAC_BLOCK_SIZE - partial; - memcpy((u8 *)sctx->block + partial, data, done); - md5_sparc64_transform(sctx->hash, (u8 *)sctx->block, 1); - } - if (len - done >= MD5_HMAC_BLOCK_SIZE) { - const unsigned int rounds = (len - done) / MD5_HMAC_BLOCK_SIZE; - - md5_sparc64_transform(sctx->hash, data + done, rounds); - done += rounds * MD5_HMAC_BLOCK_SIZE; - } - - memcpy(sctx->block, data + done, len - done); -} - static int md5_sparc64_update(struct shash_desc *desc, const u8 *data, unsigned int len) { - struct md5_state *sctx = shash_desc_ctx(desc); - unsigned int partial = sctx->byte_count % MD5_HMAC_BLOCK_SIZE; + struct sparc_md5_state *sctx = shash_desc_ctx(desc); - /* Handle the fast case right here */ - if (partial + len < MD5_HMAC_BLOCK_SIZE) { - sctx->byte_count += len; - memcpy((u8 *)sctx->block + partial, data, len); - } else - __md5_sparc64_update(sctx, data, len, partial); - - return 0; + sctx->byte_count += round_down(len, MD5_HMAC_BLOCK_SIZE); + md5_sparc64_transform(sctx->hash, data, len / MD5_HMAC_BLOCK_SIZE); + return len - round_down(len, MD5_HMAC_BLOCK_SIZE); } /* Add padding and return the message digest. */ -static int md5_sparc64_final(struct shash_desc *desc, u8 *out) +static int md5_sparc64_finup(struct shash_desc *desc, const u8 *src, + unsigned int offset, u8 *out) { - struct md5_state *sctx = shash_desc_ctx(desc); - unsigned int i, index, padlen; - u32 *dst = (u32 *)out; - __le64 bits; - static const u8 padding[MD5_HMAC_BLOCK_SIZE] = { 0x80, }; + struct sparc_md5_state *sctx = shash_desc_ctx(desc); + __le64 block[MD5_BLOCK_WORDS] = {}; + u8 *p = memcpy(block, src, offset); + __le32 *dst = (__le32 *)out; + __le64 *pbits; + int i; - bits = cpu_to_le64(sctx->byte_count << 3); - - /* Pad out to 56 mod 64 and append length */ - index = sctx->byte_count % MD5_HMAC_BLOCK_SIZE; - padlen = (index < 56) ? (56 - index) : ((MD5_HMAC_BLOCK_SIZE+56) - index); - - /* We need to fill a whole block for __md5_sparc64_update() */ - if (padlen <= 56) { - sctx->byte_count += padlen; - memcpy((u8 *)sctx->block + index, padding, padlen); - } else { - __md5_sparc64_update(sctx, padding, padlen, index); - } - __md5_sparc64_update(sctx, (const u8 *)&bits, sizeof(bits), 56); + src = p; + p += offset; + *p++ = 0x80; + sctx->byte_count += offset; + pbits = &block[(MD5_BLOCK_WORDS / (offset > 55 ? 1 : 2)) - 1]; + *pbits = cpu_to_le64(sctx->byte_count << 3); + md5_sparc64_transform(sctx->hash, src, (pbits - block + 1) / 8); + memzero_explicit(block, sizeof(block)); /* Store state in digest */ for (i = 0; i < MD5_HASH_WORDS; i++) dst[i] = sctx->hash[i]; - /* Wipe context */ - memset(sctx, 0, sizeof(*sctx)); - return 0; } static int md5_sparc64_export(struct shash_desc *desc, void *out) { - struct md5_state *sctx = shash_desc_ctx(desc); - - memcpy(out, sctx, sizeof(*sctx)); + struct sparc_md5_state *sctx = shash_desc_ctx(desc); + union { + u8 *u8; + u32 *u32; + u64 *u64; + } p = { .u8 = out }; + int i; + for (i = 0; i < MD5_HASH_WORDS; i++) + put_unaligned(le32_to_cpu(sctx->hash[i]), p.u32++); + put_unaligned(sctx->byte_count, p.u64); return 0; } static int md5_sparc64_import(struct shash_desc *desc, const void *in) { - struct md5_state *sctx = shash_desc_ctx(desc); - - memcpy(sctx, in, sizeof(*sctx)); + struct sparc_md5_state *sctx = shash_desc_ctx(desc); + union { + const u8 *u8; + const u32 *u32; + const u64 *u64; + } p = { .u8 = in }; + int i; + for (i = 0; i < MD5_HASH_WORDS; i++) + sctx->hash[i] = cpu_to_le32(get_unaligned(p.u32++)); + sctx->byte_count = get_unaligned(p.u64); return 0; } @@ -136,15 +120,16 @@ static struct shash_alg alg = { .digestsize = MD5_DIGEST_SIZE, .init = md5_sparc64_init, .update = md5_sparc64_update, - .final = md5_sparc64_final, + .finup = md5_sparc64_finup, .export = md5_sparc64_export, .import = md5_sparc64_import, - .descsize = sizeof(struct md5_state), - .statesize = sizeof(struct md5_state), + .descsize = sizeof(struct sparc_md5_state), + .statesize = sizeof(struct sparc_md5_state), .base = { .cra_name = "md5", .cra_driver_name= "md5-sparc64", .cra_priority = SPARC_CR_OPCODE_PRIORITY, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_blocksize = MD5_HMAC_BLOCK_SIZE, .cra_module = THIS_MODULE, } From 0865a89413652ecad551c9b782fa5146259640e4 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 18 Apr 2025 10:59:13 +0800 Subject: [PATCH 163/382] crypto: x86/sha1 - Use API partial block handling Use the Crypto API partial block handling. Also remove the unnecessary SIMD fallback path. Signed-off-by: Herbert Xu --- arch/x86/crypto/sha1_ssse3_glue.c | 81 ++++++++++--------------------- include/crypto/sha1.h | 1 + include/crypto/sha1_base.h | 42 ++++++++++++++-- 3 files changed, 64 insertions(+), 60 deletions(-) diff --git a/arch/x86/crypto/sha1_ssse3_glue.c b/arch/x86/crypto/sha1_ssse3_glue.c index abb793cbad01..0a912bfc86c5 100644 --- a/arch/x86/crypto/sha1_ssse3_glue.c +++ b/arch/x86/crypto/sha1_ssse3_glue.c @@ -16,16 +16,14 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -#include -#include -#include -#include -#include -#include -#include -#include #include #include +#include +#include +#include +#include +#include +#include static const struct x86_cpu_id module_cpu_ids[] = { X86_MATCH_FEATURE(X86_FEATURE_SHA_NI, NULL), @@ -36,14 +34,10 @@ static const struct x86_cpu_id module_cpu_ids[] = { }; MODULE_DEVICE_TABLE(x86cpu, module_cpu_ids); -static int sha1_update(struct shash_desc *desc, const u8 *data, - unsigned int len, sha1_block_fn *sha1_xform) +static inline int sha1_update(struct shash_desc *desc, const u8 *data, + unsigned int len, sha1_block_fn *sha1_xform) { - struct sha1_state *sctx = shash_desc_ctx(desc); - - if (!crypto_simd_usable() || - (sctx->count % SHA1_BLOCK_SIZE) + len < SHA1_BLOCK_SIZE) - return crypto_sha1_update(desc, data, len); + int remain; /* * Make sure struct sha1_state begins directly with the SHA1 @@ -52,22 +46,18 @@ static int sha1_update(struct shash_desc *desc, const u8 *data, BUILD_BUG_ON(offsetof(struct sha1_state, state) != 0); kernel_fpu_begin(); - sha1_base_do_update(desc, data, len, sha1_xform); + remain = sha1_base_do_update_blocks(desc, data, len, sha1_xform); kernel_fpu_end(); - return 0; + return remain; } -static int sha1_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out, sha1_block_fn *sha1_xform) +static inline int sha1_finup(struct shash_desc *desc, const u8 *data, + unsigned int len, u8 *out, + sha1_block_fn *sha1_xform) { - if (!crypto_simd_usable()) - return crypto_sha1_finup(desc, data, len, out); - kernel_fpu_begin(); - if (len) - sha1_base_do_update(desc, data, len, sha1_xform); - sha1_base_do_finalize(desc, sha1_xform); + sha1_base_do_finup(desc, data, len, sha1_xform); kernel_fpu_end(); return sha1_base_finish(desc, out); @@ -88,23 +78,17 @@ static int sha1_ssse3_finup(struct shash_desc *desc, const u8 *data, return sha1_finup(desc, data, len, out, sha1_transform_ssse3); } -/* Add padding and return the message digest. */ -static int sha1_ssse3_final(struct shash_desc *desc, u8 *out) -{ - return sha1_ssse3_finup(desc, NULL, 0, out); -} - static struct shash_alg sha1_ssse3_alg = { .digestsize = SHA1_DIGEST_SIZE, .init = sha1_base_init, .update = sha1_ssse3_update, - .final = sha1_ssse3_final, .finup = sha1_ssse3_finup, - .descsize = sizeof(struct sha1_state), + .descsize = SHA1_STATE_SIZE, .base = { .cra_name = "sha1", .cra_driver_name = "sha1-ssse3", .cra_priority = 150, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_blocksize = SHA1_BLOCK_SIZE, .cra_module = THIS_MODULE, } @@ -138,22 +122,17 @@ static int sha1_avx_finup(struct shash_desc *desc, const u8 *data, return sha1_finup(desc, data, len, out, sha1_transform_avx); } -static int sha1_avx_final(struct shash_desc *desc, u8 *out) -{ - return sha1_avx_finup(desc, NULL, 0, out); -} - static struct shash_alg sha1_avx_alg = { .digestsize = SHA1_DIGEST_SIZE, .init = sha1_base_init, .update = sha1_avx_update, - .final = sha1_avx_final, .finup = sha1_avx_finup, - .descsize = sizeof(struct sha1_state), + .descsize = SHA1_STATE_SIZE, .base = { .cra_name = "sha1", .cra_driver_name = "sha1-avx", .cra_priority = 160, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_blocksize = SHA1_BLOCK_SIZE, .cra_module = THIS_MODULE, } @@ -198,8 +177,8 @@ static bool avx2_usable(void) return false; } -static void sha1_apply_transform_avx2(struct sha1_state *state, - const u8 *data, int blocks) +static inline void sha1_apply_transform_avx2(struct sha1_state *state, + const u8 *data, int blocks) { /* Select the optimal transform based on data block size */ if (blocks >= SHA1_AVX2_BLOCK_OPTSIZE) @@ -220,22 +199,17 @@ static int sha1_avx2_finup(struct shash_desc *desc, const u8 *data, return sha1_finup(desc, data, len, out, sha1_apply_transform_avx2); } -static int sha1_avx2_final(struct shash_desc *desc, u8 *out) -{ - return sha1_avx2_finup(desc, NULL, 0, out); -} - static struct shash_alg sha1_avx2_alg = { .digestsize = SHA1_DIGEST_SIZE, .init = sha1_base_init, .update = sha1_avx2_update, - .final = sha1_avx2_final, .finup = sha1_avx2_finup, - .descsize = sizeof(struct sha1_state), + .descsize = SHA1_STATE_SIZE, .base = { .cra_name = "sha1", .cra_driver_name = "sha1-avx2", .cra_priority = 170, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_blocksize = SHA1_BLOCK_SIZE, .cra_module = THIS_MODULE, } @@ -269,22 +243,17 @@ static int sha1_ni_finup(struct shash_desc *desc, const u8 *data, return sha1_finup(desc, data, len, out, sha1_ni_transform); } -static int sha1_ni_final(struct shash_desc *desc, u8 *out) -{ - return sha1_ni_finup(desc, NULL, 0, out); -} - static struct shash_alg sha1_ni_alg = { .digestsize = SHA1_DIGEST_SIZE, .init = sha1_base_init, .update = sha1_ni_update, - .final = sha1_ni_final, .finup = sha1_ni_finup, - .descsize = sizeof(struct sha1_state), + .descsize = SHA1_STATE_SIZE, .base = { .cra_name = "sha1", .cra_driver_name = "sha1-ni", .cra_priority = 250, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_blocksize = SHA1_BLOCK_SIZE, .cra_module = THIS_MODULE, } diff --git a/include/crypto/sha1.h b/include/crypto/sha1.h index 044ecea60ac8..dd6de4a4d6e6 100644 --- a/include/crypto/sha1.h +++ b/include/crypto/sha1.h @@ -10,6 +10,7 @@ #define SHA1_DIGEST_SIZE 20 #define SHA1_BLOCK_SIZE 64 +#define SHA1_STATE_SIZE offsetof(struct sha1_state, buffer) #define SHA1_H0 0x67452301UL #define SHA1_H1 0xefcdab89UL diff --git a/include/crypto/sha1_base.h b/include/crypto/sha1_base.h index 0c342ed0d038..b23cfad18ce2 100644 --- a/include/crypto/sha1_base.h +++ b/include/crypto/sha1_base.h @@ -10,10 +10,9 @@ #include #include -#include -#include +#include #include - +#include #include typedef void (sha1_block_fn)(struct sha1_state *sst, u8 const *src, int blocks); @@ -70,6 +69,19 @@ static inline int sha1_base_do_update(struct shash_desc *desc, return 0; } +static inline int sha1_base_do_update_blocks(struct shash_desc *desc, + const u8 *data, + unsigned int len, + sha1_block_fn *block_fn) +{ + unsigned int remain = len - round_down(len, SHA1_BLOCK_SIZE); + struct sha1_state *sctx = shash_desc_ctx(desc); + + sctx->count += len - remain; + block_fn(sctx, data, len / SHA1_BLOCK_SIZE); + return remain; +} + static inline int sha1_base_do_finalize(struct shash_desc *desc, sha1_block_fn *block_fn) { @@ -93,6 +105,29 @@ static inline int sha1_base_do_finalize(struct shash_desc *desc, return 0; } +static inline int sha1_base_do_finup(struct shash_desc *desc, + const u8 *src, unsigned int len, + sha1_block_fn *block_fn) +{ + unsigned int bit_offset = SHA1_BLOCK_SIZE / 8 - 1; + struct sha1_state *sctx = shash_desc_ctx(desc); + union { + __be64 b64[SHA1_BLOCK_SIZE / 4]; + u8 u8[SHA1_BLOCK_SIZE * 2]; + } block = {}; + + if (len >= bit_offset * 8) + bit_offset += SHA1_BLOCK_SIZE / 8; + memcpy(&block, src, len); + block.u8[len] = 0x80; + sctx->count += len; + block.b64[bit_offset] = cpu_to_be64(sctx->count << 3); + block_fn(sctx, block.u8, (bit_offset + 1) * 8 / SHA1_BLOCK_SIZE); + memzero_explicit(&block, sizeof(block)); + + return 0; +} + static inline int sha1_base_finish(struct shash_desc *desc, u8 *out) { struct sha1_state *sctx = shash_desc_ctx(desc); @@ -102,7 +137,6 @@ static inline int sha1_base_finish(struct shash_desc *desc, u8 *out) for (i = 0; i < SHA1_DIGEST_SIZE / sizeof(__be32); i++) put_unaligned_be32(sctx->state[i], digest++); - memzero_explicit(sctx, sizeof(*sctx)); return 0; } From b97d31100e36aae7f9e39543326dae31ee8cdae7 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 18 Apr 2025 10:59:15 +0800 Subject: [PATCH 164/382] crypto: arm64/sha1 - Use API partial block handling Use the Crypto API partial block handling. Also remove the unnecessary SIMD fallback path. Signed-off-by: Herbert Xu --- arch/arm64/crypto/sha1-ce-glue.c | 66 ++++++++------------------------ 1 file changed, 17 insertions(+), 49 deletions(-) diff --git a/arch/arm64/crypto/sha1-ce-glue.c b/arch/arm64/crypto/sha1-ce-glue.c index cbd14f208f83..1f8c93fe1e64 100644 --- a/arch/arm64/crypto/sha1-ce-glue.c +++ b/arch/arm64/crypto/sha1-ce-glue.c @@ -7,14 +7,14 @@ #include #include -#include #include #include #include #include #include -#include +#include #include +#include MODULE_DESCRIPTION("SHA1 secure hash using ARMv8 Crypto Extensions"); MODULE_AUTHOR("Ard Biesheuvel "); @@ -56,79 +56,47 @@ static int sha1_ce_update(struct shash_desc *desc, const u8 *data, { struct sha1_ce_state *sctx = shash_desc_ctx(desc); - if (!crypto_simd_usable()) - return crypto_sha1_update(desc, data, len); - sctx->finalize = 0; - sha1_base_do_update(desc, data, len, sha1_ce_transform); - - return 0; + return sha1_base_do_update_blocks(desc, data, len, sha1_ce_transform); } static int sha1_ce_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) { struct sha1_ce_state *sctx = shash_desc_ctx(desc); - bool finalize = !sctx->sst.count && !(len % SHA1_BLOCK_SIZE) && len; - - if (!crypto_simd_usable()) - return crypto_sha1_finup(desc, data, len, out); + bool finalized = false; /* * Allow the asm code to perform the finalization if there is no * partial data and the input is a round multiple of the block size. */ - sctx->finalize = finalize; + if (len >= SHA1_BLOCK_SIZE) { + unsigned int remain = len - round_down(len, SHA1_BLOCK_SIZE); - sha1_base_do_update(desc, data, len, sha1_ce_transform); - if (!finalize) - sha1_base_do_finalize(desc, sha1_ce_transform); + finalized = !remain; + sctx->finalize = finalized; + sha1_base_do_update_blocks(desc, data, len, sha1_ce_transform); + data += len - remain; + len = remain; + } + if (!finalized) + sha1_base_do_finup(desc, data, len, sha1_ce_transform); return sha1_base_finish(desc, out); } -static int sha1_ce_final(struct shash_desc *desc, u8 *out) -{ - struct sha1_ce_state *sctx = shash_desc_ctx(desc); - - if (!crypto_simd_usable()) - return crypto_sha1_finup(desc, NULL, 0, out); - - sctx->finalize = 0; - sha1_base_do_finalize(desc, sha1_ce_transform); - return sha1_base_finish(desc, out); -} - -static int sha1_ce_export(struct shash_desc *desc, void *out) -{ - struct sha1_ce_state *sctx = shash_desc_ctx(desc); - - memcpy(out, &sctx->sst, sizeof(struct sha1_state)); - return 0; -} - -static int sha1_ce_import(struct shash_desc *desc, const void *in) -{ - struct sha1_ce_state *sctx = shash_desc_ctx(desc); - - memcpy(&sctx->sst, in, sizeof(struct sha1_state)); - sctx->finalize = 0; - return 0; -} - static struct shash_alg alg = { .init = sha1_base_init, .update = sha1_ce_update, - .final = sha1_ce_final, .finup = sha1_ce_finup, - .import = sha1_ce_import, - .export = sha1_ce_export, .descsize = sizeof(struct sha1_ce_state), - .statesize = sizeof(struct sha1_state), + .statesize = SHA1_STATE_SIZE, .digestsize = SHA1_DIGEST_SIZE, .base = { .cra_name = "sha1", .cra_driver_name = "sha1-ce", .cra_priority = 200, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA1_BLOCK_SIZE, .cra_module = THIS_MODULE, } From 62a59a464fb1add17ba64663d4eac7fb4302665c Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 18 Apr 2025 10:59:18 +0800 Subject: [PATCH 165/382] crypto: mips/octeon-sha1 - Use API partial block handling Use the Crypto API partial block handling. Signed-off-by: Herbert Xu --- arch/mips/cavium-octeon/crypto/octeon-sha1.c | 136 ++++--------------- 1 file changed, 30 insertions(+), 106 deletions(-) diff --git a/arch/mips/cavium-octeon/crypto/octeon-sha1.c b/arch/mips/cavium-octeon/crypto/octeon-sha1.c index 37a07b3c4568..e70f21a473da 100644 --- a/arch/mips/cavium-octeon/crypto/octeon-sha1.c +++ b/arch/mips/cavium-octeon/crypto/octeon-sha1.c @@ -13,15 +13,13 @@ * Copyright (c) Jean-Francois Dive */ -#include -#include -#include -#include -#include -#include -#include #include #include +#include +#include +#include +#include +#include #include "octeon-crypto.h" @@ -58,49 +56,23 @@ static void octeon_sha1_read_hash(struct sha1_state *sctx) memzero_explicit(&hash_tail.dword, sizeof(hash_tail.dword)); } -static void octeon_sha1_transform(const void *_block) +static void octeon_sha1_transform(struct sha1_state *sctx, const u8 *src, + int blocks) { - const u64 *block = _block; + do { + const u64 *block = (const u64 *)src; - write_octeon_64bit_block_dword(block[0], 0); - write_octeon_64bit_block_dword(block[1], 1); - write_octeon_64bit_block_dword(block[2], 2); - write_octeon_64bit_block_dword(block[3], 3); - write_octeon_64bit_block_dword(block[4], 4); - write_octeon_64bit_block_dword(block[5], 5); - write_octeon_64bit_block_dword(block[6], 6); - octeon_sha1_start(block[7]); -} + write_octeon_64bit_block_dword(block[0], 0); + write_octeon_64bit_block_dword(block[1], 1); + write_octeon_64bit_block_dword(block[2], 2); + write_octeon_64bit_block_dword(block[3], 3); + write_octeon_64bit_block_dword(block[4], 4); + write_octeon_64bit_block_dword(block[5], 5); + write_octeon_64bit_block_dword(block[6], 6); + octeon_sha1_start(block[7]); -static void __octeon_sha1_update(struct sha1_state *sctx, const u8 *data, - unsigned int len) -{ - unsigned int partial; - unsigned int done; - const u8 *src; - - partial = sctx->count % SHA1_BLOCK_SIZE; - sctx->count += len; - done = 0; - src = data; - - if ((partial + len) >= SHA1_BLOCK_SIZE) { - if (partial) { - done = -partial; - memcpy(sctx->buffer + partial, data, - done + SHA1_BLOCK_SIZE); - src = sctx->buffer; - } - - do { - octeon_sha1_transform(src); - done += SHA1_BLOCK_SIZE; - src = data + done; - } while (done + SHA1_BLOCK_SIZE <= len); - - partial = 0; - } - memcpy(sctx->buffer + partial, src, len - done); + src += SHA1_BLOCK_SIZE; + } while (--blocks); } static int octeon_sha1_update(struct shash_desc *desc, const u8 *data, @@ -109,95 +81,47 @@ static int octeon_sha1_update(struct shash_desc *desc, const u8 *data, struct sha1_state *sctx = shash_desc_ctx(desc); struct octeon_cop2_state state; unsigned long flags; - - /* - * Small updates never reach the crypto engine, so the generic sha1 is - * faster because of the heavyweight octeon_crypto_enable() / - * octeon_crypto_disable(). - */ - if ((sctx->count % SHA1_BLOCK_SIZE) + len < SHA1_BLOCK_SIZE) - return crypto_sha1_update(desc, data, len); + int remain; flags = octeon_crypto_enable(&state); octeon_sha1_store_hash(sctx); - __octeon_sha1_update(sctx, data, len); + remain = sha1_base_do_update_blocks(desc, data, len, + octeon_sha1_transform); octeon_sha1_read_hash(sctx); octeon_crypto_disable(&state, flags); - - return 0; + return remain; } -static int octeon_sha1_final(struct shash_desc *desc, u8 *out) +static int octeon_sha1_finup(struct shash_desc *desc, const u8 *src, + unsigned int len, u8 *out) { struct sha1_state *sctx = shash_desc_ctx(desc); - static const u8 padding[64] = { 0x80, }; struct octeon_cop2_state state; - __be32 *dst = (__be32 *)out; - unsigned int pad_len; unsigned long flags; - unsigned int index; - __be64 bits; - int i; - - /* Save number of bits. */ - bits = cpu_to_be64(sctx->count << 3); - - /* Pad out to 56 mod 64. */ - index = sctx->count & 0x3f; - pad_len = (index < 56) ? (56 - index) : ((64+56) - index); flags = octeon_crypto_enable(&state); octeon_sha1_store_hash(sctx); - __octeon_sha1_update(sctx, padding, pad_len); - - /* Append length (before padding). */ - __octeon_sha1_update(sctx, (const u8 *)&bits, sizeof(bits)); + sha1_base_do_finup(desc, src, len, octeon_sha1_transform); octeon_sha1_read_hash(sctx); octeon_crypto_disable(&state, flags); - - /* Store state in digest */ - for (i = 0; i < 5; i++) - dst[i] = cpu_to_be32(sctx->state[i]); - - /* Zeroize sensitive information. */ - memset(sctx, 0, sizeof(*sctx)); - - return 0; -} - -static int octeon_sha1_export(struct shash_desc *desc, void *out) -{ - struct sha1_state *sctx = shash_desc_ctx(desc); - - memcpy(out, sctx, sizeof(*sctx)); - return 0; -} - -static int octeon_sha1_import(struct shash_desc *desc, const void *in) -{ - struct sha1_state *sctx = shash_desc_ctx(desc); - - memcpy(sctx, in, sizeof(*sctx)); - return 0; + return sha1_base_finish(desc, out); } static struct shash_alg octeon_sha1_alg = { .digestsize = SHA1_DIGEST_SIZE, .init = sha1_base_init, .update = octeon_sha1_update, - .final = octeon_sha1_final, - .export = octeon_sha1_export, - .import = octeon_sha1_import, - .descsize = sizeof(struct sha1_state), - .statesize = sizeof(struct sha1_state), + .finup = octeon_sha1_finup, + .descsize = SHA1_STATE_SIZE, .base = { .cra_name = "sha1", .cra_driver_name= "octeon-sha1", .cra_priority = OCTEON_CR_OPCODE_PRIORITY, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_blocksize = SHA1_BLOCK_SIZE, .cra_module = THIS_MODULE, } From a2d910b8466af942d34478b4c480360f0b3d29ae Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 18 Apr 2025 10:59:20 +0800 Subject: [PATCH 166/382] crypto: sha1-generic - Use API partial block handling Use the Crypto API partial block handling. Signed-off-by: Herbert Xu --- crypto/sha1_generic.c | 33 ++++++++++++--------------------- include/crypto/sha1.h | 8 -------- 2 files changed, 12 insertions(+), 29 deletions(-) diff --git a/crypto/sha1_generic.c b/crypto/sha1_generic.c index 325b57fe28dc..7a3c837923b5 100644 --- a/crypto/sha1_generic.c +++ b/crypto/sha1_generic.c @@ -12,13 +12,11 @@ * Copyright (c) Jean-Francois Dive */ #include -#include -#include -#include -#include #include #include -#include +#include +#include +#include const u8 sha1_zero_message_hash[SHA1_DIGEST_SIZE] = { 0xda, 0x39, 0xa3, 0xee, 0x5e, 0x6b, 0x4b, 0x0d, @@ -39,38 +37,31 @@ static void sha1_generic_block_fn(struct sha1_state *sst, u8 const *src, memzero_explicit(temp, sizeof(temp)); } -int crypto_sha1_update(struct shash_desc *desc, const u8 *data, - unsigned int len) +static int crypto_sha1_update(struct shash_desc *desc, const u8 *data, + unsigned int len) { - return sha1_base_do_update(desc, data, len, sha1_generic_block_fn); + return sha1_base_do_update_blocks(desc, data, len, + sha1_generic_block_fn); } -EXPORT_SYMBOL(crypto_sha1_update); -static int sha1_final(struct shash_desc *desc, u8 *out) +static int crypto_sha1_finup(struct shash_desc *desc, const u8 *data, + unsigned int len, u8 *out) { - sha1_base_do_finalize(desc, sha1_generic_block_fn); + sha1_base_do_finup(desc, data, len, sha1_generic_block_fn); return sha1_base_finish(desc, out); } -int crypto_sha1_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - sha1_base_do_update(desc, data, len, sha1_generic_block_fn); - return sha1_final(desc, out); -} -EXPORT_SYMBOL(crypto_sha1_finup); - static struct shash_alg alg = { .digestsize = SHA1_DIGEST_SIZE, .init = sha1_base_init, .update = crypto_sha1_update, - .final = sha1_final, .finup = crypto_sha1_finup, - .descsize = sizeof(struct sha1_state), + .descsize = SHA1_STATE_SIZE, .base = { .cra_name = "sha1", .cra_driver_name= "sha1-generic", .cra_priority = 100, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_blocksize = SHA1_BLOCK_SIZE, .cra_module = THIS_MODULE, } diff --git a/include/crypto/sha1.h b/include/crypto/sha1.h index dd6de4a4d6e6..f48230b1413c 100644 --- a/include/crypto/sha1.h +++ b/include/crypto/sha1.h @@ -26,14 +26,6 @@ struct sha1_state { u8 buffer[SHA1_BLOCK_SIZE]; }; -struct shash_desc; - -extern int crypto_sha1_update(struct shash_desc *desc, const u8 *data, - unsigned int len); - -extern int crypto_sha1_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *hash); - /* * An implementation of SHA-1's compression function. Don't use in new code! * You shouldn't be using SHA-1, and even if you *have* to use SHA-1, this isn't From d52ad050430a18bac6e2137f3de602538ec5616a Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 18 Apr 2025 10:59:22 +0800 Subject: [PATCH 167/382] crypto: arm/sha1-ce - Use API partial block handling Use the Crypto API partial block handling. Also remove the unnecessary SIMD fallback path. Signed-off-by: Herbert Xu --- arch/arm/crypto/sha1-ce-glue.c | 36 ++++++++-------------------------- 1 file changed, 8 insertions(+), 28 deletions(-) diff --git a/arch/arm/crypto/sha1-ce-glue.c b/arch/arm/crypto/sha1-ce-glue.c index de9100c67b37..fac07a4799de 100644 --- a/arch/arm/crypto/sha1-ce-glue.c +++ b/arch/arm/crypto/sha1-ce-glue.c @@ -5,20 +5,14 @@ * Copyright (C) 2015 Linaro Ltd */ +#include #include -#include #include #include #include -#include +#include #include -#include -#include -#include - -#include "sha1.h" - MODULE_DESCRIPTION("SHA1 secure hash using ARMv8 Crypto Extensions"); MODULE_AUTHOR("Ard Biesheuvel "); MODULE_LICENSE("GPL v2"); @@ -29,50 +23,36 @@ asmlinkage void sha1_ce_transform(struct sha1_state *sst, u8 const *src, static int sha1_ce_update(struct shash_desc *desc, const u8 *data, unsigned int len) { - struct sha1_state *sctx = shash_desc_ctx(desc); - - if (!crypto_simd_usable() || - (sctx->count % SHA1_BLOCK_SIZE) + len < SHA1_BLOCK_SIZE) - return sha1_update_arm(desc, data, len); + int remain; kernel_neon_begin(); - sha1_base_do_update(desc, data, len, sha1_ce_transform); + remain = sha1_base_do_update_blocks(desc, data, len, sha1_ce_transform); kernel_neon_end(); - return 0; + return remain; } static int sha1_ce_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) { - if (!crypto_simd_usable()) - return sha1_finup_arm(desc, data, len, out); - kernel_neon_begin(); - if (len) - sha1_base_do_update(desc, data, len, sha1_ce_transform); - sha1_base_do_finalize(desc, sha1_ce_transform); + sha1_base_do_finup(desc, data, len, sha1_ce_transform); kernel_neon_end(); return sha1_base_finish(desc, out); } -static int sha1_ce_final(struct shash_desc *desc, u8 *out) -{ - return sha1_ce_finup(desc, NULL, 0, out); -} - static struct shash_alg alg = { .init = sha1_base_init, .update = sha1_ce_update, - .final = sha1_ce_final, .finup = sha1_ce_finup, - .descsize = sizeof(struct sha1_state), + .descsize = SHA1_STATE_SIZE, .digestsize = SHA1_DIGEST_SIZE, .base = { .cra_name = "sha1", .cra_driver_name = "sha1-ce", .cra_priority = 200, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_blocksize = SHA1_BLOCK_SIZE, .cra_module = THIS_MODULE, } From adfb79164c7959ae43f93f54430e3dee0ffab404 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 18 Apr 2025 10:59:25 +0800 Subject: [PATCH 168/382] crypto: arm/sha1-neon - Use API partial block handling Use the Crypto API partial block handling. Also remove the unnecessary SIMD fallback path. Signed-off-by: Herbert Xu --- arch/arm/crypto/sha1_neon_glue.c | 39 ++++++++------------------------ 1 file changed, 10 insertions(+), 29 deletions(-) diff --git a/arch/arm/crypto/sha1_neon_glue.c b/arch/arm/crypto/sha1_neon_glue.c index 9c70b87e69f7..d321850f22a6 100644 --- a/arch/arm/crypto/sha1_neon_glue.c +++ b/arch/arm/crypto/sha1_neon_glue.c @@ -13,18 +13,12 @@ * Copyright (c) Chandramouli Narayanan */ +#include #include -#include -#include -#include -#include -#include #include #include -#include -#include - -#include "sha1.h" +#include +#include asmlinkage void sha1_transform_neon(struct sha1_state *state_h, const u8 *data, int rounds); @@ -32,50 +26,37 @@ asmlinkage void sha1_transform_neon(struct sha1_state *state_h, static int sha1_neon_update(struct shash_desc *desc, const u8 *data, unsigned int len) { - struct sha1_state *sctx = shash_desc_ctx(desc); - - if (!crypto_simd_usable() || - (sctx->count % SHA1_BLOCK_SIZE) + len < SHA1_BLOCK_SIZE) - return sha1_update_arm(desc, data, len); + int remain; kernel_neon_begin(); - sha1_base_do_update(desc, data, len, sha1_transform_neon); + remain = sha1_base_do_update_blocks(desc, data, len, + sha1_transform_neon); kernel_neon_end(); - return 0; + return remain; } static int sha1_neon_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) { - if (!crypto_simd_usable()) - return sha1_finup_arm(desc, data, len, out); - kernel_neon_begin(); - if (len) - sha1_base_do_update(desc, data, len, sha1_transform_neon); - sha1_base_do_finalize(desc, sha1_transform_neon); + sha1_base_do_finup(desc, data, len, sha1_transform_neon); kernel_neon_end(); return sha1_base_finish(desc, out); } -static int sha1_neon_final(struct shash_desc *desc, u8 *out) -{ - return sha1_neon_finup(desc, NULL, 0, out); -} - static struct shash_alg alg = { .digestsize = SHA1_DIGEST_SIZE, .init = sha1_base_init, .update = sha1_neon_update, - .final = sha1_neon_final, .finup = sha1_neon_finup, - .descsize = sizeof(struct sha1_state), + .descsize = SHA1_STATE_SIZE, .base = { .cra_name = "sha1", .cra_driver_name = "sha1-neon", .cra_priority = 250, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_blocksize = SHA1_BLOCK_SIZE, .cra_module = THIS_MODULE, } From e7396043a0a0f0a9589236db1d6902ebec5a9534 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 18 Apr 2025 10:59:27 +0800 Subject: [PATCH 169/382] crypto: arm/sha1-asm - Use API partial block handling Use the Crypto API partial block handling. Signed-off-by: Herbert Xu --- arch/arm/crypto/sha1.h | 14 -------------- arch/arm/crypto/sha1_glue.c | 33 +++++++++++---------------------- 2 files changed, 11 insertions(+), 36 deletions(-) delete mode 100644 arch/arm/crypto/sha1.h diff --git a/arch/arm/crypto/sha1.h b/arch/arm/crypto/sha1.h deleted file mode 100644 index b1b7e21da2c3..000000000000 --- a/arch/arm/crypto/sha1.h +++ /dev/null @@ -1,14 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef ASM_ARM_CRYPTO_SHA1_H -#define ASM_ARM_CRYPTO_SHA1_H - -#include -#include - -extern int sha1_update_arm(struct shash_desc *desc, const u8 *data, - unsigned int len); - -extern int sha1_finup_arm(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out); - -#endif diff --git a/arch/arm/crypto/sha1_glue.c b/arch/arm/crypto/sha1_glue.c index 95a727bcd664..255da00c7d98 100644 --- a/arch/arm/crypto/sha1_glue.c +++ b/arch/arm/crypto/sha1_glue.c @@ -12,53 +12,42 @@ */ #include -#include -#include -#include #include #include -#include - -#include "sha1.h" +#include +#include asmlinkage void sha1_block_data_order(struct sha1_state *digest, const u8 *data, int rounds); -int sha1_update_arm(struct shash_desc *desc, const u8 *data, - unsigned int len) +static int sha1_update_arm(struct shash_desc *desc, const u8 *data, + unsigned int len) { /* make sure signature matches sha1_block_fn() */ BUILD_BUG_ON(offsetof(struct sha1_state, state) != 0); - return sha1_base_do_update(desc, data, len, sha1_block_data_order); + return sha1_base_do_update_blocks(desc, data, len, + sha1_block_data_order); } -EXPORT_SYMBOL_GPL(sha1_update_arm); -static int sha1_final(struct shash_desc *desc, u8 *out) +static int sha1_finup_arm(struct shash_desc *desc, const u8 *data, + unsigned int len, u8 *out) { - sha1_base_do_finalize(desc, sha1_block_data_order); + sha1_base_do_finup(desc, data, len, sha1_block_data_order); return sha1_base_finish(desc, out); } -int sha1_finup_arm(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - sha1_base_do_update(desc, data, len, sha1_block_data_order); - return sha1_final(desc, out); -} -EXPORT_SYMBOL_GPL(sha1_finup_arm); - static struct shash_alg alg = { .digestsize = SHA1_DIGEST_SIZE, .init = sha1_base_init, .update = sha1_update_arm, - .final = sha1_final, .finup = sha1_finup_arm, - .descsize = sizeof(struct sha1_state), + .descsize = SHA1_STATE_SIZE, .base = { .cra_name = "sha1", .cra_driver_name= "sha1-asm", .cra_priority = 150, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_blocksize = SHA1_BLOCK_SIZE, .cra_module = THIS_MODULE, } From c0f07da681b1e6f9e3acd90a423c6966be8e701c Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 18 Apr 2025 10:59:29 +0800 Subject: [PATCH 170/382] crypto: powerpc/sha1 - Use API partial block handling Use the Crypto API partial block handling. Signed-off-by: Herbert Xu --- arch/powerpc/crypto/sha1.c | 101 ++++++++----------------------------- 1 file changed, 20 insertions(+), 81 deletions(-) diff --git a/arch/powerpc/crypto/sha1.c b/arch/powerpc/crypto/sha1.c index f283bbd3f121..4593946aa9b3 100644 --- a/arch/powerpc/crypto/sha1.c +++ b/arch/powerpc/crypto/sha1.c @@ -13,107 +13,46 @@ * Copyright (c) Jean-Francois Dive */ #include -#include -#include -#include -#include #include #include -#include +#include +#include -void powerpc_sha_transform(u32 *state, const u8 *src); +asmlinkage void powerpc_sha_transform(u32 *state, const u8 *src); + +static void powerpc_sha_block(struct sha1_state *sctx, const u8 *data, + int blocks) +{ + do { + powerpc_sha_transform(sctx->state, data); + data += 64; + } while (--blocks); +} static int powerpc_sha1_update(struct shash_desc *desc, const u8 *data, unsigned int len) { - struct sha1_state *sctx = shash_desc_ctx(desc); - unsigned int partial, done; - const u8 *src; - - partial = sctx->count & 0x3f; - sctx->count += len; - done = 0; - src = data; - - if ((partial + len) > 63) { - - if (partial) { - done = -partial; - memcpy(sctx->buffer + partial, data, done + 64); - src = sctx->buffer; - } - - do { - powerpc_sha_transform(sctx->state, src); - done += 64; - src = data + done; - } while (done + 63 < len); - - partial = 0; - } - memcpy(sctx->buffer + partial, src, len - done); - - return 0; + return sha1_base_do_update_blocks(desc, data, len, powerpc_sha_block); } - /* Add padding and return the message digest. */ -static int powerpc_sha1_final(struct shash_desc *desc, u8 *out) +static int powerpc_sha1_finup(struct shash_desc *desc, const u8 *src, + unsigned int len, u8 *out) { - struct sha1_state *sctx = shash_desc_ctx(desc); - __be32 *dst = (__be32 *)out; - u32 i, index, padlen; - __be64 bits; - static const u8 padding[64] = { 0x80, }; - - bits = cpu_to_be64(sctx->count << 3); - - /* Pad out to 56 mod 64 */ - index = sctx->count & 0x3f; - padlen = (index < 56) ? (56 - index) : ((64+56) - index); - powerpc_sha1_update(desc, padding, padlen); - - /* Append length */ - powerpc_sha1_update(desc, (const u8 *)&bits, sizeof(bits)); - - /* Store state in digest */ - for (i = 0; i < 5; i++) - dst[i] = cpu_to_be32(sctx->state[i]); - - /* Wipe context */ - memset(sctx, 0, sizeof *sctx); - - return 0; -} - -static int powerpc_sha1_export(struct shash_desc *desc, void *out) -{ - struct sha1_state *sctx = shash_desc_ctx(desc); - - memcpy(out, sctx, sizeof(*sctx)); - return 0; -} - -static int powerpc_sha1_import(struct shash_desc *desc, const void *in) -{ - struct sha1_state *sctx = shash_desc_ctx(desc); - - memcpy(sctx, in, sizeof(*sctx)); - return 0; + sha1_base_do_finup(desc, src, len, powerpc_sha_block); + return sha1_base_finish(desc, out); } static struct shash_alg alg = { .digestsize = SHA1_DIGEST_SIZE, .init = sha1_base_init, .update = powerpc_sha1_update, - .final = powerpc_sha1_final, - .export = powerpc_sha1_export, - .import = powerpc_sha1_import, - .descsize = sizeof(struct sha1_state), - .statesize = sizeof(struct sha1_state), + .finup = powerpc_sha1_finup, + .descsize = SHA1_STATE_SIZE, .base = { .cra_name = "sha1", .cra_driver_name= "sha1-powerpc", + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_blocksize = SHA1_BLOCK_SIZE, .cra_module = THIS_MODULE, } From 52c78783063215c41d55831e6e969b339bca1ef7 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 18 Apr 2025 10:59:31 +0800 Subject: [PATCH 171/382] crypto: powerpc/sha1-spe - Use API partial block handling Use the Crypto API partial block handling. Signed-off-by: Herbert Xu --- arch/powerpc/crypto/sha1-spe-glue.c | 132 +++++----------------------- 1 file changed, 24 insertions(+), 108 deletions(-) diff --git a/arch/powerpc/crypto/sha1-spe-glue.c b/arch/powerpc/crypto/sha1-spe-glue.c index 9170892a8557..04c88e173ce1 100644 --- a/arch/powerpc/crypto/sha1-spe-glue.c +++ b/arch/powerpc/crypto/sha1-spe-glue.c @@ -7,16 +7,13 @@ * Copyright (c) 2015 Markus Stockhausen */ +#include #include -#include -#include -#include -#include #include #include -#include -#include -#include +#include +#include +#include /* * MAX_BYTES defines the number of bytes that are allowed to be processed @@ -30,7 +27,7 @@ */ #define MAX_BYTES 2048 -extern void ppc_spe_sha1_transform(u32 *state, const u8 *src, u32 blocks); +asmlinkage void ppc_spe_sha1_transform(u32 *state, const u8 *src, u32 blocks); static void spe_begin(void) { @@ -46,126 +43,45 @@ static void spe_end(void) preempt_enable(); } -static inline void ppc_sha1_clear_context(struct sha1_state *sctx) +static void ppc_spe_sha1_block(struct sha1_state *sctx, const u8 *src, + int blocks) { - int count = sizeof(struct sha1_state) >> 2; - u32 *ptr = (u32 *)sctx; + do { + int unit = min(blocks, MAX_BYTES / SHA1_BLOCK_SIZE); - /* make sure we can clear the fast way */ - BUILD_BUG_ON(sizeof(struct sha1_state) % 4); - do { *ptr++ = 0; } while (--count); + spe_begin(); + ppc_spe_sha1_transform(sctx->state, src, unit); + spe_end(); + + src += unit * SHA1_BLOCK_SIZE; + blocks -= unit; + } while (blocks); } static int ppc_spe_sha1_update(struct shash_desc *desc, const u8 *data, unsigned int len) { - struct sha1_state *sctx = shash_desc_ctx(desc); - const unsigned int offset = sctx->count & 0x3f; - const unsigned int avail = 64 - offset; - unsigned int bytes; - const u8 *src = data; - - if (avail > len) { - sctx->count += len; - memcpy((char *)sctx->buffer + offset, src, len); - return 0; - } - - sctx->count += len; - - if (offset) { - memcpy((char *)sctx->buffer + offset, src, avail); - - spe_begin(); - ppc_spe_sha1_transform(sctx->state, (const u8 *)sctx->buffer, 1); - spe_end(); - - len -= avail; - src += avail; - } - - while (len > 63) { - bytes = (len > MAX_BYTES) ? MAX_BYTES : len; - bytes = bytes & ~0x3f; - - spe_begin(); - ppc_spe_sha1_transform(sctx->state, src, bytes >> 6); - spe_end(); - - src += bytes; - len -= bytes; - } - - memcpy((char *)sctx->buffer, src, len); - return 0; + return sha1_base_do_update_blocks(desc, data, len, ppc_spe_sha1_block); } -static int ppc_spe_sha1_final(struct shash_desc *desc, u8 *out) +static int ppc_spe_sha1_finup(struct shash_desc *desc, const u8 *src, + unsigned int len, u8 *out) { - struct sha1_state *sctx = shash_desc_ctx(desc); - const unsigned int offset = sctx->count & 0x3f; - char *p = (char *)sctx->buffer + offset; - int padlen; - __be64 *pbits = (__be64 *)(((char *)&sctx->buffer) + 56); - __be32 *dst = (__be32 *)out; - - padlen = 55 - offset; - *p++ = 0x80; - - spe_begin(); - - if (padlen < 0) { - memset(p, 0x00, padlen + sizeof (u64)); - ppc_spe_sha1_transform(sctx->state, sctx->buffer, 1); - p = (char *)sctx->buffer; - padlen = 56; - } - - memset(p, 0, padlen); - *pbits = cpu_to_be64(sctx->count << 3); - ppc_spe_sha1_transform(sctx->state, sctx->buffer, 1); - - spe_end(); - - dst[0] = cpu_to_be32(sctx->state[0]); - dst[1] = cpu_to_be32(sctx->state[1]); - dst[2] = cpu_to_be32(sctx->state[2]); - dst[3] = cpu_to_be32(sctx->state[3]); - dst[4] = cpu_to_be32(sctx->state[4]); - - ppc_sha1_clear_context(sctx); - return 0; -} - -static int ppc_spe_sha1_export(struct shash_desc *desc, void *out) -{ - struct sha1_state *sctx = shash_desc_ctx(desc); - - memcpy(out, sctx, sizeof(*sctx)); - return 0; -} - -static int ppc_spe_sha1_import(struct shash_desc *desc, const void *in) -{ - struct sha1_state *sctx = shash_desc_ctx(desc); - - memcpy(sctx, in, sizeof(*sctx)); - return 0; + sha1_base_do_finup(desc, src, len, ppc_spe_sha1_block); + return sha1_base_finish(desc, out); } static struct shash_alg alg = { .digestsize = SHA1_DIGEST_SIZE, .init = sha1_base_init, .update = ppc_spe_sha1_update, - .final = ppc_spe_sha1_final, - .export = ppc_spe_sha1_export, - .import = ppc_spe_sha1_import, - .descsize = sizeof(struct sha1_state), - .statesize = sizeof(struct sha1_state), + .finup = ppc_spe_sha1_finup, + .descsize = SHA1_STATE_SIZE, .base = { .cra_name = "sha1", .cra_driver_name= "sha1-ppc-spe", .cra_priority = 300, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_blocksize = SHA1_BLOCK_SIZE, .cra_module = THIS_MODULE, } From 7b83638f962c30cb6271b5698dc52cdf9b638b48 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 18 Apr 2025 10:59:34 +0800 Subject: [PATCH 172/382] crypto: s390/sha1 - Use API partial block handling Use the Crypto API partial block handling. Signed-off-by: Herbert Xu --- arch/s390/crypto/sha.h | 13 +++--- arch/s390/crypto/sha1_s390.c | 22 +++++------ arch/s390/crypto/sha_common.c | 74 +++++++++++++++++++++++++++++++++++ 3 files changed, 93 insertions(+), 16 deletions(-) diff --git a/arch/s390/crypto/sha.h b/arch/s390/crypto/sha.h index 2bb22db54c31..b8aeb51b2f3d 100644 --- a/arch/s390/crypto/sha.h +++ b/arch/s390/crypto/sha.h @@ -10,27 +10,30 @@ #ifndef _CRYPTO_ARCH_S390_SHA_H #define _CRYPTO_ARCH_S390_SHA_H -#include -#include -#include #include +#include /* must be big enough for the largest SHA variant */ #define SHA3_STATE_SIZE 200 #define CPACF_MAX_PARMBLOCK_SIZE SHA3_STATE_SIZE #define SHA_MAX_BLOCK_SIZE SHA3_224_BLOCK_SIZE +#define S390_SHA_CTX_SIZE offsetof(struct s390_sha_ctx, buf) struct s390_sha_ctx { u64 count; /* message length in bytes */ u32 state[CPACF_MAX_PARMBLOCK_SIZE / sizeof(u32)]; - u8 buf[SHA_MAX_BLOCK_SIZE]; int func; /* KIMD function to use */ - int first_message_part; + bool first_message_part; + u8 buf[SHA_MAX_BLOCK_SIZE]; }; struct shash_desc; int s390_sha_update(struct shash_desc *desc, const u8 *data, unsigned int len); +int s390_sha_update_blocks(struct shash_desc *desc, const u8 *data, + unsigned int len); int s390_sha_final(struct shash_desc *desc, u8 *out); +int s390_sha_finup(struct shash_desc *desc, const u8 *src, unsigned int len, + u8 *out); #endif diff --git a/arch/s390/crypto/sha1_s390.c b/arch/s390/crypto/sha1_s390.c index bc3a22704e09..d229cbd2ba22 100644 --- a/arch/s390/crypto/sha1_s390.c +++ b/arch/s390/crypto/sha1_s390.c @@ -18,12 +18,12 @@ * Copyright (c) Andrew McDonald * Copyright (c) Jean-Francois Dive */ -#include -#include -#include -#include -#include #include +#include +#include +#include +#include +#include #include "sha.h" @@ -49,7 +49,6 @@ static int s390_sha1_export(struct shash_desc *desc, void *out) octx->count = sctx->count; memcpy(octx->state, sctx->state, sizeof(octx->state)); - memcpy(octx->buffer, sctx->buf, sizeof(octx->buffer)); return 0; } @@ -60,7 +59,6 @@ static int s390_sha1_import(struct shash_desc *desc, const void *in) sctx->count = ictx->count; memcpy(sctx->state, ictx->state, sizeof(ictx->state)); - memcpy(sctx->buf, ictx->buffer, sizeof(ictx->buffer)); sctx->func = CPACF_KIMD_SHA_1; return 0; } @@ -68,16 +66,18 @@ static int s390_sha1_import(struct shash_desc *desc, const void *in) static struct shash_alg alg = { .digestsize = SHA1_DIGEST_SIZE, .init = s390_sha1_init, - .update = s390_sha_update, - .final = s390_sha_final, + .update = s390_sha_update_blocks, + .finup = s390_sha_finup, .export = s390_sha1_export, .import = s390_sha1_import, - .descsize = sizeof(struct s390_sha_ctx), - .statesize = sizeof(struct sha1_state), + .descsize = S390_SHA_CTX_SIZE, + .statesize = SHA1_STATE_SIZE, .base = { .cra_name = "sha1", .cra_driver_name= "sha1-s390", .cra_priority = 300, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA1_BLOCK_SIZE, .cra_module = THIS_MODULE, } diff --git a/arch/s390/crypto/sha_common.c b/arch/s390/crypto/sha_common.c index 961d7d522af1..013bb37ad3ef 100644 --- a/arch/s390/crypto/sha_common.c +++ b/arch/s390/crypto/sha_common.c @@ -58,6 +58,27 @@ store: } EXPORT_SYMBOL_GPL(s390_sha_update); +int s390_sha_update_blocks(struct shash_desc *desc, const u8 *data, + unsigned int len) +{ + unsigned int bsize = crypto_shash_blocksize(desc->tfm); + struct s390_sha_ctx *ctx = shash_desc_ctx(desc); + unsigned int n; + int fc; + + fc = ctx->func; + if (ctx->first_message_part) + fc |= test_facility(86) ? CPACF_KIMD_NIP : 0; + + /* process as many blocks as possible */ + n = (len / bsize) * bsize; + ctx->count += n; + cpacf_kimd(fc, ctx->state, data, n); + ctx->first_message_part = 0; + return len - n; +} +EXPORT_SYMBOL_GPL(s390_sha_update_blocks); + static int s390_crypto_shash_parmsize(int func) { switch (func) { @@ -132,5 +153,58 @@ int s390_sha_final(struct shash_desc *desc, u8 *out) } EXPORT_SYMBOL_GPL(s390_sha_final); +int s390_sha_finup(struct shash_desc *desc, const u8 *src, unsigned int len, + u8 *out) +{ + struct s390_sha_ctx *ctx = shash_desc_ctx(desc); + int mbl_offset, fc; + u64 bits; + + ctx->count += len; + + bits = ctx->count * 8; + mbl_offset = s390_crypto_shash_parmsize(ctx->func); + if (mbl_offset < 0) + return -EINVAL; + + mbl_offset = mbl_offset / sizeof(u32); + + /* set total msg bit length (mbl) in CPACF parmblock */ + switch (ctx->func) { + case CPACF_KLMD_SHA_1: + case CPACF_KLMD_SHA_256: + memcpy(ctx->state + mbl_offset, &bits, sizeof(bits)); + break; + case CPACF_KLMD_SHA_512: + /* + * the SHA512 parmblock has a 128-bit mbl field, clear + * high-order u64 field, copy bits to low-order u64 field + */ + memset(ctx->state + mbl_offset, 0x00, sizeof(bits)); + mbl_offset += sizeof(u64) / sizeof(u32); + memcpy(ctx->state + mbl_offset, &bits, sizeof(bits)); + break; + case CPACF_KLMD_SHA3_224: + case CPACF_KLMD_SHA3_256: + case CPACF_KLMD_SHA3_384: + case CPACF_KLMD_SHA3_512: + break; + default: + return -EINVAL; + } + + fc = ctx->func; + fc |= test_facility(86) ? CPACF_KLMD_DUFOP : 0; + if (ctx->first_message_part) + fc |= CPACF_KLMD_NIP; + cpacf_klmd(fc, ctx->state, src, len); + + /* copy digest to out */ + memcpy(out, ctx->state, crypto_shash_digestsize(desc->tfm)); + + return 0; +} +EXPORT_SYMBOL_GPL(s390_sha_finup); + MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("s390 SHA cipher common functions"); From fd00e76ac67c23317252f1e93e078c0da9102741 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 18 Apr 2025 10:59:36 +0800 Subject: [PATCH 173/382] crypto: sparc/sha1 - Use API partial block handling Use the Crypto API partial block handling. Signed-off-by: Herbert Xu --- arch/sparc/crypto/sha1_glue.c | 109 +++++----------------------------- 1 file changed, 15 insertions(+), 94 deletions(-) diff --git a/arch/sparc/crypto/sha1_glue.c b/arch/sparc/crypto/sha1_glue.c index 06b7becfcb21..ec5a06948e0d 100644 --- a/arch/sparc/crypto/sha1_glue.c +++ b/arch/sparc/crypto/sha1_glue.c @@ -11,124 +11,45 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt +#include +#include #include -#include -#include -#include -#include #include #include - -#include -#include +#include +#include #include "opcodes.h" -asmlinkage void sha1_sparc64_transform(u32 *digest, const char *data, - unsigned int rounds); - -static void __sha1_sparc64_update(struct sha1_state *sctx, const u8 *data, - unsigned int len, unsigned int partial) -{ - unsigned int done = 0; - - sctx->count += len; - if (partial) { - done = SHA1_BLOCK_SIZE - partial; - memcpy(sctx->buffer + partial, data, done); - sha1_sparc64_transform(sctx->state, sctx->buffer, 1); - } - if (len - done >= SHA1_BLOCK_SIZE) { - const unsigned int rounds = (len - done) / SHA1_BLOCK_SIZE; - - sha1_sparc64_transform(sctx->state, data + done, rounds); - done += rounds * SHA1_BLOCK_SIZE; - } - - memcpy(sctx->buffer, data + done, len - done); -} +asmlinkage void sha1_sparc64_transform(struct sha1_state *digest, + const u8 *data, int rounds); static int sha1_sparc64_update(struct shash_desc *desc, const u8 *data, unsigned int len) { - struct sha1_state *sctx = shash_desc_ctx(desc); - unsigned int partial = sctx->count % SHA1_BLOCK_SIZE; - - /* Handle the fast case right here */ - if (partial + len < SHA1_BLOCK_SIZE) { - sctx->count += len; - memcpy(sctx->buffer + partial, data, len); - } else - __sha1_sparc64_update(sctx, data, len, partial); - - return 0; + return sha1_base_do_update_blocks(desc, data, len, + sha1_sparc64_transform); } /* Add padding and return the message digest. */ -static int sha1_sparc64_final(struct shash_desc *desc, u8 *out) +static int sha1_sparc64_finup(struct shash_desc *desc, const u8 *src, + unsigned int len, u8 *out) { - struct sha1_state *sctx = shash_desc_ctx(desc); - unsigned int i, index, padlen; - __be32 *dst = (__be32 *)out; - __be64 bits; - static const u8 padding[SHA1_BLOCK_SIZE] = { 0x80, }; - - bits = cpu_to_be64(sctx->count << 3); - - /* Pad out to 56 mod 64 and append length */ - index = sctx->count % SHA1_BLOCK_SIZE; - padlen = (index < 56) ? (56 - index) : ((SHA1_BLOCK_SIZE+56) - index); - - /* We need to fill a whole block for __sha1_sparc64_update() */ - if (padlen <= 56) { - sctx->count += padlen; - memcpy(sctx->buffer + index, padding, padlen); - } else { - __sha1_sparc64_update(sctx, padding, padlen, index); - } - __sha1_sparc64_update(sctx, (const u8 *)&bits, sizeof(bits), 56); - - /* Store state in digest */ - for (i = 0; i < 5; i++) - dst[i] = cpu_to_be32(sctx->state[i]); - - /* Wipe context */ - memset(sctx, 0, sizeof(*sctx)); - - return 0; -} - -static int sha1_sparc64_export(struct shash_desc *desc, void *out) -{ - struct sha1_state *sctx = shash_desc_ctx(desc); - - memcpy(out, sctx, sizeof(*sctx)); - - return 0; -} - -static int sha1_sparc64_import(struct shash_desc *desc, const void *in) -{ - struct sha1_state *sctx = shash_desc_ctx(desc); - - memcpy(sctx, in, sizeof(*sctx)); - - return 0; + sha1_base_do_finup(desc, src, len, sha1_sparc64_transform); + return sha1_base_finish(desc, out); } static struct shash_alg alg = { .digestsize = SHA1_DIGEST_SIZE, .init = sha1_base_init, .update = sha1_sparc64_update, - .final = sha1_sparc64_final, - .export = sha1_sparc64_export, - .import = sha1_sparc64_import, - .descsize = sizeof(struct sha1_state), - .statesize = sizeof(struct sha1_state), + .finup = sha1_sparc64_finup, + .descsize = SHA1_STATE_SIZE, .base = { .cra_name = "sha1", .cra_driver_name= "sha1-sparc64", .cra_priority = SPARC_CR_OPCODE_PRIORITY, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_blocksize = SHA1_BLOCK_SIZE, .cra_module = THIS_MODULE, } From a86ee2d74013e071efaa18631f9aa313262e5b1f Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 18 Apr 2025 10:59:38 +0800 Subject: [PATCH 174/382] crypto: sha1_base - Remove partial block helpers Now that all sha1_base users have been converted to use the API partial block handling, remove the partial block helpers. Signed-off-by: Herbert Xu --- include/crypto/sha1_base.h | 61 -------------------------------------- 1 file changed, 61 deletions(-) diff --git a/include/crypto/sha1_base.h b/include/crypto/sha1_base.h index b23cfad18ce2..62701d136c79 100644 --- a/include/crypto/sha1_base.h +++ b/include/crypto/sha1_base.h @@ -31,44 +31,6 @@ static inline int sha1_base_init(struct shash_desc *desc) return 0; } -static inline int sha1_base_do_update(struct shash_desc *desc, - const u8 *data, - unsigned int len, - sha1_block_fn *block_fn) -{ - struct sha1_state *sctx = shash_desc_ctx(desc); - unsigned int partial = sctx->count % SHA1_BLOCK_SIZE; - - sctx->count += len; - - if (unlikely((partial + len) >= SHA1_BLOCK_SIZE)) { - int blocks; - - if (partial) { - int p = SHA1_BLOCK_SIZE - partial; - - memcpy(sctx->buffer + partial, data, p); - data += p; - len -= p; - - block_fn(sctx, sctx->buffer, 1); - } - - blocks = len / SHA1_BLOCK_SIZE; - len %= SHA1_BLOCK_SIZE; - - if (blocks) { - block_fn(sctx, data, blocks); - data += blocks * SHA1_BLOCK_SIZE; - } - partial = 0; - } - if (len) - memcpy(sctx->buffer + partial, data, len); - - return 0; -} - static inline int sha1_base_do_update_blocks(struct shash_desc *desc, const u8 *data, unsigned int len, @@ -82,29 +44,6 @@ static inline int sha1_base_do_update_blocks(struct shash_desc *desc, return remain; } -static inline int sha1_base_do_finalize(struct shash_desc *desc, - sha1_block_fn *block_fn) -{ - const int bit_offset = SHA1_BLOCK_SIZE - sizeof(__be64); - struct sha1_state *sctx = shash_desc_ctx(desc); - __be64 *bits = (__be64 *)(sctx->buffer + bit_offset); - unsigned int partial = sctx->count % SHA1_BLOCK_SIZE; - - sctx->buffer[partial++] = 0x80; - if (partial > bit_offset) { - memset(sctx->buffer + partial, 0x0, SHA1_BLOCK_SIZE - partial); - partial = 0; - - block_fn(sctx, sctx->buffer, 1); - } - - memset(sctx->buffer + partial, 0x0, bit_offset - partial); - *bits = cpu_to_be64(sctx->count << 3); - block_fn(sctx, sctx->buffer, 1); - - return 0; -} - static inline int sha1_base_do_finup(struct shash_desc *desc, const u8 *src, unsigned int len, sha1_block_fn *block_fn) From eba187a6e7141a1166a68c4d27b4ee5a27670b3b Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 18 Apr 2025 10:59:41 +0800 Subject: [PATCH 175/382] crypto: x86/sha256 - Use API partial block handling Use the Crypto API partial block handling. Also remove the unnecessary SIMD fallback path. Signed-off-by: Herbert Xu --- arch/x86/crypto/sha256_ssse3_glue.c | 107 +++++++++++----------------- include/crypto/sha2.h | 5 ++ include/crypto/sha256_base.h | 80 +++++++++++++++++++-- 3 files changed, 119 insertions(+), 73 deletions(-) diff --git a/arch/x86/crypto/sha256_ssse3_glue.c b/arch/x86/crypto/sha256_ssse3_glue.c index 429a3cefbab4..367ce4830fa4 100644 --- a/arch/x86/crypto/sha256_ssse3_glue.c +++ b/arch/x86/crypto/sha256_ssse3_glue.c @@ -29,19 +29,15 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt +#include +#include #include -#include -#include -#include -#include -#include #include #include -#include -#include -#include +#include +#include -asmlinkage void sha256_transform_ssse3(struct sha256_state *state, +asmlinkage void sha256_transform_ssse3(struct crypto_sha256_state *state, const u8 *data, int blocks); static const struct x86_cpu_id module_cpu_ids[] = { @@ -54,37 +50,29 @@ static const struct x86_cpu_id module_cpu_ids[] = { MODULE_DEVICE_TABLE(x86cpu, module_cpu_ids); static int _sha256_update(struct shash_desc *desc, const u8 *data, - unsigned int len, sha256_block_fn *sha256_xform) + unsigned int len, + crypto_sha256_block_fn *sha256_xform) { - struct sha256_state *sctx = shash_desc_ctx(desc); - - if (!crypto_simd_usable() || - (sctx->count % SHA256_BLOCK_SIZE) + len < SHA256_BLOCK_SIZE) - return crypto_sha256_update(desc, data, len); + int remain; /* - * Make sure struct sha256_state begins directly with the SHA256 + * Make sure struct crypto_sha256_state begins directly with the SHA256 * 256-bit internal state, as this is what the asm functions expect. */ - BUILD_BUG_ON(offsetof(struct sha256_state, state) != 0); + BUILD_BUG_ON(offsetof(struct crypto_sha256_state, state) != 0); kernel_fpu_begin(); - sha256_base_do_update(desc, data, len, sha256_xform); + remain = sha256_base_do_update_blocks(desc, data, len, sha256_xform); kernel_fpu_end(); - return 0; + return remain; } static int sha256_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out, sha256_block_fn *sha256_xform) + unsigned int len, u8 *out, crypto_sha256_block_fn *sha256_xform) { - if (!crypto_simd_usable()) - return crypto_sha256_finup(desc, data, len, out); - kernel_fpu_begin(); - if (len) - sha256_base_do_update(desc, data, len, sha256_xform); - sha256_base_do_finalize(desc, sha256_xform); + sha256_base_do_finup(desc, data, len, sha256_xform); kernel_fpu_end(); return sha256_base_finish(desc, out); @@ -102,12 +90,6 @@ static int sha256_ssse3_finup(struct shash_desc *desc, const u8 *data, return sha256_finup(desc, data, len, out, sha256_transform_ssse3); } -/* Add padding and return the message digest. */ -static int sha256_ssse3_final(struct shash_desc *desc, u8 *out) -{ - return sha256_ssse3_finup(desc, NULL, 0, out); -} - static int sha256_ssse3_digest(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) { @@ -119,14 +101,15 @@ static struct shash_alg sha256_ssse3_algs[] = { { .digestsize = SHA256_DIGEST_SIZE, .init = sha256_base_init, .update = sha256_ssse3_update, - .final = sha256_ssse3_final, .finup = sha256_ssse3_finup, .digest = sha256_ssse3_digest, - .descsize = sizeof(struct sha256_state), + .descsize = sizeof(struct crypto_sha256_state), .base = { .cra_name = "sha256", .cra_driver_name = "sha256-ssse3", .cra_priority = 150, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA256_BLOCK_SIZE, .cra_module = THIS_MODULE, } @@ -134,13 +117,14 @@ static struct shash_alg sha256_ssse3_algs[] = { { .digestsize = SHA224_DIGEST_SIZE, .init = sha224_base_init, .update = sha256_ssse3_update, - .final = sha256_ssse3_final, .finup = sha256_ssse3_finup, - .descsize = sizeof(struct sha256_state), + .descsize = sizeof(struct crypto_sha256_state), .base = { .cra_name = "sha224", .cra_driver_name = "sha224-ssse3", .cra_priority = 150, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA224_BLOCK_SIZE, .cra_module = THIS_MODULE, } @@ -161,7 +145,7 @@ static void unregister_sha256_ssse3(void) ARRAY_SIZE(sha256_ssse3_algs)); } -asmlinkage void sha256_transform_avx(struct sha256_state *state, +asmlinkage void sha256_transform_avx(struct crypto_sha256_state *state, const u8 *data, int blocks); static int sha256_avx_update(struct shash_desc *desc, const u8 *data, @@ -176,11 +160,6 @@ static int sha256_avx_finup(struct shash_desc *desc, const u8 *data, return sha256_finup(desc, data, len, out, sha256_transform_avx); } -static int sha256_avx_final(struct shash_desc *desc, u8 *out) -{ - return sha256_avx_finup(desc, NULL, 0, out); -} - static int sha256_avx_digest(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) { @@ -192,14 +171,15 @@ static struct shash_alg sha256_avx_algs[] = { { .digestsize = SHA256_DIGEST_SIZE, .init = sha256_base_init, .update = sha256_avx_update, - .final = sha256_avx_final, .finup = sha256_avx_finup, .digest = sha256_avx_digest, - .descsize = sizeof(struct sha256_state), + .descsize = sizeof(struct crypto_sha256_state), .base = { .cra_name = "sha256", .cra_driver_name = "sha256-avx", .cra_priority = 160, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA256_BLOCK_SIZE, .cra_module = THIS_MODULE, } @@ -207,13 +187,14 @@ static struct shash_alg sha256_avx_algs[] = { { .digestsize = SHA224_DIGEST_SIZE, .init = sha224_base_init, .update = sha256_avx_update, - .final = sha256_avx_final, .finup = sha256_avx_finup, - .descsize = sizeof(struct sha256_state), + .descsize = sizeof(struct crypto_sha256_state), .base = { .cra_name = "sha224", .cra_driver_name = "sha224-avx", .cra_priority = 160, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA224_BLOCK_SIZE, .cra_module = THIS_MODULE, } @@ -245,7 +226,7 @@ static void unregister_sha256_avx(void) ARRAY_SIZE(sha256_avx_algs)); } -asmlinkage void sha256_transform_rorx(struct sha256_state *state, +asmlinkage void sha256_transform_rorx(struct crypto_sha256_state *state, const u8 *data, int blocks); static int sha256_avx2_update(struct shash_desc *desc, const u8 *data, @@ -260,11 +241,6 @@ static int sha256_avx2_finup(struct shash_desc *desc, const u8 *data, return sha256_finup(desc, data, len, out, sha256_transform_rorx); } -static int sha256_avx2_final(struct shash_desc *desc, u8 *out) -{ - return sha256_avx2_finup(desc, NULL, 0, out); -} - static int sha256_avx2_digest(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) { @@ -276,14 +252,15 @@ static struct shash_alg sha256_avx2_algs[] = { { .digestsize = SHA256_DIGEST_SIZE, .init = sha256_base_init, .update = sha256_avx2_update, - .final = sha256_avx2_final, .finup = sha256_avx2_finup, .digest = sha256_avx2_digest, - .descsize = sizeof(struct sha256_state), + .descsize = sizeof(struct crypto_sha256_state), .base = { .cra_name = "sha256", .cra_driver_name = "sha256-avx2", .cra_priority = 170, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA256_BLOCK_SIZE, .cra_module = THIS_MODULE, } @@ -291,13 +268,14 @@ static struct shash_alg sha256_avx2_algs[] = { { .digestsize = SHA224_DIGEST_SIZE, .init = sha224_base_init, .update = sha256_avx2_update, - .final = sha256_avx2_final, .finup = sha256_avx2_finup, - .descsize = sizeof(struct sha256_state), + .descsize = sizeof(struct crypto_sha256_state), .base = { .cra_name = "sha224", .cra_driver_name = "sha224-avx2", .cra_priority = 170, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA224_BLOCK_SIZE, .cra_module = THIS_MODULE, } @@ -327,7 +305,7 @@ static void unregister_sha256_avx2(void) ARRAY_SIZE(sha256_avx2_algs)); } -asmlinkage void sha256_ni_transform(struct sha256_state *digest, +asmlinkage void sha256_ni_transform(struct crypto_sha256_state *digest, const u8 *data, int rounds); static int sha256_ni_update(struct shash_desc *desc, const u8 *data, @@ -342,11 +320,6 @@ static int sha256_ni_finup(struct shash_desc *desc, const u8 *data, return sha256_finup(desc, data, len, out, sha256_ni_transform); } -static int sha256_ni_final(struct shash_desc *desc, u8 *out) -{ - return sha256_ni_finup(desc, NULL, 0, out); -} - static int sha256_ni_digest(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) { @@ -358,14 +331,15 @@ static struct shash_alg sha256_ni_algs[] = { { .digestsize = SHA256_DIGEST_SIZE, .init = sha256_base_init, .update = sha256_ni_update, - .final = sha256_ni_final, .finup = sha256_ni_finup, .digest = sha256_ni_digest, - .descsize = sizeof(struct sha256_state), + .descsize = sizeof(struct crypto_sha256_state), .base = { .cra_name = "sha256", .cra_driver_name = "sha256-ni", .cra_priority = 250, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA256_BLOCK_SIZE, .cra_module = THIS_MODULE, } @@ -373,13 +347,14 @@ static struct shash_alg sha256_ni_algs[] = { { .digestsize = SHA224_DIGEST_SIZE, .init = sha224_base_init, .update = sha256_ni_update, - .final = sha256_ni_final, .finup = sha256_ni_finup, - .descsize = sizeof(struct sha256_state), + .descsize = sizeof(struct crypto_sha256_state), .base = { .cra_name = "sha224", .cra_driver_name = "sha224-ni", .cra_priority = 250, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA224_BLOCK_SIZE, .cra_module = THIS_MODULE, } diff --git a/include/crypto/sha2.h b/include/crypto/sha2.h index b9e9281d76c9..d9b1b9932393 100644 --- a/include/crypto/sha2.h +++ b/include/crypto/sha2.h @@ -64,6 +64,11 @@ extern const u8 sha384_zero_message_hash[SHA384_DIGEST_SIZE]; extern const u8 sha512_zero_message_hash[SHA512_DIGEST_SIZE]; +struct crypto_sha256_state { + u32 state[SHA256_DIGEST_SIZE / 4]; + u64 count; +}; + struct sha256_state { u32 state[SHA256_DIGEST_SIZE / 4]; u64 count; diff --git a/include/crypto/sha256_base.h b/include/crypto/sha256_base.h index e0418818d63c..727a1b63e1e9 100644 --- a/include/crypto/sha256_base.h +++ b/include/crypto/sha256_base.h @@ -8,15 +8,17 @@ #ifndef _CRYPTO_SHA256_BASE_H #define _CRYPTO_SHA256_BASE_H -#include -#include #include #include +#include #include #include +#include typedef void (sha256_block_fn)(struct sha256_state *sst, u8 const *src, int blocks); +typedef void (crypto_sha256_block_fn)(struct crypto_sha256_state *sst, + u8 const *src, int blocks); static inline int sha224_base_init(struct shash_desc *desc) { @@ -81,6 +83,64 @@ static inline int sha256_base_do_update(struct shash_desc *desc, return lib_sha256_base_do_update(sctx, data, len, block_fn); } +static inline int lib_sha256_base_do_update_blocks( + struct crypto_sha256_state *sctx, const u8 *data, unsigned int len, + crypto_sha256_block_fn *block_fn) +{ + unsigned int remain = len - round_down(len, SHA256_BLOCK_SIZE); + + sctx->count += len - remain; + block_fn(sctx, data, len / SHA256_BLOCK_SIZE); + return remain; +} + +static inline int sha256_base_do_update_blocks( + struct shash_desc *desc, const u8 *data, unsigned int len, + crypto_sha256_block_fn *block_fn) +{ + return lib_sha256_base_do_update_blocks(shash_desc_ctx(desc), data, + len, block_fn); +} + +static inline int lib_sha256_base_do_finup(struct crypto_sha256_state *sctx, + const u8 *src, unsigned int len, + crypto_sha256_block_fn *block_fn) +{ + unsigned int bit_offset = SHA256_BLOCK_SIZE / 8 - 1; + union { + __be64 b64[SHA256_BLOCK_SIZE / 4]; + u8 u8[SHA256_BLOCK_SIZE * 2]; + } block = {}; + + if (len >= bit_offset * 8) + bit_offset += SHA256_BLOCK_SIZE / 8; + memcpy(&block, src, len); + block.u8[len] = 0x80; + sctx->count += len; + block.b64[bit_offset] = cpu_to_be64(sctx->count << 3); + block_fn(sctx, block.u8, (bit_offset + 1) * 8 / SHA256_BLOCK_SIZE); + memzero_explicit(&block, sizeof(block)); + + return 0; +} + +static inline int sha256_base_do_finup(struct shash_desc *desc, + const u8 *src, unsigned int len, + crypto_sha256_block_fn *block_fn) +{ + struct crypto_sha256_state *sctx = shash_desc_ctx(desc); + + if (len >= SHA256_BLOCK_SIZE) { + int remain; + + remain = lib_sha256_base_do_update_blocks(sctx, src, len, + block_fn); + src += len - remain; + len = remain; + } + return lib_sha256_base_do_finup(sctx, src, len, block_fn); +} + static inline int lib_sha256_base_do_finalize(struct sha256_state *sctx, sha256_block_fn *block_fn) { @@ -111,15 +171,21 @@ static inline int sha256_base_do_finalize(struct shash_desc *desc, return lib_sha256_base_do_finalize(sctx, block_fn); } -static inline int lib_sha256_base_finish(struct sha256_state *sctx, u8 *out, - unsigned int digest_size) +static inline int __sha256_base_finish(u32 state[SHA256_DIGEST_SIZE / 4], + u8 *out, unsigned int digest_size) { __be32 *digest = (__be32 *)out; int i; for (i = 0; digest_size > 0; i++, digest_size -= sizeof(__be32)) - put_unaligned_be32(sctx->state[i], digest++); + put_unaligned_be32(state[i], digest++); + return 0; +} +static inline int lib_sha256_base_finish(struct sha256_state *sctx, u8 *out, + unsigned int digest_size) +{ + __sha256_base_finish(sctx->state, out, digest_size); memzero_explicit(sctx, sizeof(*sctx)); return 0; } @@ -127,9 +193,9 @@ static inline int lib_sha256_base_finish(struct sha256_state *sctx, u8 *out, static inline int sha256_base_finish(struct shash_desc *desc, u8 *out) { unsigned int digest_size = crypto_shash_digestsize(desc->tfm); - struct sha256_state *sctx = shash_desc_ctx(desc); + struct crypto_sha256_state *sctx = shash_desc_ctx(desc); - return lib_sha256_base_finish(sctx, out, digest_size); + return __sha256_base_finish(sctx->state, out, digest_size); } #endif /* _CRYPTO_SHA256_BASE_H */ From 5f804244d0ba8bf07c7eb6caeb63bd38a2bc910e Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 18 Apr 2025 10:59:43 +0800 Subject: [PATCH 176/382] crypto: mips/octeon-sha256 - Use API partial block handling Use the Crypto API partial block handling. Signed-off-by: Herbert Xu --- .../mips/cavium-octeon/crypto/octeon-sha256.c | 161 ++++-------------- 1 file changed, 37 insertions(+), 124 deletions(-) diff --git a/arch/mips/cavium-octeon/crypto/octeon-sha256.c b/arch/mips/cavium-octeon/crypto/octeon-sha256.c index 435e4a6e7f13..8e85ea65387c 100644 --- a/arch/mips/cavium-octeon/crypto/octeon-sha256.c +++ b/arch/mips/cavium-octeon/crypto/octeon-sha256.c @@ -14,15 +14,12 @@ * SHA224 Support Copyright 2007 Intel Corporation */ -#include -#include -#include -#include -#include -#include -#include #include #include +#include +#include +#include +#include #include "octeon-crypto.h" @@ -30,7 +27,7 @@ * We pass everything as 64-bit. OCTEON can handle misaligned data. */ -static void octeon_sha256_store_hash(struct sha256_state *sctx) +static void octeon_sha256_store_hash(struct crypto_sha256_state *sctx) { u64 *hash = (u64 *)sctx->state; @@ -40,7 +37,7 @@ static void octeon_sha256_store_hash(struct sha256_state *sctx) write_octeon_64bit_hash_dword(hash[3], 3); } -static void octeon_sha256_read_hash(struct sha256_state *sctx) +static void octeon_sha256_read_hash(struct crypto_sha256_state *sctx) { u64 *hash = (u64 *)sctx->state; @@ -50,158 +47,72 @@ static void octeon_sha256_read_hash(struct sha256_state *sctx) hash[3] = read_octeon_64bit_hash_dword(3); } -static void octeon_sha256_transform(const void *_block) +static void octeon_sha256_transform(struct crypto_sha256_state *sctx, + const u8 *src, int blocks) { - const u64 *block = _block; + do { + const u64 *block = (const u64 *)src; - write_octeon_64bit_block_dword(block[0], 0); - write_octeon_64bit_block_dword(block[1], 1); - write_octeon_64bit_block_dword(block[2], 2); - write_octeon_64bit_block_dword(block[3], 3); - write_octeon_64bit_block_dword(block[4], 4); - write_octeon_64bit_block_dword(block[5], 5); - write_octeon_64bit_block_dword(block[6], 6); - octeon_sha256_start(block[7]); -} + write_octeon_64bit_block_dword(block[0], 0); + write_octeon_64bit_block_dword(block[1], 1); + write_octeon_64bit_block_dword(block[2], 2); + write_octeon_64bit_block_dword(block[3], 3); + write_octeon_64bit_block_dword(block[4], 4); + write_octeon_64bit_block_dword(block[5], 5); + write_octeon_64bit_block_dword(block[6], 6); + octeon_sha256_start(block[7]); -static void __octeon_sha256_update(struct sha256_state *sctx, const u8 *data, - unsigned int len) -{ - unsigned int partial; - unsigned int done; - const u8 *src; - - partial = sctx->count % SHA256_BLOCK_SIZE; - sctx->count += len; - done = 0; - src = data; - - if ((partial + len) >= SHA256_BLOCK_SIZE) { - if (partial) { - done = -partial; - memcpy(sctx->buf + partial, data, - done + SHA256_BLOCK_SIZE); - src = sctx->buf; - } - - do { - octeon_sha256_transform(src); - done += SHA256_BLOCK_SIZE; - src = data + done; - } while (done + SHA256_BLOCK_SIZE <= len); - - partial = 0; - } - memcpy(sctx->buf + partial, src, len - done); + src += SHA256_BLOCK_SIZE; + } while (--blocks); } static int octeon_sha256_update(struct shash_desc *desc, const u8 *data, unsigned int len) { - struct sha256_state *sctx = shash_desc_ctx(desc); + struct crypto_sha256_state *sctx = shash_desc_ctx(desc); struct octeon_cop2_state state; unsigned long flags; - - /* - * Small updates never reach the crypto engine, so the generic sha256 is - * faster because of the heavyweight octeon_crypto_enable() / - * octeon_crypto_disable(). - */ - if ((sctx->count % SHA256_BLOCK_SIZE) + len < SHA256_BLOCK_SIZE) - return crypto_sha256_update(desc, data, len); + int remain; flags = octeon_crypto_enable(&state); octeon_sha256_store_hash(sctx); - __octeon_sha256_update(sctx, data, len); + remain = sha256_base_do_update_blocks(desc, data, len, + octeon_sha256_transform); octeon_sha256_read_hash(sctx); octeon_crypto_disable(&state, flags); - - return 0; + return remain; } -static int octeon_sha256_final(struct shash_desc *desc, u8 *out) +static int octeon_sha256_finup(struct shash_desc *desc, const u8 *src, + unsigned int len, u8 *out) { - struct sha256_state *sctx = shash_desc_ctx(desc); - static const u8 padding[64] = { 0x80, }; + struct crypto_sha256_state *sctx = shash_desc_ctx(desc); struct octeon_cop2_state state; - __be32 *dst = (__be32 *)out; - unsigned int pad_len; unsigned long flags; - unsigned int index; - __be64 bits; - int i; - - /* Save number of bits. */ - bits = cpu_to_be64(sctx->count << 3); - - /* Pad out to 56 mod 64. */ - index = sctx->count & 0x3f; - pad_len = (index < 56) ? (56 - index) : ((64+56) - index); flags = octeon_crypto_enable(&state); octeon_sha256_store_hash(sctx); - __octeon_sha256_update(sctx, padding, pad_len); - - /* Append length (before padding). */ - __octeon_sha256_update(sctx, (const u8 *)&bits, sizeof(bits)); + sha256_base_do_finup(desc, src, len, octeon_sha256_transform); octeon_sha256_read_hash(sctx); octeon_crypto_disable(&state, flags); - - /* Store state in digest */ - for (i = 0; i < 8; i++) - dst[i] = cpu_to_be32(sctx->state[i]); - - /* Zeroize sensitive information. */ - memset(sctx, 0, sizeof(*sctx)); - - return 0; -} - -static int octeon_sha224_final(struct shash_desc *desc, u8 *hash) -{ - u8 D[SHA256_DIGEST_SIZE]; - - octeon_sha256_final(desc, D); - - memcpy(hash, D, SHA224_DIGEST_SIZE); - memzero_explicit(D, SHA256_DIGEST_SIZE); - - return 0; -} - -static int octeon_sha256_export(struct shash_desc *desc, void *out) -{ - struct sha256_state *sctx = shash_desc_ctx(desc); - - memcpy(out, sctx, sizeof(*sctx)); - return 0; -} - -static int octeon_sha256_import(struct shash_desc *desc, const void *in) -{ - struct sha256_state *sctx = shash_desc_ctx(desc); - - memcpy(sctx, in, sizeof(*sctx)); - return 0; + return sha256_base_finish(desc, out); } static struct shash_alg octeon_sha256_algs[2] = { { .digestsize = SHA256_DIGEST_SIZE, .init = sha256_base_init, .update = octeon_sha256_update, - .final = octeon_sha256_final, - .export = octeon_sha256_export, - .import = octeon_sha256_import, - .descsize = sizeof(struct sha256_state), - .statesize = sizeof(struct sha256_state), + .finup = octeon_sha256_finup, + .descsize = sizeof(struct crypto_sha256_state), .base = { .cra_name = "sha256", .cra_driver_name= "octeon-sha256", .cra_priority = OCTEON_CR_OPCODE_PRIORITY, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_blocksize = SHA256_BLOCK_SIZE, .cra_module = THIS_MODULE, } @@ -209,11 +120,13 @@ static struct shash_alg octeon_sha256_algs[2] = { { .digestsize = SHA224_DIGEST_SIZE, .init = sha224_base_init, .update = octeon_sha256_update, - .final = octeon_sha224_final, - .descsize = sizeof(struct sha256_state), + .finup = octeon_sha256_finup, + .descsize = sizeof(struct crypto_sha256_state), .base = { .cra_name = "sha224", .cra_driver_name= "octeon-sha224", + .cra_priority = OCTEON_CR_OPCODE_PRIORITY, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_blocksize = SHA224_BLOCK_SIZE, .cra_module = THIS_MODULE, } From e6c5597badf2998980639fb83e60696d0f2ce0dd Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 18 Apr 2025 10:59:45 +0800 Subject: [PATCH 177/382] crypto: riscv/sha256 - Use API partial block handling Use the Crypto API partial block handling. Signed-off-by: Herbert Xu --- arch/riscv/crypto/sha256-riscv64-glue.c | 68 ++++++++++--------------- include/crypto/sha256_base.h | 3 ++ lib/crypto/sha256.c | 18 +++++-- 3 files changed, 44 insertions(+), 45 deletions(-) diff --git a/arch/riscv/crypto/sha256-riscv64-glue.c b/arch/riscv/crypto/sha256-riscv64-glue.c index 71e051e40a64..c998300ab843 100644 --- a/arch/riscv/crypto/sha256-riscv64-glue.c +++ b/arch/riscv/crypto/sha256-riscv64-glue.c @@ -14,7 +14,7 @@ #include #include #include -#include +#include #include /* @@ -22,50 +22,36 @@ * It is assumed to be the first field. */ asmlinkage void sha256_transform_zvknha_or_zvknhb_zvkb( - struct sha256_state *state, const u8 *data, int num_blocks); + struct crypto_sha256_state *state, const u8 *data, int num_blocks); + +static void sha256_block(struct crypto_sha256_state *state, const u8 *data, + int num_blocks) +{ + /* + * Ensure struct crypto_sha256_state begins directly with the SHA-256 + * 256-bit internal state, as this is what the asm function expects. + */ + BUILD_BUG_ON(offsetof(struct crypto_sha256_state, state) != 0); + + if (crypto_simd_usable()) { + kernel_vector_begin(); + sha256_transform_zvknha_or_zvknhb_zvkb(state, data, num_blocks); + kernel_vector_end(); + } else + sha256_transform_blocks(state, data, num_blocks); +} static int riscv64_sha256_update(struct shash_desc *desc, const u8 *data, unsigned int len) { - /* - * Ensure struct sha256_state begins directly with the SHA-256 - * 256-bit internal state, as this is what the asm function expects. - */ - BUILD_BUG_ON(offsetof(struct sha256_state, state) != 0); - - if (crypto_simd_usable()) { - kernel_vector_begin(); - sha256_base_do_update(desc, data, len, - sha256_transform_zvknha_or_zvknhb_zvkb); - kernel_vector_end(); - } else { - crypto_sha256_update(desc, data, len); - } - return 0; + return sha256_base_do_update_blocks(desc, data, len, sha256_block); } static int riscv64_sha256_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) { - if (crypto_simd_usable()) { - kernel_vector_begin(); - if (len) - sha256_base_do_update( - desc, data, len, - sha256_transform_zvknha_or_zvknhb_zvkb); - sha256_base_do_finalize( - desc, sha256_transform_zvknha_or_zvknhb_zvkb); - kernel_vector_end(); - - return sha256_base_finish(desc, out); - } - - return crypto_sha256_finup(desc, data, len, out); -} - -static int riscv64_sha256_final(struct shash_desc *desc, u8 *out) -{ - return riscv64_sha256_finup(desc, NULL, 0, out); + sha256_base_do_finup(desc, data, len, sha256_block); + return sha256_base_finish(desc, out); } static int riscv64_sha256_digest(struct shash_desc *desc, const u8 *data, @@ -79,13 +65,14 @@ static struct shash_alg riscv64_sha256_algs[] = { { .init = sha256_base_init, .update = riscv64_sha256_update, - .final = riscv64_sha256_final, .finup = riscv64_sha256_finup, .digest = riscv64_sha256_digest, - .descsize = sizeof(struct sha256_state), + .descsize = sizeof(struct crypto_sha256_state), .digestsize = SHA256_DIGEST_SIZE, .base = { .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_priority = 300, .cra_name = "sha256", .cra_driver_name = "sha256-riscv64-zvknha_or_zvknhb-zvkb", @@ -94,12 +81,13 @@ static struct shash_alg riscv64_sha256_algs[] = { }, { .init = sha224_base_init, .update = riscv64_sha256_update, - .final = riscv64_sha256_final, .finup = riscv64_sha256_finup, - .descsize = sizeof(struct sha256_state), + .descsize = sizeof(struct crypto_sha256_state), .digestsize = SHA224_DIGEST_SIZE, .base = { .cra_blocksize = SHA224_BLOCK_SIZE, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_priority = 300, .cra_name = "sha224", .cra_driver_name = "sha224-riscv64-zvknha_or_zvknhb-zvkb", diff --git a/include/crypto/sha256_base.h b/include/crypto/sha256_base.h index 727a1b63e1e9..b9d3583b6256 100644 --- a/include/crypto/sha256_base.h +++ b/include/crypto/sha256_base.h @@ -198,4 +198,7 @@ static inline int sha256_base_finish(struct shash_desc *desc, u8 *out) return __sha256_base_finish(sctx->state, out, digest_size); } +void sha256_transform_blocks(struct crypto_sha256_state *sst, + const u8 *input, int blocks); + #endif /* _CRYPTO_SHA256_BASE_H */ diff --git a/lib/crypto/sha256.c b/lib/crypto/sha256.c index 04c1f2557e6c..39ead0222937 100644 --- a/lib/crypto/sha256.c +++ b/lib/crypto/sha256.c @@ -118,28 +118,36 @@ static void sha256_transform(u32 *state, const u8 *input, u32 *W) state[4] += e; state[5] += f; state[6] += g; state[7] += h; } -static void sha256_transform_blocks(struct sha256_state *sctx, - const u8 *input, int blocks) +void sha256_transform_blocks(struct crypto_sha256_state *sst, + const u8 *input, int blocks) { u32 W[64]; do { - sha256_transform(sctx->state, input, W); + sha256_transform(sst->state, input, W); input += SHA256_BLOCK_SIZE; } while (--blocks); memzero_explicit(W, sizeof(W)); } +EXPORT_SYMBOL_GPL(sha256_transform_blocks); + +static void lib_sha256_transform_blocks(struct sha256_state *sctx, + const u8 *input, int blocks) +{ + sha256_transform_blocks((struct crypto_sha256_state *)sctx, input, + blocks); +} void sha256_update(struct sha256_state *sctx, const u8 *data, unsigned int len) { - lib_sha256_base_do_update(sctx, data, len, sha256_transform_blocks); + lib_sha256_base_do_update(sctx, data, len, lib_sha256_transform_blocks); } EXPORT_SYMBOL(sha256_update); static void __sha256_final(struct sha256_state *sctx, u8 *out, int digest_size) { - lib_sha256_base_do_finalize(sctx, sha256_transform_blocks); + lib_sha256_base_do_finalize(sctx, lib_sha256_transform_blocks); lib_sha256_base_finish(sctx, out, digest_size); } From 9adeea13ed7f5218028cdc5b11df53e69c39a60f Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 18 Apr 2025 10:59:48 +0800 Subject: [PATCH 178/382] crypto: sha256-generic - Use API partial block handling Use the Crypto API partial block handling. Signed-off-by: Herbert Xu --- crypto/sha256_generic.c | 50 +++++++++++++++++------------------------ include/crypto/sha2.h | 6 ----- 2 files changed, 21 insertions(+), 35 deletions(-) diff --git a/crypto/sha256_generic.c b/crypto/sha256_generic.c index b00521f1a6d4..05084e5bbaec 100644 --- a/crypto/sha256_generic.c +++ b/crypto/sha256_generic.c @@ -8,14 +8,10 @@ * SHA224 Support Copyright 2007 Intel Corporation */ #include -#include -#include -#include -#include #include #include -#include -#include +#include +#include const u8 sha224_zero_message_hash[SHA224_DIGEST_SIZE] = { 0xd1, 0x4a, 0x02, 0x8c, 0x2a, 0x3a, 0x2b, 0xc9, 0x47, @@ -33,42 +29,37 @@ const u8 sha256_zero_message_hash[SHA256_DIGEST_SIZE] = { }; EXPORT_SYMBOL_GPL(sha256_zero_message_hash); -int crypto_sha256_update(struct shash_desc *desc, const u8 *data, - unsigned int len) +static void sha256_block(struct crypto_sha256_state *sctx, const u8 *input, + int blocks) { - sha256_update(shash_desc_ctx(desc), data, len); - return 0; -} -EXPORT_SYMBOL(crypto_sha256_update); - -static int crypto_sha256_final(struct shash_desc *desc, u8 *out) -{ - if (crypto_shash_digestsize(desc->tfm) == SHA224_DIGEST_SIZE) - sha224_final(shash_desc_ctx(desc), out); - else - sha256_final(shash_desc_ctx(desc), out); - return 0; + sha256_transform_blocks(sctx, input, blocks); } -int crypto_sha256_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *hash) +static int crypto_sha256_update(struct shash_desc *desc, const u8 *data, + unsigned int len) { - sha256_update(shash_desc_ctx(desc), data, len); - return crypto_sha256_final(desc, hash); + return sha256_base_do_update_blocks(desc, data, len, sha256_block); +} + +static int crypto_sha256_finup(struct shash_desc *desc, const u8 *data, + unsigned int len, u8 *hash) +{ + sha256_base_do_finup(desc, data, len, sha256_block); + return sha256_base_finish(desc, hash); } -EXPORT_SYMBOL(crypto_sha256_finup); static struct shash_alg sha256_algs[2] = { { .digestsize = SHA256_DIGEST_SIZE, .init = sha256_base_init, .update = crypto_sha256_update, - .final = crypto_sha256_final, .finup = crypto_sha256_finup, - .descsize = sizeof(struct sha256_state), + .descsize = sizeof(struct crypto_sha256_state), .base = { .cra_name = "sha256", .cra_driver_name= "sha256-generic", .cra_priority = 100, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA256_BLOCK_SIZE, .cra_module = THIS_MODULE, } @@ -76,13 +67,14 @@ static struct shash_alg sha256_algs[2] = { { .digestsize = SHA224_DIGEST_SIZE, .init = sha224_base_init, .update = crypto_sha256_update, - .final = crypto_sha256_final, .finup = crypto_sha256_finup, - .descsize = sizeof(struct sha256_state), + .descsize = sizeof(struct crypto_sha256_state), .base = { .cra_name = "sha224", .cra_driver_name= "sha224-generic", .cra_priority = 100, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA224_BLOCK_SIZE, .cra_module = THIS_MODULE, } diff --git a/include/crypto/sha2.h b/include/crypto/sha2.h index d9b1b9932393..a913bad5dd3b 100644 --- a/include/crypto/sha2.h +++ b/include/crypto/sha2.h @@ -83,12 +83,6 @@ struct sha512_state { struct shash_desc; -extern int crypto_sha256_update(struct shash_desc *desc, const u8 *data, - unsigned int len); - -extern int crypto_sha256_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *hash); - extern int crypto_sha512_update(struct shash_desc *desc, const u8 *data, unsigned int len); From e2d0c746700e3a1eec4db34054a3e65ec6c55ce0 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 18 Apr 2025 10:59:50 +0800 Subject: [PATCH 179/382] crypto: arm/sha256-ce - Use API partial block handling Use the Crypto API partial block handling. Also remove the unnecessary SIMD fallback path. Signed-off-by: Herbert Xu --- arch/arm/crypto/sha2-ce-glue.c | 52 ++++++++++------------------------ 1 file changed, 15 insertions(+), 37 deletions(-) diff --git a/arch/arm/crypto/sha2-ce-glue.c b/arch/arm/crypto/sha2-ce-glue.c index aeac45bfbf9f..1e9d16f79678 100644 --- a/arch/arm/crypto/sha2-ce-glue.c +++ b/arch/arm/crypto/sha2-ce-glue.c @@ -5,91 +5,69 @@ * Copyright (C) 2015 Linaro Ltd */ +#include #include -#include #include #include #include -#include +#include #include -#include -#include -#include -#include - -#include "sha256_glue.h" - MODULE_DESCRIPTION("SHA-224/SHA-256 secure hash using ARMv8 Crypto Extensions"); MODULE_AUTHOR("Ard Biesheuvel "); MODULE_LICENSE("GPL v2"); -asmlinkage void sha2_ce_transform(struct sha256_state *sst, u8 const *src, - int blocks); +asmlinkage void sha2_ce_transform(struct crypto_sha256_state *sst, + u8 const *src, int blocks); static int sha2_ce_update(struct shash_desc *desc, const u8 *data, unsigned int len) { - struct sha256_state *sctx = shash_desc_ctx(desc); - - if (!crypto_simd_usable() || - (sctx->count % SHA256_BLOCK_SIZE) + len < SHA256_BLOCK_SIZE) - return crypto_sha256_arm_update(desc, data, len); + int remain; kernel_neon_begin(); - sha256_base_do_update(desc, data, len, - (sha256_block_fn *)sha2_ce_transform); + remain = sha256_base_do_update_blocks(desc, data, len, + sha2_ce_transform); kernel_neon_end(); - - return 0; + return remain; } static int sha2_ce_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) { - if (!crypto_simd_usable()) - return crypto_sha256_arm_finup(desc, data, len, out); - kernel_neon_begin(); - if (len) - sha256_base_do_update(desc, data, len, - (sha256_block_fn *)sha2_ce_transform); - sha256_base_do_finalize(desc, (sha256_block_fn *)sha2_ce_transform); + sha256_base_do_finup(desc, data, len, sha2_ce_transform); kernel_neon_end(); - return sha256_base_finish(desc, out); } -static int sha2_ce_final(struct shash_desc *desc, u8 *out) -{ - return sha2_ce_finup(desc, NULL, 0, out); -} - static struct shash_alg algs[] = { { .init = sha224_base_init, .update = sha2_ce_update, - .final = sha2_ce_final, .finup = sha2_ce_finup, - .descsize = sizeof(struct sha256_state), + .descsize = sizeof(struct crypto_sha256_state), .digestsize = SHA224_DIGEST_SIZE, .base = { .cra_name = "sha224", .cra_driver_name = "sha224-ce", .cra_priority = 300, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA256_BLOCK_SIZE, .cra_module = THIS_MODULE, } }, { .init = sha256_base_init, .update = sha2_ce_update, - .final = sha2_ce_final, .finup = sha2_ce_finup, - .descsize = sizeof(struct sha256_state), + .descsize = sizeof(struct crypto_sha256_state), .digestsize = SHA256_DIGEST_SIZE, .base = { .cra_name = "sha256", .cra_driver_name = "sha256-ce", .cra_priority = 300, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA256_BLOCK_SIZE, .cra_module = THIS_MODULE, } From 83498139b63f8312e1be480a10111dec87d1e450 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 18 Apr 2025 10:59:52 +0800 Subject: [PATCH 180/382] crypto: arm/sha256-neon - Use API partial block handling Use the Crypto API partial block handling. Also remove the unnecessary SIMD fallback path. Signed-off-by: Herbert Xu --- arch/arm/crypto/sha256_neon_glue.c | 49 ++++++++++-------------------- 1 file changed, 16 insertions(+), 33 deletions(-) diff --git a/arch/arm/crypto/sha256_neon_glue.c b/arch/arm/crypto/sha256_neon_glue.c index ccdcfff71910..76eb3cdc21c9 100644 --- a/arch/arm/crypto/sha256_neon_glue.c +++ b/arch/arm/crypto/sha256_neon_glue.c @@ -9,69 +9,51 @@ * Copyright © 2014 Jussi Kivilinna */ +#include #include -#include -#include -#include #include #include -#include -#include -#include +#include +#include #include "sha256_glue.h" -asmlinkage void sha256_block_data_order_neon(struct sha256_state *digest, - const u8 *data, int num_blks); +asmlinkage void sha256_block_data_order_neon( + struct crypto_sha256_state *digest, const u8 *data, int num_blks); static int crypto_sha256_neon_update(struct shash_desc *desc, const u8 *data, unsigned int len) { - struct sha256_state *sctx = shash_desc_ctx(desc); - - if (!crypto_simd_usable() || - (sctx->count % SHA256_BLOCK_SIZE) + len < SHA256_BLOCK_SIZE) - return crypto_sha256_arm_update(desc, data, len); + int remain; kernel_neon_begin(); - sha256_base_do_update(desc, data, len, sha256_block_data_order_neon); + remain = sha256_base_do_update_blocks(desc, data, len, + sha256_block_data_order_neon); kernel_neon_end(); - - return 0; + return remain; } static int crypto_sha256_neon_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) { - if (!crypto_simd_usable()) - return crypto_sha256_arm_finup(desc, data, len, out); - kernel_neon_begin(); - if (len) - sha256_base_do_update(desc, data, len, - sha256_block_data_order_neon); - sha256_base_do_finalize(desc, sha256_block_data_order_neon); + sha256_base_do_finup(desc, data, len, sha256_block_data_order_neon); kernel_neon_end(); - return sha256_base_finish(desc, out); } -static int crypto_sha256_neon_final(struct shash_desc *desc, u8 *out) -{ - return crypto_sha256_neon_finup(desc, NULL, 0, out); -} - struct shash_alg sha256_neon_algs[] = { { .digestsize = SHA256_DIGEST_SIZE, .init = sha256_base_init, .update = crypto_sha256_neon_update, - .final = crypto_sha256_neon_final, .finup = crypto_sha256_neon_finup, - .descsize = sizeof(struct sha256_state), + .descsize = sizeof(struct crypto_sha256_state), .base = { .cra_name = "sha256", .cra_driver_name = "sha256-neon", .cra_priority = 250, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA256_BLOCK_SIZE, .cra_module = THIS_MODULE, } @@ -79,13 +61,14 @@ struct shash_alg sha256_neon_algs[] = { { .digestsize = SHA224_DIGEST_SIZE, .init = sha224_base_init, .update = crypto_sha256_neon_update, - .final = crypto_sha256_neon_final, .finup = crypto_sha256_neon_finup, - .descsize = sizeof(struct sha256_state), + .descsize = sizeof(struct crypto_sha256_state), .base = { .cra_name = "sha224", .cra_driver_name = "sha224-neon", .cra_priority = 250, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA224_BLOCK_SIZE, .cra_module = THIS_MODULE, } From 5cd5d39e7b0e77497ef3872518ecde32c94eec2c Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 18 Apr 2025 10:59:55 +0800 Subject: [PATCH 181/382] crypto: arm/sha256-asm - Use API partial block handling Use the Crypto API partial block handling. Signed-off-by: Herbert Xu --- arch/arm/crypto/sha256_glue.c | 46 ++++++++++++++--------------------- arch/arm/crypto/sha256_glue.h | 8 +----- 2 files changed, 19 insertions(+), 35 deletions(-) diff --git a/arch/arm/crypto/sha256_glue.c b/arch/arm/crypto/sha256_glue.c index f85933fdec75..d04c4e6bae6d 100644 --- a/arch/arm/crypto/sha256_glue.c +++ b/arch/arm/crypto/sha256_glue.c @@ -10,58 +10,47 @@ * Author: Tim Chen */ +#include #include -#include -#include -#include -#include -#include -#include #include #include -#include -#include +#include +#include #include "sha256_glue.h" -asmlinkage void sha256_block_data_order(struct sha256_state *state, +asmlinkage void sha256_block_data_order(struct crypto_sha256_state *state, const u8 *data, int num_blks); -int crypto_sha256_arm_update(struct shash_desc *desc, const u8 *data, - unsigned int len) +static int crypto_sha256_arm_update(struct shash_desc *desc, const u8 *data, + unsigned int len) { /* make sure casting to sha256_block_fn() is safe */ - BUILD_BUG_ON(offsetof(struct sha256_state, state) != 0); + BUILD_BUG_ON(offsetof(struct crypto_sha256_state, state) != 0); - return sha256_base_do_update(desc, data, len, sha256_block_data_order); + return sha256_base_do_update_blocks(desc, data, len, + sha256_block_data_order); } -EXPORT_SYMBOL(crypto_sha256_arm_update); -static int crypto_sha256_arm_final(struct shash_desc *desc, u8 *out) +static int crypto_sha256_arm_finup(struct shash_desc *desc, const u8 *data, + unsigned int len, u8 *out) { - sha256_base_do_finalize(desc, sha256_block_data_order); + sha256_base_do_finup(desc, data, len, sha256_block_data_order); return sha256_base_finish(desc, out); } -int crypto_sha256_arm_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - sha256_base_do_update(desc, data, len, sha256_block_data_order); - return crypto_sha256_arm_final(desc, out); -} -EXPORT_SYMBOL(crypto_sha256_arm_finup); - static struct shash_alg algs[] = { { .digestsize = SHA256_DIGEST_SIZE, .init = sha256_base_init, .update = crypto_sha256_arm_update, - .final = crypto_sha256_arm_final, .finup = crypto_sha256_arm_finup, - .descsize = sizeof(struct sha256_state), + .descsize = sizeof(struct crypto_sha256_state), .base = { .cra_name = "sha256", .cra_driver_name = "sha256-asm", .cra_priority = 150, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA256_BLOCK_SIZE, .cra_module = THIS_MODULE, } @@ -69,13 +58,14 @@ static struct shash_alg algs[] = { { .digestsize = SHA224_DIGEST_SIZE, .init = sha224_base_init, .update = crypto_sha256_arm_update, - .final = crypto_sha256_arm_final, .finup = crypto_sha256_arm_finup, - .descsize = sizeof(struct sha256_state), + .descsize = sizeof(struct crypto_sha256_state), .base = { .cra_name = "sha224", .cra_driver_name = "sha224-asm", .cra_priority = 150, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA224_BLOCK_SIZE, .cra_module = THIS_MODULE, } diff --git a/arch/arm/crypto/sha256_glue.h b/arch/arm/crypto/sha256_glue.h index 9f0d578bab5f..9881c9a115d1 100644 --- a/arch/arm/crypto/sha256_glue.h +++ b/arch/arm/crypto/sha256_glue.h @@ -2,14 +2,8 @@ #ifndef _CRYPTO_SHA256_GLUE_H #define _CRYPTO_SHA256_GLUE_H -#include +#include extern struct shash_alg sha256_neon_algs[2]; -int crypto_sha256_arm_update(struct shash_desc *desc, const u8 *data, - unsigned int len); - -int crypto_sha256_arm_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *hash); - #endif /* _CRYPTO_SHA256_GLUE_H */ From be32039547e4b367afe1a2738ded32b3ff390f3a Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 18 Apr 2025 10:59:57 +0800 Subject: [PATCH 182/382] crypto: arm64/sha256-ce - Use API partial block handling Use the Crypto API partial block handling. Also remove the unnecessary SIMD fallback path. Signed-off-by: Herbert Xu --- arch/arm64/crypto/sha2-ce-glue.c | 90 +++++++------------------------- 1 file changed, 18 insertions(+), 72 deletions(-) diff --git a/arch/arm64/crypto/sha2-ce-glue.c b/arch/arm64/crypto/sha2-ce-glue.c index 6b4866a88ded..912c215101eb 100644 --- a/arch/arm64/crypto/sha2-ce-glue.c +++ b/arch/arm64/crypto/sha2-ce-glue.c @@ -6,15 +6,13 @@ */ #include -#include -#include #include -#include #include #include #include -#include +#include #include +#include MODULE_DESCRIPTION("SHA-224/SHA-256 secure hash using ARMv8 Crypto Extensions"); MODULE_AUTHOR("Ard Biesheuvel "); @@ -23,7 +21,7 @@ MODULE_ALIAS_CRYPTO("sha224"); MODULE_ALIAS_CRYPTO("sha256"); struct sha256_ce_state { - struct sha256_state sst; + struct crypto_sha256_state sst; u32 finalize; }; @@ -33,7 +31,7 @@ extern const u32 sha256_ce_offsetof_finalize; asmlinkage int __sha256_ce_transform(struct sha256_ce_state *sst, u8 const *src, int blocks); -static void sha256_ce_transform(struct sha256_state *sst, u8 const *src, +static void sha256_ce_transform(struct crypto_sha256_state *sst, u8 const *src, int blocks) { while (blocks) { @@ -54,42 +52,21 @@ const u32 sha256_ce_offsetof_count = offsetof(struct sha256_ce_state, const u32 sha256_ce_offsetof_finalize = offsetof(struct sha256_ce_state, finalize); -asmlinkage void sha256_block_data_order(u32 *digest, u8 const *src, int blocks); - -static void sha256_arm64_transform(struct sha256_state *sst, u8 const *src, - int blocks) -{ - sha256_block_data_order(sst->state, src, blocks); -} - static int sha256_ce_update(struct shash_desc *desc, const u8 *data, unsigned int len) { struct sha256_ce_state *sctx = shash_desc_ctx(desc); - if (!crypto_simd_usable()) - return sha256_base_do_update(desc, data, len, - sha256_arm64_transform); - sctx->finalize = 0; - sha256_base_do_update(desc, data, len, sha256_ce_transform); - - return 0; + return sha256_base_do_update_blocks(desc, data, len, + sha256_ce_transform); } static int sha256_ce_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) { struct sha256_ce_state *sctx = shash_desc_ctx(desc); - bool finalize = !sctx->sst.count && !(len % SHA256_BLOCK_SIZE) && len; - - if (!crypto_simd_usable()) { - if (len) - sha256_base_do_update(desc, data, len, - sha256_arm64_transform); - sha256_base_do_finalize(desc, sha256_arm64_transform); - return sha256_base_finish(desc, out); - } + bool finalize = !(len % SHA256_BLOCK_SIZE) && len; /* * Allow the asm code to perform the finalization if there is no @@ -97,23 +74,11 @@ static int sha256_ce_finup(struct shash_desc *desc, const u8 *data, */ sctx->finalize = finalize; - sha256_base_do_update(desc, data, len, sha256_ce_transform); - if (!finalize) - sha256_base_do_finalize(desc, sha256_ce_transform); - return sha256_base_finish(desc, out); -} - -static int sha256_ce_final(struct shash_desc *desc, u8 *out) -{ - struct sha256_ce_state *sctx = shash_desc_ctx(desc); - - if (!crypto_simd_usable()) { - sha256_base_do_finalize(desc, sha256_arm64_transform); - return sha256_base_finish(desc, out); - } - - sctx->finalize = 0; - sha256_base_do_finalize(desc, sha256_ce_transform); + if (finalize) + sha256_base_do_update_blocks(desc, data, len, + sha256_ce_transform); + else + sha256_base_do_finup(desc, data, len, sha256_ce_transform); return sha256_base_finish(desc, out); } @@ -124,55 +89,36 @@ static int sha256_ce_digest(struct shash_desc *desc, const u8 *data, return sha256_ce_finup(desc, data, len, out); } -static int sha256_ce_export(struct shash_desc *desc, void *out) -{ - struct sha256_ce_state *sctx = shash_desc_ctx(desc); - - memcpy(out, &sctx->sst, sizeof(struct sha256_state)); - return 0; -} - -static int sha256_ce_import(struct shash_desc *desc, const void *in) -{ - struct sha256_ce_state *sctx = shash_desc_ctx(desc); - - memcpy(&sctx->sst, in, sizeof(struct sha256_state)); - sctx->finalize = 0; - return 0; -} - static struct shash_alg algs[] = { { .init = sha224_base_init, .update = sha256_ce_update, - .final = sha256_ce_final, .finup = sha256_ce_finup, - .export = sha256_ce_export, - .import = sha256_ce_import, .descsize = sizeof(struct sha256_ce_state), - .statesize = sizeof(struct sha256_state), + .statesize = sizeof(struct crypto_sha256_state), .digestsize = SHA224_DIGEST_SIZE, .base = { .cra_name = "sha224", .cra_driver_name = "sha224-ce", .cra_priority = 200, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA256_BLOCK_SIZE, .cra_module = THIS_MODULE, } }, { .init = sha256_base_init, .update = sha256_ce_update, - .final = sha256_ce_final, .finup = sha256_ce_finup, .digest = sha256_ce_digest, - .export = sha256_ce_export, - .import = sha256_ce_import, .descsize = sizeof(struct sha256_ce_state), - .statesize = sizeof(struct sha256_state), + .statesize = sizeof(struct crypto_sha256_state), .digestsize = SHA256_DIGEST_SIZE, .base = { .cra_name = "sha256", .cra_driver_name = "sha256-ce", .cra_priority = 200, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA256_BLOCK_SIZE, .cra_module = THIS_MODULE, } From a417f16f885d753115bc6a3735b1d8afa5f3b7c6 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 18 Apr 2025 10:59:59 +0800 Subject: [PATCH 183/382] crypto: arm64/sha256 - Use API partial block handling Use the Crypto API partial block handling. Also remove the unnecessary SIMD fallback path. Signed-off-by: Herbert Xu --- arch/arm64/crypto/sha256-glue.c | 97 +++++++++++++-------------------- 1 file changed, 37 insertions(+), 60 deletions(-) diff --git a/arch/arm64/crypto/sha256-glue.c b/arch/arm64/crypto/sha256-glue.c index 35356987cc1e..26f9fdfae87b 100644 --- a/arch/arm64/crypto/sha256-glue.c +++ b/arch/arm64/crypto/sha256-glue.c @@ -5,16 +5,13 @@ * Copyright (c) 2016 Linaro Ltd. */ -#include #include -#include #include -#include #include #include +#include +#include #include -#include -#include MODULE_DESCRIPTION("SHA-224/SHA-256 secure hash for arm64"); MODULE_AUTHOR("Andy Polyakov "); @@ -27,8 +24,8 @@ asmlinkage void sha256_block_data_order(u32 *digest, const void *data, unsigned int num_blks); EXPORT_SYMBOL(sha256_block_data_order); -static void sha256_arm64_transform(struct sha256_state *sst, u8 const *src, - int blocks) +static void sha256_arm64_transform(struct crypto_sha256_state *sst, + u8 const *src, int blocks) { sha256_block_data_order(sst->state, src, blocks); } @@ -36,55 +33,52 @@ static void sha256_arm64_transform(struct sha256_state *sst, u8 const *src, asmlinkage void sha256_block_neon(u32 *digest, const void *data, unsigned int num_blks); -static void sha256_neon_transform(struct sha256_state *sst, u8 const *src, - int blocks) +static void sha256_neon_transform(struct crypto_sha256_state *sst, + u8 const *src, int blocks) { + kernel_neon_begin(); sha256_block_neon(sst->state, src, blocks); + kernel_neon_end(); } static int crypto_sha256_arm64_update(struct shash_desc *desc, const u8 *data, unsigned int len) { - return sha256_base_do_update(desc, data, len, sha256_arm64_transform); + return sha256_base_do_update_blocks(desc, data, len, + sha256_arm64_transform); } static int crypto_sha256_arm64_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) { - if (len) - sha256_base_do_update(desc, data, len, sha256_arm64_transform); - sha256_base_do_finalize(desc, sha256_arm64_transform); - + sha256_base_do_finup(desc, data, len, sha256_arm64_transform); return sha256_base_finish(desc, out); } -static int crypto_sha256_arm64_final(struct shash_desc *desc, u8 *out) -{ - return crypto_sha256_arm64_finup(desc, NULL, 0, out); -} - static struct shash_alg algs[] = { { .digestsize = SHA256_DIGEST_SIZE, .init = sha256_base_init, .update = crypto_sha256_arm64_update, - .final = crypto_sha256_arm64_final, .finup = crypto_sha256_arm64_finup, - .descsize = sizeof(struct sha256_state), + .descsize = sizeof(struct crypto_sha256_state), .base.cra_name = "sha256", .base.cra_driver_name = "sha256-arm64", .base.cra_priority = 125, + .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .base.cra_blocksize = SHA256_BLOCK_SIZE, .base.cra_module = THIS_MODULE, }, { .digestsize = SHA224_DIGEST_SIZE, .init = sha224_base_init, .update = crypto_sha256_arm64_update, - .final = crypto_sha256_arm64_final, .finup = crypto_sha256_arm64_finup, - .descsize = sizeof(struct sha256_state), + .descsize = sizeof(struct crypto_sha256_state), .base.cra_name = "sha224", .base.cra_driver_name = "sha224-arm64", .base.cra_priority = 125, + .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .base.cra_blocksize = SHA224_BLOCK_SIZE, .base.cra_module = THIS_MODULE, } }; @@ -92,13 +86,7 @@ static struct shash_alg algs[] = { { static int sha256_update_neon(struct shash_desc *desc, const u8 *data, unsigned int len) { - struct sha256_state *sctx = shash_desc_ctx(desc); - - if (!crypto_simd_usable()) - return sha256_base_do_update(desc, data, len, - sha256_arm64_transform); - - while (len > 0) { + do { unsigned int chunk = len; /* @@ -106,65 +94,54 @@ static int sha256_update_neon(struct shash_desc *desc, const u8 *data, * input when running on a preemptible kernel, but process the * data block by block instead. */ - if (IS_ENABLED(CONFIG_PREEMPTION) && - chunk + sctx->count % SHA256_BLOCK_SIZE > SHA256_BLOCK_SIZE) - chunk = SHA256_BLOCK_SIZE - - sctx->count % SHA256_BLOCK_SIZE; + if (IS_ENABLED(CONFIG_PREEMPTION)) + chunk = SHA256_BLOCK_SIZE; - kernel_neon_begin(); - sha256_base_do_update(desc, data, chunk, sha256_neon_transform); - kernel_neon_end(); + chunk -= sha256_base_do_update_blocks(desc, data, chunk, + sha256_neon_transform); data += chunk; len -= chunk; - } - return 0; + } while (len >= SHA256_BLOCK_SIZE); + return len; } static int sha256_finup_neon(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) { - if (!crypto_simd_usable()) { - if (len) - sha256_base_do_update(desc, data, len, - sha256_arm64_transform); - sha256_base_do_finalize(desc, sha256_arm64_transform); - } else { - if (len) - sha256_update_neon(desc, data, len); - kernel_neon_begin(); - sha256_base_do_finalize(desc, sha256_neon_transform); - kernel_neon_end(); - } - return sha256_base_finish(desc, out); -} + if (len >= SHA256_BLOCK_SIZE) { + int remain = sha256_update_neon(desc, data, len); -static int sha256_final_neon(struct shash_desc *desc, u8 *out) -{ - return sha256_finup_neon(desc, NULL, 0, out); + data += len - remain; + len = remain; + } + sha256_base_do_finup(desc, data, len, sha256_neon_transform); + return sha256_base_finish(desc, out); } static struct shash_alg neon_algs[] = { { .digestsize = SHA256_DIGEST_SIZE, .init = sha256_base_init, .update = sha256_update_neon, - .final = sha256_final_neon, .finup = sha256_finup_neon, - .descsize = sizeof(struct sha256_state), + .descsize = sizeof(struct crypto_sha256_state), .base.cra_name = "sha256", .base.cra_driver_name = "sha256-arm64-neon", .base.cra_priority = 150, + .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .base.cra_blocksize = SHA256_BLOCK_SIZE, .base.cra_module = THIS_MODULE, }, { .digestsize = SHA224_DIGEST_SIZE, .init = sha224_base_init, .update = sha256_update_neon, - .final = sha256_final_neon, .finup = sha256_finup_neon, - .descsize = sizeof(struct sha256_state), + .descsize = sizeof(struct crypto_sha256_state), .base.cra_name = "sha224", .base.cra_driver_name = "sha224-arm64-neon", .base.cra_priority = 150, + .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .base.cra_blocksize = SHA224_BLOCK_SIZE, .base.cra_module = THIS_MODULE, } }; From 9cd9c99be1211b2b40cd8708e3aeae8afcedd371 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 18 Apr 2025 11:00:01 +0800 Subject: [PATCH 184/382] crypto: powerpc/sha256-spe - Use API partial block handling Use the Crypto API partial block handling. Signed-off-by: Herbert Xu --- arch/powerpc/crypto/sha256-spe-glue.c | 167 +++++--------------------- 1 file changed, 30 insertions(+), 137 deletions(-) diff --git a/arch/powerpc/crypto/sha256-spe-glue.c b/arch/powerpc/crypto/sha256-spe-glue.c index 2997d13236e0..42c76bf8062d 100644 --- a/arch/powerpc/crypto/sha256-spe-glue.c +++ b/arch/powerpc/crypto/sha256-spe-glue.c @@ -8,16 +8,13 @@ * Copyright (c) 2015 Markus Stockhausen */ +#include #include -#include -#include -#include -#include #include #include -#include -#include -#include +#include +#include +#include /* * MAX_BYTES defines the number of bytes that are allowed to be processed @@ -47,151 +44,48 @@ static void spe_end(void) preempt_enable(); } -static inline void ppc_sha256_clear_context(struct sha256_state *sctx) +static void ppc_spe_sha256_block(struct crypto_sha256_state *sctx, + const u8 *src, int blocks) { - int count = sizeof(struct sha256_state) >> 2; - u32 *ptr = (u32 *)sctx; + do { + /* cut input data into smaller blocks */ + int unit = min(blocks, MAX_BYTES / SHA256_BLOCK_SIZE); - /* make sure we can clear the fast way */ - BUILD_BUG_ON(sizeof(struct sha256_state) % 4); - do { *ptr++ = 0; } while (--count); + spe_begin(); + ppc_spe_sha256_transform(sctx->state, src, unit); + spe_end(); + + src += unit * SHA256_BLOCK_SIZE; + blocks -= unit; + } while (blocks); } static int ppc_spe_sha256_update(struct shash_desc *desc, const u8 *data, unsigned int len) { - struct sha256_state *sctx = shash_desc_ctx(desc); - const unsigned int offset = sctx->count & 0x3f; - const unsigned int avail = 64 - offset; - unsigned int bytes; - const u8 *src = data; - - if (avail > len) { - sctx->count += len; - memcpy((char *)sctx->buf + offset, src, len); - return 0; - } - - sctx->count += len; - - if (offset) { - memcpy((char *)sctx->buf + offset, src, avail); - - spe_begin(); - ppc_spe_sha256_transform(sctx->state, (const u8 *)sctx->buf, 1); - spe_end(); - - len -= avail; - src += avail; - } - - while (len > 63) { - /* cut input data into smaller blocks */ - bytes = (len > MAX_BYTES) ? MAX_BYTES : len; - bytes = bytes & ~0x3f; - - spe_begin(); - ppc_spe_sha256_transform(sctx->state, src, bytes >> 6); - spe_end(); - - src += bytes; - len -= bytes; - } - - memcpy((char *)sctx->buf, src, len); - return 0; + return sha256_base_do_update_blocks(desc, data, len, + ppc_spe_sha256_block); } -static int ppc_spe_sha256_final(struct shash_desc *desc, u8 *out) +static int ppc_spe_sha256_finup(struct shash_desc *desc, const u8 *src, + unsigned int len, u8 *out) { - struct sha256_state *sctx = shash_desc_ctx(desc); - const unsigned int offset = sctx->count & 0x3f; - char *p = (char *)sctx->buf + offset; - int padlen; - __be64 *pbits = (__be64 *)(((char *)&sctx->buf) + 56); - __be32 *dst = (__be32 *)out; - - padlen = 55 - offset; - *p++ = 0x80; - - spe_begin(); - - if (padlen < 0) { - memset(p, 0x00, padlen + sizeof (u64)); - ppc_spe_sha256_transform(sctx->state, sctx->buf, 1); - p = (char *)sctx->buf; - padlen = 56; - } - - memset(p, 0, padlen); - *pbits = cpu_to_be64(sctx->count << 3); - ppc_spe_sha256_transform(sctx->state, sctx->buf, 1); - - spe_end(); - - dst[0] = cpu_to_be32(sctx->state[0]); - dst[1] = cpu_to_be32(sctx->state[1]); - dst[2] = cpu_to_be32(sctx->state[2]); - dst[3] = cpu_to_be32(sctx->state[3]); - dst[4] = cpu_to_be32(sctx->state[4]); - dst[5] = cpu_to_be32(sctx->state[5]); - dst[6] = cpu_to_be32(sctx->state[6]); - dst[7] = cpu_to_be32(sctx->state[7]); - - ppc_sha256_clear_context(sctx); - return 0; -} - -static int ppc_spe_sha224_final(struct shash_desc *desc, u8 *out) -{ - __be32 D[SHA256_DIGEST_SIZE >> 2]; - __be32 *dst = (__be32 *)out; - - ppc_spe_sha256_final(desc, (u8 *)D); - - /* avoid bytewise memcpy */ - dst[0] = D[0]; - dst[1] = D[1]; - dst[2] = D[2]; - dst[3] = D[3]; - dst[4] = D[4]; - dst[5] = D[5]; - dst[6] = D[6]; - - /* clear sensitive data */ - memzero_explicit(D, SHA256_DIGEST_SIZE); - return 0; -} - -static int ppc_spe_sha256_export(struct shash_desc *desc, void *out) -{ - struct sha256_state *sctx = shash_desc_ctx(desc); - - memcpy(out, sctx, sizeof(*sctx)); - return 0; -} - -static int ppc_spe_sha256_import(struct shash_desc *desc, const void *in) -{ - struct sha256_state *sctx = shash_desc_ctx(desc); - - memcpy(sctx, in, sizeof(*sctx)); - return 0; + sha256_base_do_finup(desc, src, len, ppc_spe_sha256_block); + return sha256_base_finish(desc, out); } static struct shash_alg algs[2] = { { .digestsize = SHA256_DIGEST_SIZE, .init = sha256_base_init, .update = ppc_spe_sha256_update, - .final = ppc_spe_sha256_final, - .export = ppc_spe_sha256_export, - .import = ppc_spe_sha256_import, - .descsize = sizeof(struct sha256_state), - .statesize = sizeof(struct sha256_state), + .finup = ppc_spe_sha256_finup, + .descsize = sizeof(struct crypto_sha256_state), .base = { .cra_name = "sha256", .cra_driver_name= "sha256-ppc-spe", .cra_priority = 300, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA256_BLOCK_SIZE, .cra_module = THIS_MODULE, } @@ -199,15 +93,14 @@ static struct shash_alg algs[2] = { { .digestsize = SHA224_DIGEST_SIZE, .init = sha224_base_init, .update = ppc_spe_sha256_update, - .final = ppc_spe_sha224_final, - .export = ppc_spe_sha256_export, - .import = ppc_spe_sha256_import, - .descsize = sizeof(struct sha256_state), - .statesize = sizeof(struct sha256_state), + .finup = ppc_spe_sha256_finup, + .descsize = sizeof(struct crypto_sha256_state), .base = { .cra_name = "sha224", .cra_driver_name= "sha224-ppc-spe", .cra_priority = 300, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA224_BLOCK_SIZE, .cra_module = THIS_MODULE, } From 1340113bdb827a00ba031d04a2510e23175a42b2 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 18 Apr 2025 11:00:04 +0800 Subject: [PATCH 185/382] crypto: s390/sha256 - Use API partial block handling Use the Crypto API partial block handling. Signed-off-by: Herbert Xu --- arch/s390/crypto/sha256_s390.c | 35 +++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/arch/s390/crypto/sha256_s390.c b/arch/s390/crypto/sha256_s390.c index 6f1ccdf93d3e..e6876c49414d 100644 --- a/arch/s390/crypto/sha256_s390.c +++ b/arch/s390/crypto/sha256_s390.c @@ -8,12 +8,13 @@ * Copyright IBM Corp. 2005, 2011 * Author(s): Jan Glauber (jang@de.ibm.com) */ -#include -#include -#include -#include -#include #include +#include +#include +#include +#include +#include +#include #include "sha.h" @@ -38,22 +39,20 @@ static int s390_sha256_init(struct shash_desc *desc) static int sha256_export(struct shash_desc *desc, void *out) { struct s390_sha_ctx *sctx = shash_desc_ctx(desc); - struct sha256_state *octx = out; + struct crypto_sha256_state *octx = out; octx->count = sctx->count; memcpy(octx->state, sctx->state, sizeof(octx->state)); - memcpy(octx->buf, sctx->buf, sizeof(octx->buf)); return 0; } static int sha256_import(struct shash_desc *desc, const void *in) { struct s390_sha_ctx *sctx = shash_desc_ctx(desc); - const struct sha256_state *ictx = in; + const struct crypto_sha256_state *ictx = in; sctx->count = ictx->count; memcpy(sctx->state, ictx->state, sizeof(ictx->state)); - memcpy(sctx->buf, ictx->buf, sizeof(ictx->buf)); sctx->func = CPACF_KIMD_SHA_256; return 0; } @@ -61,16 +60,17 @@ static int sha256_import(struct shash_desc *desc, const void *in) static struct shash_alg sha256_alg = { .digestsize = SHA256_DIGEST_SIZE, .init = s390_sha256_init, - .update = s390_sha_update, - .final = s390_sha_final, + .update = s390_sha_update_blocks, + .finup = s390_sha_finup, .export = sha256_export, .import = sha256_import, - .descsize = sizeof(struct s390_sha_ctx), - .statesize = sizeof(struct sha256_state), + .descsize = S390_SHA_CTX_SIZE, + .statesize = sizeof(struct crypto_sha256_state), .base = { .cra_name = "sha256", .cra_driver_name= "sha256-s390", .cra_priority = 300, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_blocksize = SHA256_BLOCK_SIZE, .cra_module = THIS_MODULE, } @@ -97,16 +97,17 @@ static int s390_sha224_init(struct shash_desc *desc) static struct shash_alg sha224_alg = { .digestsize = SHA224_DIGEST_SIZE, .init = s390_sha224_init, - .update = s390_sha_update, - .final = s390_sha_final, + .update = s390_sha_update_blocks, + .finup = s390_sha_finup, .export = sha256_export, .import = sha256_import, - .descsize = sizeof(struct s390_sha_ctx), - .statesize = sizeof(struct sha256_state), + .descsize = S390_SHA_CTX_SIZE, + .statesize = sizeof(struct crypto_sha256_state), .base = { .cra_name = "sha224", .cra_driver_name= "sha224-s390", .cra_priority = 300, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_blocksize = SHA224_BLOCK_SIZE, .cra_module = THIS_MODULE, } From 4dc9479742d06cfa974a603f6a320157284c3dd4 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 18 Apr 2025 11:00:06 +0800 Subject: [PATCH 186/382] crypto: sparc/sha256 - Use API partial block handling Use the Crypto API partial block handling. Signed-off-by: Herbert Xu --- arch/sparc/crypto/sha256_glue.c | 121 ++++++-------------------------- 1 file changed, 20 insertions(+), 101 deletions(-) diff --git a/arch/sparc/crypto/sha256_glue.c b/arch/sparc/crypto/sha256_glue.c index 285561a1cde5..ddb250242faf 100644 --- a/arch/sparc/crypto/sha256_glue.c +++ b/arch/sparc/crypto/sha256_glue.c @@ -11,133 +11,50 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt +#include +#include #include -#include -#include -#include -#include #include #include - -#include -#include +#include +#include #include "opcodes.h" asmlinkage void sha256_sparc64_transform(u32 *digest, const char *data, unsigned int rounds); -static void __sha256_sparc64_update(struct sha256_state *sctx, const u8 *data, - unsigned int len, unsigned int partial) +static void sha256_block(struct crypto_sha256_state *sctx, const u8 *src, + int blocks) { - unsigned int done = 0; - - sctx->count += len; - if (partial) { - done = SHA256_BLOCK_SIZE - partial; - memcpy(sctx->buf + partial, data, done); - sha256_sparc64_transform(sctx->state, sctx->buf, 1); - } - if (len - done >= SHA256_BLOCK_SIZE) { - const unsigned int rounds = (len - done) / SHA256_BLOCK_SIZE; - - sha256_sparc64_transform(sctx->state, data + done, rounds); - done += rounds * SHA256_BLOCK_SIZE; - } - - memcpy(sctx->buf, data + done, len - done); + sha256_sparc64_transform(sctx->state, src, blocks); } static int sha256_sparc64_update(struct shash_desc *desc, const u8 *data, unsigned int len) { - struct sha256_state *sctx = shash_desc_ctx(desc); - unsigned int partial = sctx->count % SHA256_BLOCK_SIZE; - - /* Handle the fast case right here */ - if (partial + len < SHA256_BLOCK_SIZE) { - sctx->count += len; - memcpy(sctx->buf + partial, data, len); - } else - __sha256_sparc64_update(sctx, data, len, partial); - - return 0; + return sha256_base_do_update_blocks(desc, data, len, sha256_block); } -static int sha256_sparc64_final(struct shash_desc *desc, u8 *out) +static int sha256_sparc64_finup(struct shash_desc *desc, const u8 *src, + unsigned int len, u8 *out) { - struct sha256_state *sctx = shash_desc_ctx(desc); - unsigned int i, index, padlen; - __be32 *dst = (__be32 *)out; - __be64 bits; - static const u8 padding[SHA256_BLOCK_SIZE] = { 0x80, }; - - bits = cpu_to_be64(sctx->count << 3); - - /* Pad out to 56 mod 64 and append length */ - index = sctx->count % SHA256_BLOCK_SIZE; - padlen = (index < 56) ? (56 - index) : ((SHA256_BLOCK_SIZE+56) - index); - - /* We need to fill a whole block for __sha256_sparc64_update() */ - if (padlen <= 56) { - sctx->count += padlen; - memcpy(sctx->buf + index, padding, padlen); - } else { - __sha256_sparc64_update(sctx, padding, padlen, index); - } - __sha256_sparc64_update(sctx, (const u8 *)&bits, sizeof(bits), 56); - - /* Store state in digest */ - for (i = 0; i < 8; i++) - dst[i] = cpu_to_be32(sctx->state[i]); - - /* Wipe context */ - memset(sctx, 0, sizeof(*sctx)); - - return 0; -} - -static int sha224_sparc64_final(struct shash_desc *desc, u8 *hash) -{ - u8 D[SHA256_DIGEST_SIZE]; - - sha256_sparc64_final(desc, D); - - memcpy(hash, D, SHA224_DIGEST_SIZE); - memzero_explicit(D, SHA256_DIGEST_SIZE); - - return 0; -} - -static int sha256_sparc64_export(struct shash_desc *desc, void *out) -{ - struct sha256_state *sctx = shash_desc_ctx(desc); - - memcpy(out, sctx, sizeof(*sctx)); - return 0; -} - -static int sha256_sparc64_import(struct shash_desc *desc, const void *in) -{ - struct sha256_state *sctx = shash_desc_ctx(desc); - - memcpy(sctx, in, sizeof(*sctx)); - return 0; + sha256_base_do_finup(desc, src, len, sha256_block); + return sha256_base_finish(desc, out); } static struct shash_alg sha256_alg = { .digestsize = SHA256_DIGEST_SIZE, .init = sha256_base_init, .update = sha256_sparc64_update, - .final = sha256_sparc64_final, - .export = sha256_sparc64_export, - .import = sha256_sparc64_import, - .descsize = sizeof(struct sha256_state), - .statesize = sizeof(struct sha256_state), + .finup = sha256_sparc64_finup, + .descsize = sizeof(struct crypto_sha256_state), .base = { .cra_name = "sha256", .cra_driver_name= "sha256-sparc64", .cra_priority = SPARC_CR_OPCODE_PRIORITY, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA256_BLOCK_SIZE, .cra_module = THIS_MODULE, } @@ -147,12 +64,14 @@ static struct shash_alg sha224_alg = { .digestsize = SHA224_DIGEST_SIZE, .init = sha224_base_init, .update = sha256_sparc64_update, - .final = sha224_sparc64_final, - .descsize = sizeof(struct sha256_state), + .finup = sha256_sparc64_finup, + .descsize = sizeof(struct crypto_sha256_state), .base = { .cra_name = "sha224", .cra_driver_name= "sha224-sparc64", .cra_priority = SPARC_CR_OPCODE_PRIORITY, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA224_BLOCK_SIZE, .cra_module = THIS_MODULE, } From 8ba81fef400bf5dec895bc259d65c91bfece798a Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 18 Apr 2025 11:00:08 +0800 Subject: [PATCH 187/382] crypto: sha256_base - Remove partial block helpers Now that all sha256_base users have been converted to use the API partial block handling, remove the partial block helpers. Signed-off-by: Herbert Xu --- arch/x86/crypto/sha256_ssse3_glue.c | 4 +-- include/crypto/sha256_base.h | 50 ++++++++--------------------- lib/crypto/sha256.c | 11 ++----- 3 files changed, 17 insertions(+), 48 deletions(-) diff --git a/arch/x86/crypto/sha256_ssse3_glue.c b/arch/x86/crypto/sha256_ssse3_glue.c index 367ce4830fa4..a5d3be00550b 100644 --- a/arch/x86/crypto/sha256_ssse3_glue.c +++ b/arch/x86/crypto/sha256_ssse3_glue.c @@ -51,7 +51,7 @@ MODULE_DEVICE_TABLE(x86cpu, module_cpu_ids); static int _sha256_update(struct shash_desc *desc, const u8 *data, unsigned int len, - crypto_sha256_block_fn *sha256_xform) + sha256_block_fn *sha256_xform) { int remain; @@ -69,7 +69,7 @@ static int _sha256_update(struct shash_desc *desc, const u8 *data, } static int sha256_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out, crypto_sha256_block_fn *sha256_xform) + unsigned int len, u8 *out, sha256_block_fn *sha256_xform) { kernel_fpu_begin(); sha256_base_do_finup(desc, data, len, sha256_xform); diff --git a/include/crypto/sha256_base.h b/include/crypto/sha256_base.h index b9d3583b6256..08cd5e41d4fd 100644 --- a/include/crypto/sha256_base.h +++ b/include/crypto/sha256_base.h @@ -15,10 +15,8 @@ #include #include -typedef void (sha256_block_fn)(struct sha256_state *sst, u8 const *src, +typedef void (sha256_block_fn)(struct crypto_sha256_state *sst, u8 const *src, int blocks); -typedef void (crypto_sha256_block_fn)(struct crypto_sha256_state *sst, - u8 const *src, int blocks); static inline int sha224_base_init(struct shash_desc *desc) { @@ -42,6 +40,7 @@ static inline int lib_sha256_base_do_update(struct sha256_state *sctx, sha256_block_fn *block_fn) { unsigned int partial = sctx->count % SHA256_BLOCK_SIZE; + struct crypto_sha256_state *state = (void *)sctx; sctx->count += len; @@ -55,14 +54,14 @@ static inline int lib_sha256_base_do_update(struct sha256_state *sctx, data += p; len -= p; - block_fn(sctx, sctx->buf, 1); + block_fn(state, sctx->buf, 1); } blocks = len / SHA256_BLOCK_SIZE; len %= SHA256_BLOCK_SIZE; if (blocks) { - block_fn(sctx, data, blocks); + block_fn(state, data, blocks); data += blocks * SHA256_BLOCK_SIZE; } partial = 0; @@ -73,19 +72,9 @@ static inline int lib_sha256_base_do_update(struct sha256_state *sctx, return 0; } -static inline int sha256_base_do_update(struct shash_desc *desc, - const u8 *data, - unsigned int len, - sha256_block_fn *block_fn) -{ - struct sha256_state *sctx = shash_desc_ctx(desc); - - return lib_sha256_base_do_update(sctx, data, len, block_fn); -} - static inline int lib_sha256_base_do_update_blocks( struct crypto_sha256_state *sctx, const u8 *data, unsigned int len, - crypto_sha256_block_fn *block_fn) + sha256_block_fn *block_fn) { unsigned int remain = len - round_down(len, SHA256_BLOCK_SIZE); @@ -96,7 +85,7 @@ static inline int lib_sha256_base_do_update_blocks( static inline int sha256_base_do_update_blocks( struct shash_desc *desc, const u8 *data, unsigned int len, - crypto_sha256_block_fn *block_fn) + sha256_block_fn *block_fn) { return lib_sha256_base_do_update_blocks(shash_desc_ctx(desc), data, len, block_fn); @@ -104,7 +93,7 @@ static inline int sha256_base_do_update_blocks( static inline int lib_sha256_base_do_finup(struct crypto_sha256_state *sctx, const u8 *src, unsigned int len, - crypto_sha256_block_fn *block_fn) + sha256_block_fn *block_fn) { unsigned int bit_offset = SHA256_BLOCK_SIZE / 8 - 1; union { @@ -126,7 +115,7 @@ static inline int lib_sha256_base_do_finup(struct crypto_sha256_state *sctx, static inline int sha256_base_do_finup(struct shash_desc *desc, const u8 *src, unsigned int len, - crypto_sha256_block_fn *block_fn) + sha256_block_fn *block_fn) { struct crypto_sha256_state *sctx = shash_desc_ctx(desc); @@ -144,23 +133,11 @@ static inline int sha256_base_do_finup(struct shash_desc *desc, static inline int lib_sha256_base_do_finalize(struct sha256_state *sctx, sha256_block_fn *block_fn) { - const int bit_offset = SHA256_BLOCK_SIZE - sizeof(__be64); - __be64 *bits = (__be64 *)(sctx->buf + bit_offset); unsigned int partial = sctx->count % SHA256_BLOCK_SIZE; + struct crypto_sha256_state *state = (void *)sctx; - sctx->buf[partial++] = 0x80; - if (partial > bit_offset) { - memset(sctx->buf + partial, 0x0, SHA256_BLOCK_SIZE - partial); - partial = 0; - - block_fn(sctx, sctx->buf, 1); - } - - memset(sctx->buf + partial, 0x0, bit_offset - partial); - *bits = cpu_to_be64(sctx->count << 3); - block_fn(sctx, sctx->buf, 1); - - return 0; + sctx->count -= partial; + return lib_sha256_base_do_finup(state, sctx->buf, partial, block_fn); } static inline int sha256_base_do_finalize(struct shash_desc *desc, @@ -182,12 +159,11 @@ static inline int __sha256_base_finish(u32 state[SHA256_DIGEST_SIZE / 4], return 0; } -static inline int lib_sha256_base_finish(struct sha256_state *sctx, u8 *out, - unsigned int digest_size) +static inline void lib_sha256_base_finish(struct sha256_state *sctx, u8 *out, + unsigned int digest_size) { __sha256_base_finish(sctx->state, out, digest_size); memzero_explicit(sctx, sizeof(*sctx)); - return 0; } static inline int sha256_base_finish(struct shash_desc *desc, u8 *out) diff --git a/lib/crypto/sha256.c b/lib/crypto/sha256.c index 39ead0222937..a89bab377de1 100644 --- a/lib/crypto/sha256.c +++ b/lib/crypto/sha256.c @@ -132,22 +132,15 @@ void sha256_transform_blocks(struct crypto_sha256_state *sst, } EXPORT_SYMBOL_GPL(sha256_transform_blocks); -static void lib_sha256_transform_blocks(struct sha256_state *sctx, - const u8 *input, int blocks) -{ - sha256_transform_blocks((struct crypto_sha256_state *)sctx, input, - blocks); -} - void sha256_update(struct sha256_state *sctx, const u8 *data, unsigned int len) { - lib_sha256_base_do_update(sctx, data, len, lib_sha256_transform_blocks); + lib_sha256_base_do_update(sctx, data, len, sha256_transform_blocks); } EXPORT_SYMBOL(sha256_update); static void __sha256_final(struct sha256_state *sctx, u8 *out, int digest_size) { - lib_sha256_base_do_finalize(sctx, lib_sha256_transform_blocks); + lib_sha256_base_do_finalize(sctx, sha256_transform_blocks); lib_sha256_base_finish(sctx, out, digest_size); } From b333c273ab43b96ca25469770d3319ec7ababd5a Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 18 Apr 2025 11:00:11 +0800 Subject: [PATCH 188/382] crypto: arm64/sha3-ce - Use API partial block handling Use the Crypto API partial block handling. Also remove the unnecessary SIMD fallback path. Signed-off-by: Herbert Xu --- arch/arm64/crypto/sha3-ce-glue.c | 107 +++++++++++++------------------ arch/s390/crypto/sha.h | 1 - include/crypto/sha3.h | 4 +- 3 files changed, 49 insertions(+), 63 deletions(-) diff --git a/arch/arm64/crypto/sha3-ce-glue.c b/arch/arm64/crypto/sha3-ce-glue.c index 5662c3ac49e9..b4f1001046c9 100644 --- a/arch/arm64/crypto/sha3-ce-glue.c +++ b/arch/arm64/crypto/sha3-ce-glue.c @@ -12,13 +12,13 @@ #include #include #include -#include #include -#include #include #include -#include +#include #include +#include +#include MODULE_DESCRIPTION("SHA3 secure hash using ARMv8 Crypto Extensions"); MODULE_AUTHOR("Ard Biesheuvel "); @@ -35,74 +35,55 @@ static int sha3_update(struct shash_desc *desc, const u8 *data, unsigned int len) { struct sha3_state *sctx = shash_desc_ctx(desc); - unsigned int digest_size = crypto_shash_digestsize(desc->tfm); + struct crypto_shash *tfm = desc->tfm; + unsigned int bs, ds; + int blocks; - if (!crypto_simd_usable()) - return crypto_sha3_update(desc, data, len); + ds = crypto_shash_digestsize(tfm); + bs = crypto_shash_blocksize(tfm); + blocks = len / bs; + len -= blocks * bs; + do { + int rem; - if ((sctx->partial + len) >= sctx->rsiz) { - int blocks; - - if (sctx->partial) { - int p = sctx->rsiz - sctx->partial; - - memcpy(sctx->buf + sctx->partial, data, p); - kernel_neon_begin(); - sha3_ce_transform(sctx->st, sctx->buf, 1, digest_size); - kernel_neon_end(); - - data += p; - len -= p; - sctx->partial = 0; - } - - blocks = len / sctx->rsiz; - len %= sctx->rsiz; - - while (blocks) { - int rem; - - kernel_neon_begin(); - rem = sha3_ce_transform(sctx->st, data, blocks, - digest_size); - kernel_neon_end(); - data += (blocks - rem) * sctx->rsiz; - blocks = rem; - } - } - - if (len) { - memcpy(sctx->buf + sctx->partial, data, len); - sctx->partial += len; - } - return 0; + kernel_neon_begin(); + rem = sha3_ce_transform(sctx->st, data, blocks, ds); + kernel_neon_end(); + data += (blocks - rem) * bs; + blocks = rem; + } while (blocks); + return len; } -static int sha3_final(struct shash_desc *desc, u8 *out) +static int sha3_finup(struct shash_desc *desc, const u8 *src, unsigned int len, + u8 *out) { struct sha3_state *sctx = shash_desc_ctx(desc); - unsigned int digest_size = crypto_shash_digestsize(desc->tfm); + struct crypto_shash *tfm = desc->tfm; __le64 *digest = (__le64 *)out; + u8 block[SHA3_224_BLOCK_SIZE]; + unsigned int bs, ds; int i; - if (!crypto_simd_usable()) - return crypto_sha3_final(desc, out); + ds = crypto_shash_digestsize(tfm); + bs = crypto_shash_blocksize(tfm); + memcpy(block, src, len); - sctx->buf[sctx->partial++] = 0x06; - memset(sctx->buf + sctx->partial, 0, sctx->rsiz - sctx->partial); - sctx->buf[sctx->rsiz - 1] |= 0x80; + block[len++] = 0x06; + memset(block + len, 0, bs - len); + block[bs - 1] |= 0x80; kernel_neon_begin(); - sha3_ce_transform(sctx->st, sctx->buf, 1, digest_size); + sha3_ce_transform(sctx->st, block, 1, ds); kernel_neon_end(); + memzero_explicit(block , sizeof(block)); - for (i = 0; i < digest_size / 8; i++) + for (i = 0; i < ds / 8; i++) put_unaligned_le64(sctx->st[i], digest++); - if (digest_size & 4) + if (ds & 4) put_unaligned_le32(sctx->st[i], (__le32 *)digest); - memzero_explicit(sctx, sizeof(*sctx)); return 0; } @@ -110,10 +91,11 @@ static struct shash_alg algs[] = { { .digestsize = SHA3_224_DIGEST_SIZE, .init = crypto_sha3_init, .update = sha3_update, - .final = sha3_final, - .descsize = sizeof(struct sha3_state), + .finup = sha3_finup, + .descsize = SHA3_STATE_SIZE, .base.cra_name = "sha3-224", .base.cra_driver_name = "sha3-224-ce", + .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .base.cra_blocksize = SHA3_224_BLOCK_SIZE, .base.cra_module = THIS_MODULE, .base.cra_priority = 200, @@ -121,10 +103,11 @@ static struct shash_alg algs[] = { { .digestsize = SHA3_256_DIGEST_SIZE, .init = crypto_sha3_init, .update = sha3_update, - .final = sha3_final, - .descsize = sizeof(struct sha3_state), + .finup = sha3_finup, + .descsize = SHA3_STATE_SIZE, .base.cra_name = "sha3-256", .base.cra_driver_name = "sha3-256-ce", + .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .base.cra_blocksize = SHA3_256_BLOCK_SIZE, .base.cra_module = THIS_MODULE, .base.cra_priority = 200, @@ -132,10 +115,11 @@ static struct shash_alg algs[] = { { .digestsize = SHA3_384_DIGEST_SIZE, .init = crypto_sha3_init, .update = sha3_update, - .final = sha3_final, - .descsize = sizeof(struct sha3_state), + .finup = sha3_finup, + .descsize = SHA3_STATE_SIZE, .base.cra_name = "sha3-384", .base.cra_driver_name = "sha3-384-ce", + .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .base.cra_blocksize = SHA3_384_BLOCK_SIZE, .base.cra_module = THIS_MODULE, .base.cra_priority = 200, @@ -143,10 +127,11 @@ static struct shash_alg algs[] = { { .digestsize = SHA3_512_DIGEST_SIZE, .init = crypto_sha3_init, .update = sha3_update, - .final = sha3_final, - .descsize = sizeof(struct sha3_state), + .finup = sha3_finup, + .descsize = SHA3_STATE_SIZE, .base.cra_name = "sha3-512", .base.cra_driver_name = "sha3-512-ce", + .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .base.cra_blocksize = SHA3_512_BLOCK_SIZE, .base.cra_module = THIS_MODULE, .base.cra_priority = 200, diff --git a/arch/s390/crypto/sha.h b/arch/s390/crypto/sha.h index b8aeb51b2f3d..d95437ebe1ca 100644 --- a/arch/s390/crypto/sha.h +++ b/arch/s390/crypto/sha.h @@ -14,7 +14,6 @@ #include /* must be big enough for the largest SHA variant */ -#define SHA3_STATE_SIZE 200 #define CPACF_MAX_PARMBLOCK_SIZE SHA3_STATE_SIZE #define SHA_MAX_BLOCK_SIZE SHA3_224_BLOCK_SIZE #define S390_SHA_CTX_SIZE offsetof(struct s390_sha_ctx, buf) diff --git a/include/crypto/sha3.h b/include/crypto/sha3.h index 080f60c2e6b1..661f196193cf 100644 --- a/include/crypto/sha3.h +++ b/include/crypto/sha3.h @@ -17,8 +17,10 @@ #define SHA3_512_DIGEST_SIZE (512 / 8) #define SHA3_512_BLOCK_SIZE (200 - 2 * SHA3_512_DIGEST_SIZE) +#define SHA3_STATE_SIZE 200 + struct sha3_state { - u64 st[25]; + u64 st[SHA3_STATE_SIZE / 8]; unsigned int rsiz; unsigned int rsizw; From 6f90ba7065515d69b24729cf85c45b2add99e638 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 18 Apr 2025 11:00:13 +0800 Subject: [PATCH 189/382] crypto: s390/sha3 - Use API partial block handling Use the Crypto API partial block handling. Signed-off-by: Herbert Xu --- arch/s390/crypto/sha3_256_s390.c | 60 +++++++++++++--------------- arch/s390/crypto/sha3_512_s390.c | 67 +++++++++++++------------------- arch/s390/crypto/sha_common.c | 4 +- 3 files changed, 56 insertions(+), 75 deletions(-) diff --git a/arch/s390/crypto/sha3_256_s390.c b/arch/s390/crypto/sha3_256_s390.c index a84ef692f572..4a7731ac6bcd 100644 --- a/arch/s390/crypto/sha3_256_s390.c +++ b/arch/s390/crypto/sha3_256_s390.c @@ -8,12 +8,14 @@ * Copyright IBM Corp. 2019 * Author(s): Joerg Schmidbauer (jschmidb@de.ibm.com) */ -#include -#include -#include -#include -#include #include +#include +#include +#include +#include +#include +#include +#include #include "sha.h" @@ -21,11 +23,11 @@ static int sha3_256_init(struct shash_desc *desc) { struct s390_sha_ctx *sctx = shash_desc_ctx(desc); - if (!test_facility(86)) /* msa 12 */ + sctx->first_message_part = test_facility(86); + if (!sctx->first_message_part) memset(sctx->state, 0, sizeof(sctx->state)); sctx->count = 0; sctx->func = CPACF_KIMD_SHA3_256; - sctx->first_message_part = 1; return 0; } @@ -35,11 +37,11 @@ static int sha3_256_export(struct shash_desc *desc, void *out) struct s390_sha_ctx *sctx = shash_desc_ctx(desc); struct sha3_state *octx = out; - octx->rsiz = sctx->count; + if (sctx->first_message_part) { + memset(sctx->state, 0, sizeof(sctx->state)); + sctx->first_message_part = 0; + } memcpy(octx->st, sctx->state, sizeof(octx->st)); - memcpy(octx->buf, sctx->buf, sizeof(octx->buf)); - octx->partial = sctx->first_message_part; - return 0; } @@ -48,10 +50,9 @@ static int sha3_256_import(struct shash_desc *desc, const void *in) struct s390_sha_ctx *sctx = shash_desc_ctx(desc); const struct sha3_state *ictx = in; - sctx->count = ictx->rsiz; + sctx->count = 0; memcpy(sctx->state, ictx->st, sizeof(ictx->st)); - memcpy(sctx->buf, ictx->buf, sizeof(ictx->buf)); - sctx->first_message_part = ictx->partial; + sctx->first_message_part = 0; sctx->func = CPACF_KIMD_SHA3_256; return 0; @@ -60,30 +61,26 @@ static int sha3_256_import(struct shash_desc *desc, const void *in) static int sha3_224_import(struct shash_desc *desc, const void *in) { struct s390_sha_ctx *sctx = shash_desc_ctx(desc); - const struct sha3_state *ictx = in; - sctx->count = ictx->rsiz; - memcpy(sctx->state, ictx->st, sizeof(ictx->st)); - memcpy(sctx->buf, ictx->buf, sizeof(ictx->buf)); - sctx->first_message_part = ictx->partial; + sha3_256_import(desc, in); sctx->func = CPACF_KIMD_SHA3_224; - return 0; } static struct shash_alg sha3_256_alg = { .digestsize = SHA3_256_DIGEST_SIZE, /* = 32 */ .init = sha3_256_init, - .update = s390_sha_update, - .final = s390_sha_final, + .update = s390_sha_update_blocks, + .finup = s390_sha_finup, .export = sha3_256_export, .import = sha3_256_import, - .descsize = sizeof(struct s390_sha_ctx), - .statesize = sizeof(struct sha3_state), + .descsize = S390_SHA_CTX_SIZE, + .statesize = SHA3_STATE_SIZE, .base = { .cra_name = "sha3-256", .cra_driver_name = "sha3-256-s390", .cra_priority = 300, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_blocksize = SHA3_256_BLOCK_SIZE, .cra_module = THIS_MODULE, } @@ -93,28 +90,25 @@ static int sha3_224_init(struct shash_desc *desc) { struct s390_sha_ctx *sctx = shash_desc_ctx(desc); - if (!test_facility(86)) /* msa 12 */ - memset(sctx->state, 0, sizeof(sctx->state)); - sctx->count = 0; + sha3_256_init(desc); sctx->func = CPACF_KIMD_SHA3_224; - sctx->first_message_part = 1; - return 0; } static struct shash_alg sha3_224_alg = { .digestsize = SHA3_224_DIGEST_SIZE, .init = sha3_224_init, - .update = s390_sha_update, - .final = s390_sha_final, + .update = s390_sha_update_blocks, + .finup = s390_sha_finup, .export = sha3_256_export, /* same as for 256 */ .import = sha3_224_import, /* function code different! */ - .descsize = sizeof(struct s390_sha_ctx), - .statesize = sizeof(struct sha3_state), + .descsize = S390_SHA_CTX_SIZE, + .statesize = SHA3_STATE_SIZE, .base = { .cra_name = "sha3-224", .cra_driver_name = "sha3-224-s390", .cra_priority = 300, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_blocksize = SHA3_224_BLOCK_SIZE, .cra_module = THIS_MODULE, } diff --git a/arch/s390/crypto/sha3_512_s390.c b/arch/s390/crypto/sha3_512_s390.c index 07528fc98ff7..018f02fff444 100644 --- a/arch/s390/crypto/sha3_512_s390.c +++ b/arch/s390/crypto/sha3_512_s390.c @@ -7,12 +7,14 @@ * Copyright IBM Corp. 2019 * Author(s): Joerg Schmidbauer (jschmidb@de.ibm.com) */ -#include -#include -#include -#include -#include #include +#include +#include +#include +#include +#include +#include +#include #include "sha.h" @@ -20,11 +22,11 @@ static int sha3_512_init(struct shash_desc *desc) { struct s390_sha_ctx *sctx = shash_desc_ctx(desc); - if (!test_facility(86)) /* msa 12 */ + sctx->first_message_part = test_facility(86); + if (!sctx->first_message_part) memset(sctx->state, 0, sizeof(sctx->state)); sctx->count = 0; sctx->func = CPACF_KIMD_SHA3_512; - sctx->first_message_part = 1; return 0; } @@ -34,13 +36,12 @@ static int sha3_512_export(struct shash_desc *desc, void *out) struct s390_sha_ctx *sctx = shash_desc_ctx(desc); struct sha3_state *octx = out; - octx->rsiz = sctx->count; - octx->rsizw = sctx->count >> 32; + if (sctx->first_message_part) { + memset(sctx->state, 0, sizeof(sctx->state)); + sctx->first_message_part = 0; + } memcpy(octx->st, sctx->state, sizeof(octx->st)); - memcpy(octx->buf, sctx->buf, sizeof(octx->buf)); - octx->partial = sctx->first_message_part; - return 0; } @@ -49,13 +50,9 @@ static int sha3_512_import(struct shash_desc *desc, const void *in) struct s390_sha_ctx *sctx = shash_desc_ctx(desc); const struct sha3_state *ictx = in; - if (unlikely(ictx->rsizw)) - return -ERANGE; - sctx->count = ictx->rsiz; - + sctx->count = 0; memcpy(sctx->state, ictx->st, sizeof(ictx->st)); - memcpy(sctx->buf, ictx->buf, sizeof(ictx->buf)); - sctx->first_message_part = ictx->partial; + sctx->first_message_part = 0; sctx->func = CPACF_KIMD_SHA3_512; return 0; @@ -64,33 +61,26 @@ static int sha3_512_import(struct shash_desc *desc, const void *in) static int sha3_384_import(struct shash_desc *desc, const void *in) { struct s390_sha_ctx *sctx = shash_desc_ctx(desc); - const struct sha3_state *ictx = in; - if (unlikely(ictx->rsizw)) - return -ERANGE; - sctx->count = ictx->rsiz; - - memcpy(sctx->state, ictx->st, sizeof(ictx->st)); - memcpy(sctx->buf, ictx->buf, sizeof(ictx->buf)); - sctx->first_message_part = ictx->partial; + sha3_512_import(desc, in); sctx->func = CPACF_KIMD_SHA3_384; - return 0; } static struct shash_alg sha3_512_alg = { .digestsize = SHA3_512_DIGEST_SIZE, .init = sha3_512_init, - .update = s390_sha_update, - .final = s390_sha_final, + .update = s390_sha_update_blocks, + .finup = s390_sha_finup, .export = sha3_512_export, .import = sha3_512_import, - .descsize = sizeof(struct s390_sha_ctx), - .statesize = sizeof(struct sha3_state), + .descsize = S390_SHA_CTX_SIZE, + .statesize = SHA3_STATE_SIZE, .base = { .cra_name = "sha3-512", .cra_driver_name = "sha3-512-s390", .cra_priority = 300, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_blocksize = SHA3_512_BLOCK_SIZE, .cra_module = THIS_MODULE, } @@ -102,28 +92,25 @@ static int sha3_384_init(struct shash_desc *desc) { struct s390_sha_ctx *sctx = shash_desc_ctx(desc); - if (!test_facility(86)) /* msa 12 */ - memset(sctx->state, 0, sizeof(sctx->state)); - sctx->count = 0; + sha3_512_init(desc); sctx->func = CPACF_KIMD_SHA3_384; - sctx->first_message_part = 1; - return 0; } static struct shash_alg sha3_384_alg = { .digestsize = SHA3_384_DIGEST_SIZE, .init = sha3_384_init, - .update = s390_sha_update, - .final = s390_sha_final, + .update = s390_sha_update_blocks, + .finup = s390_sha_finup, .export = sha3_512_export, /* same as for 512 */ .import = sha3_384_import, /* function code different! */ - .descsize = sizeof(struct s390_sha_ctx), - .statesize = sizeof(struct sha3_state), + .descsize = S390_SHA_CTX_SIZE, + .statesize = SHA3_STATE_SIZE, .base = { .cra_name = "sha3-384", .cra_driver_name = "sha3-384-s390", .cra_priority = 300, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_blocksize = SHA3_384_BLOCK_SIZE, .cra_ctxsize = sizeof(struct s390_sha_ctx), .cra_module = THIS_MODULE, diff --git a/arch/s390/crypto/sha_common.c b/arch/s390/crypto/sha_common.c index 013bb37ad3ef..69e23e0c5394 100644 --- a/arch/s390/crypto/sha_common.c +++ b/arch/s390/crypto/sha_common.c @@ -29,7 +29,7 @@ int s390_sha_update(struct shash_desc *desc, const u8 *data, unsigned int len) fc = ctx->func; if (ctx->first_message_part) - fc |= test_facility(86) ? CPACF_KIMD_NIP : 0; + fc |= CPACF_KIMD_NIP; /* process one stored block */ if (index) { @@ -68,7 +68,7 @@ int s390_sha_update_blocks(struct shash_desc *desc, const u8 *data, fc = ctx->func; if (ctx->first_message_part) - fc |= test_facility(86) ? CPACF_KIMD_NIP : 0; + fc |= CPACF_KIMD_NIP; /* process as many blocks as possible */ n = (len / bsize) * bsize; From 0d474be2676d9262afd0cf6a416e96b9277139a7 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 18 Apr 2025 11:00:15 +0800 Subject: [PATCH 190/382] crypto: sha3-generic - Use API partial block handling Use the Crypto API partial block handling. Signed-off-by: Herbert Xu --- crypto/sha3_generic.c | 101 ++++++++++++++++++------------------------ include/crypto/sha3.h | 7 +-- 2 files changed, 47 insertions(+), 61 deletions(-) diff --git a/crypto/sha3_generic.c b/crypto/sha3_generic.c index b103642b56ea..41d1e506e6de 100644 --- a/crypto/sha3_generic.c +++ b/crypto/sha3_generic.c @@ -9,10 +9,10 @@ * Ard Biesheuvel */ #include -#include -#include -#include #include +#include +#include +#include #include /* @@ -161,68 +161,51 @@ static void keccakf(u64 st[25]) int crypto_sha3_init(struct shash_desc *desc) { struct sha3_state *sctx = shash_desc_ctx(desc); - unsigned int digest_size = crypto_shash_digestsize(desc->tfm); - - sctx->rsiz = 200 - 2 * digest_size; - sctx->rsizw = sctx->rsiz / 8; - sctx->partial = 0; memset(sctx->st, 0, sizeof(sctx->st)); return 0; } EXPORT_SYMBOL(crypto_sha3_init); -int crypto_sha3_update(struct shash_desc *desc, const u8 *data, - unsigned int len) +static int crypto_sha3_update(struct shash_desc *desc, const u8 *data, + unsigned int len) { + unsigned int rsiz = crypto_shash_blocksize(desc->tfm); struct sha3_state *sctx = shash_desc_ctx(desc); - unsigned int done; - const u8 *src; + unsigned int rsizw = rsiz / 8; - done = 0; - src = data; + do { + int i; - if ((sctx->partial + len) > (sctx->rsiz - 1)) { - if (sctx->partial) { - done = -sctx->partial; - memcpy(sctx->buf + sctx->partial, data, - done + sctx->rsiz); - src = sctx->buf; - } + for (i = 0; i < rsizw; i++) + sctx->st[i] ^= get_unaligned_le64(data + 8 * i); + keccakf(sctx->st); - do { - unsigned int i; - - for (i = 0; i < sctx->rsizw; i++) - sctx->st[i] ^= get_unaligned_le64(src + 8 * i); - keccakf(sctx->st); - - done += sctx->rsiz; - src = data + done; - } while (done + (sctx->rsiz - 1) < len); - - sctx->partial = 0; - } - memcpy(sctx->buf + sctx->partial, src, len - done); - sctx->partial += (len - done); - - return 0; + data += rsiz; + len -= rsiz; + } while (len >= rsiz); + return len; } -EXPORT_SYMBOL(crypto_sha3_update); -int crypto_sha3_final(struct shash_desc *desc, u8 *out) +static int crypto_sha3_finup(struct shash_desc *desc, const u8 *src, + unsigned int len, u8 *out) { - struct sha3_state *sctx = shash_desc_ctx(desc); - unsigned int i, inlen = sctx->partial; unsigned int digest_size = crypto_shash_digestsize(desc->tfm); + unsigned int rsiz = crypto_shash_blocksize(desc->tfm); + struct sha3_state *sctx = shash_desc_ctx(desc); + __le64 block[SHA3_224_BLOCK_SIZE / 8] = {}; __le64 *digest = (__le64 *)out; + unsigned int rsizw = rsiz / 8; + u8 *p; + int i; - sctx->buf[inlen++] = 0x06; - memset(sctx->buf + inlen, 0, sctx->rsiz - inlen); - sctx->buf[sctx->rsiz - 1] |= 0x80; + p = memcpy(block, src, len); + p[len++] = 0x06; + p[rsiz - 1] |= 0x80; - for (i = 0; i < sctx->rsizw; i++) - sctx->st[i] ^= get_unaligned_le64(sctx->buf + 8 * i); + for (i = 0; i < rsizw; i++) + sctx->st[i] ^= le64_to_cpu(block[i]); + memzero_explicit(block, sizeof(block)); keccakf(sctx->st); @@ -232,49 +215,51 @@ int crypto_sha3_final(struct shash_desc *desc, u8 *out) if (digest_size & 4) put_unaligned_le32(sctx->st[i], (__le32 *)digest); - memset(sctx, 0, sizeof(*sctx)); return 0; } -EXPORT_SYMBOL(crypto_sha3_final); static struct shash_alg algs[] = { { .digestsize = SHA3_224_DIGEST_SIZE, .init = crypto_sha3_init, .update = crypto_sha3_update, - .final = crypto_sha3_final, - .descsize = sizeof(struct sha3_state), + .finup = crypto_sha3_finup, + .descsize = SHA3_STATE_SIZE, .base.cra_name = "sha3-224", .base.cra_driver_name = "sha3-224-generic", + .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .base.cra_blocksize = SHA3_224_BLOCK_SIZE, .base.cra_module = THIS_MODULE, }, { .digestsize = SHA3_256_DIGEST_SIZE, .init = crypto_sha3_init, .update = crypto_sha3_update, - .final = crypto_sha3_final, - .descsize = sizeof(struct sha3_state), + .finup = crypto_sha3_finup, + .descsize = SHA3_STATE_SIZE, .base.cra_name = "sha3-256", .base.cra_driver_name = "sha3-256-generic", + .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .base.cra_blocksize = SHA3_256_BLOCK_SIZE, .base.cra_module = THIS_MODULE, }, { .digestsize = SHA3_384_DIGEST_SIZE, .init = crypto_sha3_init, .update = crypto_sha3_update, - .final = crypto_sha3_final, - .descsize = sizeof(struct sha3_state), + .finup = crypto_sha3_finup, + .descsize = SHA3_STATE_SIZE, .base.cra_name = "sha3-384", .base.cra_driver_name = "sha3-384-generic", + .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .base.cra_blocksize = SHA3_384_BLOCK_SIZE, .base.cra_module = THIS_MODULE, }, { .digestsize = SHA3_512_DIGEST_SIZE, .init = crypto_sha3_init, .update = crypto_sha3_update, - .final = crypto_sha3_final, - .descsize = sizeof(struct sha3_state), + .finup = crypto_sha3_finup, + .descsize = SHA3_STATE_SIZE, .base.cra_name = "sha3-512", .base.cra_driver_name = "sha3-512-generic", + .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .base.cra_blocksize = SHA3_512_BLOCK_SIZE, .base.cra_module = THIS_MODULE, } }; @@ -289,7 +274,7 @@ static void __exit sha3_generic_mod_fini(void) crypto_unregister_shashes(algs, ARRAY_SIZE(algs)); } -subsys_initcall(sha3_generic_mod_init); +module_init(sha3_generic_mod_init); module_exit(sha3_generic_mod_fini); MODULE_LICENSE("GPL"); diff --git a/include/crypto/sha3.h b/include/crypto/sha3.h index 661f196193cf..420b90c5f08a 100644 --- a/include/crypto/sha3.h +++ b/include/crypto/sha3.h @@ -5,6 +5,8 @@ #ifndef __CRYPTO_SHA3_H__ #define __CRYPTO_SHA3_H__ +#include + #define SHA3_224_DIGEST_SIZE (224 / 8) #define SHA3_224_BLOCK_SIZE (200 - 2 * SHA3_224_DIGEST_SIZE) @@ -19,6 +21,8 @@ #define SHA3_STATE_SIZE 200 +struct shash_desc; + struct sha3_state { u64 st[SHA3_STATE_SIZE / 8]; unsigned int rsiz; @@ -29,8 +33,5 @@ struct sha3_state { }; int crypto_sha3_init(struct shash_desc *desc); -int crypto_sha3_update(struct shash_desc *desc, const u8 *data, - unsigned int len); -int crypto_sha3_final(struct shash_desc *desc, u8 *out); #endif From 201e9ec3b621c9542441eb7bd34facbc1749b975 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 18 Apr 2025 11:00:18 +0800 Subject: [PATCH 191/382] crypto: zynqmp-sha - Use API partial block handling Use the Crypto API partial block handling. As this was the last user of the extra fields in struct sha3_state, remove them. Signed-off-by: Herbert Xu --- drivers/crypto/xilinx/zynqmp-sha.c | 71 +++++++++--------------------- include/crypto/sha3.h | 5 --- 2 files changed, 22 insertions(+), 54 deletions(-) diff --git a/drivers/crypto/xilinx/zynqmp-sha.c b/drivers/crypto/xilinx/zynqmp-sha.c index 4db8c717906f..67cf8d990a1d 100644 --- a/drivers/crypto/xilinx/zynqmp-sha.c +++ b/drivers/crypto/xilinx/zynqmp-sha.c @@ -3,7 +3,6 @@ * Xilinx ZynqMP SHA Driver. * Copyright (c) 2022 Xilinx Inc. */ -#include #include #include #include @@ -37,10 +36,6 @@ struct zynqmp_sha_tfm_ctx { struct crypto_shash *fbk_tfm; }; -struct zynqmp_sha_desc_ctx { - struct shash_desc fbk_req; -}; - static dma_addr_t update_dma_addr, final_dma_addr; static char *ubuf, *fbuf; @@ -64,7 +59,6 @@ static int zynqmp_sha_init_tfm(struct crypto_shash *hash) return PTR_ERR(fallback_tfm); if (crypto_shash_descsize(hash) < - sizeof(struct zynqmp_sha_desc_ctx) + crypto_shash_descsize(tfm_ctx->fbk_tfm)) { crypto_free_shash(fallback_tfm); return -EINVAL; @@ -79,58 +73,41 @@ static void zynqmp_sha_exit_tfm(struct crypto_shash *hash) { struct zynqmp_sha_tfm_ctx *tfm_ctx = crypto_shash_ctx(hash); - if (tfm_ctx->fbk_tfm) { - crypto_free_shash(tfm_ctx->fbk_tfm); - tfm_ctx->fbk_tfm = NULL; - } - - memzero_explicit(tfm_ctx, sizeof(struct zynqmp_sha_tfm_ctx)); + crypto_free_shash(tfm_ctx->fbk_tfm); } static int zynqmp_sha_init(struct shash_desc *desc) { - struct zynqmp_sha_desc_ctx *dctx = shash_desc_ctx(desc); struct zynqmp_sha_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); + struct crypto_shash *fbtfm = tctx->fbk_tfm; + SHASH_DESC_ON_STACK(fbdesc, fbtfm); - dctx->fbk_req.tfm = tctx->fbk_tfm; - return crypto_shash_init(&dctx->fbk_req); + fbdesc->tfm = fbtfm; + return crypto_shash_init(fbdesc) ?: + crypto_shash_export_core(fbdesc, shash_desc_ctx(desc)); } static int zynqmp_sha_update(struct shash_desc *desc, const u8 *data, unsigned int length) { - struct zynqmp_sha_desc_ctx *dctx = shash_desc_ctx(desc); + struct zynqmp_sha_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); + struct crypto_shash *fbtfm = tctx->fbk_tfm; + SHASH_DESC_ON_STACK(fbdesc, fbtfm); - return crypto_shash_update(&dctx->fbk_req, data, length); -} - -static int zynqmp_sha_final(struct shash_desc *desc, u8 *out) -{ - struct zynqmp_sha_desc_ctx *dctx = shash_desc_ctx(desc); - - return crypto_shash_final(&dctx->fbk_req, out); + fbdesc->tfm = fbtfm; + return crypto_shash_import_core(fbdesc, shash_desc_ctx(desc)) ?: + crypto_shash_update(fbdesc, data, length) ?: + crypto_shash_export_core(fbdesc, shash_desc_ctx(desc)); } static int zynqmp_sha_finup(struct shash_desc *desc, const u8 *data, unsigned int length, u8 *out) { - struct zynqmp_sha_desc_ctx *dctx = shash_desc_ctx(desc); - - return crypto_shash_finup(&dctx->fbk_req, data, length, out); -} - -static int zynqmp_sha_import(struct shash_desc *desc, const void *in) -{ - struct zynqmp_sha_desc_ctx *dctx = shash_desc_ctx(desc); struct zynqmp_sha_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); + struct crypto_shash *fbtfm = tctx->fbk_tfm; + SHASH_DESC_ON_STACK(fbdesc, fbtfm); - dctx->fbk_req.tfm = tctx->fbk_tfm; - return crypto_shash_import(&dctx->fbk_req, in); -} - -static int zynqmp_sha_export(struct shash_desc *desc, void *out) -{ - struct zynqmp_sha_desc_ctx *dctx = shash_desc_ctx(desc); - - return crypto_shash_export(&dctx->fbk_req, out); + fbdesc->tfm = fbtfm; + return crypto_shash_import_core(fbdesc, shash_desc_ctx(desc)) ?: + crypto_shash_finup(fbdesc, data, length, out); } static int __zynqmp_sha_digest(struct shash_desc *desc, const u8 *data, @@ -179,24 +156,20 @@ static struct zynqmp_sha_drv_ctx sha3_drv_ctx = { .sha3_384 = { .init = zynqmp_sha_init, .update = zynqmp_sha_update, - .final = zynqmp_sha_final, .finup = zynqmp_sha_finup, .digest = zynqmp_sha_digest, - .export = zynqmp_sha_export, - .import = zynqmp_sha_import, .init_tfm = zynqmp_sha_init_tfm, .exit_tfm = zynqmp_sha_exit_tfm, - .descsize = sizeof(struct zynqmp_sha_desc_ctx) + - sizeof(struct sha3_state), - .statesize = sizeof(struct sha3_state), + .descsize = sizeof(struct sha3_state), .digestsize = SHA3_384_DIGEST_SIZE, .base = { .cra_name = "sha3-384", .cra_driver_name = "zynqmp-sha3-384", .cra_priority = 300, .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | - CRYPTO_ALG_ALLOCATES_MEMORY | - CRYPTO_ALG_NEED_FALLBACK, + CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA3_384_BLOCK_SIZE, .cra_ctxsize = sizeof(struct zynqmp_sha_tfm_ctx), .cra_module = THIS_MODULE, diff --git a/include/crypto/sha3.h b/include/crypto/sha3.h index 420b90c5f08a..3c2559f51ada 100644 --- a/include/crypto/sha3.h +++ b/include/crypto/sha3.h @@ -25,11 +25,6 @@ struct shash_desc; struct sha3_state { u64 st[SHA3_STATE_SIZE / 8]; - unsigned int rsiz; - unsigned int rsizw; - - unsigned int partial; - u8 buf[SHA3_224_BLOCK_SIZE]; }; int crypto_sha3_init(struct shash_desc *desc); From ff3cb9de53baa3da7df0e08798a5638436d8aabc Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 18 Apr 2025 11:00:20 +0800 Subject: [PATCH 192/382] crypto: x86/sha512 - Use API partial block handling Use the Crypto API partial block handling. Also remove the unnecessary SIMD fallback path. Signed-off-by: Herbert Xu --- arch/x86/crypto/sha512_ssse3_glue.c | 79 ++++++++++------------------- include/crypto/sha2.h | 1 + include/crypto/sha512_base.h | 54 ++++++++++++++++++-- 3 files changed, 77 insertions(+), 57 deletions(-) diff --git a/arch/x86/crypto/sha512_ssse3_glue.c b/arch/x86/crypto/sha512_ssse3_glue.c index 6d3b85e53d0e..067684c54395 100644 --- a/arch/x86/crypto/sha512_ssse3_glue.c +++ b/arch/x86/crypto/sha512_ssse3_glue.c @@ -27,17 +27,13 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -#include -#include -#include -#include -#include -#include -#include -#include -#include #include #include +#include +#include +#include +#include +#include asmlinkage void sha512_transform_ssse3(struct sha512_state *state, const u8 *data, int blocks); @@ -45,11 +41,7 @@ asmlinkage void sha512_transform_ssse3(struct sha512_state *state, static int sha512_update(struct shash_desc *desc, const u8 *data, unsigned int len, sha512_block_fn *sha512_xform) { - struct sha512_state *sctx = shash_desc_ctx(desc); - - if (!crypto_simd_usable() || - (sctx->count[0] % SHA512_BLOCK_SIZE) + len < SHA512_BLOCK_SIZE) - return crypto_sha512_update(desc, data, len); + int remain; /* * Make sure struct sha512_state begins directly with the SHA512 @@ -58,22 +50,17 @@ static int sha512_update(struct shash_desc *desc, const u8 *data, BUILD_BUG_ON(offsetof(struct sha512_state, state) != 0); kernel_fpu_begin(); - sha512_base_do_update(desc, data, len, sha512_xform); + remain = sha512_base_do_update_blocks(desc, data, len, sha512_xform); kernel_fpu_end(); - return 0; + return remain; } static int sha512_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out, sha512_block_fn *sha512_xform) { - if (!crypto_simd_usable()) - return crypto_sha512_finup(desc, data, len, out); - kernel_fpu_begin(); - if (len) - sha512_base_do_update(desc, data, len, sha512_xform); - sha512_base_do_finalize(desc, sha512_xform); + sha512_base_do_finup(desc, data, len, sha512_xform); kernel_fpu_end(); return sha512_base_finish(desc, out); @@ -91,23 +78,18 @@ static int sha512_ssse3_finup(struct shash_desc *desc, const u8 *data, return sha512_finup(desc, data, len, out, sha512_transform_ssse3); } -/* Add padding and return the message digest. */ -static int sha512_ssse3_final(struct shash_desc *desc, u8 *out) -{ - return sha512_ssse3_finup(desc, NULL, 0, out); -} - static struct shash_alg sha512_ssse3_algs[] = { { .digestsize = SHA512_DIGEST_SIZE, .init = sha512_base_init, .update = sha512_ssse3_update, - .final = sha512_ssse3_final, .finup = sha512_ssse3_finup, - .descsize = sizeof(struct sha512_state), + .descsize = SHA512_STATE_SIZE, .base = { .cra_name = "sha512", .cra_driver_name = "sha512-ssse3", .cra_priority = 150, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA512_BLOCK_SIZE, .cra_module = THIS_MODULE, } @@ -115,13 +97,14 @@ static struct shash_alg sha512_ssse3_algs[] = { { .digestsize = SHA384_DIGEST_SIZE, .init = sha384_base_init, .update = sha512_ssse3_update, - .final = sha512_ssse3_final, .finup = sha512_ssse3_finup, - .descsize = sizeof(struct sha512_state), + .descsize = SHA512_STATE_SIZE, .base = { .cra_name = "sha384", .cra_driver_name = "sha384-ssse3", .cra_priority = 150, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA384_BLOCK_SIZE, .cra_module = THIS_MODULE, } @@ -167,23 +150,18 @@ static int sha512_avx_finup(struct shash_desc *desc, const u8 *data, return sha512_finup(desc, data, len, out, sha512_transform_avx); } -/* Add padding and return the message digest. */ -static int sha512_avx_final(struct shash_desc *desc, u8 *out) -{ - return sha512_avx_finup(desc, NULL, 0, out); -} - static struct shash_alg sha512_avx_algs[] = { { .digestsize = SHA512_DIGEST_SIZE, .init = sha512_base_init, .update = sha512_avx_update, - .final = sha512_avx_final, .finup = sha512_avx_finup, - .descsize = sizeof(struct sha512_state), + .descsize = SHA512_STATE_SIZE, .base = { .cra_name = "sha512", .cra_driver_name = "sha512-avx", .cra_priority = 160, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA512_BLOCK_SIZE, .cra_module = THIS_MODULE, } @@ -191,13 +169,14 @@ static struct shash_alg sha512_avx_algs[] = { { .digestsize = SHA384_DIGEST_SIZE, .init = sha384_base_init, .update = sha512_avx_update, - .final = sha512_avx_final, .finup = sha512_avx_finup, - .descsize = sizeof(struct sha512_state), + .descsize = SHA512_STATE_SIZE, .base = { .cra_name = "sha384", .cra_driver_name = "sha384-avx", .cra_priority = 160, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA384_BLOCK_SIZE, .cra_module = THIS_MODULE, } @@ -233,23 +212,18 @@ static int sha512_avx2_finup(struct shash_desc *desc, const u8 *data, return sha512_finup(desc, data, len, out, sha512_transform_rorx); } -/* Add padding and return the message digest. */ -static int sha512_avx2_final(struct shash_desc *desc, u8 *out) -{ - return sha512_avx2_finup(desc, NULL, 0, out); -} - static struct shash_alg sha512_avx2_algs[] = { { .digestsize = SHA512_DIGEST_SIZE, .init = sha512_base_init, .update = sha512_avx2_update, - .final = sha512_avx2_final, .finup = sha512_avx2_finup, - .descsize = sizeof(struct sha512_state), + .descsize = SHA512_STATE_SIZE, .base = { .cra_name = "sha512", .cra_driver_name = "sha512-avx2", .cra_priority = 170, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA512_BLOCK_SIZE, .cra_module = THIS_MODULE, } @@ -257,13 +231,14 @@ static struct shash_alg sha512_avx2_algs[] = { { .digestsize = SHA384_DIGEST_SIZE, .init = sha384_base_init, .update = sha512_avx2_update, - .final = sha512_avx2_final, .finup = sha512_avx2_finup, - .descsize = sizeof(struct sha512_state), + .descsize = SHA512_STATE_SIZE, .base = { .cra_name = "sha384", .cra_driver_name = "sha384-avx2", .cra_priority = 170, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA384_BLOCK_SIZE, .cra_module = THIS_MODULE, } diff --git a/include/crypto/sha2.h b/include/crypto/sha2.h index a913bad5dd3b..e9ad7ab955aa 100644 --- a/include/crypto/sha2.h +++ b/include/crypto/sha2.h @@ -19,6 +19,7 @@ #define SHA512_DIGEST_SIZE 64 #define SHA512_BLOCK_SIZE 128 +#define SHA512_STATE_SIZE 80 #define SHA224_H0 0xc1059ed8UL #define SHA224_H1 0x367cd507UL diff --git a/include/crypto/sha512_base.h b/include/crypto/sha512_base.h index 679916a84cb2..8cb172e52dc0 100644 --- a/include/crypto/sha512_base.h +++ b/include/crypto/sha512_base.h @@ -10,10 +10,7 @@ #include #include -#include -#include #include - #include typedef void (sha512_block_fn)(struct sha512_state *sst, u8 const *src, @@ -93,6 +90,55 @@ static inline int sha512_base_do_update(struct shash_desc *desc, return 0; } +static inline int sha512_base_do_update_blocks(struct shash_desc *desc, + const u8 *data, + unsigned int len, + sha512_block_fn *block_fn) +{ + unsigned int remain = len - round_down(len, SHA512_BLOCK_SIZE); + struct sha512_state *sctx = shash_desc_ctx(desc); + + len -= remain; + sctx->count[0] += len; + if (sctx->count[0] < len) + sctx->count[1]++; + block_fn(sctx, data, len / SHA512_BLOCK_SIZE); + return remain; +} + +static inline int sha512_base_do_finup(struct shash_desc *desc, const u8 *src, + unsigned int len, + sha512_block_fn *block_fn) +{ + unsigned int bit_offset = SHA512_BLOCK_SIZE / 8 - 2; + struct sha512_state *sctx = shash_desc_ctx(desc); + union { + __be64 b64[SHA512_BLOCK_SIZE / 4]; + u8 u8[SHA512_BLOCK_SIZE * 2]; + } block = {}; + + if (len >= SHA512_BLOCK_SIZE) { + int remain; + + remain = sha512_base_do_update_blocks(desc, src, len, block_fn); + src += len - remain; + len = remain; + } + + if (len >= bit_offset * 8) + bit_offset += SHA512_BLOCK_SIZE / 8; + memcpy(&block, src, len); + block.u8[len] = 0x80; + sctx->count[0] += len; + block.b64[bit_offset] = cpu_to_be64(sctx->count[1] << 3 | + sctx->count[0] >> 61); + block.b64[bit_offset + 1] = cpu_to_be64(sctx->count[0] << 3); + block_fn(sctx, block.u8, (bit_offset + 2) * 8 / SHA512_BLOCK_SIZE); + memzero_explicit(&block, sizeof(block)); + + return 0; +} + static inline int sha512_base_do_finalize(struct shash_desc *desc, sha512_block_fn *block_fn) { @@ -126,8 +172,6 @@ static inline int sha512_base_finish(struct shash_desc *desc, u8 *out) for (i = 0; digest_size > 0; i++, digest_size -= sizeof(__be64)) put_unaligned_be64(sctx->state[i], digest++); - - memzero_explicit(sctx, sizeof(*sctx)); return 0; } From 0b4aa3d10f7dbe398e4d5050f3ccb7d0ca438913 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 18 Apr 2025 11:00:22 +0800 Subject: [PATCH 193/382] crypto: mips/octeon-sha512 - Use API partial block handling Use the Crypto API partial block handling. Signed-off-by: Herbert Xu --- .../mips/cavium-octeon/crypto/octeon-sha512.c | 155 +++++------------- 1 file changed, 42 insertions(+), 113 deletions(-) diff --git a/arch/mips/cavium-octeon/crypto/octeon-sha512.c b/arch/mips/cavium-octeon/crypto/octeon-sha512.c index 2dee9354e33f..215311053db3 100644 --- a/arch/mips/cavium-octeon/crypto/octeon-sha512.c +++ b/arch/mips/cavium-octeon/crypto/octeon-sha512.c @@ -13,15 +13,12 @@ * Copyright (c) 2003 Kyle McMartin */ -#include -#include -#include -#include -#include -#include -#include #include #include +#include +#include +#include +#include #include "octeon-crypto.h" @@ -53,60 +50,31 @@ static void octeon_sha512_read_hash(struct sha512_state *sctx) sctx->state[7] = read_octeon_64bit_hash_sha512(7); } -static void octeon_sha512_transform(const void *_block) +static void octeon_sha512_transform(struct sha512_state *sctx, + const u8 *src, int blocks) { - const u64 *block = _block; + do { + const u64 *block = (const u64 *)src; - write_octeon_64bit_block_sha512(block[0], 0); - write_octeon_64bit_block_sha512(block[1], 1); - write_octeon_64bit_block_sha512(block[2], 2); - write_octeon_64bit_block_sha512(block[3], 3); - write_octeon_64bit_block_sha512(block[4], 4); - write_octeon_64bit_block_sha512(block[5], 5); - write_octeon_64bit_block_sha512(block[6], 6); - write_octeon_64bit_block_sha512(block[7], 7); - write_octeon_64bit_block_sha512(block[8], 8); - write_octeon_64bit_block_sha512(block[9], 9); - write_octeon_64bit_block_sha512(block[10], 10); - write_octeon_64bit_block_sha512(block[11], 11); - write_octeon_64bit_block_sha512(block[12], 12); - write_octeon_64bit_block_sha512(block[13], 13); - write_octeon_64bit_block_sha512(block[14], 14); - octeon_sha512_start(block[15]); -} + write_octeon_64bit_block_sha512(block[0], 0); + write_octeon_64bit_block_sha512(block[1], 1); + write_octeon_64bit_block_sha512(block[2], 2); + write_octeon_64bit_block_sha512(block[3], 3); + write_octeon_64bit_block_sha512(block[4], 4); + write_octeon_64bit_block_sha512(block[5], 5); + write_octeon_64bit_block_sha512(block[6], 6); + write_octeon_64bit_block_sha512(block[7], 7); + write_octeon_64bit_block_sha512(block[8], 8); + write_octeon_64bit_block_sha512(block[9], 9); + write_octeon_64bit_block_sha512(block[10], 10); + write_octeon_64bit_block_sha512(block[11], 11); + write_octeon_64bit_block_sha512(block[12], 12); + write_octeon_64bit_block_sha512(block[13], 13); + write_octeon_64bit_block_sha512(block[14], 14); + octeon_sha512_start(block[15]); -static void __octeon_sha512_update(struct sha512_state *sctx, const u8 *data, - unsigned int len) -{ - unsigned int part_len; - unsigned int index; - unsigned int i; - - /* Compute number of bytes mod 128. */ - index = sctx->count[0] % SHA512_BLOCK_SIZE; - - /* Update number of bytes. */ - if ((sctx->count[0] += len) < len) - sctx->count[1]++; - - part_len = SHA512_BLOCK_SIZE - index; - - /* Transform as many times as possible. */ - if (len >= part_len) { - memcpy(&sctx->buf[index], data, part_len); - octeon_sha512_transform(sctx->buf); - - for (i = part_len; i + SHA512_BLOCK_SIZE <= len; - i += SHA512_BLOCK_SIZE) - octeon_sha512_transform(&data[i]); - - index = 0; - } else { - i = 0; - } - - /* Buffer remaining input. */ - memcpy(&sctx->buf[index], &data[i], len - i); + src += SHA512_BLOCK_SIZE; + } while (--blocks); } static int octeon_sha512_update(struct shash_desc *desc, const u8 *data, @@ -115,89 +83,48 @@ static int octeon_sha512_update(struct shash_desc *desc, const u8 *data, struct sha512_state *sctx = shash_desc_ctx(desc); struct octeon_cop2_state state; unsigned long flags; - - /* - * Small updates never reach the crypto engine, so the generic sha512 is - * faster because of the heavyweight octeon_crypto_enable() / - * octeon_crypto_disable(). - */ - if ((sctx->count[0] % SHA512_BLOCK_SIZE) + len < SHA512_BLOCK_SIZE) - return crypto_sha512_update(desc, data, len); + int remain; flags = octeon_crypto_enable(&state); octeon_sha512_store_hash(sctx); - __octeon_sha512_update(sctx, data, len); + remain = sha512_base_do_update_blocks(desc, data, len, + octeon_sha512_transform); octeon_sha512_read_hash(sctx); octeon_crypto_disable(&state, flags); - - return 0; + return remain; } -static int octeon_sha512_final(struct shash_desc *desc, u8 *hash) +static int octeon_sha512_finup(struct shash_desc *desc, const u8 *src, + unsigned int len, u8 *hash) { struct sha512_state *sctx = shash_desc_ctx(desc); - static u8 padding[128] = { 0x80, }; struct octeon_cop2_state state; - __be64 *dst = (__be64 *)hash; - unsigned int pad_len; unsigned long flags; - unsigned int index; - __be64 bits[2]; - int i; - - /* Save number of bits. */ - bits[1] = cpu_to_be64(sctx->count[0] << 3); - bits[0] = cpu_to_be64(sctx->count[1] << 3 | sctx->count[0] >> 61); - - /* Pad out to 112 mod 128. */ - index = sctx->count[0] & 0x7f; - pad_len = (index < 112) ? (112 - index) : ((128+112) - index); flags = octeon_crypto_enable(&state); octeon_sha512_store_hash(sctx); - __octeon_sha512_update(sctx, padding, pad_len); - - /* Append length (before padding). */ - __octeon_sha512_update(sctx, (const u8 *)bits, sizeof(bits)); + sha512_base_do_finup(desc, src, len, octeon_sha512_transform); octeon_sha512_read_hash(sctx); octeon_crypto_disable(&state, flags); - - /* Store state in digest. */ - for (i = 0; i < 8; i++) - dst[i] = cpu_to_be64(sctx->state[i]); - - /* Zeroize sensitive information. */ - memset(sctx, 0, sizeof(struct sha512_state)); - - return 0; -} - -static int octeon_sha384_final(struct shash_desc *desc, u8 *hash) -{ - u8 D[64]; - - octeon_sha512_final(desc, D); - - memcpy(hash, D, 48); - memzero_explicit(D, 64); - - return 0; + return sha512_base_finish(desc, hash); } static struct shash_alg octeon_sha512_algs[2] = { { .digestsize = SHA512_DIGEST_SIZE, .init = sha512_base_init, .update = octeon_sha512_update, - .final = octeon_sha512_final, - .descsize = sizeof(struct sha512_state), + .finup = octeon_sha512_finup, + .descsize = SHA512_STATE_SIZE, .base = { .cra_name = "sha512", .cra_driver_name= "octeon-sha512", .cra_priority = OCTEON_CR_OPCODE_PRIORITY, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA512_BLOCK_SIZE, .cra_module = THIS_MODULE, } @@ -205,12 +132,14 @@ static struct shash_alg octeon_sha512_algs[2] = { { .digestsize = SHA384_DIGEST_SIZE, .init = sha384_base_init, .update = octeon_sha512_update, - .final = octeon_sha384_final, - .descsize = sizeof(struct sha512_state), + .finup = octeon_sha512_finup, + .descsize = SHA512_STATE_SIZE, .base = { .cra_name = "sha384", .cra_driver_name= "octeon-sha384", .cra_priority = OCTEON_CR_OPCODE_PRIORITY, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA384_BLOCK_SIZE, .cra_module = THIS_MODULE, } From 561aab1104d80083886beaaeee39fa6482d6f157 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 18 Apr 2025 11:00:25 +0800 Subject: [PATCH 194/382] crypto: riscv/sha512 - Use API partial block handling Use the Crypto API partial block handling. Signed-off-by: Herbert Xu --- arch/riscv/crypto/sha512-riscv64-glue.c | 47 ++++++++++--------------- crypto/sha512_generic.c | 5 +-- include/crypto/sha512_base.h | 6 ++++ 3 files changed, 28 insertions(+), 30 deletions(-) diff --git a/arch/riscv/crypto/sha512-riscv64-glue.c b/arch/riscv/crypto/sha512-riscv64-glue.c index 43b56a08aeb5..4634fca78ae2 100644 --- a/arch/riscv/crypto/sha512-riscv64-glue.c +++ b/arch/riscv/crypto/sha512-riscv64-glue.c @@ -14,7 +14,7 @@ #include #include #include -#include +#include #include /* @@ -24,8 +24,8 @@ asmlinkage void sha512_transform_zvknhb_zvkb( struct sha512_state *state, const u8 *data, int num_blocks); -static int riscv64_sha512_update(struct shash_desc *desc, const u8 *data, - unsigned int len) +static void sha512_block(struct sha512_state *state, const u8 *data, + int num_blocks) { /* * Ensure struct sha512_state begins directly with the SHA-512 @@ -35,35 +35,24 @@ static int riscv64_sha512_update(struct shash_desc *desc, const u8 *data, if (crypto_simd_usable()) { kernel_vector_begin(); - sha512_base_do_update(desc, data, len, - sha512_transform_zvknhb_zvkb); + sha512_transform_zvknhb_zvkb(state, data, num_blocks); kernel_vector_end(); } else { - crypto_sha512_update(desc, data, len); + sha512_generic_block_fn(state, data, num_blocks); } - return 0; +} + +static int riscv64_sha512_update(struct shash_desc *desc, const u8 *data, + unsigned int len) +{ + return sha512_base_do_update_blocks(desc, data, len, sha512_block); } static int riscv64_sha512_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) { - if (crypto_simd_usable()) { - kernel_vector_begin(); - if (len) - sha512_base_do_update(desc, data, len, - sha512_transform_zvknhb_zvkb); - sha512_base_do_finalize(desc, sha512_transform_zvknhb_zvkb); - kernel_vector_end(); - - return sha512_base_finish(desc, out); - } - - return crypto_sha512_finup(desc, data, len, out); -} - -static int riscv64_sha512_final(struct shash_desc *desc, u8 *out) -{ - return riscv64_sha512_finup(desc, NULL, 0, out); + sha512_base_do_finup(desc, data, len, sha512_block); + return sha512_base_finish(desc, out); } static int riscv64_sha512_digest(struct shash_desc *desc, const u8 *data, @@ -77,14 +66,15 @@ static struct shash_alg riscv64_sha512_algs[] = { { .init = sha512_base_init, .update = riscv64_sha512_update, - .final = riscv64_sha512_final, .finup = riscv64_sha512_finup, .digest = riscv64_sha512_digest, - .descsize = sizeof(struct sha512_state), + .descsize = SHA512_STATE_SIZE, .digestsize = SHA512_DIGEST_SIZE, .base = { .cra_blocksize = SHA512_BLOCK_SIZE, .cra_priority = 300, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_name = "sha512", .cra_driver_name = "sha512-riscv64-zvknhb-zvkb", .cra_module = THIS_MODULE, @@ -92,13 +82,14 @@ static struct shash_alg riscv64_sha512_algs[] = { }, { .init = sha384_base_init, .update = riscv64_sha512_update, - .final = riscv64_sha512_final, .finup = riscv64_sha512_finup, - .descsize = sizeof(struct sha512_state), + .descsize = SHA512_STATE_SIZE, .digestsize = SHA384_DIGEST_SIZE, .base = { .cra_blocksize = SHA384_BLOCK_SIZE, .cra_priority = 300, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_name = "sha384", .cra_driver_name = "sha384-riscv64-zvknhb-zvkb", .cra_module = THIS_MODULE, diff --git a/crypto/sha512_generic.c b/crypto/sha512_generic.c index ed81813bd420..27a7346ff143 100644 --- a/crypto/sha512_generic.c +++ b/crypto/sha512_generic.c @@ -145,14 +145,15 @@ sha512_transform(u64 *state, const u8 *input) state[4] += e; state[5] += f; state[6] += g; state[7] += h; } -static void sha512_generic_block_fn(struct sha512_state *sst, u8 const *src, - int blocks) +void sha512_generic_block_fn(struct sha512_state *sst, u8 const *src, + int blocks) { while (blocks--) { sha512_transform(sst->state, src); src += SHA512_BLOCK_SIZE; } } +EXPORT_SYMBOL_GPL(sha512_generic_block_fn); int crypto_sha512_update(struct shash_desc *desc, const u8 *data, unsigned int len) diff --git a/include/crypto/sha512_base.h b/include/crypto/sha512_base.h index 8cb172e52dc0..e9f302ec3ede 100644 --- a/include/crypto/sha512_base.h +++ b/include/crypto/sha512_base.h @@ -10,7 +10,10 @@ #include #include +#include +#include #include +#include #include typedef void (sha512_block_fn)(struct sha512_state *sst, u8 const *src, @@ -175,4 +178,7 @@ static inline int sha512_base_finish(struct shash_desc *desc, u8 *out) return 0; } +void sha512_generic_block_fn(struct sha512_state *sst, u8 const *src, + int blocks); + #endif /* _CRYPTO_SHA512_BASE_H */ From 216623af53062943ee4ef76a0b597ea6030ffd42 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 18 Apr 2025 11:00:27 +0800 Subject: [PATCH 195/382] crypto: sha512-generic - Use API partial block handling Use the Crypto API partial block handling. Signed-off-by: Herbert Xu --- crypto/sha512_generic.c | 45 ++++++++++++++++------------------------- include/crypto/sha2.h | 8 -------- 2 files changed, 17 insertions(+), 36 deletions(-) diff --git a/crypto/sha512_generic.c b/crypto/sha512_generic.c index 27a7346ff143..bfea65f4181c 100644 --- a/crypto/sha512_generic.c +++ b/crypto/sha512_generic.c @@ -6,16 +6,10 @@ * Copyright (c) 2003 Kyle McMartin */ #include -#include -#include -#include -#include -#include -#include #include #include -#include -#include +#include +#include #include const u8 sha384_zero_message_hash[SHA384_DIGEST_SIZE] = { @@ -148,45 +142,39 @@ sha512_transform(u64 *state, const u8 *input) void sha512_generic_block_fn(struct sha512_state *sst, u8 const *src, int blocks) { - while (blocks--) { + do { sha512_transform(sst->state, src); src += SHA512_BLOCK_SIZE; - } + } while (--blocks); } EXPORT_SYMBOL_GPL(sha512_generic_block_fn); -int crypto_sha512_update(struct shash_desc *desc, const u8 *data, - unsigned int len) +static int crypto_sha512_update(struct shash_desc *desc, const u8 *data, + unsigned int len) { - return sha512_base_do_update(desc, data, len, sha512_generic_block_fn); + return sha512_base_do_update_blocks(desc, data, len, + sha512_generic_block_fn); } -EXPORT_SYMBOL(crypto_sha512_update); -static int sha512_final(struct shash_desc *desc, u8 *hash) +static int crypto_sha512_finup(struct shash_desc *desc, const u8 *data, + unsigned int len, u8 *hash) { - sha512_base_do_finalize(desc, sha512_generic_block_fn); + sha512_base_do_finup(desc, data, len, sha512_generic_block_fn); return sha512_base_finish(desc, hash); } -int crypto_sha512_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *hash) -{ - sha512_base_do_update(desc, data, len, sha512_generic_block_fn); - return sha512_final(desc, hash); -} -EXPORT_SYMBOL(crypto_sha512_finup); - static struct shash_alg sha512_algs[2] = { { .digestsize = SHA512_DIGEST_SIZE, .init = sha512_base_init, .update = crypto_sha512_update, - .final = sha512_final, .finup = crypto_sha512_finup, - .descsize = sizeof(struct sha512_state), + .descsize = SHA512_STATE_SIZE, .base = { .cra_name = "sha512", .cra_driver_name = "sha512-generic", .cra_priority = 100, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA512_BLOCK_SIZE, .cra_module = THIS_MODULE, } @@ -194,13 +182,14 @@ static struct shash_alg sha512_algs[2] = { { .digestsize = SHA384_DIGEST_SIZE, .init = sha384_base_init, .update = crypto_sha512_update, - .final = sha512_final, .finup = crypto_sha512_finup, - .descsize = sizeof(struct sha512_state), + .descsize = SHA512_STATE_SIZE, .base = { .cra_name = "sha384", .cra_driver_name = "sha384-generic", .cra_priority = 100, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA384_BLOCK_SIZE, .cra_module = THIS_MODULE, } diff --git a/include/crypto/sha2.h b/include/crypto/sha2.h index e9ad7ab955aa..abbd882f7849 100644 --- a/include/crypto/sha2.h +++ b/include/crypto/sha2.h @@ -82,14 +82,6 @@ struct sha512_state { u8 buf[SHA512_BLOCK_SIZE]; }; -struct shash_desc; - -extern int crypto_sha512_update(struct shash_desc *desc, const u8 *data, - unsigned int len); - -extern int crypto_sha512_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *hash); - /* * Stand-alone implementation of the SHA256 algorithm. It is designed to * have as little dependencies as possible so it can be used in the From 022bd7c9569dbfbfab4c3a58b7d655472c189cb8 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 18 Apr 2025 11:00:29 +0800 Subject: [PATCH 196/382] crypto: arm/sha512-neon - Use API partial block handling Use the Crypto API partial block handling. Also remove the unnecessary SIMD fallback path. Signed-off-by: Herbert Xu --- arch/arm/crypto/sha512-neon-glue.c | 43 +++++++++--------------------- 1 file changed, 13 insertions(+), 30 deletions(-) diff --git a/arch/arm/crypto/sha512-neon-glue.c b/arch/arm/crypto/sha512-neon-glue.c index c6e58fe475ac..bd528077fefb 100644 --- a/arch/arm/crypto/sha512-neon-glue.c +++ b/arch/arm/crypto/sha512-neon-glue.c @@ -5,16 +5,13 @@ * Copyright (C) 2015 Linaro Ltd */ +#include #include -#include #include #include -#include +#include #include -#include -#include - #include "sha512.h" MODULE_ALIAS_CRYPTO("sha384-neon"); @@ -26,51 +23,36 @@ asmlinkage void sha512_block_data_order_neon(struct sha512_state *state, static int sha512_neon_update(struct shash_desc *desc, const u8 *data, unsigned int len) { - struct sha512_state *sctx = shash_desc_ctx(desc); - - if (!crypto_simd_usable() || - (sctx->count[0] % SHA512_BLOCK_SIZE) + len < SHA512_BLOCK_SIZE) - return sha512_arm_update(desc, data, len); + int remain; kernel_neon_begin(); - sha512_base_do_update(desc, data, len, sha512_block_data_order_neon); + remain = sha512_base_do_update_blocks(desc, data, len, + sha512_block_data_order_neon); kernel_neon_end(); - - return 0; + return remain; } static int sha512_neon_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) { - if (!crypto_simd_usable()) - return sha512_arm_finup(desc, data, len, out); - kernel_neon_begin(); - if (len) - sha512_base_do_update(desc, data, len, - sha512_block_data_order_neon); - sha512_base_do_finalize(desc, sha512_block_data_order_neon); + sha512_base_do_finup(desc, data, len, sha512_block_data_order_neon); kernel_neon_end(); - return sha512_base_finish(desc, out); } -static int sha512_neon_final(struct shash_desc *desc, u8 *out) -{ - return sha512_neon_finup(desc, NULL, 0, out); -} - struct shash_alg sha512_neon_algs[] = { { .init = sha384_base_init, .update = sha512_neon_update, - .final = sha512_neon_final, .finup = sha512_neon_finup, - .descsize = sizeof(struct sha512_state), + .descsize = SHA512_STATE_SIZE, .digestsize = SHA384_DIGEST_SIZE, .base = { .cra_name = "sha384", .cra_driver_name = "sha384-neon", .cra_priority = 300, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA384_BLOCK_SIZE, .cra_module = THIS_MODULE, @@ -78,14 +60,15 @@ struct shash_alg sha512_neon_algs[] = { { }, { .init = sha512_base_init, .update = sha512_neon_update, - .final = sha512_neon_final, .finup = sha512_neon_finup, - .descsize = sizeof(struct sha512_state), + .descsize = SHA512_STATE_SIZE, .digestsize = SHA512_DIGEST_SIZE, .base = { .cra_name = "sha512", .cra_driver_name = "sha512-neon", .cra_priority = 300, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA512_BLOCK_SIZE, .cra_module = THIS_MODULE, } From 10e6f26ec6af2521954d57d50f6deb376713b3b1 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 18 Apr 2025 11:00:31 +0800 Subject: [PATCH 197/382] crypto: arm/sha512-asm - Use API partial block handling Use the Crypto API partial block handling. Signed-off-by: Herbert Xu --- arch/arm/crypto/sha512-glue.c | 36 ++++++++++++++++------------------- arch/arm/crypto/sha512.h | 6 ------ 2 files changed, 16 insertions(+), 26 deletions(-) diff --git a/arch/arm/crypto/sha512-glue.c b/arch/arm/crypto/sha512-glue.c index 1be5bd498af3..f8a6480889b1 100644 --- a/arch/arm/crypto/sha512-glue.c +++ b/arch/arm/crypto/sha512-glue.c @@ -5,15 +5,14 @@ * Copyright (C) 2015 Linaro Ltd */ +#include +#include #include #include #include -#include +#include #include -#include -#include - #include "sha512.h" MODULE_DESCRIPTION("Accelerated SHA-384/SHA-512 secure hash for ARM"); @@ -28,50 +27,47 @@ MODULE_ALIAS_CRYPTO("sha512-arm"); asmlinkage void sha512_block_data_order(struct sha512_state *state, u8 const *src, int blocks); -int sha512_arm_update(struct shash_desc *desc, const u8 *data, - unsigned int len) +static int sha512_arm_update(struct shash_desc *desc, const u8 *data, + unsigned int len) { - return sha512_base_do_update(desc, data, len, sha512_block_data_order); + return sha512_base_do_update_blocks(desc, data, len, + sha512_block_data_order); } -static int sha512_arm_final(struct shash_desc *desc, u8 *out) +static int sha512_arm_finup(struct shash_desc *desc, const u8 *data, + unsigned int len, u8 *out) { - sha512_base_do_finalize(desc, sha512_block_data_order); + sha512_base_do_finup(desc, data, len, sha512_block_data_order); return sha512_base_finish(desc, out); } -int sha512_arm_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - sha512_base_do_update(desc, data, len, sha512_block_data_order); - return sha512_arm_final(desc, out); -} - static struct shash_alg sha512_arm_algs[] = { { .init = sha384_base_init, .update = sha512_arm_update, - .final = sha512_arm_final, .finup = sha512_arm_finup, - .descsize = sizeof(struct sha512_state), + .descsize = SHA512_STATE_SIZE, .digestsize = SHA384_DIGEST_SIZE, .base = { .cra_name = "sha384", .cra_driver_name = "sha384-arm", .cra_priority = 250, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA512_BLOCK_SIZE, .cra_module = THIS_MODULE, } }, { .init = sha512_base_init, .update = sha512_arm_update, - .final = sha512_arm_final, .finup = sha512_arm_finup, - .descsize = sizeof(struct sha512_state), + .descsize = SHA512_STATE_SIZE, .digestsize = SHA512_DIGEST_SIZE, .base = { .cra_name = "sha512", .cra_driver_name = "sha512-arm", .cra_priority = 250, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA512_BLOCK_SIZE, .cra_module = THIS_MODULE, } diff --git a/arch/arm/crypto/sha512.h b/arch/arm/crypto/sha512.h index e14572be76d1..eeaee52cda69 100644 --- a/arch/arm/crypto/sha512.h +++ b/arch/arm/crypto/sha512.h @@ -1,9 +1,3 @@ /* SPDX-License-Identifier: GPL-2.0 */ -int sha512_arm_update(struct shash_desc *desc, const u8 *data, - unsigned int len); - -int sha512_arm_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out); - extern struct shash_alg sha512_neon_algs[2]; From f294a6d9e904e99c7c4602fb31e79a9a75071b6a Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 18 Apr 2025 11:00:34 +0800 Subject: [PATCH 198/382] crypto: arm64/sha512-ce - Use API partial block handling Use the Crypto API partial block handling. Also remove the unnecessary SIMD fallback path. Signed-off-by: Herbert Xu --- arch/arm64/crypto/sha512-ce-glue.c | 49 ++++++++---------------------- 1 file changed, 12 insertions(+), 37 deletions(-) diff --git a/arch/arm64/crypto/sha512-ce-glue.c b/arch/arm64/crypto/sha512-ce-glue.c index 071f64293227..6fb3001fa2c9 100644 --- a/arch/arm64/crypto/sha512-ce-glue.c +++ b/arch/arm64/crypto/sha512-ce-glue.c @@ -10,14 +10,11 @@ */ #include -#include -#include #include -#include #include #include #include -#include +#include #include MODULE_DESCRIPTION("SHA-384/SHA-512 secure hash using ARMv8 Crypto Extensions"); @@ -29,12 +26,10 @@ MODULE_ALIAS_CRYPTO("sha512"); asmlinkage int __sha512_ce_transform(struct sha512_state *sst, u8 const *src, int blocks); -asmlinkage void sha512_block_data_order(u64 *digest, u8 const *src, int blocks); - static void sha512_ce_transform(struct sha512_state *sst, u8 const *src, int blocks) { - while (blocks) { + do { int rem; kernel_neon_begin(); @@ -42,67 +37,47 @@ static void sha512_ce_transform(struct sha512_state *sst, u8 const *src, kernel_neon_end(); src += (blocks - rem) * SHA512_BLOCK_SIZE; blocks = rem; - } -} - -static void sha512_arm64_transform(struct sha512_state *sst, u8 const *src, - int blocks) -{ - sha512_block_data_order(sst->state, src, blocks); + } while (blocks); } static int sha512_ce_update(struct shash_desc *desc, const u8 *data, unsigned int len) { - sha512_block_fn *fn = crypto_simd_usable() ? sha512_ce_transform - : sha512_arm64_transform; - - sha512_base_do_update(desc, data, len, fn); - return 0; + return sha512_base_do_update_blocks(desc, data, len, + sha512_ce_transform); } static int sha512_ce_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) { - sha512_block_fn *fn = crypto_simd_usable() ? sha512_ce_transform - : sha512_arm64_transform; - - sha512_base_do_update(desc, data, len, fn); - sha512_base_do_finalize(desc, fn); - return sha512_base_finish(desc, out); -} - -static int sha512_ce_final(struct shash_desc *desc, u8 *out) -{ - sha512_block_fn *fn = crypto_simd_usable() ? sha512_ce_transform - : sha512_arm64_transform; - - sha512_base_do_finalize(desc, fn); + sha512_base_do_finup(desc, data, len, sha512_ce_transform); return sha512_base_finish(desc, out); } static struct shash_alg algs[] = { { .init = sha384_base_init, .update = sha512_ce_update, - .final = sha512_ce_final, .finup = sha512_ce_finup, - .descsize = sizeof(struct sha512_state), + .descsize = SHA512_STATE_SIZE, .digestsize = SHA384_DIGEST_SIZE, .base.cra_name = "sha384", .base.cra_driver_name = "sha384-ce", .base.cra_priority = 200, + .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .base.cra_blocksize = SHA512_BLOCK_SIZE, .base.cra_module = THIS_MODULE, }, { .init = sha512_base_init, .update = sha512_ce_update, - .final = sha512_ce_final, .finup = sha512_ce_finup, - .descsize = sizeof(struct sha512_state), + .descsize = SHA512_STATE_SIZE, .digestsize = SHA512_DIGEST_SIZE, .base.cra_name = "sha512", .base.cra_driver_name = "sha512-ce", .base.cra_priority = 200, + .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .base.cra_blocksize = SHA512_BLOCK_SIZE, .base.cra_module = THIS_MODULE, } }; From 2d0d18d801b6efbd034725338ed1b08ad5c74e40 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 18 Apr 2025 11:00:36 +0800 Subject: [PATCH 199/382] crypto: arm/sha512 - Use API partial block handling Use the Crypto API partial block handling. Signed-off-by: Herbert Xu --- arch/arm64/crypto/sha512-glue.c | 28 +++++++++++----------------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/arch/arm64/crypto/sha512-glue.c b/arch/arm64/crypto/sha512-glue.c index f789deabefc0..ab2e1c13dfad 100644 --- a/arch/arm64/crypto/sha512-glue.c +++ b/arch/arm64/crypto/sha512-glue.c @@ -6,10 +6,10 @@ */ #include -#include -#include #include #include +#include +#include MODULE_DESCRIPTION("SHA-384/SHA-512 secure hash for arm64"); MODULE_AUTHOR("Andy Polyakov "); @@ -20,7 +20,6 @@ MODULE_ALIAS_CRYPTO("sha512"); asmlinkage void sha512_block_data_order(u64 *digest, const void *data, unsigned int num_blks); -EXPORT_SYMBOL(sha512_block_data_order); static void sha512_arm64_transform(struct sha512_state *sst, u8 const *src, int blocks) @@ -31,46 +30,41 @@ static void sha512_arm64_transform(struct sha512_state *sst, u8 const *src, static int sha512_update(struct shash_desc *desc, const u8 *data, unsigned int len) { - return sha512_base_do_update(desc, data, len, sha512_arm64_transform); + return sha512_base_do_update_blocks(desc, data, len, + sha512_arm64_transform); } static int sha512_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) { - if (len) - sha512_base_do_update(desc, data, len, sha512_arm64_transform); - sha512_base_do_finalize(desc, sha512_arm64_transform); - + sha512_base_do_finup(desc, data, len, sha512_arm64_transform); return sha512_base_finish(desc, out); } -static int sha512_final(struct shash_desc *desc, u8 *out) -{ - return sha512_finup(desc, NULL, 0, out); -} - static struct shash_alg algs[] = { { .digestsize = SHA512_DIGEST_SIZE, .init = sha512_base_init, .update = sha512_update, - .final = sha512_final, .finup = sha512_finup, - .descsize = sizeof(struct sha512_state), + .descsize = SHA512_STATE_SIZE, .base.cra_name = "sha512", .base.cra_driver_name = "sha512-arm64", .base.cra_priority = 150, + .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .base.cra_blocksize = SHA512_BLOCK_SIZE, .base.cra_module = THIS_MODULE, }, { .digestsize = SHA384_DIGEST_SIZE, .init = sha384_base_init, .update = sha512_update, - .final = sha512_final, .finup = sha512_finup, - .descsize = sizeof(struct sha512_state), + .descsize = SHA512_STATE_SIZE, .base.cra_name = "sha384", .base.cra_driver_name = "sha384-arm64", .base.cra_priority = 150, + .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .base.cra_blocksize = SHA384_BLOCK_SIZE, .base.cra_module = THIS_MODULE, } }; From 572b5c4682c77be19699ab38a2c477ec2b5a7226 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 18 Apr 2025 11:00:38 +0800 Subject: [PATCH 200/382] crypto: s390/sha512 - Use API partial block handling Use the Crypto API partial block handling. Signed-off-by: Herbert Xu --- arch/s390/crypto/sha.h | 14 ++-- arch/s390/crypto/sha512_s390.c | 45 ++++++------ arch/s390/crypto/sha_common.c | 124 ++++----------------------------- 3 files changed, 47 insertions(+), 136 deletions(-) diff --git a/arch/s390/crypto/sha.h b/arch/s390/crypto/sha.h index d95437ebe1ca..0a3cc1739144 100644 --- a/arch/s390/crypto/sha.h +++ b/arch/s390/crypto/sha.h @@ -10,28 +10,32 @@ #ifndef _CRYPTO_ARCH_S390_SHA_H #define _CRYPTO_ARCH_S390_SHA_H +#include #include #include /* must be big enough for the largest SHA variant */ #define CPACF_MAX_PARMBLOCK_SIZE SHA3_STATE_SIZE #define SHA_MAX_BLOCK_SIZE SHA3_224_BLOCK_SIZE -#define S390_SHA_CTX_SIZE offsetof(struct s390_sha_ctx, buf) +#define S390_SHA_CTX_SIZE sizeof(struct s390_sha_ctx) struct s390_sha_ctx { u64 count; /* message length in bytes */ - u32 state[CPACF_MAX_PARMBLOCK_SIZE / sizeof(u32)]; + union { + u32 state[CPACF_MAX_PARMBLOCK_SIZE / sizeof(u32)]; + struct { + u64 state[SHA512_DIGEST_SIZE]; + u64 count_hi; + } sha512; + }; int func; /* KIMD function to use */ bool first_message_part; - u8 buf[SHA_MAX_BLOCK_SIZE]; }; struct shash_desc; -int s390_sha_update(struct shash_desc *desc, const u8 *data, unsigned int len); int s390_sha_update_blocks(struct shash_desc *desc, const u8 *data, unsigned int len); -int s390_sha_final(struct shash_desc *desc, u8 *out); int s390_sha_finup(struct shash_desc *desc, const u8 *src, unsigned int len, u8 *out); diff --git a/arch/s390/crypto/sha512_s390.c b/arch/s390/crypto/sha512_s390.c index 04f11c407763..14818fcc9cd4 100644 --- a/arch/s390/crypto/sha512_s390.c +++ b/arch/s390/crypto/sha512_s390.c @@ -7,14 +7,13 @@ * Copyright IBM Corp. 2007 * Author(s): Jan Glauber (jang@de.ibm.com) */ +#include #include #include +#include #include -#include #include #include -#include -#include #include "sha.h" @@ -22,15 +21,16 @@ static int sha512_init(struct shash_desc *desc) { struct s390_sha_ctx *ctx = shash_desc_ctx(desc); - *(__u64 *)&ctx->state[0] = SHA512_H0; - *(__u64 *)&ctx->state[2] = SHA512_H1; - *(__u64 *)&ctx->state[4] = SHA512_H2; - *(__u64 *)&ctx->state[6] = SHA512_H3; - *(__u64 *)&ctx->state[8] = SHA512_H4; - *(__u64 *)&ctx->state[10] = SHA512_H5; - *(__u64 *)&ctx->state[12] = SHA512_H6; - *(__u64 *)&ctx->state[14] = SHA512_H7; + ctx->sha512.state[0] = SHA512_H0; + ctx->sha512.state[2] = SHA512_H1; + ctx->sha512.state[4] = SHA512_H2; + ctx->sha512.state[6] = SHA512_H3; + ctx->sha512.state[8] = SHA512_H4; + ctx->sha512.state[10] = SHA512_H5; + ctx->sha512.state[12] = SHA512_H6; + ctx->sha512.state[14] = SHA512_H7; ctx->count = 0; + ctx->sha512.count_hi = 0; ctx->func = CPACF_KIMD_SHA_512; return 0; @@ -42,9 +42,8 @@ static int sha512_export(struct shash_desc *desc, void *out) struct sha512_state *octx = out; octx->count[0] = sctx->count; - octx->count[1] = 0; + octx->count[1] = sctx->sha512.count_hi; memcpy(octx->state, sctx->state, sizeof(octx->state)); - memcpy(octx->buf, sctx->buf, sizeof(octx->buf)); return 0; } @@ -53,12 +52,10 @@ static int sha512_import(struct shash_desc *desc, const void *in) struct s390_sha_ctx *sctx = shash_desc_ctx(desc); const struct sha512_state *ictx = in; - if (unlikely(ictx->count[1])) - return -ERANGE; sctx->count = ictx->count[0]; + sctx->sha512.count_hi = ictx->count[1]; memcpy(sctx->state, ictx->state, sizeof(ictx->state)); - memcpy(sctx->buf, ictx->buf, sizeof(ictx->buf)); sctx->func = CPACF_KIMD_SHA_512; return 0; } @@ -66,16 +63,18 @@ static int sha512_import(struct shash_desc *desc, const void *in) static struct shash_alg sha512_alg = { .digestsize = SHA512_DIGEST_SIZE, .init = sha512_init, - .update = s390_sha_update, - .final = s390_sha_final, + .update = s390_sha_update_blocks, + .finup = s390_sha_finup, .export = sha512_export, .import = sha512_import, .descsize = sizeof(struct s390_sha_ctx), - .statesize = sizeof(struct sha512_state), + .statesize = SHA512_STATE_SIZE, .base = { .cra_name = "sha512", .cra_driver_name= "sha512-s390", .cra_priority = 300, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA512_BLOCK_SIZE, .cra_module = THIS_MODULE, } @@ -104,17 +103,19 @@ static int sha384_init(struct shash_desc *desc) static struct shash_alg sha384_alg = { .digestsize = SHA384_DIGEST_SIZE, .init = sha384_init, - .update = s390_sha_update, - .final = s390_sha_final, + .update = s390_sha_update_blocks, + .finup = s390_sha_finup, .export = sha512_export, .import = sha512_import, .descsize = sizeof(struct s390_sha_ctx), - .statesize = sizeof(struct sha512_state), + .statesize = SHA512_STATE_SIZE, .base = { .cra_name = "sha384", .cra_driver_name= "sha384-s390", .cra_priority = 300, .cra_blocksize = SHA384_BLOCK_SIZE, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_ctxsize = sizeof(struct s390_sha_ctx), .cra_module = THIS_MODULE, } diff --git a/arch/s390/crypto/sha_common.c b/arch/s390/crypto/sha_common.c index 69e23e0c5394..b5e2c365ea05 100644 --- a/arch/s390/crypto/sha_common.c +++ b/arch/s390/crypto/sha_common.c @@ -13,51 +13,6 @@ #include #include "sha.h" -int s390_sha_update(struct shash_desc *desc, const u8 *data, unsigned int len) -{ - struct s390_sha_ctx *ctx = shash_desc_ctx(desc); - unsigned int bsize = crypto_shash_blocksize(desc->tfm); - unsigned int index, n; - int fc; - - /* how much is already in the buffer? */ - index = ctx->count % bsize; - ctx->count += len; - - if ((index + len) < bsize) - goto store; - - fc = ctx->func; - if (ctx->first_message_part) - fc |= CPACF_KIMD_NIP; - - /* process one stored block */ - if (index) { - memcpy(ctx->buf + index, data, bsize - index); - cpacf_kimd(fc, ctx->state, ctx->buf, bsize); - ctx->first_message_part = 0; - fc &= ~CPACF_KIMD_NIP; - data += bsize - index; - len -= bsize - index; - index = 0; - } - - /* process as many blocks as possible */ - if (len >= bsize) { - n = (len / bsize) * bsize; - cpacf_kimd(fc, ctx->state, data, n); - ctx->first_message_part = 0; - data += n; - len -= n; - } -store: - if (len) - memcpy(ctx->buf + index , data, len); - - return 0; -} -EXPORT_SYMBOL_GPL(s390_sha_update); - int s390_sha_update_blocks(struct shash_desc *desc, const u8 *data, unsigned int len) { @@ -73,6 +28,13 @@ int s390_sha_update_blocks(struct shash_desc *desc, const u8 *data, /* process as many blocks as possible */ n = (len / bsize) * bsize; ctx->count += n; + switch (ctx->func) { + case CPACF_KLMD_SHA_512: + case CPACF_KLMD_SHA3_384: + if (ctx->count < n) + ctx->sha512.count_hi++; + break; + } cpacf_kimd(fc, ctx->state, data, n); ctx->first_message_part = 0; return len - n; @@ -98,61 +60,6 @@ static int s390_crypto_shash_parmsize(int func) } } -int s390_sha_final(struct shash_desc *desc, u8 *out) -{ - struct s390_sha_ctx *ctx = shash_desc_ctx(desc); - unsigned int bsize = crypto_shash_blocksize(desc->tfm); - u64 bits; - unsigned int n; - int mbl_offset, fc; - - n = ctx->count % bsize; - bits = ctx->count * 8; - mbl_offset = s390_crypto_shash_parmsize(ctx->func); - if (mbl_offset < 0) - return -EINVAL; - - mbl_offset = mbl_offset / sizeof(u32); - - /* set total msg bit length (mbl) in CPACF parmblock */ - switch (ctx->func) { - case CPACF_KLMD_SHA_1: - case CPACF_KLMD_SHA_256: - memcpy(ctx->state + mbl_offset, &bits, sizeof(bits)); - break; - case CPACF_KLMD_SHA_512: - /* - * the SHA512 parmblock has a 128-bit mbl field, clear - * high-order u64 field, copy bits to low-order u64 field - */ - memset(ctx->state + mbl_offset, 0x00, sizeof(bits)); - mbl_offset += sizeof(u64) / sizeof(u32); - memcpy(ctx->state + mbl_offset, &bits, sizeof(bits)); - break; - case CPACF_KLMD_SHA3_224: - case CPACF_KLMD_SHA3_256: - case CPACF_KLMD_SHA3_384: - case CPACF_KLMD_SHA3_512: - break; - default: - return -EINVAL; - } - - fc = ctx->func; - fc |= test_facility(86) ? CPACF_KLMD_DUFOP : 0; - if (ctx->first_message_part) - fc |= CPACF_KLMD_NIP; - cpacf_klmd(fc, ctx->state, ctx->buf, n); - - /* copy digest to out */ - memcpy(out, ctx->state, crypto_shash_digestsize(desc->tfm)); - /* wipe context */ - memset(ctx, 0, sizeof *ctx); - - return 0; -} -EXPORT_SYMBOL_GPL(s390_sha_final); - int s390_sha_finup(struct shash_desc *desc, const u8 *src, unsigned int len, u8 *out) { @@ -171,19 +78,18 @@ int s390_sha_finup(struct shash_desc *desc, const u8 *src, unsigned int len, /* set total msg bit length (mbl) in CPACF parmblock */ switch (ctx->func) { + case CPACF_KLMD_SHA_512: + /* The SHA512 parmblock has a 128-bit mbl field. */ + if (ctx->count < len) + ctx->sha512.count_hi++; + ctx->sha512.count_hi <<= 3; + ctx->sha512.count_hi |= ctx->count >> 61; + mbl_offset += sizeof(u64) / sizeof(u32); + fallthrough; case CPACF_KLMD_SHA_1: case CPACF_KLMD_SHA_256: memcpy(ctx->state + mbl_offset, &bits, sizeof(bits)); break; - case CPACF_KLMD_SHA_512: - /* - * the SHA512 parmblock has a 128-bit mbl field, clear - * high-order u64 field, copy bits to low-order u64 field - */ - memset(ctx->state + mbl_offset, 0x00, sizeof(bits)); - mbl_offset += sizeof(u64) / sizeof(u32); - memcpy(ctx->state + mbl_offset, &bits, sizeof(bits)); - break; case CPACF_KLMD_SHA3_224: case CPACF_KLMD_SHA3_256: case CPACF_KLMD_SHA3_384: From d8573b2cf7e2a3c22f6683add9eb239e3bd048ea Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 18 Apr 2025 11:00:41 +0800 Subject: [PATCH 201/382] crypto: sparc/sha512 - Use API partial block handling Use the Crypto API partial block handling. Signed-off-by: Herbert Xu --- arch/sparc/crypto/sha512_glue.c | 102 +++++--------------------------- 1 file changed, 15 insertions(+), 87 deletions(-) diff --git a/arch/sparc/crypto/sha512_glue.c b/arch/sparc/crypto/sha512_glue.c index d66efa4ec59a..1d0e1f98ca46 100644 --- a/arch/sparc/crypto/sha512_glue.c +++ b/arch/sparc/crypto/sha512_glue.c @@ -10,115 +10,43 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt +#include +#include #include -#include -#include -#include -#include #include #include - -#include -#include +#include +#include #include "opcodes.h" asmlinkage void sha512_sparc64_transform(u64 *digest, const char *data, unsigned int rounds); -static void __sha512_sparc64_update(struct sha512_state *sctx, const u8 *data, - unsigned int len, unsigned int partial) +static void sha512_block(struct sha512_state *sctx, const u8 *src, int blocks) { - unsigned int done = 0; - - if ((sctx->count[0] += len) < len) - sctx->count[1]++; - if (partial) { - done = SHA512_BLOCK_SIZE - partial; - memcpy(sctx->buf + partial, data, done); - sha512_sparc64_transform(sctx->state, sctx->buf, 1); - } - if (len - done >= SHA512_BLOCK_SIZE) { - const unsigned int rounds = (len - done) / SHA512_BLOCK_SIZE; - - sha512_sparc64_transform(sctx->state, data + done, rounds); - done += rounds * SHA512_BLOCK_SIZE; - } - - memcpy(sctx->buf, data + done, len - done); + sha512_sparc64_transform(sctx->state, src, blocks); } static int sha512_sparc64_update(struct shash_desc *desc, const u8 *data, unsigned int len) { - struct sha512_state *sctx = shash_desc_ctx(desc); - unsigned int partial = sctx->count[0] % SHA512_BLOCK_SIZE; - - /* Handle the fast case right here */ - if (partial + len < SHA512_BLOCK_SIZE) { - if ((sctx->count[0] += len) < len) - sctx->count[1]++; - memcpy(sctx->buf + partial, data, len); - } else - __sha512_sparc64_update(sctx, data, len, partial); - - return 0; + return sha512_base_do_update_blocks(desc, data, len, sha512_block); } -static int sha512_sparc64_final(struct shash_desc *desc, u8 *out) +static int sha512_sparc64_finup(struct shash_desc *desc, const u8 *src, + unsigned int len, u8 *out) { - struct sha512_state *sctx = shash_desc_ctx(desc); - unsigned int i, index, padlen; - __be64 *dst = (__be64 *)out; - __be64 bits[2]; - static const u8 padding[SHA512_BLOCK_SIZE] = { 0x80, }; - - /* Save number of bits */ - bits[1] = cpu_to_be64(sctx->count[0] << 3); - bits[0] = cpu_to_be64(sctx->count[1] << 3 | sctx->count[0] >> 61); - - /* Pad out to 112 mod 128 and append length */ - index = sctx->count[0] % SHA512_BLOCK_SIZE; - padlen = (index < 112) ? (112 - index) : ((SHA512_BLOCK_SIZE+112) - index); - - /* We need to fill a whole block for __sha512_sparc64_update() */ - if (padlen <= 112) { - if ((sctx->count[0] += padlen) < padlen) - sctx->count[1]++; - memcpy(sctx->buf + index, padding, padlen); - } else { - __sha512_sparc64_update(sctx, padding, padlen, index); - } - __sha512_sparc64_update(sctx, (const u8 *)&bits, sizeof(bits), 112); - - /* Store state in digest */ - for (i = 0; i < 8; i++) - dst[i] = cpu_to_be64(sctx->state[i]); - - /* Wipe context */ - memset(sctx, 0, sizeof(*sctx)); - - return 0; -} - -static int sha384_sparc64_final(struct shash_desc *desc, u8 *hash) -{ - u8 D[64]; - - sha512_sparc64_final(desc, D); - - memcpy(hash, D, 48); - memzero_explicit(D, 64); - - return 0; + sha512_base_do_finup(desc, src, len, sha512_block); + return sha512_base_finish(desc, out); } static struct shash_alg sha512 = { .digestsize = SHA512_DIGEST_SIZE, .init = sha512_base_init, .update = sha512_sparc64_update, - .final = sha512_sparc64_final, - .descsize = sizeof(struct sha512_state), + .finup = sha512_sparc64_finup, + .descsize = SHA512_STATE_SIZE, .base = { .cra_name = "sha512", .cra_driver_name= "sha512-sparc64", @@ -132,8 +60,8 @@ static struct shash_alg sha384 = { .digestsize = SHA384_DIGEST_SIZE, .init = sha384_base_init, .update = sha512_sparc64_update, - .final = sha384_sparc64_final, - .descsize = sizeof(struct sha512_state), + .finup = sha512_sparc64_finup, + .descsize = SHA512_STATE_SIZE, .base = { .cra_name = "sha384", .cra_driver_name= "sha384-sparc64", From f9fbc8bff1e19dc88ceeb24b39fd78631e5ead13 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 18 Apr 2025 11:00:43 +0800 Subject: [PATCH 202/382] crypto: sha512_base - Remove partial block helpers Now that all sha256_base users have been converted to use the API partial block handling, remove the partial block helpers. Signed-off-by: Herbert Xu --- include/crypto/sha512_base.h | 64 ------------------------------------ 1 file changed, 64 deletions(-) diff --git a/include/crypto/sha512_base.h b/include/crypto/sha512_base.h index e9f302ec3ede..aa814bab442d 100644 --- a/include/crypto/sha512_base.h +++ b/include/crypto/sha512_base.h @@ -53,46 +53,6 @@ static inline int sha512_base_init(struct shash_desc *desc) return 0; } -static inline int sha512_base_do_update(struct shash_desc *desc, - const u8 *data, - unsigned int len, - sha512_block_fn *block_fn) -{ - struct sha512_state *sctx = shash_desc_ctx(desc); - unsigned int partial = sctx->count[0] % SHA512_BLOCK_SIZE; - - sctx->count[0] += len; - if (sctx->count[0] < len) - sctx->count[1]++; - - if (unlikely((partial + len) >= SHA512_BLOCK_SIZE)) { - int blocks; - - if (partial) { - int p = SHA512_BLOCK_SIZE - partial; - - memcpy(sctx->buf + partial, data, p); - data += p; - len -= p; - - block_fn(sctx, sctx->buf, 1); - } - - blocks = len / SHA512_BLOCK_SIZE; - len %= SHA512_BLOCK_SIZE; - - if (blocks) { - block_fn(sctx, data, blocks); - data += blocks * SHA512_BLOCK_SIZE; - } - partial = 0; - } - if (len) - memcpy(sctx->buf + partial, data, len); - - return 0; -} - static inline int sha512_base_do_update_blocks(struct shash_desc *desc, const u8 *data, unsigned int len, @@ -142,30 +102,6 @@ static inline int sha512_base_do_finup(struct shash_desc *desc, const u8 *src, return 0; } -static inline int sha512_base_do_finalize(struct shash_desc *desc, - sha512_block_fn *block_fn) -{ - const int bit_offset = SHA512_BLOCK_SIZE - sizeof(__be64[2]); - struct sha512_state *sctx = shash_desc_ctx(desc); - __be64 *bits = (__be64 *)(sctx->buf + bit_offset); - unsigned int partial = sctx->count[0] % SHA512_BLOCK_SIZE; - - sctx->buf[partial++] = 0x80; - if (partial > bit_offset) { - memset(sctx->buf + partial, 0x0, SHA512_BLOCK_SIZE - partial); - partial = 0; - - block_fn(sctx, sctx->buf, 1); - } - - memset(sctx->buf + partial, 0x0, bit_offset - partial); - bits[0] = cpu_to_be64(sctx->count[1] << 3 | sctx->count[0] >> 61); - bits[1] = cpu_to_be64(sctx->count[0] << 3); - block_fn(sctx, sctx->buf, 1); - - return 0; -} - static inline int sha512_base_finish(struct shash_desc *desc, u8 *out) { unsigned int digest_size = crypto_shash_digestsize(desc->tfm); From 8266393e9ba0bee395bc256ba56427ec5493e2d1 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 18 Apr 2025 11:00:45 +0800 Subject: [PATCH 203/382] crypto: sm3-generic - Use API partial block handling Use the Crypto API partial block handling. Signed-off-by: Herbert Xu --- crypto/sm3_generic.c | 31 +++++++-------------------- include/crypto/sm3.h | 1 + include/crypto/sm3_base.h | 45 +++++++++++++++++++++++++++++++++++++-- 3 files changed, 52 insertions(+), 25 deletions(-) diff --git a/crypto/sm3_generic.c b/crypto/sm3_generic.c index a2d23a46924e..4fb6957c2f0c 100644 --- a/crypto/sm3_generic.c +++ b/crypto/sm3_generic.c @@ -9,15 +9,10 @@ */ #include -#include -#include -#include -#include #include #include -#include -#include -#include +#include +#include const u8 sm3_zero_message_hash[SM3_DIGEST_SIZE] = { 0x1A, 0xB2, 0x1D, 0x83, 0x55, 0xCF, 0xA1, 0x7F, @@ -30,38 +25,28 @@ EXPORT_SYMBOL_GPL(sm3_zero_message_hash); static int crypto_sm3_update(struct shash_desc *desc, const u8 *data, unsigned int len) { - sm3_update(shash_desc_ctx(desc), data, len); - return 0; -} - -static int crypto_sm3_final(struct shash_desc *desc, u8 *out) -{ - sm3_final(shash_desc_ctx(desc), out); - return 0; + return sm3_base_do_update_blocks(desc, data, len, sm3_block_generic); } static int crypto_sm3_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *hash) { - struct sm3_state *sctx = shash_desc_ctx(desc); - - if (len) - sm3_update(sctx, data, len); - sm3_final(sctx, hash); - return 0; + sm3_base_do_finup(desc, data, len, sm3_block_generic); + return sm3_base_finish(desc, hash); } static struct shash_alg sm3_alg = { .digestsize = SM3_DIGEST_SIZE, .init = sm3_base_init, .update = crypto_sm3_update, - .final = crypto_sm3_final, .finup = crypto_sm3_finup, - .descsize = sizeof(struct sm3_state), + .descsize = SM3_STATE_SIZE, .base = { .cra_name = "sm3", .cra_driver_name = "sm3-generic", .cra_priority = 100, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SM3_BLOCK_SIZE, .cra_module = THIS_MODULE, } diff --git a/include/crypto/sm3.h b/include/crypto/sm3.h index d49211ba9a20..6dc95264a836 100644 --- a/include/crypto/sm3.h +++ b/include/crypto/sm3.h @@ -14,6 +14,7 @@ #define SM3_DIGEST_SIZE 32 #define SM3_BLOCK_SIZE 64 +#define SM3_STATE_SIZE 40 #define SM3_T1 0x79CC4519 #define SM3_T2 0x7A879D8A diff --git a/include/crypto/sm3_base.h b/include/crypto/sm3_base.h index 835896228e8e..9460589c8cb8 100644 --- a/include/crypto/sm3_base.h +++ b/include/crypto/sm3_base.h @@ -62,6 +62,49 @@ static inline int sm3_base_do_update(struct shash_desc *desc, return 0; } +static inline int sm3_base_do_update_blocks(struct shash_desc *desc, + const u8 *data, unsigned int len, + sm3_block_fn *block_fn) +{ + unsigned int remain = len - round_down(len, SM3_BLOCK_SIZE); + struct sm3_state *sctx = shash_desc_ctx(desc); + + sctx->count += len - remain; + block_fn(sctx, data, len / SM3_BLOCK_SIZE); + return remain; +} + +static inline int sm3_base_do_finup(struct shash_desc *desc, + const u8 *src, unsigned int len, + sm3_block_fn *block_fn) +{ + unsigned int bit_offset = SM3_BLOCK_SIZE / 8 - 1; + struct sm3_state *sctx = shash_desc_ctx(desc); + union { + __be64 b64[SM3_BLOCK_SIZE / 4]; + u8 u8[SM3_BLOCK_SIZE * 2]; + } block = {}; + + if (len >= SM3_BLOCK_SIZE) { + int remain; + + remain = sm3_base_do_update_blocks(desc, src, len, block_fn); + src += len - remain; + len = remain; + } + + if (len >= bit_offset * 8) + bit_offset += SM3_BLOCK_SIZE / 8; + memcpy(&block, src, len); + block.u8[len] = 0x80; + sctx->count += len; + block.b64[bit_offset] = cpu_to_be64(sctx->count << 3); + block_fn(sctx, block.u8, (bit_offset + 1) * 8 / SM3_BLOCK_SIZE); + memzero_explicit(&block, sizeof(block)); + + return 0; +} + static inline int sm3_base_do_finalize(struct shash_desc *desc, sm3_block_fn *block_fn) { @@ -93,8 +136,6 @@ static inline int sm3_base_finish(struct shash_desc *desc, u8 *out) for (i = 0; i < SM3_DIGEST_SIZE / sizeof(__be32); i++) put_unaligned_be32(sctx->state[i], digest++); - - memzero_explicit(sctx, sizeof(*sctx)); return 0; } From 6ba8c5f5a4fdd78c7b8ee591be7a6c7a1fb7f446 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 18 Apr 2025 11:00:48 +0800 Subject: [PATCH 204/382] crypto: arm64/sm3-ce - Use API partial block handling Use the Crypto API partial block handling. Also remove the unnecessary SIMD fallback path. Signed-off-by: Herbert Xu --- arch/arm64/crypto/sm3-ce-glue.c | 48 ++++++--------------------------- 1 file changed, 8 insertions(+), 40 deletions(-) diff --git a/arch/arm64/crypto/sm3-ce-glue.c b/arch/arm64/crypto/sm3-ce-glue.c index 1a71788c4cda..eac6f5fa0abe 100644 --- a/arch/arm64/crypto/sm3-ce-glue.c +++ b/arch/arm64/crypto/sm3-ce-glue.c @@ -6,14 +6,11 @@ */ #include -#include -#include #include -#include #include #include #include -#include +#include #include MODULE_DESCRIPTION("SM3 secure hash using ARMv8 Crypto Extensions"); @@ -26,50 +23,20 @@ asmlinkage void sm3_ce_transform(struct sm3_state *sst, u8 const *src, static int sm3_ce_update(struct shash_desc *desc, const u8 *data, unsigned int len) { - if (!crypto_simd_usable()) { - sm3_update(shash_desc_ctx(desc), data, len); - return 0; - } + int remain; kernel_neon_begin(); - sm3_base_do_update(desc, data, len, sm3_ce_transform); + remain = sm3_base_do_update_blocks(desc, data, len, sm3_ce_transform); kernel_neon_end(); - - return 0; -} - -static int sm3_ce_final(struct shash_desc *desc, u8 *out) -{ - if (!crypto_simd_usable()) { - sm3_final(shash_desc_ctx(desc), out); - return 0; - } - - kernel_neon_begin(); - sm3_base_do_finalize(desc, sm3_ce_transform); - kernel_neon_end(); - - return sm3_base_finish(desc, out); + return remain; } static int sm3_ce_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) { - if (!crypto_simd_usable()) { - struct sm3_state *sctx = shash_desc_ctx(desc); - - if (len) - sm3_update(sctx, data, len); - sm3_final(sctx, out); - return 0; - } - kernel_neon_begin(); - if (len) - sm3_base_do_update(desc, data, len, sm3_ce_transform); - sm3_base_do_finalize(desc, sm3_ce_transform); + sm3_base_do_finup(desc, data, len, sm3_ce_transform); kernel_neon_end(); - return sm3_base_finish(desc, out); } @@ -77,11 +44,12 @@ static struct shash_alg sm3_alg = { .digestsize = SM3_DIGEST_SIZE, .init = sm3_base_init, .update = sm3_ce_update, - .final = sm3_ce_final, .finup = sm3_ce_finup, - .descsize = sizeof(struct sm3_state), + .descsize = SM3_STATE_SIZE, .base.cra_name = "sm3", .base.cra_driver_name = "sm3-ce", + .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .base.cra_blocksize = SM3_BLOCK_SIZE, .base.cra_module = THIS_MODULE, .base.cra_priority = 400, From 045f17b444b270eb627142008ab26ec6ede61a95 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 18 Apr 2025 11:00:50 +0800 Subject: [PATCH 205/382] crypto: arm64/sm3-neon - Use API partial block handling Use the Crypto API partial block handling. Also remove the unnecessary SIMD fallback path. Signed-off-by: Herbert Xu --- arch/arm64/crypto/sm3-neon-glue.c | 48 ++++++------------------------- 1 file changed, 8 insertions(+), 40 deletions(-) diff --git a/arch/arm64/crypto/sm3-neon-glue.c b/arch/arm64/crypto/sm3-neon-glue.c index 8dd71ce79b69..6c4611a503a3 100644 --- a/arch/arm64/crypto/sm3-neon-glue.c +++ b/arch/arm64/crypto/sm3-neon-glue.c @@ -6,14 +6,11 @@ */ #include -#include -#include #include -#include #include #include #include -#include +#include #include @@ -23,50 +20,20 @@ asmlinkage void sm3_neon_transform(struct sm3_state *sst, u8 const *src, static int sm3_neon_update(struct shash_desc *desc, const u8 *data, unsigned int len) { - if (!crypto_simd_usable()) { - sm3_update(shash_desc_ctx(desc), data, len); - return 0; - } + int remain; kernel_neon_begin(); - sm3_base_do_update(desc, data, len, sm3_neon_transform); + remain = sm3_base_do_update_blocks(desc, data, len, sm3_neon_transform); kernel_neon_end(); - - return 0; -} - -static int sm3_neon_final(struct shash_desc *desc, u8 *out) -{ - if (!crypto_simd_usable()) { - sm3_final(shash_desc_ctx(desc), out); - return 0; - } - - kernel_neon_begin(); - sm3_base_do_finalize(desc, sm3_neon_transform); - kernel_neon_end(); - - return sm3_base_finish(desc, out); + return remain; } static int sm3_neon_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) { - if (!crypto_simd_usable()) { - struct sm3_state *sctx = shash_desc_ctx(desc); - - if (len) - sm3_update(sctx, data, len); - sm3_final(sctx, out); - return 0; - } - kernel_neon_begin(); - if (len) - sm3_base_do_update(desc, data, len, sm3_neon_transform); - sm3_base_do_finalize(desc, sm3_neon_transform); + sm3_base_do_finup(desc, data, len, sm3_neon_transform); kernel_neon_end(); - return sm3_base_finish(desc, out); } @@ -74,11 +41,12 @@ static struct shash_alg sm3_alg = { .digestsize = SM3_DIGEST_SIZE, .init = sm3_base_init, .update = sm3_neon_update, - .final = sm3_neon_final, .finup = sm3_neon_finup, - .descsize = sizeof(struct sm3_state), + .descsize = SM3_STATE_SIZE, .base.cra_name = "sm3", .base.cra_driver_name = "sm3-neon", + .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .base.cra_blocksize = SM3_BLOCK_SIZE, .base.cra_module = THIS_MODULE, .base.cra_priority = 200, From 1523eaed0ac5e8dda3751d6a4bf4071ed23af8bc Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 18 Apr 2025 11:00:52 +0800 Subject: [PATCH 206/382] crypto: riscv/sm3 - Use API partial block handling Use the Crypto API partial block handling. Signed-off-by: Herbert Xu --- arch/riscv/crypto/sm3-riscv64-glue.c | 49 ++++++++++------------------ 1 file changed, 17 insertions(+), 32 deletions(-) diff --git a/arch/riscv/crypto/sm3-riscv64-glue.c b/arch/riscv/crypto/sm3-riscv64-glue.c index e1737a970c7c..abdfe4a63a27 100644 --- a/arch/riscv/crypto/sm3-riscv64-glue.c +++ b/arch/riscv/crypto/sm3-riscv64-glue.c @@ -13,8 +13,9 @@ #include #include #include +#include #include -#include +#include #include /* @@ -24,8 +25,8 @@ asmlinkage void sm3_transform_zvksh_zvkb( struct sm3_state *state, const u8 *data, int num_blocks); -static int riscv64_sm3_update(struct shash_desc *desc, const u8 *data, - unsigned int len) +static void sm3_block(struct sm3_state *state, const u8 *data, + int num_blocks) { /* * Ensure struct sm3_state begins directly with the SM3 @@ -35,52 +36,36 @@ static int riscv64_sm3_update(struct shash_desc *desc, const u8 *data, if (crypto_simd_usable()) { kernel_vector_begin(); - sm3_base_do_update(desc, data, len, sm3_transform_zvksh_zvkb); + sm3_transform_zvksh_zvkb(state, data, num_blocks); kernel_vector_end(); } else { - sm3_update(shash_desc_ctx(desc), data, len); + sm3_block_generic(state, data, num_blocks); } - return 0; +} + +static int riscv64_sm3_update(struct shash_desc *desc, const u8 *data, + unsigned int len) +{ + return sm3_base_do_update_blocks(desc, data, len, sm3_block); } static int riscv64_sm3_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) { - struct sm3_state *ctx; - - if (crypto_simd_usable()) { - kernel_vector_begin(); - if (len) - sm3_base_do_update(desc, data, len, - sm3_transform_zvksh_zvkb); - sm3_base_do_finalize(desc, sm3_transform_zvksh_zvkb); - kernel_vector_end(); - - return sm3_base_finish(desc, out); - } - - ctx = shash_desc_ctx(desc); - if (len) - sm3_update(ctx, data, len); - sm3_final(ctx, out); - - return 0; -} - -static int riscv64_sm3_final(struct shash_desc *desc, u8 *out) -{ - return riscv64_sm3_finup(desc, NULL, 0, out); + sm3_base_do_finup(desc, data, len, sm3_block); + return sm3_base_finish(desc, out); } static struct shash_alg riscv64_sm3_alg = { .init = sm3_base_init, .update = riscv64_sm3_update, - .final = riscv64_sm3_final, .finup = riscv64_sm3_finup, - .descsize = sizeof(struct sm3_state), + .descsize = SM3_STATE_SIZE, .digestsize = SM3_DIGEST_SIZE, .base = { .cra_blocksize = SM3_BLOCK_SIZE, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_priority = 300, .cra_name = "sm3", .cra_driver_name = "sm3-riscv64-zvksh-zvkb", From 68932c6be344bce60fe6672c50358340ac8dcea6 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 18 Apr 2025 11:00:55 +0800 Subject: [PATCH 207/382] crypto: x86/sm3 - Use API partial block handling Use the Crypto API partial block handling. Also remove the unnecessary SIMD fallback path. Signed-off-by: Herbert Xu --- arch/x86/crypto/sm3_avx_glue.c | 54 +++++++--------------------------- 1 file changed, 10 insertions(+), 44 deletions(-) diff --git a/arch/x86/crypto/sm3_avx_glue.c b/arch/x86/crypto/sm3_avx_glue.c index 661b6f22ffcd..6e8c42b9dc8e 100644 --- a/arch/x86/crypto/sm3_avx_glue.c +++ b/arch/x86/crypto/sm3_avx_glue.c @@ -10,12 +10,11 @@ #include #include -#include -#include -#include #include #include -#include +#include +#include +#include asmlinkage void sm3_transform_avx(struct sm3_state *state, const u8 *data, int nblocks); @@ -23,13 +22,7 @@ asmlinkage void sm3_transform_avx(struct sm3_state *state, static int sm3_avx_update(struct shash_desc *desc, const u8 *data, unsigned int len) { - struct sm3_state *sctx = shash_desc_ctx(desc); - - if (!crypto_simd_usable() || - (sctx->count % SM3_BLOCK_SIZE) + len < SM3_BLOCK_SIZE) { - sm3_update(sctx, data, len); - return 0; - } + int remain; /* * Make sure struct sm3_state begins directly with the SM3 @@ -38,45 +31,17 @@ static int sm3_avx_update(struct shash_desc *desc, const u8 *data, BUILD_BUG_ON(offsetof(struct sm3_state, state) != 0); kernel_fpu_begin(); - sm3_base_do_update(desc, data, len, sm3_transform_avx); + remain = sm3_base_do_update_blocks(desc, data, len, sm3_transform_avx); kernel_fpu_end(); - - return 0; + return remain; } static int sm3_avx_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) { - if (!crypto_simd_usable()) { - struct sm3_state *sctx = shash_desc_ctx(desc); - - if (len) - sm3_update(sctx, data, len); - - sm3_final(sctx, out); - return 0; - } - kernel_fpu_begin(); - if (len) - sm3_base_do_update(desc, data, len, sm3_transform_avx); - sm3_base_do_finalize(desc, sm3_transform_avx); + sm3_base_do_finup(desc, data, len, sm3_transform_avx); kernel_fpu_end(); - - return sm3_base_finish(desc, out); -} - -static int sm3_avx_final(struct shash_desc *desc, u8 *out) -{ - if (!crypto_simd_usable()) { - sm3_final(shash_desc_ctx(desc), out); - return 0; - } - - kernel_fpu_begin(); - sm3_base_do_finalize(desc, sm3_transform_avx); - kernel_fpu_end(); - return sm3_base_finish(desc, out); } @@ -84,13 +49,14 @@ static struct shash_alg sm3_avx_alg = { .digestsize = SM3_DIGEST_SIZE, .init = sm3_base_init, .update = sm3_avx_update, - .final = sm3_avx_final, .finup = sm3_avx_finup, - .descsize = sizeof(struct sm3_state), + .descsize = SM3_STATE_SIZE, .base = { .cra_name = "sm3", .cra_driver_name = "sm3-avx", .cra_priority = 300, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SM3_BLOCK_SIZE, .cra_module = THIS_MODULE, } From 9939049085bb80205b5e7e5b2b1d594f2731917f Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 18 Apr 2025 11:00:57 +0800 Subject: [PATCH 208/382] crypto: lib/sm3 - Remove partial block helpers Now that all sm3_base users have been converted to use the API partial block handling, remove the partial block helpers as well as the lib/crypto functions. Signed-off-by: Herbert Xu --- include/crypto/sm3.h | 2 -- include/crypto/sm3_base.h | 64 ++---------------------------------- lib/crypto/sm3.c | 68 +++------------------------------------ 3 files changed, 6 insertions(+), 128 deletions(-) diff --git a/include/crypto/sm3.h b/include/crypto/sm3.h index 6dc95264a836..c8d02c86c298 100644 --- a/include/crypto/sm3.h +++ b/include/crypto/sm3.h @@ -60,7 +60,5 @@ static inline void sm3_init(struct sm3_state *sctx) } void sm3_block_generic(struct sm3_state *sctx, u8 const *data, int blocks); -void sm3_update(struct sm3_state *sctx, const u8 *data, unsigned int len); -void sm3_final(struct sm3_state *sctx, u8 *out); #endif diff --git a/include/crypto/sm3_base.h b/include/crypto/sm3_base.h index 9460589c8cb8..7c53570bc05e 100644 --- a/include/crypto/sm3_base.h +++ b/include/crypto/sm3_base.h @@ -11,9 +11,10 @@ #include #include -#include +#include #include #include +#include #include typedef void (sm3_block_fn)(struct sm3_state *sst, u8 const *src, int blocks); @@ -24,44 +25,6 @@ static inline int sm3_base_init(struct shash_desc *desc) return 0; } -static inline int sm3_base_do_update(struct shash_desc *desc, - const u8 *data, - unsigned int len, - sm3_block_fn *block_fn) -{ - struct sm3_state *sctx = shash_desc_ctx(desc); - unsigned int partial = sctx->count % SM3_BLOCK_SIZE; - - sctx->count += len; - - if (unlikely((partial + len) >= SM3_BLOCK_SIZE)) { - int blocks; - - if (partial) { - int p = SM3_BLOCK_SIZE - partial; - - memcpy(sctx->buffer + partial, data, p); - data += p; - len -= p; - - block_fn(sctx, sctx->buffer, 1); - } - - blocks = len / SM3_BLOCK_SIZE; - len %= SM3_BLOCK_SIZE; - - if (blocks) { - block_fn(sctx, data, blocks); - data += blocks * SM3_BLOCK_SIZE; - } - partial = 0; - } - if (len) - memcpy(sctx->buffer + partial, data, len); - - return 0; -} - static inline int sm3_base_do_update_blocks(struct shash_desc *desc, const u8 *data, unsigned int len, sm3_block_fn *block_fn) @@ -105,29 +68,6 @@ static inline int sm3_base_do_finup(struct shash_desc *desc, return 0; } -static inline int sm3_base_do_finalize(struct shash_desc *desc, - sm3_block_fn *block_fn) -{ - const int bit_offset = SM3_BLOCK_SIZE - sizeof(__be64); - struct sm3_state *sctx = shash_desc_ctx(desc); - __be64 *bits = (__be64 *)(sctx->buffer + bit_offset); - unsigned int partial = sctx->count % SM3_BLOCK_SIZE; - - sctx->buffer[partial++] = 0x80; - if (partial > bit_offset) { - memset(sctx->buffer + partial, 0x0, SM3_BLOCK_SIZE - partial); - partial = 0; - - block_fn(sctx, sctx->buffer, 1); - } - - memset(sctx->buffer + partial, 0x0, bit_offset - partial); - *bits = cpu_to_be64(sctx->count << 3); - block_fn(sctx, sctx->buffer, 1); - - return 0; -} - static inline int sm3_base_finish(struct shash_desc *desc, u8 *out) { struct sm3_state *sctx = shash_desc_ctx(desc); diff --git a/lib/crypto/sm3.c b/lib/crypto/sm3.c index de64aa913280..efff0e267d84 100644 --- a/lib/crypto/sm3.c +++ b/lib/crypto/sm3.c @@ -8,9 +8,11 @@ * Copyright (C) 2021 Tianjia Zhang */ -#include -#include #include +#include +#include +#include +#include static const u32 ____cacheline_aligned K[64] = { 0x79cc4519, 0xf3988a32, 0xe7311465, 0xce6228cb, @@ -179,67 +181,5 @@ void sm3_block_generic(struct sm3_state *sctx, u8 const *data, int blocks) } EXPORT_SYMBOL_GPL(sm3_block_generic); -void sm3_update(struct sm3_state *sctx, const u8 *data, unsigned int len) -{ - unsigned int partial = sctx->count % SM3_BLOCK_SIZE; - - sctx->count += len; - - if ((partial + len) >= SM3_BLOCK_SIZE) { - int blocks; - - if (partial) { - int p = SM3_BLOCK_SIZE - partial; - - memcpy(sctx->buffer + partial, data, p); - data += p; - len -= p; - - sm3_block_generic(sctx, sctx->buffer, 1); - } - - blocks = len / SM3_BLOCK_SIZE; - len %= SM3_BLOCK_SIZE; - - if (blocks) { - sm3_block_generic(sctx, data, blocks); - data += blocks * SM3_BLOCK_SIZE; - } - - partial = 0; - } - if (len) - memcpy(sctx->buffer + partial, data, len); -} -EXPORT_SYMBOL_GPL(sm3_update); - -void sm3_final(struct sm3_state *sctx, u8 *out) -{ - const int bit_offset = SM3_BLOCK_SIZE - sizeof(u64); - __be64 *bits = (__be64 *)(sctx->buffer + bit_offset); - __be32 *digest = (__be32 *)out; - unsigned int partial = sctx->count % SM3_BLOCK_SIZE; - int i; - - sctx->buffer[partial++] = 0x80; - if (partial > bit_offset) { - memset(sctx->buffer + partial, 0, SM3_BLOCK_SIZE - partial); - partial = 0; - - sm3_block_generic(sctx, sctx->buffer, 1); - } - - memset(sctx->buffer + partial, 0, bit_offset - partial); - *bits = cpu_to_be64(sctx->count << 3); - sm3_block_generic(sctx, sctx->buffer, 1); - - for (i = 0; i < 8; i++) - put_unaligned_be32(sctx->state[i], digest++); - - /* Zeroize sensitive information. */ - memzero_explicit(sctx, sizeof(*sctx)); -} -EXPORT_SYMBOL_GPL(sm3_final); - MODULE_DESCRIPTION("Generic SM3 library"); MODULE_LICENSE("GPL v2"); From ca5d7d5f7aeb2f8d3edb5d3ee9ed3344ca3b1c12 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 18 Apr 2025 11:00:59 +0800 Subject: [PATCH 209/382] crypto: cbcmac - Use API partial block handling Use the Crypto API partial block handling. Signed-off-by: Herbert Xu --- crypto/ccm.c | 59 +++++++++++++++++++++------------------------------- 1 file changed, 24 insertions(+), 35 deletions(-) diff --git a/crypto/ccm.c b/crypto/ccm.c index a0610ff6ce02..f3f455e4908b 100644 --- a/crypto/ccm.c +++ b/crypto/ccm.c @@ -10,11 +10,12 @@ #include #include #include +#include #include -#include #include #include #include +#include struct ccm_instance_ctx { struct crypto_skcipher_spawn ctr; @@ -54,11 +55,6 @@ struct cbcmac_tfm_ctx { struct crypto_cipher *child; }; -struct cbcmac_desc_ctx { - unsigned int len; - u8 dg[]; -}; - static inline struct crypto_ccm_req_priv_ctx *crypto_ccm_reqctx( struct aead_request *req) { @@ -783,12 +779,10 @@ static int crypto_cbcmac_digest_setkey(struct crypto_shash *parent, static int crypto_cbcmac_digest_init(struct shash_desc *pdesc) { - struct cbcmac_desc_ctx *ctx = shash_desc_ctx(pdesc); int bs = crypto_shash_digestsize(pdesc->tfm); + u8 *dg = shash_desc_ctx(pdesc); - ctx->len = 0; - memset(ctx->dg, 0, bs); - + memset(dg, 0, bs); return 0; } @@ -797,39 +791,34 @@ static int crypto_cbcmac_digest_update(struct shash_desc *pdesc, const u8 *p, { struct crypto_shash *parent = pdesc->tfm; struct cbcmac_tfm_ctx *tctx = crypto_shash_ctx(parent); - struct cbcmac_desc_ctx *ctx = shash_desc_ctx(pdesc); struct crypto_cipher *tfm = tctx->child; int bs = crypto_shash_digestsize(parent); + u8 *dg = shash_desc_ctx(pdesc); - while (len > 0) { - unsigned int l = min(len, bs - ctx->len); - - crypto_xor(&ctx->dg[ctx->len], p, l); - ctx->len +=l; - len -= l; - p += l; - - if (ctx->len == bs) { - crypto_cipher_encrypt_one(tfm, ctx->dg, ctx->dg); - ctx->len = 0; - } - } - - return 0; + do { + crypto_xor(dg, p, bs); + crypto_cipher_encrypt_one(tfm, dg, dg); + p += bs; + len -= bs; + } while (len >= bs); + return len; } -static int crypto_cbcmac_digest_final(struct shash_desc *pdesc, u8 *out) +static int crypto_cbcmac_digest_finup(struct shash_desc *pdesc, const u8 *src, + unsigned int len, u8 *out) { struct crypto_shash *parent = pdesc->tfm; struct cbcmac_tfm_ctx *tctx = crypto_shash_ctx(parent); - struct cbcmac_desc_ctx *ctx = shash_desc_ctx(pdesc); struct crypto_cipher *tfm = tctx->child; int bs = crypto_shash_digestsize(parent); + u8 *dg = shash_desc_ctx(pdesc); - if (ctx->len) - crypto_cipher_encrypt_one(tfm, ctx->dg, ctx->dg); - - memcpy(out, ctx->dg, bs); + if (len) { + crypto_xor(dg, src, len); + crypto_cipher_encrypt_one(tfm, out, dg); + return 0; + } + memcpy(out, dg, bs); return 0; } @@ -886,16 +875,16 @@ static int cbcmac_create(struct crypto_template *tmpl, struct rtattr **tb) inst->alg.base.cra_blocksize = alg->cra_blocksize; inst->alg.digestsize = alg->cra_blocksize; - inst->alg.descsize = sizeof(struct cbcmac_desc_ctx) + - alg->cra_blocksize; + inst->alg.descsize = alg->cra_blocksize; + inst->alg.base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY; inst->alg.base.cra_ctxsize = sizeof(struct cbcmac_tfm_ctx); inst->alg.base.cra_init = cbcmac_init_tfm; inst->alg.base.cra_exit = cbcmac_exit_tfm; inst->alg.init = crypto_cbcmac_digest_init; inst->alg.update = crypto_cbcmac_digest_update; - inst->alg.final = crypto_cbcmac_digest_final; + inst->alg.finup = crypto_cbcmac_digest_finup; inst->alg.setkey = crypto_cbcmac_digest_setkey; inst->free = shash_free_singlespawn_instance; From f4bb31367e1019f088f6c5665460e53f907a7e92 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 18 Apr 2025 11:01:02 +0800 Subject: [PATCH 210/382] crypto: cmac - Use API partial block handling Use the Crypto API partial block handling. Signed-off-by: Herbert Xu --- crypto/cmac.c | 92 ++++++++++----------------------------------------- 1 file changed, 18 insertions(+), 74 deletions(-) diff --git a/crypto/cmac.c b/crypto/cmac.c index c66a0f4d8808..f297042a324b 100644 --- a/crypto/cmac.c +++ b/crypto/cmac.c @@ -13,9 +13,12 @@ #include #include +#include #include #include #include +#include +#include /* * +------------------------ @@ -31,22 +34,6 @@ struct cmac_tfm_ctx { __be64 consts[]; }; -/* - * +------------------------ - * | - * +------------------------ - * | cmac_desc_ctx - * +------------------------ - * | odds (block size) - * +------------------------ - * | prev (block size) - * +------------------------ - */ -struct cmac_desc_ctx { - unsigned int len; - u8 odds[]; -}; - static int crypto_cmac_digest_setkey(struct crypto_shash *parent, const u8 *inkey, unsigned int keylen) { @@ -102,13 +89,10 @@ static int crypto_cmac_digest_setkey(struct crypto_shash *parent, static int crypto_cmac_digest_init(struct shash_desc *pdesc) { - struct cmac_desc_ctx *ctx = shash_desc_ctx(pdesc); int bs = crypto_shash_blocksize(pdesc->tfm); - u8 *prev = &ctx->odds[bs]; + u8 *prev = shash_desc_ctx(pdesc); - ctx->len = 0; memset(prev, 0, bs); - return 0; } @@ -117,77 +101,36 @@ static int crypto_cmac_digest_update(struct shash_desc *pdesc, const u8 *p, { struct crypto_shash *parent = pdesc->tfm; struct cmac_tfm_ctx *tctx = crypto_shash_ctx(parent); - struct cmac_desc_ctx *ctx = shash_desc_ctx(pdesc); struct crypto_cipher *tfm = tctx->child; int bs = crypto_shash_blocksize(parent); - u8 *odds = ctx->odds; - u8 *prev = odds + bs; + u8 *prev = shash_desc_ctx(pdesc); - /* checking the data can fill the block */ - if ((ctx->len + len) <= bs) { - memcpy(odds + ctx->len, p, len); - ctx->len += len; - return 0; - } - - /* filling odds with new data and encrypting it */ - memcpy(odds + ctx->len, p, bs - ctx->len); - len -= bs - ctx->len; - p += bs - ctx->len; - - crypto_xor(prev, odds, bs); - crypto_cipher_encrypt_one(tfm, prev, prev); - - /* clearing the length */ - ctx->len = 0; - - /* encrypting the rest of data */ - while (len > bs) { + do { crypto_xor(prev, p, bs); crypto_cipher_encrypt_one(tfm, prev, prev); p += bs; len -= bs; - } - - /* keeping the surplus of blocksize */ - if (len) { - memcpy(odds, p, len); - ctx->len = len; - } - - return 0; + } while (len >= bs); + return len; } -static int crypto_cmac_digest_final(struct shash_desc *pdesc, u8 *out) +static int crypto_cmac_digest_finup(struct shash_desc *pdesc, const u8 *src, + unsigned int len, u8 *out) { struct crypto_shash *parent = pdesc->tfm; struct cmac_tfm_ctx *tctx = crypto_shash_ctx(parent); - struct cmac_desc_ctx *ctx = shash_desc_ctx(pdesc); struct crypto_cipher *tfm = tctx->child; int bs = crypto_shash_blocksize(parent); - u8 *odds = ctx->odds; - u8 *prev = odds + bs; + u8 *prev = shash_desc_ctx(pdesc); unsigned int offset = 0; - if (ctx->len != bs) { - unsigned int rlen; - u8 *p = odds + ctx->len; - - *p = 0x80; - p++; - - rlen = bs - ctx->len - 1; - if (rlen) - memset(p, 0, rlen); - + crypto_xor(prev, src, len); + if (len != bs) { + prev[len] ^= 0x80; offset += bs; } - - crypto_xor(prev, odds, bs); crypto_xor(prev, (const u8 *)tctx->consts + offset, bs); - crypto_cipher_encrypt_one(tfm, out, prev); - return 0; } @@ -269,13 +212,14 @@ static int cmac_create(struct crypto_template *tmpl, struct rtattr **tb) inst->alg.base.cra_blocksize = alg->cra_blocksize; inst->alg.base.cra_ctxsize = sizeof(struct cmac_tfm_ctx) + alg->cra_blocksize * 2; + inst->alg.base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINAL_NONZERO; inst->alg.digestsize = alg->cra_blocksize; - inst->alg.descsize = sizeof(struct cmac_desc_ctx) + - alg->cra_blocksize * 2; + inst->alg.descsize = alg->cra_blocksize; inst->alg.init = crypto_cmac_digest_init; inst->alg.update = crypto_cmac_digest_update; - inst->alg.final = crypto_cmac_digest_final; + inst->alg.finup = crypto_cmac_digest_finup; inst->alg.setkey = crypto_cmac_digest_setkey; inst->alg.init_tfm = cmac_init_tfm; inst->alg.clone_tfm = cmac_clone_tfm; From 566ec9adfef56dcb23e9cd94a5046bae4c3694ab Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 18 Apr 2025 11:01:04 +0800 Subject: [PATCH 211/382] crypto: xcbc - Use API partial block handling Use the Crypto API partial block handling. Signed-off-by: Herbert Xu --- crypto/xcbc.c | 92 ++++++++++----------------------------------------- 1 file changed, 18 insertions(+), 74 deletions(-) diff --git a/crypto/xcbc.c b/crypto/xcbc.c index fc785667b134..970ff581dc58 100644 --- a/crypto/xcbc.c +++ b/crypto/xcbc.c @@ -8,9 +8,12 @@ #include #include +#include #include #include #include +#include +#include static u_int32_t ks[12] = {0x01010101, 0x01010101, 0x01010101, 0x01010101, 0x02020202, 0x02020202, 0x02020202, 0x02020202, @@ -30,22 +33,6 @@ struct xcbc_tfm_ctx { u8 consts[]; }; -/* - * +------------------------ - * | - * +------------------------ - * | xcbc_desc_ctx - * +------------------------ - * | odds (block size) - * +------------------------ - * | prev (block size) - * +------------------------ - */ -struct xcbc_desc_ctx { - unsigned int len; - u8 odds[]; -}; - #define XCBC_BLOCKSIZE 16 static int crypto_xcbc_digest_setkey(struct crypto_shash *parent, @@ -70,13 +57,10 @@ static int crypto_xcbc_digest_setkey(struct crypto_shash *parent, static int crypto_xcbc_digest_init(struct shash_desc *pdesc) { - struct xcbc_desc_ctx *ctx = shash_desc_ctx(pdesc); int bs = crypto_shash_blocksize(pdesc->tfm); - u8 *prev = &ctx->odds[bs]; + u8 *prev = shash_desc_ctx(pdesc); - ctx->len = 0; memset(prev, 0, bs); - return 0; } @@ -85,77 +69,36 @@ static int crypto_xcbc_digest_update(struct shash_desc *pdesc, const u8 *p, { struct crypto_shash *parent = pdesc->tfm; struct xcbc_tfm_ctx *tctx = crypto_shash_ctx(parent); - struct xcbc_desc_ctx *ctx = shash_desc_ctx(pdesc); struct crypto_cipher *tfm = tctx->child; int bs = crypto_shash_blocksize(parent); - u8 *odds = ctx->odds; - u8 *prev = odds + bs; + u8 *prev = shash_desc_ctx(pdesc); - /* checking the data can fill the block */ - if ((ctx->len + len) <= bs) { - memcpy(odds + ctx->len, p, len); - ctx->len += len; - return 0; - } - - /* filling odds with new data and encrypting it */ - memcpy(odds + ctx->len, p, bs - ctx->len); - len -= bs - ctx->len; - p += bs - ctx->len; - - crypto_xor(prev, odds, bs); - crypto_cipher_encrypt_one(tfm, prev, prev); - - /* clearing the length */ - ctx->len = 0; - - /* encrypting the rest of data */ - while (len > bs) { + do { crypto_xor(prev, p, bs); crypto_cipher_encrypt_one(tfm, prev, prev); p += bs; len -= bs; - } - - /* keeping the surplus of blocksize */ - if (len) { - memcpy(odds, p, len); - ctx->len = len; - } - - return 0; + } while (len >= bs); + return len; } -static int crypto_xcbc_digest_final(struct shash_desc *pdesc, u8 *out) +static int crypto_xcbc_digest_finup(struct shash_desc *pdesc, const u8 *src, + unsigned int len, u8 *out) { struct crypto_shash *parent = pdesc->tfm; struct xcbc_tfm_ctx *tctx = crypto_shash_ctx(parent); - struct xcbc_desc_ctx *ctx = shash_desc_ctx(pdesc); struct crypto_cipher *tfm = tctx->child; int bs = crypto_shash_blocksize(parent); - u8 *odds = ctx->odds; - u8 *prev = odds + bs; + u8 *prev = shash_desc_ctx(pdesc); unsigned int offset = 0; - if (ctx->len != bs) { - unsigned int rlen; - u8 *p = odds + ctx->len; - - *p = 0x80; - p++; - - rlen = bs - ctx->len -1; - if (rlen) - memset(p, 0, rlen); - + crypto_xor(prev, src, len); + if (len != bs) { + prev[len] ^= 0x80; offset += bs; } - - crypto_xor(prev, odds, bs); crypto_xor(prev, &tctx->consts[offset], bs); - crypto_cipher_encrypt_one(tfm, out, prev); - return 0; } @@ -216,17 +159,18 @@ static int xcbc_create(struct crypto_template *tmpl, struct rtattr **tb) inst->alg.base.cra_blocksize = alg->cra_blocksize; inst->alg.base.cra_ctxsize = sizeof(struct xcbc_tfm_ctx) + alg->cra_blocksize * 2; + inst->alg.base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINAL_NONZERO; inst->alg.digestsize = alg->cra_blocksize; - inst->alg.descsize = sizeof(struct xcbc_desc_ctx) + - alg->cra_blocksize * 2; + inst->alg.descsize = alg->cra_blocksize; inst->alg.base.cra_init = xcbc_init_tfm; inst->alg.base.cra_exit = xcbc_exit_tfm; inst->alg.init = crypto_xcbc_digest_init; inst->alg.update = crypto_xcbc_digest_update; - inst->alg.final = crypto_xcbc_digest_final; + inst->alg.finup = crypto_xcbc_digest_finup; inst->alg.setkey = crypto_xcbc_digest_setkey; inst->free = shash_free_singlespawn_instance; From 4dc3c40c4daea76c28d1a8a1de727410bd765431 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 18 Apr 2025 11:01:06 +0800 Subject: [PATCH 212/382] crypto: arm64/aes - Use API partial block handling Use the Crypto API partial block handling. Also remove the unnecessary SIMD fallback path. Signed-off-by: Herbert Xu --- arch/arm64/crypto/aes-glue.c | 122 ++++++++++++----------------------- 1 file changed, 41 insertions(+), 81 deletions(-) diff --git a/arch/arm64/crypto/aes-glue.c b/arch/arm64/crypto/aes-glue.c index 5ca3b5661749..81560f722b9d 100644 --- a/arch/arm64/crypto/aes-glue.c +++ b/arch/arm64/crypto/aes-glue.c @@ -5,19 +5,20 @@ * Copyright (C) 2013 - 2017 Linaro Ltd */ -#include #include -#include +#include #include #include -#include #include -#include #include #include -#include -#include +#include +#include #include +#include +#include +#include +#include #include "aes-ce-setkey.h" @@ -130,7 +131,6 @@ struct mac_tfm_ctx { }; struct mac_desc_ctx { - unsigned int len; u8 dg[AES_BLOCK_SIZE]; }; @@ -869,109 +869,64 @@ static int mac_init(struct shash_desc *desc) struct mac_desc_ctx *ctx = shash_desc_ctx(desc); memset(ctx->dg, 0, AES_BLOCK_SIZE); - ctx->len = 0; - return 0; } static void mac_do_update(struct crypto_aes_ctx *ctx, u8 const in[], int blocks, - u8 dg[], int enc_before, int enc_after) + u8 dg[], int enc_before) { int rounds = 6 + ctx->key_length / 4; + int rem; - if (crypto_simd_usable()) { - int rem; - - do { - kernel_neon_begin(); - rem = aes_mac_update(in, ctx->key_enc, rounds, blocks, - dg, enc_before, enc_after); - kernel_neon_end(); - in += (blocks - rem) * AES_BLOCK_SIZE; - blocks = rem; - enc_before = 0; - } while (blocks); - } else { - if (enc_before) - aes_encrypt(ctx, dg, dg); - - while (blocks--) { - crypto_xor(dg, in, AES_BLOCK_SIZE); - in += AES_BLOCK_SIZE; - - if (blocks || enc_after) - aes_encrypt(ctx, dg, dg); - } - } + do { + kernel_neon_begin(); + rem = aes_mac_update(in, ctx->key_enc, rounds, blocks, + dg, enc_before, !enc_before); + kernel_neon_end(); + in += (blocks - rem) * AES_BLOCK_SIZE; + blocks = rem; + } while (blocks); } static int mac_update(struct shash_desc *desc, const u8 *p, unsigned int len) { struct mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); struct mac_desc_ctx *ctx = shash_desc_ctx(desc); + int blocks = len / AES_BLOCK_SIZE; - while (len > 0) { - unsigned int l; - - if ((ctx->len % AES_BLOCK_SIZE) == 0 && - (ctx->len + len) > AES_BLOCK_SIZE) { - - int blocks = len / AES_BLOCK_SIZE; - - len %= AES_BLOCK_SIZE; - - mac_do_update(&tctx->key, p, blocks, ctx->dg, - (ctx->len != 0), (len != 0)); - - p += blocks * AES_BLOCK_SIZE; - - if (!len) { - ctx->len = AES_BLOCK_SIZE; - break; - } - ctx->len = 0; - } - - l = min(len, AES_BLOCK_SIZE - ctx->len); - - if (l <= AES_BLOCK_SIZE) { - crypto_xor(ctx->dg + ctx->len, p, l); - ctx->len += l; - len -= l; - p += l; - } - } - - return 0; + len %= AES_BLOCK_SIZE; + mac_do_update(&tctx->key, p, blocks, ctx->dg, 0); + return len; } -static int cbcmac_final(struct shash_desc *desc, u8 *out) +static int cbcmac_finup(struct shash_desc *desc, const u8 *src, + unsigned int len, u8 *out) { struct mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); struct mac_desc_ctx *ctx = shash_desc_ctx(desc); - mac_do_update(&tctx->key, NULL, 0, ctx->dg, (ctx->len != 0), 0); - + if (len) { + crypto_xor(ctx->dg, src, len); + mac_do_update(&tctx->key, NULL, 0, ctx->dg, 1); + } memcpy(out, ctx->dg, AES_BLOCK_SIZE); - return 0; } -static int cmac_final(struct shash_desc *desc, u8 *out) +static int cmac_finup(struct shash_desc *desc, const u8 *src, unsigned int len, + u8 *out) { struct mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); struct mac_desc_ctx *ctx = shash_desc_ctx(desc); u8 *consts = tctx->consts; - if (ctx->len != AES_BLOCK_SIZE) { - ctx->dg[ctx->len] ^= 0x80; + crypto_xor(ctx->dg, src, len); + if (len != AES_BLOCK_SIZE) { + ctx->dg[len] ^= 0x80; consts += AES_BLOCK_SIZE; } - - mac_do_update(&tctx->key, consts, 1, ctx->dg, 0, 1); - + mac_do_update(&tctx->key, consts, 1, ctx->dg, 0); memcpy(out, ctx->dg, AES_BLOCK_SIZE); - return 0; } @@ -979,6 +934,8 @@ static struct shash_alg mac_algs[] = { { .base.cra_name = "cmac(aes)", .base.cra_driver_name = "cmac-aes-" MODE, .base.cra_priority = PRIO, + .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINAL_NONZERO, .base.cra_blocksize = AES_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct mac_tfm_ctx) + 2 * AES_BLOCK_SIZE, @@ -987,13 +944,15 @@ static struct shash_alg mac_algs[] = { { .digestsize = AES_BLOCK_SIZE, .init = mac_init, .update = mac_update, - .final = cmac_final, + .finup = cmac_finup, .setkey = cmac_setkey, .descsize = sizeof(struct mac_desc_ctx), }, { .base.cra_name = "xcbc(aes)", .base.cra_driver_name = "xcbc-aes-" MODE, .base.cra_priority = PRIO, + .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINAL_NONZERO, .base.cra_blocksize = AES_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct mac_tfm_ctx) + 2 * AES_BLOCK_SIZE, @@ -1002,13 +961,14 @@ static struct shash_alg mac_algs[] = { { .digestsize = AES_BLOCK_SIZE, .init = mac_init, .update = mac_update, - .final = cmac_final, + .finup = cmac_finup, .setkey = xcbc_setkey, .descsize = sizeof(struct mac_desc_ctx), }, { .base.cra_name = "cbcmac(aes)", .base.cra_driver_name = "cbcmac-aes-" MODE, .base.cra_priority = PRIO, + .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .base.cra_blocksize = AES_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct mac_tfm_ctx), .base.cra_module = THIS_MODULE, @@ -1016,7 +976,7 @@ static struct shash_alg mac_algs[] = { { .digestsize = AES_BLOCK_SIZE, .init = mac_init, .update = mac_update, - .final = cbcmac_final, + .finup = cbcmac_finup, .setkey = cbcmac_setkey, .descsize = sizeof(struct mac_desc_ctx), } }; From ef17008481a17bbf89229982fd2d32aa48e27f35 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 18 Apr 2025 11:01:09 +0800 Subject: [PATCH 213/382] crypto: arm64/sm4 - Use API partial block handling Use the Crypto API partial block handling. Signed-off-by: Herbert Xu --- arch/arm64/crypto/sm4-ce-glue.c | 98 +++++++++++---------------------- 1 file changed, 31 insertions(+), 67 deletions(-) diff --git a/arch/arm64/crypto/sm4-ce-glue.c b/arch/arm64/crypto/sm4-ce-glue.c index f11cf26e5a20..7a60e7b559dc 100644 --- a/arch/arm64/crypto/sm4-ce-glue.c +++ b/arch/arm64/crypto/sm4-ce-glue.c @@ -8,19 +8,18 @@ * Copyright (C) 2022 Tianjia Zhang */ -#include -#include -#include -#include #include -#include #include -#include -#include #include +#include #include -#include #include +#include +#include +#include +#include +#include +#include #define BYTES2BLKS(nbytes) ((nbytes) >> 4) @@ -64,7 +63,6 @@ struct sm4_mac_tfm_ctx { }; struct sm4_mac_desc_ctx { - unsigned int len; u8 digest[SM4_BLOCK_SIZE]; }; @@ -591,8 +589,6 @@ static int sm4_mac_init(struct shash_desc *desc) struct sm4_mac_desc_ctx *ctx = shash_desc_ctx(desc); memset(ctx->digest, 0, SM4_BLOCK_SIZE); - ctx->len = 0; - return 0; } @@ -601,87 +597,50 @@ static int sm4_mac_update(struct shash_desc *desc, const u8 *p, { struct sm4_mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); struct sm4_mac_desc_ctx *ctx = shash_desc_ctx(desc); - unsigned int l, nblocks; + unsigned int nblocks = len / SM4_BLOCK_SIZE; - if (len == 0) - return 0; - - if (ctx->len || ctx->len + len < SM4_BLOCK_SIZE) { - l = min(len, SM4_BLOCK_SIZE - ctx->len); - - crypto_xor(ctx->digest + ctx->len, p, l); - ctx->len += l; - len -= l; - p += l; - } - - if (len && (ctx->len % SM4_BLOCK_SIZE) == 0) { - kernel_neon_begin(); - - if (len < SM4_BLOCK_SIZE && ctx->len == SM4_BLOCK_SIZE) { - sm4_ce_crypt_block(tctx->key.rkey_enc, - ctx->digest, ctx->digest); - ctx->len = 0; - } else { - nblocks = len / SM4_BLOCK_SIZE; - len %= SM4_BLOCK_SIZE; - - sm4_ce_mac_update(tctx->key.rkey_enc, ctx->digest, p, - nblocks, (ctx->len == SM4_BLOCK_SIZE), - (len != 0)); - - p += nblocks * SM4_BLOCK_SIZE; - - if (len == 0) - ctx->len = SM4_BLOCK_SIZE; - } - - kernel_neon_end(); - - if (len) { - crypto_xor(ctx->digest, p, len); - ctx->len = len; - } - } - - return 0; + len %= SM4_BLOCK_SIZE; + kernel_neon_begin(); + sm4_ce_mac_update(tctx->key.rkey_enc, ctx->digest, p, + nblocks, false, true); + kernel_neon_end(); + return len; } -static int sm4_cmac_final(struct shash_desc *desc, u8 *out) +static int sm4_cmac_finup(struct shash_desc *desc, const u8 *src, + unsigned int len, u8 *out) { struct sm4_mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); struct sm4_mac_desc_ctx *ctx = shash_desc_ctx(desc); const u8 *consts = tctx->consts; - if (ctx->len != SM4_BLOCK_SIZE) { - ctx->digest[ctx->len] ^= 0x80; + crypto_xor(ctx->digest, src, len); + if (len != SM4_BLOCK_SIZE) { + ctx->digest[len] ^= 0x80; consts += SM4_BLOCK_SIZE; } - kernel_neon_begin(); sm4_ce_mac_update(tctx->key.rkey_enc, ctx->digest, consts, 1, false, true); kernel_neon_end(); - memcpy(out, ctx->digest, SM4_BLOCK_SIZE); - return 0; } -static int sm4_cbcmac_final(struct shash_desc *desc, u8 *out) +static int sm4_cbcmac_finup(struct shash_desc *desc, const u8 *src, + unsigned int len, u8 *out) { struct sm4_mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); struct sm4_mac_desc_ctx *ctx = shash_desc_ctx(desc); - if (ctx->len) { + if (len) { + crypto_xor(ctx->digest, src, len); kernel_neon_begin(); sm4_ce_crypt_block(tctx->key.rkey_enc, ctx->digest, ctx->digest); kernel_neon_end(); } - memcpy(out, ctx->digest, SM4_BLOCK_SIZE); - return 0; } @@ -691,6 +650,8 @@ static struct shash_alg sm4_mac_algs[] = { .cra_name = "cmac(sm4)", .cra_driver_name = "cmac-sm4-ce", .cra_priority = 400, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINAL_NONZERO, .cra_blocksize = SM4_BLOCK_SIZE, .cra_ctxsize = sizeof(struct sm4_mac_tfm_ctx) + SM4_BLOCK_SIZE * 2, @@ -699,7 +660,7 @@ static struct shash_alg sm4_mac_algs[] = { .digestsize = SM4_BLOCK_SIZE, .init = sm4_mac_init, .update = sm4_mac_update, - .final = sm4_cmac_final, + .finup = sm4_cmac_finup, .setkey = sm4_cmac_setkey, .descsize = sizeof(struct sm4_mac_desc_ctx), }, { @@ -707,6 +668,8 @@ static struct shash_alg sm4_mac_algs[] = { .cra_name = "xcbc(sm4)", .cra_driver_name = "xcbc-sm4-ce", .cra_priority = 400, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINAL_NONZERO, .cra_blocksize = SM4_BLOCK_SIZE, .cra_ctxsize = sizeof(struct sm4_mac_tfm_ctx) + SM4_BLOCK_SIZE * 2, @@ -715,7 +678,7 @@ static struct shash_alg sm4_mac_algs[] = { .digestsize = SM4_BLOCK_SIZE, .init = sm4_mac_init, .update = sm4_mac_update, - .final = sm4_cmac_final, + .finup = sm4_cmac_finup, .setkey = sm4_xcbc_setkey, .descsize = sizeof(struct sm4_mac_desc_ctx), }, { @@ -723,6 +686,7 @@ static struct shash_alg sm4_mac_algs[] = { .cra_name = "cbcmac(sm4)", .cra_driver_name = "cbcmac-sm4-ce", .cra_priority = 400, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_blocksize = SM4_BLOCK_SIZE, .cra_ctxsize = sizeof(struct sm4_mac_tfm_ctx), .cra_module = THIS_MODULE, @@ -730,7 +694,7 @@ static struct shash_alg sm4_mac_algs[] = { .digestsize = SM4_BLOCK_SIZE, .init = sm4_mac_init, .update = sm4_mac_update, - .final = sm4_cbcmac_final, + .finup = sm4_cbcmac_finup, .setkey = sm4_cbcmac_setkey, .descsize = sizeof(struct sm4_mac_desc_ctx), } From 9420e628e7d8d0d41c45ceed8aac84c5e2e1f5c1 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 18 Apr 2025 11:01:11 +0800 Subject: [PATCH 214/382] crypto: nx - Use API partial block handling Use the Crypto API partial block handling. Also switch to the generic export format. Signed-off-by: Herbert Xu --- drivers/crypto/nx/nx-aes-xcbc.c | 128 +++++++++------------------- drivers/crypto/nx/nx-sha256.c | 130 ++++++++++++----------------- drivers/crypto/nx/nx-sha512.c | 143 +++++++++++++------------------- drivers/crypto/nx/nx.c | 15 ++-- drivers/crypto/nx/nx.h | 6 +- 5 files changed, 161 insertions(+), 261 deletions(-) diff --git a/drivers/crypto/nx/nx-aes-xcbc.c b/drivers/crypto/nx/nx-aes-xcbc.c index eb5c8f689360..bf465d824e2c 100644 --- a/drivers/crypto/nx/nx-aes-xcbc.c +++ b/drivers/crypto/nx/nx-aes-xcbc.c @@ -7,13 +7,14 @@ * Author: Kent Yoder */ -#include #include -#include +#include +#include +#include +#include #include -#include -#include -#include +#include +#include #include "nx_csbcpb.h" #include "nx.h" @@ -21,8 +22,6 @@ struct xcbc_state { u8 state[AES_BLOCK_SIZE]; - unsigned int count; - u8 buffer[AES_BLOCK_SIZE]; }; static int nx_xcbc_set_key(struct crypto_shash *desc, @@ -58,7 +57,7 @@ static int nx_xcbc_set_key(struct crypto_shash *desc, */ static int nx_xcbc_empty(struct shash_desc *desc, u8 *out) { - struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base); + struct nx_crypto_ctx *nx_ctx = crypto_shash_ctx(desc->tfm); struct nx_csbcpb *csbcpb = nx_ctx->csbcpb; struct nx_sg *in_sg, *out_sg; u8 keys[2][AES_BLOCK_SIZE]; @@ -135,9 +134,9 @@ out: return rc; } -static int nx_crypto_ctx_aes_xcbc_init2(struct crypto_tfm *tfm) +static int nx_crypto_ctx_aes_xcbc_init2(struct crypto_shash *tfm) { - struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(tfm); + struct nx_crypto_ctx *nx_ctx = crypto_shash_ctx(tfm); struct nx_csbcpb *csbcpb = nx_ctx->csbcpb; int err; @@ -166,31 +165,24 @@ static int nx_xcbc_update(struct shash_desc *desc, const u8 *data, unsigned int len) { + struct nx_crypto_ctx *nx_ctx = crypto_shash_ctx(desc->tfm); struct xcbc_state *sctx = shash_desc_ctx(desc); - struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base); struct nx_csbcpb *csbcpb = nx_ctx->csbcpb; struct nx_sg *in_sg; struct nx_sg *out_sg; - u32 to_process = 0, leftover, total; unsigned int max_sg_len; unsigned long irq_flags; + u32 to_process, total; int rc = 0; int data_len; spin_lock_irqsave(&nx_ctx->lock, irq_flags); + memcpy(csbcpb->cpb.aes_xcbc.out_cv_mac, sctx->state, AES_BLOCK_SIZE); + NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE; + NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION; - total = sctx->count + len; - - /* 2 cases for total data len: - * 1: <= AES_BLOCK_SIZE: copy into state, return 0 - * 2: > AES_BLOCK_SIZE: process X blocks, copy in leftover - */ - if (total <= AES_BLOCK_SIZE) { - memcpy(sctx->buffer + sctx->count, data, len); - sctx->count += len; - goto out; - } + total = len; in_sg = nx_ctx->in_sg; max_sg_len = min_t(u64, nx_driver.of.max_sg_len/sizeof(struct nx_sg), @@ -200,7 +192,7 @@ static int nx_xcbc_update(struct shash_desc *desc, data_len = AES_BLOCK_SIZE; out_sg = nx_build_sg_list(nx_ctx->out_sg, (u8 *)sctx->state, - &len, nx_ctx->ap->sglen); + &data_len, nx_ctx->ap->sglen); if (data_len != AES_BLOCK_SIZE) { rc = -EINVAL; @@ -210,56 +202,21 @@ static int nx_xcbc_update(struct shash_desc *desc, nx_ctx->op.outlen = (nx_ctx->out_sg - out_sg) * sizeof(struct nx_sg); do { - to_process = total - to_process; - to_process = to_process & ~(AES_BLOCK_SIZE - 1); + to_process = total & ~(AES_BLOCK_SIZE - 1); - leftover = total - to_process; - - /* the hardware will not accept a 0 byte operation for this - * algorithm and the operation MUST be finalized to be correct. - * So if we happen to get an update that falls on a block sized - * boundary, we must save off the last block to finalize with - * later. */ - if (!leftover) { - to_process -= AES_BLOCK_SIZE; - leftover = AES_BLOCK_SIZE; - } - - if (sctx->count) { - data_len = sctx->count; - in_sg = nx_build_sg_list(nx_ctx->in_sg, - (u8 *) sctx->buffer, - &data_len, - max_sg_len); - if (data_len != sctx->count) { - rc = -EINVAL; - goto out; - } - } - - data_len = to_process - sctx->count; in_sg = nx_build_sg_list(in_sg, (u8 *) data, - &data_len, + &to_process, max_sg_len); - if (data_len != to_process - sctx->count) { - rc = -EINVAL; - goto out; - } - nx_ctx->op.inlen = (nx_ctx->in_sg - in_sg) * sizeof(struct nx_sg); /* we've hit the nx chip previously and we're updating again, * so copy over the partial digest */ - if (NX_CPB_FDM(csbcpb) & NX_FDM_CONTINUATION) { - memcpy(csbcpb->cpb.aes_xcbc.cv, - csbcpb->cpb.aes_xcbc.out_cv_mac, - AES_BLOCK_SIZE); - } + memcpy(csbcpb->cpb.aes_xcbc.cv, + csbcpb->cpb.aes_xcbc.out_cv_mac, AES_BLOCK_SIZE); - NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE; if (!nx_ctx->op.inlen || !nx_ctx->op.outlen) { rc = -EINVAL; goto out; @@ -271,28 +228,24 @@ static int nx_xcbc_update(struct shash_desc *desc, atomic_inc(&(nx_ctx->stats->aes_ops)); - /* everything after the first update is continuation */ - NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION; - total -= to_process; - data += to_process - sctx->count; - sctx->count = 0; + data += to_process; in_sg = nx_ctx->in_sg; - } while (leftover > AES_BLOCK_SIZE); + } while (total >= AES_BLOCK_SIZE); - /* copy the leftover back into the state struct */ - memcpy(sctx->buffer, data, leftover); - sctx->count = leftover; + rc = total; + memcpy(sctx->state, csbcpb->cpb.aes_xcbc.out_cv_mac, AES_BLOCK_SIZE); out: spin_unlock_irqrestore(&nx_ctx->lock, irq_flags); return rc; } -static int nx_xcbc_final(struct shash_desc *desc, u8 *out) +static int nx_xcbc_finup(struct shash_desc *desc, const u8 *src, + unsigned int nbytes, u8 *out) { + struct nx_crypto_ctx *nx_ctx = crypto_shash_ctx(desc->tfm); struct xcbc_state *sctx = shash_desc_ctx(desc); - struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base); struct nx_csbcpb *csbcpb = nx_ctx->csbcpb; struct nx_sg *in_sg, *out_sg; unsigned long irq_flags; @@ -301,12 +254,10 @@ static int nx_xcbc_final(struct shash_desc *desc, u8 *out) spin_lock_irqsave(&nx_ctx->lock, irq_flags); - if (NX_CPB_FDM(csbcpb) & NX_FDM_CONTINUATION) { - /* we've hit the nx chip previously, now we're finalizing, - * so copy over the partial digest */ - memcpy(csbcpb->cpb.aes_xcbc.cv, - csbcpb->cpb.aes_xcbc.out_cv_mac, AES_BLOCK_SIZE); - } else if (sctx->count == 0) { + if (nbytes) { + /* non-zero final, so copy over the partial digest */ + memcpy(csbcpb->cpb.aes_xcbc.cv, sctx->state, AES_BLOCK_SIZE); + } else { /* * we've never seen an update, so this is a 0 byte op. The * hardware cannot handle a 0 byte op, so just ECB to @@ -320,11 +271,11 @@ static int nx_xcbc_final(struct shash_desc *desc, u8 *out) * this is not an intermediate operation */ NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE; - len = sctx->count; - in_sg = nx_build_sg_list(nx_ctx->in_sg, (u8 *)sctx->buffer, - &len, nx_ctx->ap->sglen); + len = nbytes; + in_sg = nx_build_sg_list(nx_ctx->in_sg, (u8 *)src, &len, + nx_ctx->ap->sglen); - if (len != sctx->count) { + if (len != nbytes) { rc = -EINVAL; goto out; } @@ -362,18 +313,19 @@ struct shash_alg nx_shash_aes_xcbc_alg = { .digestsize = AES_BLOCK_SIZE, .init = nx_xcbc_init, .update = nx_xcbc_update, - .final = nx_xcbc_final, + .finup = nx_xcbc_finup, .setkey = nx_xcbc_set_key, .descsize = sizeof(struct xcbc_state), - .statesize = sizeof(struct xcbc_state), + .init_tfm = nx_crypto_ctx_aes_xcbc_init2, + .exit_tfm = nx_crypto_ctx_shash_exit, .base = { .cra_name = "xcbc(aes)", .cra_driver_name = "xcbc-aes-nx", .cra_priority = 300, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINAL_NONZERO, .cra_blocksize = AES_BLOCK_SIZE, .cra_module = THIS_MODULE, .cra_ctxsize = sizeof(struct nx_crypto_ctx), - .cra_init = nx_crypto_ctx_aes_xcbc_init2, - .cra_exit = nx_crypto_ctx_exit, } }; diff --git a/drivers/crypto/nx/nx-sha256.c b/drivers/crypto/nx/nx-sha256.c index c3bebf0feabe..5b29dd026df2 100644 --- a/drivers/crypto/nx/nx-sha256.c +++ b/drivers/crypto/nx/nx-sha256.c @@ -9,9 +9,12 @@ #include #include +#include +#include #include -#include -#include +#include +#include +#include #include "nx_csbcpb.h" #include "nx.h" @@ -19,12 +22,11 @@ struct sha256_state_be { __be32 state[SHA256_DIGEST_SIZE / 4]; u64 count; - u8 buf[SHA256_BLOCK_SIZE]; }; -static int nx_crypto_ctx_sha256_init(struct crypto_tfm *tfm) +static int nx_crypto_ctx_sha256_init(struct crypto_shash *tfm) { - struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(tfm); + struct nx_crypto_ctx *nx_ctx = crypto_shash_ctx(tfm); int err; err = nx_crypto_ctx_sha_init(tfm); @@ -40,11 +42,10 @@ static int nx_crypto_ctx_sha256_init(struct crypto_tfm *tfm) return 0; } -static int nx_sha256_init(struct shash_desc *desc) { +static int nx_sha256_init(struct shash_desc *desc) +{ struct sha256_state_be *sctx = shash_desc_ctx(desc); - memset(sctx, 0, sizeof *sctx); - sctx->state[0] = __cpu_to_be32(SHA256_H0); sctx->state[1] = __cpu_to_be32(SHA256_H1); sctx->state[2] = __cpu_to_be32(SHA256_H2); @@ -61,30 +62,18 @@ static int nx_sha256_init(struct shash_desc *desc) { static int nx_sha256_update(struct shash_desc *desc, const u8 *data, unsigned int len) { + struct nx_crypto_ctx *nx_ctx = crypto_shash_ctx(desc->tfm); struct sha256_state_be *sctx = shash_desc_ctx(desc); - struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base); struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb; + u64 to_process, leftover, total = len; struct nx_sg *out_sg; - u64 to_process = 0, leftover, total; unsigned long irq_flags; int rc = 0; int data_len; u32 max_sg_len; - u64 buf_len = (sctx->count % SHA256_BLOCK_SIZE); spin_lock_irqsave(&nx_ctx->lock, irq_flags); - /* 2 cases for total data len: - * 1: < SHA256_BLOCK_SIZE: copy into state, return 0 - * 2: >= SHA256_BLOCK_SIZE: process X blocks, copy in leftover - */ - total = (sctx->count % SHA256_BLOCK_SIZE) + len; - if (total < SHA256_BLOCK_SIZE) { - memcpy(sctx->buf + buf_len, data, len); - sctx->count += len; - goto out; - } - memcpy(csbcpb->cpb.sha256.message_digest, sctx->state, SHA256_DIGEST_SIZE); NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE; NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION; @@ -105,41 +94,17 @@ static int nx_sha256_update(struct shash_desc *desc, const u8 *data, } do { - int used_sgs = 0; struct nx_sg *in_sg = nx_ctx->in_sg; - if (buf_len) { - data_len = buf_len; - in_sg = nx_build_sg_list(in_sg, - (u8 *) sctx->buf, - &data_len, - max_sg_len); + to_process = total & ~(SHA256_BLOCK_SIZE - 1); - if (data_len != buf_len) { - rc = -EINVAL; - goto out; - } - used_sgs = in_sg - nx_ctx->in_sg; - } - - /* to_process: SHA256_BLOCK_SIZE aligned chunk to be - * processed in this iteration. This value is restricted - * by sg list limits and number of sgs we already used - * for leftover data. (see above) - * In ideal case, we could allow NX_PAGE_SIZE * max_sg_len, - * but because data may not be aligned, we need to account - * for that too. */ - to_process = min_t(u64, total, - (max_sg_len - 1 - used_sgs) * NX_PAGE_SIZE); - to_process = to_process & ~(SHA256_BLOCK_SIZE - 1); - - data_len = to_process - buf_len; + data_len = to_process; in_sg = nx_build_sg_list(in_sg, (u8 *) data, &data_len, max_sg_len); nx_ctx->op.inlen = (nx_ctx->in_sg - in_sg) * sizeof(struct nx_sg); - to_process = data_len + buf_len; + to_process = data_len; leftover = total - to_process; /* @@ -162,26 +127,22 @@ static int nx_sha256_update(struct shash_desc *desc, const u8 *data, atomic_inc(&(nx_ctx->stats->sha256_ops)); total -= to_process; - data += to_process - buf_len; - buf_len = 0; - + data += to_process; + sctx->count += to_process; } while (leftover >= SHA256_BLOCK_SIZE); - /* copy the leftover back into the state struct */ - if (leftover) - memcpy(sctx->buf, data, leftover); - - sctx->count += len; + rc = leftover; memcpy(sctx->state, csbcpb->cpb.sha256.message_digest, SHA256_DIGEST_SIZE); out: spin_unlock_irqrestore(&nx_ctx->lock, irq_flags); return rc; } -static int nx_sha256_final(struct shash_desc *desc, u8 *out) +static int nx_sha256_finup(struct shash_desc *desc, const u8 *src, + unsigned int nbytes, u8 *out) { + struct nx_crypto_ctx *nx_ctx = crypto_shash_ctx(desc->tfm); struct sha256_state_be *sctx = shash_desc_ctx(desc); - struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base); struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb; struct nx_sg *in_sg, *out_sg; unsigned long irq_flags; @@ -197,25 +158,19 @@ static int nx_sha256_final(struct shash_desc *desc, u8 *out) nx_ctx->ap->databytelen/NX_PAGE_SIZE); /* final is represented by continuing the operation and indicating that - * this is not an intermediate operation */ - if (sctx->count >= SHA256_BLOCK_SIZE) { - /* we've hit the nx chip previously, now we're finalizing, - * so copy over the partial digest */ - memcpy(csbcpb->cpb.sha256.input_partial_digest, sctx->state, SHA256_DIGEST_SIZE); - NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE; - NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION; - } else { - NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE; - NX_CPB_FDM(csbcpb) &= ~NX_FDM_CONTINUATION; - } + * this is not an intermediate operation + * copy over the partial digest */ + memcpy(csbcpb->cpb.sha256.input_partial_digest, sctx->state, SHA256_DIGEST_SIZE); + NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE; + NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION; + sctx->count += nbytes; csbcpb->cpb.sha256.message_bit_length = (u64) (sctx->count * 8); - len = sctx->count & (SHA256_BLOCK_SIZE - 1); - in_sg = nx_build_sg_list(nx_ctx->in_sg, (u8 *) sctx->buf, - &len, max_sg_len); + len = nbytes; + in_sg = nx_build_sg_list(nx_ctx->in_sg, (u8 *)src, &len, max_sg_len); - if (len != (sctx->count & (SHA256_BLOCK_SIZE - 1))) { + if (len != nbytes) { rc = -EINVAL; goto out; } @@ -251,18 +206,34 @@ out: static int nx_sha256_export(struct shash_desc *desc, void *out) { struct sha256_state_be *sctx = shash_desc_ctx(desc); + union { + u8 *u8; + u32 *u32; + u64 *u64; + } p = { .u8 = out }; + int i; - memcpy(out, sctx, sizeof(*sctx)); + for (i = 0; i < SHA256_DIGEST_SIZE / sizeof(*p.u32); i++) + put_unaligned(be32_to_cpu(sctx->state[i]), p.u32++); + put_unaligned(sctx->count, p.u64++); return 0; } static int nx_sha256_import(struct shash_desc *desc, const void *in) { struct sha256_state_be *sctx = shash_desc_ctx(desc); + union { + const u8 *u8; + const u32 *u32; + const u64 *u64; + } p = { .u8 = in }; + int i; - memcpy(sctx, in, sizeof(*sctx)); + for (i = 0; i < SHA256_DIGEST_SIZE / sizeof(*p.u32); i++) + sctx->state[i] = cpu_to_be32(get_unaligned(p.u32++)); + sctx->count = get_unaligned(p.u64++); return 0; } @@ -270,19 +241,20 @@ struct shash_alg nx_shash_sha256_alg = { .digestsize = SHA256_DIGEST_SIZE, .init = nx_sha256_init, .update = nx_sha256_update, - .final = nx_sha256_final, + .finup = nx_sha256_finup, .export = nx_sha256_export, .import = nx_sha256_import, + .init_tfm = nx_crypto_ctx_sha256_init, + .exit_tfm = nx_crypto_ctx_shash_exit, .descsize = sizeof(struct sha256_state_be), .statesize = sizeof(struct sha256_state_be), .base = { .cra_name = "sha256", .cra_driver_name = "sha256-nx", .cra_priority = 300, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_blocksize = SHA256_BLOCK_SIZE, .cra_module = THIS_MODULE, .cra_ctxsize = sizeof(struct nx_crypto_ctx), - .cra_init = nx_crypto_ctx_sha256_init, - .cra_exit = nx_crypto_ctx_exit, } }; diff --git a/drivers/crypto/nx/nx-sha512.c b/drivers/crypto/nx/nx-sha512.c index 1ffb40d2c324..f74776b7d7d7 100644 --- a/drivers/crypto/nx/nx-sha512.c +++ b/drivers/crypto/nx/nx-sha512.c @@ -9,8 +9,12 @@ #include #include +#include +#include #include -#include +#include +#include +#include #include "nx_csbcpb.h" #include "nx.h" @@ -18,12 +22,11 @@ struct sha512_state_be { __be64 state[SHA512_DIGEST_SIZE / 8]; u64 count[2]; - u8 buf[SHA512_BLOCK_SIZE]; }; -static int nx_crypto_ctx_sha512_init(struct crypto_tfm *tfm) +static int nx_crypto_ctx_sha512_init(struct crypto_shash *tfm) { - struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(tfm); + struct nx_crypto_ctx *nx_ctx = crypto_shash_ctx(tfm); int err; err = nx_crypto_ctx_sha_init(tfm); @@ -43,8 +46,6 @@ static int nx_sha512_init(struct shash_desc *desc) { struct sha512_state_be *sctx = shash_desc_ctx(desc); - memset(sctx, 0, sizeof *sctx); - sctx->state[0] = __cpu_to_be64(SHA512_H0); sctx->state[1] = __cpu_to_be64(SHA512_H1); sctx->state[2] = __cpu_to_be64(SHA512_H2); @@ -54,6 +55,7 @@ static int nx_sha512_init(struct shash_desc *desc) sctx->state[6] = __cpu_to_be64(SHA512_H6); sctx->state[7] = __cpu_to_be64(SHA512_H7); sctx->count[0] = 0; + sctx->count[1] = 0; return 0; } @@ -61,30 +63,18 @@ static int nx_sha512_init(struct shash_desc *desc) static int nx_sha512_update(struct shash_desc *desc, const u8 *data, unsigned int len) { + struct nx_crypto_ctx *nx_ctx = crypto_shash_ctx(desc->tfm); struct sha512_state_be *sctx = shash_desc_ctx(desc); - struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base); struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb; + u64 to_process, leftover, total = len; struct nx_sg *out_sg; - u64 to_process, leftover = 0, total; unsigned long irq_flags; int rc = 0; int data_len; u32 max_sg_len; - u64 buf_len = (sctx->count[0] % SHA512_BLOCK_SIZE); spin_lock_irqsave(&nx_ctx->lock, irq_flags); - /* 2 cases for total data len: - * 1: < SHA512_BLOCK_SIZE: copy into state, return 0 - * 2: >= SHA512_BLOCK_SIZE: process X blocks, copy in leftover - */ - total = (sctx->count[0] % SHA512_BLOCK_SIZE) + len; - if (total < SHA512_BLOCK_SIZE) { - memcpy(sctx->buf + buf_len, data, len); - sctx->count[0] += len; - goto out; - } - memcpy(csbcpb->cpb.sha512.message_digest, sctx->state, SHA512_DIGEST_SIZE); NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE; NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION; @@ -105,45 +95,17 @@ static int nx_sha512_update(struct shash_desc *desc, const u8 *data, } do { - int used_sgs = 0; struct nx_sg *in_sg = nx_ctx->in_sg; - if (buf_len) { - data_len = buf_len; - in_sg = nx_build_sg_list(in_sg, - (u8 *) sctx->buf, - &data_len, max_sg_len); + to_process = total & ~(SHA512_BLOCK_SIZE - 1); - if (data_len != buf_len) { - rc = -EINVAL; - goto out; - } - used_sgs = in_sg - nx_ctx->in_sg; - } - - /* to_process: SHA512_BLOCK_SIZE aligned chunk to be - * processed in this iteration. This value is restricted - * by sg list limits and number of sgs we already used - * for leftover data. (see above) - * In ideal case, we could allow NX_PAGE_SIZE * max_sg_len, - * but because data may not be aligned, we need to account - * for that too. */ - to_process = min_t(u64, total, - (max_sg_len - 1 - used_sgs) * NX_PAGE_SIZE); - to_process = to_process & ~(SHA512_BLOCK_SIZE - 1); - - data_len = to_process - buf_len; + data_len = to_process; in_sg = nx_build_sg_list(in_sg, (u8 *) data, &data_len, max_sg_len); nx_ctx->op.inlen = (nx_ctx->in_sg - in_sg) * sizeof(struct nx_sg); - if (data_len != (to_process - buf_len)) { - rc = -EINVAL; - goto out; - } - - to_process = data_len + buf_len; + to_process = data_len; leftover = total - to_process; /* @@ -166,30 +128,29 @@ static int nx_sha512_update(struct shash_desc *desc, const u8 *data, atomic_inc(&(nx_ctx->stats->sha512_ops)); total -= to_process; - data += to_process - buf_len; - buf_len = 0; - + data += to_process; + sctx->count[0] += to_process; + if (sctx->count[0] < to_process) + sctx->count[1]++; } while (leftover >= SHA512_BLOCK_SIZE); - /* copy the leftover back into the state struct */ - if (leftover) - memcpy(sctx->buf, data, leftover); - sctx->count[0] += len; + rc = leftover; memcpy(sctx->state, csbcpb->cpb.sha512.message_digest, SHA512_DIGEST_SIZE); out: spin_unlock_irqrestore(&nx_ctx->lock, irq_flags); return rc; } -static int nx_sha512_final(struct shash_desc *desc, u8 *out) +static int nx_sha512_finup(struct shash_desc *desc, const u8 *src, + unsigned int nbytes, u8 *out) { struct sha512_state_be *sctx = shash_desc_ctx(desc); - struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base); + struct nx_crypto_ctx *nx_ctx = crypto_shash_ctx(desc->tfm); struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb; struct nx_sg *in_sg, *out_sg; u32 max_sg_len; - u64 count0; unsigned long irq_flags; + u64 count0, count1; int rc = 0; int len; @@ -201,30 +162,23 @@ static int nx_sha512_final(struct shash_desc *desc, u8 *out) nx_ctx->ap->databytelen/NX_PAGE_SIZE); /* final is represented by continuing the operation and indicating that - * this is not an intermediate operation */ - if (sctx->count[0] >= SHA512_BLOCK_SIZE) { - /* we've hit the nx chip previously, now we're finalizing, - * so copy over the partial digest */ - memcpy(csbcpb->cpb.sha512.input_partial_digest, sctx->state, - SHA512_DIGEST_SIZE); - NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE; - NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION; - } else { - NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE; - NX_CPB_FDM(csbcpb) &= ~NX_FDM_CONTINUATION; - } - + * this is not an intermediate operation + * copy over the partial digest */ + memcpy(csbcpb->cpb.sha512.input_partial_digest, sctx->state, SHA512_DIGEST_SIZE); NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE; + NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION; - count0 = sctx->count[0] * 8; + count0 = sctx->count[0] + nbytes; + count1 = sctx->count[1]; - csbcpb->cpb.sha512.message_bit_length_lo = count0; + csbcpb->cpb.sha512.message_bit_length_lo = count0 << 3; + csbcpb->cpb.sha512.message_bit_length_hi = (count1 << 3) | + (count0 >> 61); - len = sctx->count[0] & (SHA512_BLOCK_SIZE - 1); - in_sg = nx_build_sg_list(nx_ctx->in_sg, sctx->buf, &len, - max_sg_len); + len = nbytes; + in_sg = nx_build_sg_list(nx_ctx->in_sg, (u8 *)src, &len, max_sg_len); - if (len != (sctx->count[0] & (SHA512_BLOCK_SIZE - 1))) { + if (len != nbytes) { rc = -EINVAL; goto out; } @@ -246,7 +200,7 @@ static int nx_sha512_final(struct shash_desc *desc, u8 *out) goto out; atomic_inc(&(nx_ctx->stats->sha512_ops)); - atomic64_add(sctx->count[0], &(nx_ctx->stats->sha512_bytes)); + atomic64_add(count0, &(nx_ctx->stats->sha512_bytes)); memcpy(out, csbcpb->cpb.sha512.message_digest, SHA512_DIGEST_SIZE); out: @@ -257,18 +211,34 @@ out: static int nx_sha512_export(struct shash_desc *desc, void *out) { struct sha512_state_be *sctx = shash_desc_ctx(desc); + union { + u8 *u8; + u64 *u64; + } p = { .u8 = out }; + int i; - memcpy(out, sctx, sizeof(*sctx)); + for (i = 0; i < SHA512_DIGEST_SIZE / sizeof(*p.u64); i++) + put_unaligned(be64_to_cpu(sctx->state[i]), p.u64++); + put_unaligned(sctx->count[0], p.u64++); + put_unaligned(sctx->count[1], p.u64++); return 0; } static int nx_sha512_import(struct shash_desc *desc, const void *in) { struct sha512_state_be *sctx = shash_desc_ctx(desc); + union { + const u8 *u8; + const u64 *u64; + } p = { .u8 = in }; + int i; - memcpy(sctx, in, sizeof(*sctx)); + for (i = 0; i < SHA512_DIGEST_SIZE / sizeof(*p.u64); i++) + sctx->state[i] = cpu_to_be64(get_unaligned(p.u64++)); + sctx->count[0] = get_unaligned(p.u64++); + sctx->count[1] = get_unaligned(p.u64++); return 0; } @@ -276,19 +246,20 @@ struct shash_alg nx_shash_sha512_alg = { .digestsize = SHA512_DIGEST_SIZE, .init = nx_sha512_init, .update = nx_sha512_update, - .final = nx_sha512_final, + .finup = nx_sha512_finup, .export = nx_sha512_export, .import = nx_sha512_import, + .init_tfm = nx_crypto_ctx_sha512_init, + .exit_tfm = nx_crypto_ctx_shash_exit, .descsize = sizeof(struct sha512_state_be), .statesize = sizeof(struct sha512_state_be), .base = { .cra_name = "sha512", .cra_driver_name = "sha512-nx", .cra_priority = 300, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_blocksize = SHA512_BLOCK_SIZE, .cra_module = THIS_MODULE, .cra_ctxsize = sizeof(struct nx_crypto_ctx), - .cra_init = nx_crypto_ctx_sha512_init, - .cra_exit = nx_crypto_ctx_exit, } }; diff --git a/drivers/crypto/nx/nx.c b/drivers/crypto/nx/nx.c index 4e4a371ba390..78135fb13f5c 100644 --- a/drivers/crypto/nx/nx.c +++ b/drivers/crypto/nx/nx.c @@ -124,8 +124,6 @@ struct nx_sg *nx_build_sg_list(struct nx_sg *sg_head, } if ((sg - sg_head) == sgmax) { - pr_err("nx: scatter/gather list overflow, pid: %d\n", - current->pid); sg++; break; } @@ -702,14 +700,14 @@ int nx_crypto_ctx_aes_ecb_init(struct crypto_skcipher *tfm) NX_MODE_AES_ECB); } -int nx_crypto_ctx_sha_init(struct crypto_tfm *tfm) +int nx_crypto_ctx_sha_init(struct crypto_shash *tfm) { - return nx_crypto_ctx_init(crypto_tfm_ctx(tfm), NX_FC_SHA, NX_MODE_SHA); + return nx_crypto_ctx_init(crypto_shash_ctx(tfm), NX_FC_SHA, NX_MODE_SHA); } -int nx_crypto_ctx_aes_xcbc_init(struct crypto_tfm *tfm) +int nx_crypto_ctx_aes_xcbc_init(struct crypto_shash *tfm) { - return nx_crypto_ctx_init(crypto_tfm_ctx(tfm), NX_FC_AES, + return nx_crypto_ctx_init(crypto_shash_ctx(tfm), NX_FC_AES, NX_MODE_AES_XCBC_MAC); } @@ -744,6 +742,11 @@ void nx_crypto_ctx_aead_exit(struct crypto_aead *tfm) kfree_sensitive(nx_ctx->kmem); } +void nx_crypto_ctx_shash_exit(struct crypto_shash *tfm) +{ + nx_crypto_ctx_exit(crypto_shash_ctx(tfm)); +} + static int nx_probe(struct vio_dev *viodev, const struct vio_device_id *id) { dev_dbg(&viodev->dev, "driver probed: %s resource id: 0x%x\n", diff --git a/drivers/crypto/nx/nx.h b/drivers/crypto/nx/nx.h index b1f6634a1644..36974f08490a 100644 --- a/drivers/crypto/nx/nx.h +++ b/drivers/crypto/nx/nx.h @@ -3,6 +3,7 @@ #ifndef __NX_H__ #define __NX_H__ +#include #include #include #include @@ -147,14 +148,15 @@ struct scatterlist; /* prototypes */ int nx_crypto_ctx_aes_ccm_init(struct crypto_aead *tfm); int nx_crypto_ctx_aes_gcm_init(struct crypto_aead *tfm); -int nx_crypto_ctx_aes_xcbc_init(struct crypto_tfm *tfm); +int nx_crypto_ctx_aes_xcbc_init(struct crypto_shash *tfm); int nx_crypto_ctx_aes_ctr_init(struct crypto_skcipher *tfm); int nx_crypto_ctx_aes_cbc_init(struct crypto_skcipher *tfm); int nx_crypto_ctx_aes_ecb_init(struct crypto_skcipher *tfm); -int nx_crypto_ctx_sha_init(struct crypto_tfm *tfm); +int nx_crypto_ctx_sha_init(struct crypto_shash *tfm); void nx_crypto_ctx_exit(struct crypto_tfm *tfm); void nx_crypto_ctx_skcipher_exit(struct crypto_skcipher *tfm); void nx_crypto_ctx_aead_exit(struct crypto_aead *tfm); +void nx_crypto_ctx_shash_exit(struct crypto_shash *tfm); void nx_ctx_init(struct nx_crypto_ctx *nx_ctx, unsigned int function); int nx_hcall_sync(struct nx_crypto_ctx *ctx, struct vio_pfo_op *op, u32 may_sleep); From 63dc06cd12f980e304158888485a26ae2be7f371 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 18 Apr 2025 11:01:13 +0800 Subject: [PATCH 215/382] crypto: padlock-sha - Use API partial block handling Use the Crypto API partial block handling. Signed-off-by: Herbert Xu --- drivers/crypto/padlock-sha.c | 464 +++++++++++------------------------ 1 file changed, 138 insertions(+), 326 deletions(-) diff --git a/drivers/crypto/padlock-sha.c b/drivers/crypto/padlock-sha.c index 466493907d48..c89b9c6b5f4c 100644 --- a/drivers/crypto/padlock-sha.c +++ b/drivers/crypto/padlock-sha.c @@ -7,59 +7,83 @@ * Copyright (c) 2006 Michal Ludvig */ +#include #include #include #include #include +#include #include -#include -#include -#include -#include #include -#include -#include -#include +#include -struct padlock_sha_desc { - struct shash_desc fallback; -}; +#define PADLOCK_SHA_DESCSIZE (128 + ((PADLOCK_ALIGNMENT - 1) & \ + ~(CRYPTO_MINALIGN - 1))) struct padlock_sha_ctx { - struct crypto_shash *fallback; + struct crypto_ahash *fallback; }; -static int padlock_sha_init(struct shash_desc *desc) +static inline void *padlock_shash_desc_ctx(struct shash_desc *desc) { - struct padlock_sha_desc *dctx = shash_desc_ctx(desc); - struct padlock_sha_ctx *ctx = crypto_shash_ctx(desc->tfm); + return PTR_ALIGN(shash_desc_ctx(desc), PADLOCK_ALIGNMENT); +} - dctx->fallback.tfm = ctx->fallback; - return crypto_shash_init(&dctx->fallback); +static int padlock_sha1_init(struct shash_desc *desc) +{ + struct sha1_state *sctx = padlock_shash_desc_ctx(desc); + + *sctx = (struct sha1_state){ + .state = { SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4 }, + }; + + return 0; +} + +static int padlock_sha256_init(struct shash_desc *desc) +{ + struct sha256_state *sctx = padlock_shash_desc_ctx(desc); + + sha256_init(sctx); + return 0; } static int padlock_sha_update(struct shash_desc *desc, const u8 *data, unsigned int length) { - struct padlock_sha_desc *dctx = shash_desc_ctx(desc); + struct padlock_sha_ctx *ctx = crypto_shash_ctx(desc->tfm); + u8 *state = padlock_shash_desc_ctx(desc); + HASH_REQUEST_ON_STACK(req, ctx->fallback); + int remain; - return crypto_shash_update(&dctx->fallback, data, length); + ahash_request_set_callback(req, 0, NULL, NULL); + ahash_request_set_virt(req, data, NULL, length); + remain = crypto_ahash_import(req, state) ?: + crypto_ahash_update(req); + if (remain < 0) + return remain; + return crypto_ahash_export(req, state) ?: remain; } static int padlock_sha_export(struct shash_desc *desc, void *out) { - struct padlock_sha_desc *dctx = shash_desc_ctx(desc); - - return crypto_shash_export(&dctx->fallback, out); + memcpy(out, padlock_shash_desc_ctx(desc), + crypto_shash_coresize(desc->tfm)); + return 0; } static int padlock_sha_import(struct shash_desc *desc, const void *in) { - struct padlock_sha_desc *dctx = shash_desc_ctx(desc); - struct padlock_sha_ctx *ctx = crypto_shash_ctx(desc->tfm); + unsigned int bs = crypto_shash_blocksize(desc->tfm); + unsigned int ss = crypto_shash_coresize(desc->tfm); + u64 *state = padlock_shash_desc_ctx(desc); - dctx->fallback.tfm = ctx->fallback; - return crypto_shash_import(&dctx->fallback, in); + memcpy(state, in, ss); + + /* Stop evil imports from generating a fault. */ + state[ss / 8 - 1] &= ~(bs - 1); + + return 0; } static inline void padlock_output_block(uint32_t *src, @@ -69,65 +93,38 @@ static inline void padlock_output_block(uint32_t *src, *dst++ = swab32(*src++); } +static int padlock_sha_finup(struct shash_desc *desc, const u8 *in, + unsigned int count, u8 *out) +{ + struct padlock_sha_ctx *ctx = crypto_shash_ctx(desc->tfm); + HASH_REQUEST_ON_STACK(req, ctx->fallback); + + ahash_request_set_callback(req, 0, NULL, NULL); + ahash_request_set_virt(req, in, out, count); + return crypto_ahash_import(req, padlock_shash_desc_ctx(desc)) ?: + crypto_ahash_finup(req); +} + static int padlock_sha1_finup(struct shash_desc *desc, const u8 *in, unsigned int count, u8 *out) { /* We can't store directly to *out as it may be unaligned. */ /* BTW Don't reduce the buffer size below 128 Bytes! * PadLock microcode needs it that big. */ - char buf[128 + PADLOCK_ALIGNMENT - STACK_ALIGN] __attribute__ - ((aligned(STACK_ALIGN))); - char *result = PTR_ALIGN(&buf[0], PADLOCK_ALIGNMENT); - struct padlock_sha_desc *dctx = shash_desc_ctx(desc); - struct sha1_state state; - unsigned int space; - unsigned int leftover; - int err; + struct sha1_state *state = padlock_shash_desc_ctx(desc); + u64 start = state->count; - err = crypto_shash_export(&dctx->fallback, &state); - if (err) - goto out; - - if (state.count + count > ULONG_MAX) - return crypto_shash_finup(&dctx->fallback, in, count, out); - - leftover = ((state.count - 1) & (SHA1_BLOCK_SIZE - 1)) + 1; - space = SHA1_BLOCK_SIZE - leftover; - if (space) { - if (count > space) { - err = crypto_shash_update(&dctx->fallback, in, space) ?: - crypto_shash_export(&dctx->fallback, &state); - if (err) - goto out; - count -= space; - in += space; - } else { - memcpy(state.buffer + leftover, in, count); - in = state.buffer; - count += leftover; - state.count &= ~(SHA1_BLOCK_SIZE - 1); - } - } - - memcpy(result, &state.state, SHA1_DIGEST_SIZE); + if (start + count > ULONG_MAX) + return padlock_sha_finup(desc, in, count, out); asm volatile (".byte 0xf3,0x0f,0xa6,0xc8" /* rep xsha1 */ : \ - : "c"((unsigned long)state.count + count), \ - "a"((unsigned long)state.count), \ - "S"(in), "D"(result)); + : "c"((unsigned long)start + count), \ + "a"((unsigned long)start), \ + "S"(in), "D"(state)); - padlock_output_block((uint32_t *)result, (uint32_t *)out, 5); - -out: - return err; -} - -static int padlock_sha1_final(struct shash_desc *desc, u8 *out) -{ - const u8 *buf = (void *)desc; - - return padlock_sha1_finup(desc, buf, 0, out); + padlock_output_block(state->state, (uint32_t *)out, 5); + return 0; } static int padlock_sha256_finup(struct shash_desc *desc, const u8 *in, @@ -136,79 +133,41 @@ static int padlock_sha256_finup(struct shash_desc *desc, const u8 *in, /* We can't store directly to *out as it may be unaligned. */ /* BTW Don't reduce the buffer size below 128 Bytes! * PadLock microcode needs it that big. */ - char buf[128 + PADLOCK_ALIGNMENT - STACK_ALIGN] __attribute__ - ((aligned(STACK_ALIGN))); - char *result = PTR_ALIGN(&buf[0], PADLOCK_ALIGNMENT); - struct padlock_sha_desc *dctx = shash_desc_ctx(desc); - struct sha256_state state; - unsigned int space; - unsigned int leftover; - int err; + struct sha256_state *state = padlock_shash_desc_ctx(desc); + u64 start = state->count; - err = crypto_shash_export(&dctx->fallback, &state); - if (err) - goto out; - - if (state.count + count > ULONG_MAX) - return crypto_shash_finup(&dctx->fallback, in, count, out); - - leftover = ((state.count - 1) & (SHA256_BLOCK_SIZE - 1)) + 1; - space = SHA256_BLOCK_SIZE - leftover; - if (space) { - if (count > space) { - err = crypto_shash_update(&dctx->fallback, in, space) ?: - crypto_shash_export(&dctx->fallback, &state); - if (err) - goto out; - count -= space; - in += space; - } else { - memcpy(state.buf + leftover, in, count); - in = state.buf; - count += leftover; - state.count &= ~(SHA1_BLOCK_SIZE - 1); - } - } - - memcpy(result, &state.state, SHA256_DIGEST_SIZE); + if (start + count > ULONG_MAX) + return padlock_sha_finup(desc, in, count, out); asm volatile (".byte 0xf3,0x0f,0xa6,0xd0" /* rep xsha256 */ : \ - : "c"((unsigned long)state.count + count), \ - "a"((unsigned long)state.count), \ - "S"(in), "D"(result)); + : "c"((unsigned long)start + count), \ + "a"((unsigned long)start), \ + "S"(in), "D"(state)); - padlock_output_block((uint32_t *)result, (uint32_t *)out, 8); - -out: - return err; -} - -static int padlock_sha256_final(struct shash_desc *desc, u8 *out) -{ - const u8 *buf = (void *)desc; - - return padlock_sha256_finup(desc, buf, 0, out); + padlock_output_block(state->state, (uint32_t *)out, 8); + return 0; } static int padlock_init_tfm(struct crypto_shash *hash) { const char *fallback_driver_name = crypto_shash_alg_name(hash); struct padlock_sha_ctx *ctx = crypto_shash_ctx(hash); - struct crypto_shash *fallback_tfm; + struct crypto_ahash *fallback_tfm; /* Allocate a fallback and abort if it failed. */ - fallback_tfm = crypto_alloc_shash(fallback_driver_name, 0, - CRYPTO_ALG_NEED_FALLBACK); + fallback_tfm = crypto_alloc_ahash(fallback_driver_name, 0, + CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_ALG_ASYNC); if (IS_ERR(fallback_tfm)) { printk(KERN_WARNING PFX "Fallback driver '%s' could not be loaded!\n", fallback_driver_name); return PTR_ERR(fallback_tfm); } - if (crypto_shash_descsize(hash) < sizeof(struct padlock_sha_desc) + - crypto_shash_descsize(fallback_tfm)) { - crypto_free_shash(fallback_tfm); + if (crypto_shash_statesize(hash) < + crypto_ahash_statesize(fallback_tfm)) { + crypto_free_ahash(fallback_tfm); return -EINVAL; } @@ -221,27 +180,27 @@ static void padlock_exit_tfm(struct crypto_shash *hash) { struct padlock_sha_ctx *ctx = crypto_shash_ctx(hash); - crypto_free_shash(ctx->fallback); + crypto_free_ahash(ctx->fallback); } static struct shash_alg sha1_alg = { .digestsize = SHA1_DIGEST_SIZE, - .init = padlock_sha_init, + .init = padlock_sha1_init, .update = padlock_sha_update, .finup = padlock_sha1_finup, - .final = padlock_sha1_final, .export = padlock_sha_export, .import = padlock_sha_import, .init_tfm = padlock_init_tfm, .exit_tfm = padlock_exit_tfm, - .descsize = sizeof(struct padlock_sha_desc) + - sizeof(struct sha1_state), - .statesize = sizeof(struct sha1_state), + .descsize = PADLOCK_SHA_DESCSIZE, + .statesize = SHA1_STATE_SIZE, .base = { .cra_name = "sha1", .cra_driver_name = "sha1-padlock", .cra_priority = PADLOCK_CRA_PRIORITY, - .cra_flags = CRYPTO_ALG_NEED_FALLBACK, + .cra_flags = CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA1_BLOCK_SIZE, .cra_ctxsize = sizeof(struct padlock_sha_ctx), .cra_module = THIS_MODULE, @@ -250,22 +209,22 @@ static struct shash_alg sha1_alg = { static struct shash_alg sha256_alg = { .digestsize = SHA256_DIGEST_SIZE, - .init = padlock_sha_init, + .init = padlock_sha256_init, .update = padlock_sha_update, .finup = padlock_sha256_finup, - .final = padlock_sha256_final, + .init_tfm = padlock_init_tfm, .export = padlock_sha_export, .import = padlock_sha_import, - .init_tfm = padlock_init_tfm, .exit_tfm = padlock_exit_tfm, - .descsize = sizeof(struct padlock_sha_desc) + - sizeof(struct sha256_state), - .statesize = sizeof(struct sha256_state), + .descsize = PADLOCK_SHA_DESCSIZE, + .statesize = sizeof(struct crypto_sha256_state), .base = { .cra_name = "sha256", .cra_driver_name = "sha256-padlock", .cra_priority = PADLOCK_CRA_PRIORITY, - .cra_flags = CRYPTO_ALG_NEED_FALLBACK, + .cra_flags = CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA256_BLOCK_SIZE, .cra_ctxsize = sizeof(struct padlock_sha_ctx), .cra_module = THIS_MODULE, @@ -274,207 +233,58 @@ static struct shash_alg sha256_alg = { /* Add two shash_alg instance for hardware-implemented * * multiple-parts hash supported by VIA Nano Processor.*/ -static int padlock_sha1_init_nano(struct shash_desc *desc) -{ - struct sha1_state *sctx = shash_desc_ctx(desc); - - *sctx = (struct sha1_state){ - .state = { SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4 }, - }; - - return 0; -} static int padlock_sha1_update_nano(struct shash_desc *desc, - const u8 *data, unsigned int len) + const u8 *src, unsigned int len) { - struct sha1_state *sctx = shash_desc_ctx(desc); - unsigned int partial, done; - const u8 *src; /*The PHE require the out buffer must 128 bytes and 16-bytes aligned*/ - u8 buf[128 + PADLOCK_ALIGNMENT - STACK_ALIGN] __attribute__ - ((aligned(STACK_ALIGN))); - u8 *dst = PTR_ALIGN(&buf[0], PADLOCK_ALIGNMENT); + struct sha1_state *state = padlock_shash_desc_ctx(desc); + int blocks = len / SHA1_BLOCK_SIZE; - partial = sctx->count & 0x3f; - sctx->count += len; - done = 0; - src = data; - memcpy(dst, (u8 *)(sctx->state), SHA1_DIGEST_SIZE); + len -= blocks * SHA1_BLOCK_SIZE; + state->count += blocks * SHA1_BLOCK_SIZE; - if ((partial + len) >= SHA1_BLOCK_SIZE) { - - /* Append the bytes in state's buffer to a block to handle */ - if (partial) { - done = -partial; - memcpy(sctx->buffer + partial, data, - done + SHA1_BLOCK_SIZE); - src = sctx->buffer; - asm volatile (".byte 0xf3,0x0f,0xa6,0xc8" - : "+S"(src), "+D"(dst) \ - : "a"((long)-1), "c"((unsigned long)1)); - done += SHA1_BLOCK_SIZE; - src = data + done; - } - - /* Process the left bytes from the input data */ - if (len - done >= SHA1_BLOCK_SIZE) { - asm volatile (".byte 0xf3,0x0f,0xa6,0xc8" - : "+S"(src), "+D"(dst) - : "a"((long)-1), - "c"((unsigned long)((len - done) / SHA1_BLOCK_SIZE))); - done += ((len - done) - (len - done) % SHA1_BLOCK_SIZE); - src = data + done; - } - partial = 0; - } - memcpy((u8 *)(sctx->state), dst, SHA1_DIGEST_SIZE); - memcpy(sctx->buffer + partial, src, len - done); - - return 0; + /* Process the left bytes from the input data */ + asm volatile (".byte 0xf3,0x0f,0xa6,0xc8" + : "+S"(src), "+D"(state) + : "a"((long)-1), + "c"((unsigned long)blocks)); + return len; } -static int padlock_sha1_final_nano(struct shash_desc *desc, u8 *out) -{ - struct sha1_state *state = (struct sha1_state *)shash_desc_ctx(desc); - unsigned int partial, padlen; - __be64 bits; - static const u8 padding[64] = { 0x80, }; - - bits = cpu_to_be64(state->count << 3); - - /* Pad out to 56 mod 64 */ - partial = state->count & 0x3f; - padlen = (partial < 56) ? (56 - partial) : ((64+56) - partial); - padlock_sha1_update_nano(desc, padding, padlen); - - /* Append length field bytes */ - padlock_sha1_update_nano(desc, (const u8 *)&bits, sizeof(bits)); - - /* Swap to output */ - padlock_output_block((uint32_t *)(state->state), (uint32_t *)out, 5); - - return 0; -} - -static int padlock_sha256_init_nano(struct shash_desc *desc) -{ - struct sha256_state *sctx = shash_desc_ctx(desc); - - *sctx = (struct sha256_state){ - .state = { SHA256_H0, SHA256_H1, SHA256_H2, SHA256_H3, \ - SHA256_H4, SHA256_H5, SHA256_H6, SHA256_H7}, - }; - - return 0; -} - -static int padlock_sha256_update_nano(struct shash_desc *desc, const u8 *data, +static int padlock_sha256_update_nano(struct shash_desc *desc, const u8 *src, unsigned int len) { - struct sha256_state *sctx = shash_desc_ctx(desc); - unsigned int partial, done; - const u8 *src; /*The PHE require the out buffer must 128 bytes and 16-bytes aligned*/ - u8 buf[128 + PADLOCK_ALIGNMENT - STACK_ALIGN] __attribute__ - ((aligned(STACK_ALIGN))); - u8 *dst = PTR_ALIGN(&buf[0], PADLOCK_ALIGNMENT); + struct crypto_sha256_state *state = padlock_shash_desc_ctx(desc); + int blocks = len / SHA256_BLOCK_SIZE; - partial = sctx->count & 0x3f; - sctx->count += len; - done = 0; - src = data; - memcpy(dst, (u8 *)(sctx->state), SHA256_DIGEST_SIZE); + len -= blocks * SHA256_BLOCK_SIZE; + state->count += blocks * SHA256_BLOCK_SIZE; - if ((partial + len) >= SHA256_BLOCK_SIZE) { - - /* Append the bytes in state's buffer to a block to handle */ - if (partial) { - done = -partial; - memcpy(sctx->buf + partial, data, - done + SHA256_BLOCK_SIZE); - src = sctx->buf; - asm volatile (".byte 0xf3,0x0f,0xa6,0xd0" - : "+S"(src), "+D"(dst) - : "a"((long)-1), "c"((unsigned long)1)); - done += SHA256_BLOCK_SIZE; - src = data + done; - } - - /* Process the left bytes from input data*/ - if (len - done >= SHA256_BLOCK_SIZE) { - asm volatile (".byte 0xf3,0x0f,0xa6,0xd0" - : "+S"(src), "+D"(dst) - : "a"((long)-1), - "c"((unsigned long)((len - done) / 64))); - done += ((len - done) - (len - done) % 64); - src = data + done; - } - partial = 0; - } - memcpy((u8 *)(sctx->state), dst, SHA256_DIGEST_SIZE); - memcpy(sctx->buf + partial, src, len - done); - - return 0; -} - -static int padlock_sha256_final_nano(struct shash_desc *desc, u8 *out) -{ - struct sha256_state *state = - (struct sha256_state *)shash_desc_ctx(desc); - unsigned int partial, padlen; - __be64 bits; - static const u8 padding[64] = { 0x80, }; - - bits = cpu_to_be64(state->count << 3); - - /* Pad out to 56 mod 64 */ - partial = state->count & 0x3f; - padlen = (partial < 56) ? (56 - partial) : ((64+56) - partial); - padlock_sha256_update_nano(desc, padding, padlen); - - /* Append length field bytes */ - padlock_sha256_update_nano(desc, (const u8 *)&bits, sizeof(bits)); - - /* Swap to output */ - padlock_output_block((uint32_t *)(state->state), (uint32_t *)out, 8); - - return 0; -} - -static int padlock_sha_export_nano(struct shash_desc *desc, - void *out) -{ - int statesize = crypto_shash_statesize(desc->tfm); - void *sctx = shash_desc_ctx(desc); - - memcpy(out, sctx, statesize); - return 0; -} - -static int padlock_sha_import_nano(struct shash_desc *desc, - const void *in) -{ - int statesize = crypto_shash_statesize(desc->tfm); - void *sctx = shash_desc_ctx(desc); - - memcpy(sctx, in, statesize); - return 0; + /* Process the left bytes from input data*/ + asm volatile (".byte 0xf3,0x0f,0xa6,0xd0" + : "+S"(src), "+D"(state) + : "a"((long)-1), + "c"((unsigned long)blocks)); + return len; } static struct shash_alg sha1_alg_nano = { .digestsize = SHA1_DIGEST_SIZE, - .init = padlock_sha1_init_nano, + .init = padlock_sha1_init, .update = padlock_sha1_update_nano, - .final = padlock_sha1_final_nano, - .export = padlock_sha_export_nano, - .import = padlock_sha_import_nano, - .descsize = sizeof(struct sha1_state), - .statesize = sizeof(struct sha1_state), + .finup = padlock_sha1_finup, + .export = padlock_sha_export, + .import = padlock_sha_import, + .descsize = PADLOCK_SHA_DESCSIZE, + .statesize = SHA1_STATE_SIZE, .base = { .cra_name = "sha1", .cra_driver_name = "sha1-padlock-nano", .cra_priority = PADLOCK_CRA_PRIORITY, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA1_BLOCK_SIZE, .cra_module = THIS_MODULE, } @@ -482,17 +292,19 @@ static struct shash_alg sha1_alg_nano = { static struct shash_alg sha256_alg_nano = { .digestsize = SHA256_DIGEST_SIZE, - .init = padlock_sha256_init_nano, + .init = padlock_sha256_init, .update = padlock_sha256_update_nano, - .final = padlock_sha256_final_nano, - .export = padlock_sha_export_nano, - .import = padlock_sha_import_nano, - .descsize = sizeof(struct sha256_state), - .statesize = sizeof(struct sha256_state), + .finup = padlock_sha256_finup, + .export = padlock_sha_export, + .import = padlock_sha_import, + .descsize = PADLOCK_SHA_DESCSIZE, + .statesize = sizeof(struct crypto_sha256_state), .base = { .cra_name = "sha256", .cra_driver_name = "sha256-padlock-nano", .cra_priority = PADLOCK_CRA_PRIORITY, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA256_BLOCK_SIZE, .cra_module = THIS_MODULE, } From aa38d87ec85273e3cee5ad60342930612315e540 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Thu, 24 Apr 2025 22:42:51 +0800 Subject: [PATCH 216/382] crypto: engine - Remove CRYPTO_ALG_ENGINE bit Remove the private and obsolete CRYPTO_ALG_ENGINE bit which is conflicting with the new CRYPTO_ALG_DUP_FIRST bit. Reported-by: Corentin Labbe Fixes: f1440a90465b ("crypto: api - Add support for duplicating algorithms before registration") Signed-off-by: Herbert Xu --- crypto/crypto_engine.c | 31 +++---------------------------- 1 file changed, 3 insertions(+), 28 deletions(-) diff --git a/crypto/crypto_engine.c b/crypto/crypto_engine.c index c7c16da5e649..445d3c113ee1 100644 --- a/crypto/crypto_engine.c +++ b/crypto/crypto_engine.c @@ -23,9 +23,6 @@ #define CRYPTO_ENGINE_MAX_QLEN 10 -/* Temporary algorithm flag used to indicate an updated driver. */ -#define CRYPTO_ALG_ENGINE 0x200 - struct crypto_engine_alg { struct crypto_alg base; struct crypto_engine_op op; @@ -148,16 +145,9 @@ start_request: } } - if (async_req->tfm->__crt_alg->cra_flags & CRYPTO_ALG_ENGINE) { - alg = container_of(async_req->tfm->__crt_alg, - struct crypto_engine_alg, base); - op = &alg->op; - } else { - dev_err(engine->dev, "failed to do request\n"); - ret = -EINVAL; - goto req_err_1; - } - + alg = container_of(async_req->tfm->__crt_alg, + struct crypto_engine_alg, base); + op = &alg->op; ret = op->do_one_request(engine, async_req); /* Request unsuccessfully executed by hardware */ @@ -569,9 +559,6 @@ int crypto_engine_register_aead(struct aead_engine_alg *alg) { if (!alg->op.do_one_request) return -EINVAL; - - alg->base.base.cra_flags |= CRYPTO_ALG_ENGINE; - return crypto_register_aead(&alg->base); } EXPORT_SYMBOL_GPL(crypto_engine_register_aead); @@ -614,9 +601,6 @@ int crypto_engine_register_ahash(struct ahash_engine_alg *alg) { if (!alg->op.do_one_request) return -EINVAL; - - alg->base.halg.base.cra_flags |= CRYPTO_ALG_ENGINE; - return crypto_register_ahash(&alg->base); } EXPORT_SYMBOL_GPL(crypto_engine_register_ahash); @@ -660,9 +644,6 @@ int crypto_engine_register_akcipher(struct akcipher_engine_alg *alg) { if (!alg->op.do_one_request) return -EINVAL; - - alg->base.base.cra_flags |= CRYPTO_ALG_ENGINE; - return crypto_register_akcipher(&alg->base); } EXPORT_SYMBOL_GPL(crypto_engine_register_akcipher); @@ -677,9 +658,6 @@ int crypto_engine_register_kpp(struct kpp_engine_alg *alg) { if (!alg->op.do_one_request) return -EINVAL; - - alg->base.base.cra_flags |= CRYPTO_ALG_ENGINE; - return crypto_register_kpp(&alg->base); } EXPORT_SYMBOL_GPL(crypto_engine_register_kpp); @@ -694,9 +672,6 @@ int crypto_engine_register_skcipher(struct skcipher_engine_alg *alg) { if (!alg->op.do_one_request) return -EINVAL; - - alg->base.base.cra_flags |= CRYPTO_ALG_ENGINE; - return crypto_register_skcipher(&alg->base); } EXPORT_SYMBOL_GPL(crypto_engine_register_skcipher); From bf0b4f15267ca4047bbc533c83d9f7b7f447b5a1 Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Thu, 24 Apr 2025 17:23:22 -0700 Subject: [PATCH 217/382] crypto: riscv - Use SYM_FUNC_START for functions only called directly After some recent changes to the RISC-V crypto code that turned some indirect function calls into direct ones, builds with CONFIG_CFI_CLANG fail with: ld.lld: error: undefined symbol: __kcfi_typeid_sm3_transform_zvksh_zvkb >>> referenced by arch/riscv/crypto/sm3-riscv64-zvksh-zvkb.o:(.text+0x2) in archive vmlinux.a ld.lld: error: undefined symbol: __kcfi_typeid_sha512_transform_zvknhb_zvkb >>> referenced by arch/riscv/crypto/sha512-riscv64-zvknhb-zvkb.o:(.text+0x2) in archive vmlinux.a ld.lld: error: undefined symbol: __kcfi_typeid_sha256_transform_zvknha_or_zvknhb_zvkb >>> referenced by arch/riscv/crypto/sha256-riscv64-zvknha_or_zvknhb-zvkb.o:(.text+0x2) in archive vmlinux.a As these functions are no longer indirectly called (i.e., have their address taken), the compiler will not emit __kcfi_typeid symbols for them but SYM_TYPED_FUNC_START expects these to exist at link time. Switch the definitions of these functions to use SYM_FUNC_START, as they no longer need kCFI type information since they are only called directly. Fixes: 1523eaed0ac5 ("crypto: riscv/sm3 - Use API partial block handling") Fixes: 561aab1104d8 ("crypto: riscv/sha512 - Use API partial block handling") Fixes: e6c5597badf2 ("crypto: riscv/sha256 - Use API partial block handling") Signed-off-by: Nathan Chancellor Signed-off-by: Herbert Xu --- arch/riscv/crypto/sha256-riscv64-zvknha_or_zvknhb-zvkb.S | 4 ++-- arch/riscv/crypto/sha512-riscv64-zvknhb-zvkb.S | 4 ++-- arch/riscv/crypto/sm3-riscv64-zvksh-zvkb.S | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/arch/riscv/crypto/sha256-riscv64-zvknha_or_zvknhb-zvkb.S b/arch/riscv/crypto/sha256-riscv64-zvknha_or_zvknhb-zvkb.S index 8ebcc17de4dc..f1f5779e4732 100644 --- a/arch/riscv/crypto/sha256-riscv64-zvknha_or_zvknhb-zvkb.S +++ b/arch/riscv/crypto/sha256-riscv64-zvknha_or_zvknhb-zvkb.S @@ -43,7 +43,7 @@ // - RISC-V Vector SHA-2 Secure Hash extension ('Zvknha' or 'Zvknhb') // - RISC-V Vector Cryptography Bit-manipulation extension ('Zvkb') -#include +#include .text .option arch, +zvknha, +zvkb @@ -108,7 +108,7 @@ // void sha256_transform_zvknha_or_zvknhb_zvkb(u32 state[8], const u8 *data, // int num_blocks); -SYM_TYPED_FUNC_START(sha256_transform_zvknha_or_zvknhb_zvkb) +SYM_FUNC_START(sha256_transform_zvknha_or_zvknhb_zvkb) // Load the round constants into K0-K15. vsetivli zero, 4, e32, m1, ta, ma diff --git a/arch/riscv/crypto/sha512-riscv64-zvknhb-zvkb.S b/arch/riscv/crypto/sha512-riscv64-zvknhb-zvkb.S index 3a9ae210f915..89f4a10d12dd 100644 --- a/arch/riscv/crypto/sha512-riscv64-zvknhb-zvkb.S +++ b/arch/riscv/crypto/sha512-riscv64-zvknhb-zvkb.S @@ -43,7 +43,7 @@ // - RISC-V Vector SHA-2 Secure Hash extension ('Zvknhb') // - RISC-V Vector Cryptography Bit-manipulation extension ('Zvkb') -#include +#include .text .option arch, +zvknhb, +zvkb @@ -95,7 +95,7 @@ // void sha512_transform_zvknhb_zvkb(u64 state[8], const u8 *data, // int num_blocks); -SYM_TYPED_FUNC_START(sha512_transform_zvknhb_zvkb) +SYM_FUNC_START(sha512_transform_zvknhb_zvkb) // Setup mask for the vmerge to replace the first word (idx==0) in // message scheduling. There are 4 words, so an 8-bit mask suffices. diff --git a/arch/riscv/crypto/sm3-riscv64-zvksh-zvkb.S b/arch/riscv/crypto/sm3-riscv64-zvksh-zvkb.S index a2b65d961c04..4fe754846f65 100644 --- a/arch/riscv/crypto/sm3-riscv64-zvksh-zvkb.S +++ b/arch/riscv/crypto/sm3-riscv64-zvksh-zvkb.S @@ -43,7 +43,7 @@ // - RISC-V Vector SM3 Secure Hash extension ('Zvksh') // - RISC-V Vector Cryptography Bit-manipulation extension ('Zvkb') -#include +#include .text .option arch, +zvksh, +zvkb @@ -81,7 +81,7 @@ .endm // void sm3_transform_zvksh_zvkb(u32 state[8], const u8 *data, int num_blocks); -SYM_TYPED_FUNC_START(sm3_transform_zvksh_zvkb) +SYM_FUNC_START(sm3_transform_zvksh_zvkb) // Load the state and endian-swap each 32-bit word. vsetivli zero, 8, e32, m2, ta, ma From 08094a8057864fde3703ed5fc7798786f69f2559 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 25 Apr 2025 11:33:39 +0800 Subject: [PATCH 218/382] crypto: arm/blake2b - Set FINAL_NONZERO Set FINAL_NONZERO as blake2b expects to have at least one byte for finalisation. Reported-by: Corentin LABBE Fixes: cc28260ab4fb ("crypto: arm/blake2b - Use API partial block handling") Signed-off-by: Herbert Xu Tested-by: Corentin LABBE Signed-off-by: Herbert Xu --- arch/arm/crypto/blake2b-neon-glue.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/arch/arm/crypto/blake2b-neon-glue.c b/arch/arm/crypto/blake2b-neon-glue.c index 7ae4ba0afe06..2ff443a91724 100644 --- a/arch/arm/crypto/blake2b-neon-glue.c +++ b/arch/arm/crypto/blake2b-neon-glue.c @@ -52,7 +52,8 @@ static int crypto_blake2b_finup_neon(struct shash_desc *desc, const u8 *in, .base.cra_driver_name = driver_name, \ .base.cra_priority = 200, \ .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY | \ - CRYPTO_AHASH_ALG_BLOCK_ONLY, \ + CRYPTO_AHASH_ALG_BLOCK_ONLY | \ + CRYPTO_AHASH_ALG_FINAL_NONZERO, \ .base.cra_blocksize = BLAKE2B_BLOCK_SIZE, \ .base.cra_ctxsize = sizeof(struct blake2b_tfm_ctx), \ .base.cra_module = THIS_MODULE, \ From 432f98cf56cf1e0f812f79a713847b37ae86cf31 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 25 Apr 2025 18:58:13 +0800 Subject: [PATCH 219/382] crypto: arm64/sha1 - Set finalize for short finup Always set sctx->finalize before calling finup as it may not have been set previously on a short final. Reported-by: Corentin LABBE Fixes: b97d31100e36 ("crypto: arm64/sha1 - Use API partial block handling") Signed-off-by: Herbert Xu Tested-by: Corentin LABBE Signed-off-by: Herbert Xu --- arch/arm64/crypto/sha1-ce-glue.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/arch/arm64/crypto/sha1-ce-glue.c b/arch/arm64/crypto/sha1-ce-glue.c index 1f8c93fe1e64..65b6980817e5 100644 --- a/arch/arm64/crypto/sha1-ce-glue.c +++ b/arch/arm64/crypto/sha1-ce-glue.c @@ -79,8 +79,10 @@ static int sha1_ce_finup(struct shash_desc *desc, const u8 *data, data += len - remain; len = remain; } - if (!finalized) + if (!finalized) { + sctx->finalize = 0; sha1_base_do_finup(desc, data, len, sha1_ce_transform); + } return sha1_base_finish(desc, out); } From e2df5fb770c0c11d02bbcb5df6b4070b3dee844c Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Tue, 22 Apr 2025 08:27:04 -0700 Subject: [PATCH 220/382] crypto: arm64 - drop redundant dependencies on ARM64 arch/arm64/crypto/Kconfig is sourced only when CONFIG_ARM64=y, so there is no need for the symbols defined inside it to depend on ARM64. Acked-by: Ard Biesheuvel Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/arm64/crypto/Kconfig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/arm64/crypto/Kconfig b/arch/arm64/crypto/Kconfig index 7c2f63f2e307..704d0b7e1d13 100644 --- a/arch/arm64/crypto/Kconfig +++ b/arch/arm64/crypto/Kconfig @@ -137,7 +137,7 @@ config CRYPTO_AES_ARM64 config CRYPTO_AES_ARM64_CE tristate "Ciphers: AES (ARMv8 Crypto Extensions)" - depends on ARM64 && KERNEL_MODE_NEON + depends on KERNEL_MODE_NEON select CRYPTO_ALGAPI select CRYPTO_LIB_AES help @@ -254,7 +254,7 @@ config CRYPTO_SM4_ARM64_NEON_BLK config CRYPTO_AES_ARM64_CE_CCM tristate "AEAD cipher: AES in CCM mode (ARMv8 Crypto Extensions)" - depends on ARM64 && KERNEL_MODE_NEON + depends on KERNEL_MODE_NEON select CRYPTO_ALGAPI select CRYPTO_AES_ARM64_CE select CRYPTO_AES_ARM64_CE_BLK From d58a46df96b687f166aa99deb2bc32b94a2f47e8 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Tue, 22 Apr 2025 08:27:05 -0700 Subject: [PATCH 221/382] crypto: powerpc - drop redundant dependencies on PPC arch/powerpc/crypto/Kconfig is sourced only when CONFIG_PPC=y, so there is no need for the symbols defined inside it to depend on PPC. Acked-by: Ard Biesheuvel Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/powerpc/crypto/Kconfig | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/arch/powerpc/crypto/Kconfig b/arch/powerpc/crypto/Kconfig index cbcf76953d83..8bcc69013464 100644 --- a/arch/powerpc/crypto/Kconfig +++ b/arch/powerpc/crypto/Kconfig @@ -17,7 +17,6 @@ config CRYPTO_CURVE25519_PPC64 config CRYPTO_MD5_PPC tristate "Digests: MD5" - depends on PPC select CRYPTO_HASH help MD5 message digest algorithm (RFC1321) @@ -26,7 +25,6 @@ config CRYPTO_MD5_PPC config CRYPTO_SHA1_PPC tristate "Hash functions: SHA-1" - depends on PPC help SHA-1 secure hash algorithm (FIPS 180) @@ -34,7 +32,7 @@ config CRYPTO_SHA1_PPC config CRYPTO_SHA1_PPC_SPE tristate "Hash functions: SHA-1 (SPE)" - depends on PPC && SPE + depends on SPE help SHA-1 secure hash algorithm (FIPS 180) @@ -43,7 +41,7 @@ config CRYPTO_SHA1_PPC_SPE config CRYPTO_SHA256_PPC_SPE tristate "Hash functions: SHA-224 and SHA-256 (SPE)" - depends on PPC && SPE + depends on SPE select CRYPTO_SHA256 select CRYPTO_HASH help @@ -54,7 +52,7 @@ config CRYPTO_SHA256_PPC_SPE config CRYPTO_AES_PPC_SPE tristate "Ciphers: AES, modes: ECB/CBC/CTR/XTS (SPE)" - depends on PPC && SPE + depends on SPE select CRYPTO_SKCIPHER help Block ciphers: AES cipher algorithms (FIPS-197) From 4cf0e759161d28ce44b97bbdb051b176f31c647d Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Tue, 22 Apr 2025 08:27:06 -0700 Subject: [PATCH 222/382] crypto: s390 - drop redundant dependencies on S390 arch/s390/crypto/Kconfig is sourced only when CONFIG_S390=y, so there is no need for the symbols defined inside it to depend on S390. Acked-by: Ard Biesheuvel Acked-by: Heiko Carstens Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/s390/crypto/Kconfig | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/arch/s390/crypto/Kconfig b/arch/s390/crypto/Kconfig index 055b08f259ab..e88d9cd256ef 100644 --- a/arch/s390/crypto/Kconfig +++ b/arch/s390/crypto/Kconfig @@ -4,7 +4,6 @@ menu "Accelerated Cryptographic Algorithms for CPU (s390)" config CRYPTO_SHA512_S390 tristate "Hash functions: SHA-384 and SHA-512" - depends on S390 select CRYPTO_HASH help SHA-384 and SHA-512 secure hash algorithms (FIPS 180) @@ -15,7 +14,6 @@ config CRYPTO_SHA512_S390 config CRYPTO_SHA1_S390 tristate "Hash functions: SHA-1" - depends on S390 select CRYPTO_HASH help SHA-1 secure hash algorithm (FIPS 180) @@ -26,7 +24,6 @@ config CRYPTO_SHA1_S390 config CRYPTO_SHA256_S390 tristate "Hash functions: SHA-224 and SHA-256" - depends on S390 select CRYPTO_HASH help SHA-224 and SHA-256 secure hash algorithms (FIPS 180) @@ -37,7 +34,6 @@ config CRYPTO_SHA256_S390 config CRYPTO_SHA3_256_S390 tristate "Hash functions: SHA3-224 and SHA3-256" - depends on S390 select CRYPTO_HASH help SHA3-224 and SHA3-256 secure hash algorithms (FIPS 202) @@ -48,7 +44,6 @@ config CRYPTO_SHA3_256_S390 config CRYPTO_SHA3_512_S390 tristate "Hash functions: SHA3-384 and SHA3-512" - depends on S390 select CRYPTO_HASH help SHA3-384 and SHA3-512 secure hash algorithms (FIPS 202) @@ -59,7 +54,6 @@ config CRYPTO_SHA3_512_S390 config CRYPTO_GHASH_S390 tristate "Hash functions: GHASH" - depends on S390 select CRYPTO_HASH help GCM GHASH hash function (NIST SP800-38D) @@ -70,7 +64,6 @@ config CRYPTO_GHASH_S390 config CRYPTO_AES_S390 tristate "Ciphers: AES, modes: ECB, CBC, CTR, XTS, GCM" - depends on S390 select CRYPTO_ALGAPI select CRYPTO_SKCIPHER help @@ -92,7 +85,6 @@ config CRYPTO_AES_S390 config CRYPTO_DES_S390 tristate "Ciphers: DES and Triple DES EDE, modes: ECB, CBC, CTR" - depends on S390 select CRYPTO_ALGAPI select CRYPTO_SKCIPHER select CRYPTO_LIB_DES @@ -109,14 +101,12 @@ config CRYPTO_DES_S390 config CRYPTO_CHACHA_S390 tristate - depends on S390 select CRYPTO_LIB_CHACHA_GENERIC select CRYPTO_ARCH_HAVE_LIB_CHACHA default CRYPTO_LIB_CHACHA_INTERNAL config CRYPTO_HMAC_S390 tristate "Keyed-hash message authentication code: HMAC" - depends on S390 select CRYPTO_HASH help s390 specific HMAC hardware support for SHA224, SHA256, SHA384 and From 67128a90b3541298988bfc3a0bad88178d6a7fb7 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Tue, 22 Apr 2025 08:27:07 -0700 Subject: [PATCH 223/382] crypto: x86 - drop redundant dependencies on X86 arch/x86/crypto/Kconfig is sourced only when CONFIG_X86=y, so there is no need for the symbols defined inside it to depend on X86. In the case of CRYPTO_TWOFISH_586 and CRYPTO_TWOFISH_X86_64, the dependency was actually on '(X86 || UML_X86)', which suggests that these two symbols were intended to be available under user-mode Linux as well. Yet, again these symbols were defined only when CONFIG_X86=y, so that was not the case. Just remove this redundant dependency. Acked-by: Ard Biesheuvel Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/x86/crypto/Kconfig | 67 ++++++++++++++++++++--------------------- 1 file changed, 33 insertions(+), 34 deletions(-) diff --git a/arch/x86/crypto/Kconfig b/arch/x86/crypto/Kconfig index 9a3cade49f63..b4df6cf37e0e 100644 --- a/arch/x86/crypto/Kconfig +++ b/arch/x86/crypto/Kconfig @@ -4,7 +4,7 @@ menu "Accelerated Cryptographic Algorithms for CPU (x86)" config CRYPTO_CURVE25519_X86 tristate - depends on X86 && 64BIT + depends on 64BIT select CRYPTO_KPP select CRYPTO_LIB_CURVE25519_GENERIC select CRYPTO_ARCH_HAVE_LIB_CURVE25519 @@ -17,7 +17,6 @@ config CRYPTO_CURVE25519_X86 config CRYPTO_AES_NI_INTEL tristate "Ciphers: AES, modes: ECB, CBC, CTS, CTR, XCTR, XTS, GCM (AES-NI/VAES)" - depends on X86 select CRYPTO_AEAD select CRYPTO_LIB_AES select CRYPTO_LIB_GF128MUL @@ -37,7 +36,7 @@ config CRYPTO_AES_NI_INTEL config CRYPTO_BLOWFISH_X86_64 tristate "Ciphers: Blowfish, modes: ECB, CBC" - depends on X86 && 64BIT + depends on 64BIT select CRYPTO_SKCIPHER select CRYPTO_BLOWFISH_COMMON imply CRYPTO_CTR @@ -49,7 +48,7 @@ config CRYPTO_BLOWFISH_X86_64 config CRYPTO_CAMELLIA_X86_64 tristate "Ciphers: Camellia with modes: ECB, CBC" - depends on X86 && 64BIT + depends on 64BIT select CRYPTO_SKCIPHER imply CRYPTO_CTR help @@ -60,7 +59,7 @@ config CRYPTO_CAMELLIA_X86_64 config CRYPTO_CAMELLIA_AESNI_AVX_X86_64 tristate "Ciphers: Camellia with modes: ECB, CBC (AES-NI/AVX)" - depends on X86 && 64BIT + depends on 64BIT select CRYPTO_SKCIPHER select CRYPTO_CAMELLIA_X86_64 imply CRYPTO_XTS @@ -73,7 +72,7 @@ config CRYPTO_CAMELLIA_AESNI_AVX_X86_64 config CRYPTO_CAMELLIA_AESNI_AVX2_X86_64 tristate "Ciphers: Camellia with modes: ECB, CBC (AES-NI/AVX2)" - depends on X86 && 64BIT + depends on 64BIT select CRYPTO_CAMELLIA_AESNI_AVX_X86_64 help Length-preserving ciphers: Camellia with ECB and CBC modes @@ -84,7 +83,7 @@ config CRYPTO_CAMELLIA_AESNI_AVX2_X86_64 config CRYPTO_CAST5_AVX_X86_64 tristate "Ciphers: CAST5 with modes: ECB, CBC (AVX)" - depends on X86 && 64BIT + depends on 64BIT select CRYPTO_SKCIPHER select CRYPTO_CAST5 select CRYPTO_CAST_COMMON @@ -100,7 +99,7 @@ config CRYPTO_CAST5_AVX_X86_64 config CRYPTO_CAST6_AVX_X86_64 tristate "Ciphers: CAST6 with modes: ECB, CBC (AVX)" - depends on X86 && 64BIT + depends on 64BIT select CRYPTO_SKCIPHER select CRYPTO_CAST6 select CRYPTO_CAST_COMMON @@ -117,7 +116,7 @@ config CRYPTO_CAST6_AVX_X86_64 config CRYPTO_DES3_EDE_X86_64 tristate "Ciphers: Triple DES EDE with modes: ECB, CBC" - depends on X86 && 64BIT + depends on 64BIT select CRYPTO_SKCIPHER select CRYPTO_LIB_DES imply CRYPTO_CTR @@ -131,7 +130,7 @@ config CRYPTO_DES3_EDE_X86_64 config CRYPTO_SERPENT_SSE2_X86_64 tristate "Ciphers: Serpent with modes: ECB, CBC (SSE2)" - depends on X86 && 64BIT + depends on 64BIT select CRYPTO_SKCIPHER select CRYPTO_SERPENT imply CRYPTO_CTR @@ -146,7 +145,7 @@ config CRYPTO_SERPENT_SSE2_X86_64 config CRYPTO_SERPENT_SSE2_586 tristate "Ciphers: Serpent with modes: ECB, CBC (32-bit with SSE2)" - depends on X86 && !64BIT + depends on !64BIT select CRYPTO_SKCIPHER select CRYPTO_SERPENT imply CRYPTO_CTR @@ -161,7 +160,7 @@ config CRYPTO_SERPENT_SSE2_586 config CRYPTO_SERPENT_AVX_X86_64 tristate "Ciphers: Serpent with modes: ECB, CBC (AVX)" - depends on X86 && 64BIT + depends on 64BIT select CRYPTO_SKCIPHER select CRYPTO_SERPENT imply CRYPTO_XTS @@ -177,7 +176,7 @@ config CRYPTO_SERPENT_AVX_X86_64 config CRYPTO_SERPENT_AVX2_X86_64 tristate "Ciphers: Serpent with modes: ECB, CBC (AVX2)" - depends on X86 && 64BIT + depends on 64BIT select CRYPTO_SERPENT_AVX_X86_64 help Length-preserving ciphers: Serpent cipher algorithm @@ -190,7 +189,7 @@ config CRYPTO_SERPENT_AVX2_X86_64 config CRYPTO_SM4_AESNI_AVX_X86_64 tristate "Ciphers: SM4 with modes: ECB, CBC, CTR (AES-NI/AVX)" - depends on X86 && 64BIT + depends on 64BIT select CRYPTO_SKCIPHER select CRYPTO_ALGAPI select CRYPTO_SM4 @@ -210,7 +209,7 @@ config CRYPTO_SM4_AESNI_AVX_X86_64 config CRYPTO_SM4_AESNI_AVX2_X86_64 tristate "Ciphers: SM4 with modes: ECB, CBC, CTR (AES-NI/AVX2)" - depends on X86 && 64BIT + depends on 64BIT select CRYPTO_SKCIPHER select CRYPTO_ALGAPI select CRYPTO_SM4 @@ -231,7 +230,7 @@ config CRYPTO_SM4_AESNI_AVX2_X86_64 config CRYPTO_TWOFISH_586 tristate "Ciphers: Twofish (32-bit)" - depends on (X86 || UML_X86) && !64BIT + depends on !64BIT select CRYPTO_ALGAPI select CRYPTO_TWOFISH_COMMON imply CRYPTO_CTR @@ -242,7 +241,7 @@ config CRYPTO_TWOFISH_586 config CRYPTO_TWOFISH_X86_64 tristate "Ciphers: Twofish" - depends on (X86 || UML_X86) && 64BIT + depends on 64BIT select CRYPTO_ALGAPI select CRYPTO_TWOFISH_COMMON imply CRYPTO_CTR @@ -253,7 +252,7 @@ config CRYPTO_TWOFISH_X86_64 config CRYPTO_TWOFISH_X86_64_3WAY tristate "Ciphers: Twofish with modes: ECB, CBC (3-way parallel)" - depends on X86 && 64BIT + depends on 64BIT select CRYPTO_SKCIPHER select CRYPTO_TWOFISH_COMMON select CRYPTO_TWOFISH_X86_64 @@ -268,7 +267,7 @@ config CRYPTO_TWOFISH_X86_64_3WAY config CRYPTO_TWOFISH_AVX_X86_64 tristate "Ciphers: Twofish with modes: ECB, CBC (AVX)" - depends on X86 && 64BIT + depends on 64BIT select CRYPTO_SKCIPHER select CRYPTO_TWOFISH_COMMON select CRYPTO_TWOFISH_X86_64 @@ -285,7 +284,7 @@ config CRYPTO_TWOFISH_AVX_X86_64 config CRYPTO_ARIA_AESNI_AVX_X86_64 tristate "Ciphers: ARIA with modes: ECB, CTR (AES-NI/AVX/GFNI)" - depends on X86 && 64BIT + depends on 64BIT select CRYPTO_SKCIPHER select CRYPTO_ALGAPI select CRYPTO_ARIA @@ -302,7 +301,7 @@ config CRYPTO_ARIA_AESNI_AVX_X86_64 config CRYPTO_ARIA_AESNI_AVX2_X86_64 tristate "Ciphers: ARIA with modes: ECB, CTR (AES-NI/AVX2/GFNI)" - depends on X86 && 64BIT + depends on 64BIT select CRYPTO_SKCIPHER select CRYPTO_ALGAPI select CRYPTO_ARIA @@ -320,7 +319,7 @@ config CRYPTO_ARIA_AESNI_AVX2_X86_64 config CRYPTO_ARIA_GFNI_AVX512_X86_64 tristate "Ciphers: ARIA with modes: ECB, CTR (AVX512/GFNI)" - depends on X86 && 64BIT && AS_GFNI + depends on 64BIT && AS_GFNI select CRYPTO_SKCIPHER select CRYPTO_ALGAPI select CRYPTO_ARIA @@ -338,14 +337,14 @@ config CRYPTO_ARIA_GFNI_AVX512_X86_64 config CRYPTO_CHACHA20_X86_64 tristate - depends on X86 && 64BIT + depends on 64BIT select CRYPTO_LIB_CHACHA_GENERIC select CRYPTO_ARCH_HAVE_LIB_CHACHA default CRYPTO_LIB_CHACHA_INTERNAL config CRYPTO_AEGIS128_AESNI_SSE2 tristate "AEAD ciphers: AEGIS-128 (AES-NI/SSE4.1)" - depends on X86 && 64BIT + depends on 64BIT select CRYPTO_AEAD help AEGIS-128 AEAD algorithm @@ -356,7 +355,7 @@ config CRYPTO_AEGIS128_AESNI_SSE2 config CRYPTO_NHPOLY1305_SSE2 tristate "Hash functions: NHPoly1305 (SSE2)" - depends on X86 && 64BIT + depends on 64BIT select CRYPTO_NHPOLY1305 help NHPoly1305 hash function for Adiantum @@ -366,7 +365,7 @@ config CRYPTO_NHPOLY1305_SSE2 config CRYPTO_NHPOLY1305_AVX2 tristate "Hash functions: NHPoly1305 (AVX2)" - depends on X86 && 64BIT + depends on 64BIT select CRYPTO_NHPOLY1305 help NHPoly1305 hash function for Adiantum @@ -376,7 +375,7 @@ config CRYPTO_NHPOLY1305_AVX2 config CRYPTO_BLAKE2S_X86 bool "Hash functions: BLAKE2s (SSSE3/AVX-512)" - depends on X86 && 64BIT + depends on 64BIT select CRYPTO_LIB_BLAKE2S_GENERIC select CRYPTO_ARCH_HAVE_LIB_BLAKE2S help @@ -388,7 +387,7 @@ config CRYPTO_BLAKE2S_X86 config CRYPTO_POLYVAL_CLMUL_NI tristate "Hash functions: POLYVAL (CLMUL-NI)" - depends on X86 && 64BIT + depends on 64BIT select CRYPTO_POLYVAL help POLYVAL hash function for HCTR2 @@ -398,13 +397,13 @@ config CRYPTO_POLYVAL_CLMUL_NI config CRYPTO_POLY1305_X86_64 tristate - depends on X86 && 64BIT + depends on 64BIT select CRYPTO_ARCH_HAVE_LIB_POLY1305 default CRYPTO_LIB_POLY1305_INTERNAL config CRYPTO_SHA1_SSSE3 tristate "Hash functions: SHA-1 (SSSE3/AVX/AVX2/SHA-NI)" - depends on X86 && 64BIT + depends on 64BIT select CRYPTO_SHA1 select CRYPTO_HASH help @@ -418,7 +417,7 @@ config CRYPTO_SHA1_SSSE3 config CRYPTO_SHA256_SSSE3 tristate "Hash functions: SHA-224 and SHA-256 (SSSE3/AVX/AVX2/SHA-NI)" - depends on X86 && 64BIT + depends on 64BIT select CRYPTO_SHA256 select CRYPTO_HASH help @@ -432,7 +431,7 @@ config CRYPTO_SHA256_SSSE3 config CRYPTO_SHA512_SSSE3 tristate "Hash functions: SHA-384 and SHA-512 (SSSE3/AVX/AVX2)" - depends on X86 && 64BIT + depends on 64BIT select CRYPTO_SHA512 select CRYPTO_HASH help @@ -445,7 +444,7 @@ config CRYPTO_SHA512_SSSE3 config CRYPTO_SM3_AVX_X86_64 tristate "Hash functions: SM3 (AVX)" - depends on X86 && 64BIT + depends on 64BIT select CRYPTO_HASH select CRYPTO_LIB_SM3 help @@ -458,7 +457,7 @@ config CRYPTO_SM3_AVX_X86_64 config CRYPTO_GHASH_CLMUL_NI_INTEL tristate "Hash functions: GHASH (CLMUL-NI)" - depends on X86 && 64BIT + depends on 64BIT select CRYPTO_CRYPTD help GCM GHASH hash function (NIST SP800-38D) From 714656a84697f9615b9488b490c99edb3ecfcd3d Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Tue, 22 Apr 2025 08:27:08 -0700 Subject: [PATCH 224/382] crypto: arm - move library functions to arch/arm/lib/crypto/ Continue disentangling the crypto library functions from the generic crypto infrastructure by moving the arm BLAKE2s, ChaCha, and Poly1305 library functions into a new directory arch/arm/lib/crypto/ that does not depend on CRYPTO. This mirrors the distinction between crypto/ and lib/crypto/. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- MAINTAINERS | 1 + arch/arm/crypto/Kconfig | 23 ---------------- arch/arm/crypto/Makefile | 14 +--------- arch/arm/lib/Makefile | 2 ++ arch/arm/lib/crypto/.gitignore | 2 ++ arch/arm/lib/crypto/Kconfig | 24 +++++++++++++++++ arch/arm/lib/crypto/Makefile | 26 +++++++++++++++++++ arch/arm/{ => lib}/crypto/blake2s-core.S | 0 arch/arm/{ => lib}/crypto/blake2s-glue.c | 0 arch/arm/{ => lib}/crypto/chacha-glue.c | 0 arch/arm/{ => lib}/crypto/chacha-neon-core.S | 0 .../arm/{ => lib}/crypto/chacha-scalar-core.S | 0 arch/arm/{ => lib}/crypto/poly1305-armv4.pl | 0 arch/arm/{ => lib}/crypto/poly1305-glue.c | 0 lib/crypto/Kconfig | 6 +++++ 15 files changed, 62 insertions(+), 36 deletions(-) create mode 100644 arch/arm/lib/crypto/.gitignore create mode 100644 arch/arm/lib/crypto/Kconfig create mode 100644 arch/arm/lib/crypto/Makefile rename arch/arm/{ => lib}/crypto/blake2s-core.S (100%) rename arch/arm/{ => lib}/crypto/blake2s-glue.c (100%) rename arch/arm/{ => lib}/crypto/chacha-glue.c (100%) rename arch/arm/{ => lib}/crypto/chacha-neon-core.S (100%) rename arch/arm/{ => lib}/crypto/chacha-scalar-core.S (100%) rename arch/arm/{ => lib}/crypto/poly1305-armv4.pl (100%) rename arch/arm/{ => lib}/crypto/poly1305-glue.c (100%) diff --git a/MAINTAINERS b/MAINTAINERS index a2604c13f11b..d0d1968e323a 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -6288,6 +6288,7 @@ T: git git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6.git F: Documentation/crypto/ F: Documentation/devicetree/bindings/crypto/ F: arch/*/crypto/ +F: arch/*/lib/crypto/ F: crypto/ F: drivers/crypto/ F: include/crypto/ diff --git a/arch/arm/crypto/Kconfig b/arch/arm/crypto/Kconfig index 3530e7c80793..1f889d6bab77 100644 --- a/arch/arm/crypto/Kconfig +++ b/arch/arm/crypto/Kconfig @@ -46,24 +46,6 @@ config CRYPTO_NHPOLY1305_NEON Architecture: arm using: - NEON (Advanced SIMD) extensions -config CRYPTO_POLY1305_ARM - tristate - select CRYPTO_ARCH_HAVE_LIB_POLY1305 - default CRYPTO_LIB_POLY1305_INTERNAL - -config CRYPTO_BLAKE2S_ARM - bool "Hash functions: BLAKE2s" - select CRYPTO_ARCH_HAVE_LIB_BLAKE2S - help - BLAKE2s cryptographic hash function (RFC 7693) - - Architecture: arm - - This is faster than the generic implementations of BLAKE2s and - BLAKE2b, but slower than the NEON implementation of BLAKE2b. - There is no NEON implementation of BLAKE2s, since NEON doesn't - really help with it. - config CRYPTO_BLAKE2B_NEON tristate "Hash functions: BLAKE2b (NEON)" depends on KERNEL_MODE_NEON @@ -206,10 +188,5 @@ config CRYPTO_AES_ARM_CE Architecture: arm using: - ARMv8 Crypto Extensions -config CRYPTO_CHACHA20_NEON - tristate - select CRYPTO_ARCH_HAVE_LIB_CHACHA - default CRYPTO_LIB_CHACHA_INTERNAL - endmenu diff --git a/arch/arm/crypto/Makefile b/arch/arm/crypto/Makefile index 3d0e23ff9e74..ecabe6603e08 100644 --- a/arch/arm/crypto/Makefile +++ b/arch/arm/crypto/Makefile @@ -9,10 +9,7 @@ obj-$(CONFIG_CRYPTO_SHA1_ARM) += sha1-arm.o obj-$(CONFIG_CRYPTO_SHA1_ARM_NEON) += sha1-arm-neon.o obj-$(CONFIG_CRYPTO_SHA256_ARM) += sha256-arm.o obj-$(CONFIG_CRYPTO_SHA512_ARM) += sha512-arm.o -obj-$(CONFIG_CRYPTO_BLAKE2S_ARM) += libblake2s-arm.o obj-$(CONFIG_CRYPTO_BLAKE2B_NEON) += blake2b-neon.o -obj-$(CONFIG_CRYPTO_CHACHA20_NEON) += chacha-neon.o -obj-$(CONFIG_CRYPTO_POLY1305_ARM) += poly1305-arm.o obj-$(CONFIG_CRYPTO_NHPOLY1305_NEON) += nhpoly1305-neon.o obj-$(CONFIG_CRYPTO_CURVE25519_NEON) += curve25519-neon.o @@ -29,15 +26,11 @@ sha256-arm-neon-$(CONFIG_KERNEL_MODE_NEON) := sha256_neon_glue.o sha256-arm-y := sha256-core.o sha256_glue.o $(sha256-arm-neon-y) sha512-arm-neon-$(CONFIG_KERNEL_MODE_NEON) := sha512-neon-glue.o sha512-arm-y := sha512-core.o sha512-glue.o $(sha512-arm-neon-y) -libblake2s-arm-y:= blake2s-core.o blake2s-glue.o blake2b-neon-y := blake2b-neon-core.o blake2b-neon-glue.o sha1-arm-ce-y := sha1-ce-core.o sha1-ce-glue.o sha2-arm-ce-y := sha2-ce-core.o sha2-ce-glue.o aes-arm-ce-y := aes-ce-core.o aes-ce-glue.o ghash-arm-ce-y := ghash-ce-core.o ghash-ce-glue.o -chacha-neon-y := chacha-scalar-core.o chacha-glue.o -chacha-neon-$(CONFIG_KERNEL_MODE_NEON) += chacha-neon-core.o -poly1305-arm-y := poly1305-core.o poly1305-glue.o nhpoly1305-neon-y := nh-neon-core.o nhpoly1305-neon-glue.o curve25519-neon-y := curve25519-core.o curve25519-glue.o @@ -47,14 +40,9 @@ quiet_cmd_perl = PERL $@ $(obj)/%-core.S: $(src)/%-armv4.pl $(call cmd,perl) -clean-files += poly1305-core.S sha256-core.S sha512-core.S +clean-files += sha256-core.S sha512-core.S aflags-thumb2-$(CONFIG_THUMB2_KERNEL) := -U__thumb2__ -D__thumb2__=1 AFLAGS_sha256-core.o += $(aflags-thumb2-y) AFLAGS_sha512-core.o += $(aflags-thumb2-y) - -# massage the perlasm code a bit so we only get the NEON routine if we need it -poly1305-aflags-$(CONFIG_CPU_V7) := -U__LINUX_ARM_ARCH__ -D__LINUX_ARM_ARCH__=5 -poly1305-aflags-$(CONFIG_KERNEL_MODE_NEON) := -U__LINUX_ARM_ARCH__ -D__LINUX_ARM_ARCH__=7 -AFLAGS_poly1305-core.o += $(poly1305-aflags-y) $(aflags-thumb2-y) diff --git a/arch/arm/lib/Makefile b/arch/arm/lib/Makefile index 007874320937..39ffabd3cf2a 100644 --- a/arch/arm/lib/Makefile +++ b/arch/arm/lib/Makefile @@ -5,6 +5,8 @@ # Copyright (C) 1995-2000 Russell King # +obj-y += crypto/ + lib-y := changebit.o csumipv6.o csumpartial.o \ csumpartialcopy.o csumpartialcopyuser.o clearbit.o \ delay.o delay-loop.o findbit.o memchr.o memcpy.o \ diff --git a/arch/arm/lib/crypto/.gitignore b/arch/arm/lib/crypto/.gitignore new file mode 100644 index 000000000000..0d47d4f21c6d --- /dev/null +++ b/arch/arm/lib/crypto/.gitignore @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0-only +poly1305-core.S diff --git a/arch/arm/lib/crypto/Kconfig b/arch/arm/lib/crypto/Kconfig new file mode 100644 index 000000000000..181f138d563b --- /dev/null +++ b/arch/arm/lib/crypto/Kconfig @@ -0,0 +1,24 @@ +# SPDX-License-Identifier: GPL-2.0-only + +config CRYPTO_BLAKE2S_ARM + bool "Hash functions: BLAKE2s" + select CRYPTO_ARCH_HAVE_LIB_BLAKE2S + help + BLAKE2s cryptographic hash function (RFC 7693) + + Architecture: arm + + This is faster than the generic implementations of BLAKE2s and + BLAKE2b, but slower than the NEON implementation of BLAKE2b. + There is no NEON implementation of BLAKE2s, since NEON doesn't + really help with it. + +config CRYPTO_CHACHA20_NEON + tristate + default CRYPTO_LIB_CHACHA_INTERNAL + select CRYPTO_ARCH_HAVE_LIB_CHACHA + +config CRYPTO_POLY1305_ARM + tristate + default CRYPTO_LIB_POLY1305_INTERNAL + select CRYPTO_ARCH_HAVE_LIB_POLY1305 diff --git a/arch/arm/lib/crypto/Makefile b/arch/arm/lib/crypto/Makefile new file mode 100644 index 000000000000..4c042a4c77ed --- /dev/null +++ b/arch/arm/lib/crypto/Makefile @@ -0,0 +1,26 @@ +# SPDX-License-Identifier: GPL-2.0-only + +obj-$(CONFIG_CRYPTO_BLAKE2S_ARM) += libblake2s-arm.o +libblake2s-arm-y := blake2s-core.o blake2s-glue.o + +obj-$(CONFIG_CRYPTO_CHACHA20_NEON) += chacha-neon.o +chacha-neon-y := chacha-scalar-core.o chacha-glue.o +chacha-neon-$(CONFIG_KERNEL_MODE_NEON) += chacha-neon-core.o + +obj-$(CONFIG_CRYPTO_POLY1305_ARM) += poly1305-arm.o +poly1305-arm-y := poly1305-core.o poly1305-glue.o + +quiet_cmd_perl = PERL $@ + cmd_perl = $(PERL) $(<) > $(@) + +$(obj)/%-core.S: $(src)/%-armv4.pl + $(call cmd,perl) + +clean-files += poly1305-core.S + +aflags-thumb2-$(CONFIG_THUMB2_KERNEL) := -U__thumb2__ -D__thumb2__=1 + +# massage the perlasm code a bit so we only get the NEON routine if we need it +poly1305-aflags-$(CONFIG_CPU_V7) := -U__LINUX_ARM_ARCH__ -D__LINUX_ARM_ARCH__=5 +poly1305-aflags-$(CONFIG_KERNEL_MODE_NEON) := -U__LINUX_ARM_ARCH__ -D__LINUX_ARM_ARCH__=7 +AFLAGS_poly1305-core.o += $(poly1305-aflags-y) $(aflags-thumb2-y) diff --git a/arch/arm/crypto/blake2s-core.S b/arch/arm/lib/crypto/blake2s-core.S similarity index 100% rename from arch/arm/crypto/blake2s-core.S rename to arch/arm/lib/crypto/blake2s-core.S diff --git a/arch/arm/crypto/blake2s-glue.c b/arch/arm/lib/crypto/blake2s-glue.c similarity index 100% rename from arch/arm/crypto/blake2s-glue.c rename to arch/arm/lib/crypto/blake2s-glue.c diff --git a/arch/arm/crypto/chacha-glue.c b/arch/arm/lib/crypto/chacha-glue.c similarity index 100% rename from arch/arm/crypto/chacha-glue.c rename to arch/arm/lib/crypto/chacha-glue.c diff --git a/arch/arm/crypto/chacha-neon-core.S b/arch/arm/lib/crypto/chacha-neon-core.S similarity index 100% rename from arch/arm/crypto/chacha-neon-core.S rename to arch/arm/lib/crypto/chacha-neon-core.S diff --git a/arch/arm/crypto/chacha-scalar-core.S b/arch/arm/lib/crypto/chacha-scalar-core.S similarity index 100% rename from arch/arm/crypto/chacha-scalar-core.S rename to arch/arm/lib/crypto/chacha-scalar-core.S diff --git a/arch/arm/crypto/poly1305-armv4.pl b/arch/arm/lib/crypto/poly1305-armv4.pl similarity index 100% rename from arch/arm/crypto/poly1305-armv4.pl rename to arch/arm/lib/crypto/poly1305-armv4.pl diff --git a/arch/arm/crypto/poly1305-glue.c b/arch/arm/lib/crypto/poly1305-glue.c similarity index 100% rename from arch/arm/crypto/poly1305-glue.c rename to arch/arm/lib/crypto/poly1305-glue.c diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index 2c6ab80e0cdc..59135009e4f0 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -155,4 +155,10 @@ config CRYPTO_LIB_SHA256 config CRYPTO_LIB_SM3 tristate +if !KMSAN # avoid false positives from assembly +if ARM +source "arch/arm/lib/crypto/Kconfig" +endif +endif + endmenu From cc16e228a205ffe26c6f71a9ed165fe964f3d638 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Tue, 22 Apr 2025 08:27:09 -0700 Subject: [PATCH 225/382] crypto: arm64 - move library functions to arch/arm64/lib/crypto/ Continue disentangling the crypto library functions from the generic crypto infrastructure by moving the arm64 ChaCha and Poly1305 library functions into a new directory arch/arm64/lib/crypto/ that does not depend on CRYPTO. This mirrors the distinction between crypto/ and lib/crypto/. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/arm64/crypto/Kconfig | 13 ------------- arch/arm64/crypto/Makefile | 9 +-------- arch/arm64/lib/Makefile | 3 +++ arch/arm64/lib/crypto/.gitignore | 2 ++ arch/arm64/lib/crypto/Kconfig | 14 ++++++++++++++ arch/arm64/lib/crypto/Makefile | 16 ++++++++++++++++ arch/arm64/{ => lib}/crypto/chacha-neon-core.S | 0 arch/arm64/{ => lib}/crypto/chacha-neon-glue.c | 0 arch/arm64/{ => lib}/crypto/poly1305-armv8.pl | 0 arch/arm64/{ => lib}/crypto/poly1305-glue.c | 0 lib/crypto/Kconfig | 3 +++ 11 files changed, 39 insertions(+), 21 deletions(-) create mode 100644 arch/arm64/lib/crypto/.gitignore create mode 100644 arch/arm64/lib/crypto/Kconfig create mode 100644 arch/arm64/lib/crypto/Makefile rename arch/arm64/{ => lib}/crypto/chacha-neon-core.S (100%) rename arch/arm64/{ => lib}/crypto/chacha-neon-glue.c (100%) rename arch/arm64/{ => lib}/crypto/poly1305-armv8.pl (100%) rename arch/arm64/{ => lib}/crypto/poly1305-glue.c (100%) diff --git a/arch/arm64/crypto/Kconfig b/arch/arm64/crypto/Kconfig index 704d0b7e1d13..55a7d87a6769 100644 --- a/arch/arm64/crypto/Kconfig +++ b/arch/arm64/crypto/Kconfig @@ -25,12 +25,6 @@ config CRYPTO_NHPOLY1305_NEON Architecture: arm64 using: - NEON (Advanced SIMD) extensions -config CRYPTO_POLY1305_NEON - tristate - depends on KERNEL_MODE_NEON - select CRYPTO_ARCH_HAVE_LIB_POLY1305 - default CRYPTO_LIB_POLY1305_INTERNAL - config CRYPTO_SHA1_ARM64_CE tristate "Hash functions: SHA-1 (ARMv8 Crypto Extensions)" depends on KERNEL_MODE_NEON @@ -180,13 +174,6 @@ config CRYPTO_AES_ARM64_NEON_BLK Architecture: arm64 using: - NEON (Advanced SIMD) extensions -config CRYPTO_CHACHA20_NEON - tristate - depends on KERNEL_MODE_NEON - select CRYPTO_LIB_CHACHA_GENERIC - select CRYPTO_ARCH_HAVE_LIB_CHACHA - default CRYPTO_LIB_CHACHA_INTERNAL - config CRYPTO_AES_ARM64_BS tristate "Ciphers: AES, modes: ECB/CBC/CTR/XCTR/XTS modes (bit-sliced NEON)" depends on KERNEL_MODE_NEON diff --git a/arch/arm64/crypto/Makefile b/arch/arm64/crypto/Makefile index e7139c4768ce..089ae3ddde81 100644 --- a/arch/arm64/crypto/Makefile +++ b/arch/arm64/crypto/Makefile @@ -62,13 +62,6 @@ sha256-arm64-y := sha256-glue.o sha256-core.o obj-$(CONFIG_CRYPTO_SHA512_ARM64) += sha512-arm64.o sha512-arm64-y := sha512-glue.o sha512-core.o -obj-$(CONFIG_CRYPTO_CHACHA20_NEON) += chacha-neon.o -chacha-neon-y := chacha-neon-core.o chacha-neon-glue.o - -obj-$(CONFIG_CRYPTO_POLY1305_NEON) += poly1305-neon.o -poly1305-neon-y := poly1305-core.o poly1305-glue.o -AFLAGS_poly1305-core.o += -Dpoly1305_init=poly1305_init_arm64 - obj-$(CONFIG_CRYPTO_NHPOLY1305_NEON) += nhpoly1305-neon.o nhpoly1305-neon-y := nh-neon-core.o nhpoly1305-neon-glue.o @@ -87,4 +80,4 @@ $(obj)/%-core.S: $(src)/%-armv8.pl $(obj)/sha256-core.S: $(src)/sha512-armv8.pl $(call cmd,perlasm) -clean-files += poly1305-core.S sha256-core.S sha512-core.S +clean-files += sha256-core.S sha512-core.S diff --git a/arch/arm64/lib/Makefile b/arch/arm64/lib/Makefile index 4d49dff721a8..25be7825f28d 100644 --- a/arch/arm64/lib/Makefile +++ b/arch/arm64/lib/Makefile @@ -1,4 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 + +obj-y += crypto/ + lib-y := clear_user.o delay.o copy_from_user.o \ copy_to_user.o copy_page.o \ clear_page.o csum.o insn.o memchr.o memcpy.o \ diff --git a/arch/arm64/lib/crypto/.gitignore b/arch/arm64/lib/crypto/.gitignore new file mode 100644 index 000000000000..0d47d4f21c6d --- /dev/null +++ b/arch/arm64/lib/crypto/.gitignore @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0-only +poly1305-core.S diff --git a/arch/arm64/lib/crypto/Kconfig b/arch/arm64/lib/crypto/Kconfig new file mode 100644 index 000000000000..169311547efe --- /dev/null +++ b/arch/arm64/lib/crypto/Kconfig @@ -0,0 +1,14 @@ +# SPDX-License-Identifier: GPL-2.0-only + +config CRYPTO_CHACHA20_NEON + tristate + depends on KERNEL_MODE_NEON + default CRYPTO_LIB_CHACHA_INTERNAL + select CRYPTO_LIB_CHACHA_GENERIC + select CRYPTO_ARCH_HAVE_LIB_CHACHA + +config CRYPTO_POLY1305_NEON + tristate + depends on KERNEL_MODE_NEON + default CRYPTO_LIB_POLY1305_INTERNAL + select CRYPTO_ARCH_HAVE_LIB_POLY1305 diff --git a/arch/arm64/lib/crypto/Makefile b/arch/arm64/lib/crypto/Makefile new file mode 100644 index 000000000000..ac624c3effda --- /dev/null +++ b/arch/arm64/lib/crypto/Makefile @@ -0,0 +1,16 @@ +# SPDX-License-Identifier: GPL-2.0-only + +obj-$(CONFIG_CRYPTO_CHACHA20_NEON) += chacha-neon.o +chacha-neon-y := chacha-neon-core.o chacha-neon-glue.o + +obj-$(CONFIG_CRYPTO_POLY1305_NEON) += poly1305-neon.o +poly1305-neon-y := poly1305-core.o poly1305-glue.o +AFLAGS_poly1305-core.o += -Dpoly1305_init=poly1305_init_arm64 + +quiet_cmd_perlasm = PERLASM $@ + cmd_perlasm = $(PERL) $(<) void $(@) + +$(obj)/%-core.S: $(src)/%-armv8.pl + $(call cmd,perlasm) + +clean-files += poly1305-core.S diff --git a/arch/arm64/crypto/chacha-neon-core.S b/arch/arm64/lib/crypto/chacha-neon-core.S similarity index 100% rename from arch/arm64/crypto/chacha-neon-core.S rename to arch/arm64/lib/crypto/chacha-neon-core.S diff --git a/arch/arm64/crypto/chacha-neon-glue.c b/arch/arm64/lib/crypto/chacha-neon-glue.c similarity index 100% rename from arch/arm64/crypto/chacha-neon-glue.c rename to arch/arm64/lib/crypto/chacha-neon-glue.c diff --git a/arch/arm64/crypto/poly1305-armv8.pl b/arch/arm64/lib/crypto/poly1305-armv8.pl similarity index 100% rename from arch/arm64/crypto/poly1305-armv8.pl rename to arch/arm64/lib/crypto/poly1305-armv8.pl diff --git a/arch/arm64/crypto/poly1305-glue.c b/arch/arm64/lib/crypto/poly1305-glue.c similarity index 100% rename from arch/arm64/crypto/poly1305-glue.c rename to arch/arm64/lib/crypto/poly1305-glue.c diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index 59135009e4f0..7395234d654b 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -159,6 +159,9 @@ if !KMSAN # avoid false positives from assembly if ARM source "arch/arm/lib/crypto/Kconfig" endif +if ARM64 +source "arch/arm64/lib/crypto/Kconfig" +endif endif endmenu From 939a54ac073808db15ed411d563dfadb3ef12798 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Tue, 22 Apr 2025 08:27:10 -0700 Subject: [PATCH 226/382] crypto: mips - move library functions to arch/mips/lib/crypto/ Continue disentangling the crypto library functions from the generic crypto infrastructure by moving the mips ChaCha and Poly1305 library functions into a new directory arch/mips/lib/crypto/ that does not depend on CRYPTO. This mirrors the distinction between crypto/ and lib/crypto/. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/mips/crypto/Kconfig | 11 ----------- arch/mips/crypto/Makefile | 17 ----------------- arch/mips/lib/Makefile | 2 ++ arch/mips/lib/crypto/.gitignore | 2 ++ arch/mips/lib/crypto/Kconfig | 12 ++++++++++++ arch/mips/lib/crypto/Makefile | 19 +++++++++++++++++++ arch/mips/{ => lib}/crypto/chacha-core.S | 0 arch/mips/{ => lib}/crypto/chacha-glue.c | 0 arch/mips/{ => lib}/crypto/poly1305-glue.c | 0 arch/mips/{ => lib}/crypto/poly1305-mips.pl | 0 lib/crypto/Kconfig | 3 +++ 11 files changed, 38 insertions(+), 28 deletions(-) create mode 100644 arch/mips/lib/crypto/.gitignore create mode 100644 arch/mips/lib/crypto/Kconfig create mode 100644 arch/mips/lib/crypto/Makefile rename arch/mips/{ => lib}/crypto/chacha-core.S (100%) rename arch/mips/{ => lib}/crypto/chacha-glue.c (100%) rename arch/mips/{ => lib}/crypto/poly1305-glue.c (100%) rename arch/mips/{ => lib}/crypto/poly1305-mips.pl (100%) diff --git a/arch/mips/crypto/Kconfig b/arch/mips/crypto/Kconfig index 8283664a1f24..9db1fd6d9f0e 100644 --- a/arch/mips/crypto/Kconfig +++ b/arch/mips/crypto/Kconfig @@ -2,11 +2,6 @@ menu "Accelerated Cryptographic Algorithms for CPU (mips)" -config CRYPTO_POLY1305_MIPS - tristate - select CRYPTO_ARCH_HAVE_LIB_POLY1305 - default CRYPTO_LIB_POLY1305_INTERNAL - config CRYPTO_MD5_OCTEON tristate "Digests: MD5 (OCTEON)" depends on CPU_CAVIUM_OCTEON @@ -47,10 +42,4 @@ config CRYPTO_SHA512_OCTEON Architecture: mips OCTEON using crypto instructions, when available -config CRYPTO_CHACHA_MIPS - tristate - depends on CPU_MIPS32_R2 - select CRYPTO_ARCH_HAVE_LIB_CHACHA - default CRYPTO_LIB_CHACHA_INTERNAL - endmenu diff --git a/arch/mips/crypto/Makefile b/arch/mips/crypto/Makefile index fddc88281412..5adb631a69c1 100644 --- a/arch/mips/crypto/Makefile +++ b/arch/mips/crypto/Makefile @@ -3,20 +3,3 @@ # Makefile for MIPS crypto files.. # -obj-$(CONFIG_CRYPTO_CHACHA_MIPS) += chacha-mips.o -chacha-mips-y := chacha-core.o chacha-glue.o -AFLAGS_chacha-core.o += -O2 # needed to fill branch delay slots - -obj-$(CONFIG_CRYPTO_POLY1305_MIPS) += poly1305-mips.o -poly1305-mips-y := poly1305-core.o poly1305-glue.o - -perlasm-flavour-$(CONFIG_32BIT) := o32 -perlasm-flavour-$(CONFIG_64BIT) := 64 - -quiet_cmd_perlasm = PERLASM $@ - cmd_perlasm = $(PERL) $(<) $(perlasm-flavour-y) $(@) - -$(obj)/poly1305-core.S: $(src)/poly1305-mips.pl FORCE - $(call if_changed,perlasm) - -targets += poly1305-core.S diff --git a/arch/mips/lib/Makefile b/arch/mips/lib/Makefile index 9c024e6d5e54..9d75845ef78e 100644 --- a/arch/mips/lib/Makefile +++ b/arch/mips/lib/Makefile @@ -3,6 +3,8 @@ # Makefile for MIPS-specific library files.. # +obj-y += crypto/ + lib-y += bitops.o csum_partial.o delay.o memcpy.o memset.o \ mips-atomic.o strncpy_user.o \ strnlen_user.o uncached.o diff --git a/arch/mips/lib/crypto/.gitignore b/arch/mips/lib/crypto/.gitignore new file mode 100644 index 000000000000..0d47d4f21c6d --- /dev/null +++ b/arch/mips/lib/crypto/.gitignore @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0-only +poly1305-core.S diff --git a/arch/mips/lib/crypto/Kconfig b/arch/mips/lib/crypto/Kconfig new file mode 100644 index 000000000000..5b82ba753c55 --- /dev/null +++ b/arch/mips/lib/crypto/Kconfig @@ -0,0 +1,12 @@ +# SPDX-License-Identifier: GPL-2.0-only + +config CRYPTO_CHACHA_MIPS + tristate + depends on CPU_MIPS32_R2 + default CRYPTO_LIB_CHACHA_INTERNAL + select CRYPTO_ARCH_HAVE_LIB_CHACHA + +config CRYPTO_POLY1305_MIPS + tristate + default CRYPTO_LIB_POLY1305_INTERNAL + select CRYPTO_ARCH_HAVE_LIB_POLY1305 diff --git a/arch/mips/lib/crypto/Makefile b/arch/mips/lib/crypto/Makefile new file mode 100644 index 000000000000..804488c7aded --- /dev/null +++ b/arch/mips/lib/crypto/Makefile @@ -0,0 +1,19 @@ +# SPDX-License-Identifier: GPL-2.0-only + +obj-$(CONFIG_CRYPTO_CHACHA_MIPS) += chacha-mips.o +chacha-mips-y := chacha-core.o chacha-glue.o +AFLAGS_chacha-core.o += -O2 # needed to fill branch delay slots + +obj-$(CONFIG_CRYPTO_POLY1305_MIPS) += poly1305-mips.o +poly1305-mips-y := poly1305-core.o poly1305-glue.o + +perlasm-flavour-$(CONFIG_32BIT) := o32 +perlasm-flavour-$(CONFIG_64BIT) := 64 + +quiet_cmd_perlasm = PERLASM $@ + cmd_perlasm = $(PERL) $(<) $(perlasm-flavour-y) $(@) + +$(obj)/poly1305-core.S: $(src)/poly1305-mips.pl FORCE + $(call if_changed,perlasm) + +targets += poly1305-core.S diff --git a/arch/mips/crypto/chacha-core.S b/arch/mips/lib/crypto/chacha-core.S similarity index 100% rename from arch/mips/crypto/chacha-core.S rename to arch/mips/lib/crypto/chacha-core.S diff --git a/arch/mips/crypto/chacha-glue.c b/arch/mips/lib/crypto/chacha-glue.c similarity index 100% rename from arch/mips/crypto/chacha-glue.c rename to arch/mips/lib/crypto/chacha-glue.c diff --git a/arch/mips/crypto/poly1305-glue.c b/arch/mips/lib/crypto/poly1305-glue.c similarity index 100% rename from arch/mips/crypto/poly1305-glue.c rename to arch/mips/lib/crypto/poly1305-glue.c diff --git a/arch/mips/crypto/poly1305-mips.pl b/arch/mips/lib/crypto/poly1305-mips.pl similarity index 100% rename from arch/mips/crypto/poly1305-mips.pl rename to arch/mips/lib/crypto/poly1305-mips.pl diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index 7395234d654b..c5c01bc3569d 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -162,6 +162,9 @@ endif if ARM64 source "arch/arm64/lib/crypto/Kconfig" endif +if MIPS +source "arch/mips/lib/crypto/Kconfig" +endif endif endmenu From f9f86c03efebf81274b1ec66840410d58255fb84 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Tue, 22 Apr 2025 08:27:11 -0700 Subject: [PATCH 227/382] crypto: powerpc - move library functions to arch/powerpc/lib/crypto/ Continue disentangling the crypto library functions from the generic crypto infrastructure by moving the powerpc ChaCha and Poly1305 library functions into a new directory arch/powerpc/lib/crypto/ that does not depend on CRYPTO. This mirrors the distinction between crypto/ and lib/crypto/. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/powerpc/crypto/Kconfig | 14 -------------- arch/powerpc/crypto/Makefile | 4 ---- arch/powerpc/lib/Makefile | 2 ++ arch/powerpc/lib/crypto/Kconfig | 15 +++++++++++++++ arch/powerpc/lib/crypto/Makefile | 7 +++++++ arch/powerpc/{ => lib}/crypto/chacha-p10-glue.c | 0 arch/powerpc/{ => lib}/crypto/chacha-p10le-8x.S | 0 arch/powerpc/{ => lib}/crypto/poly1305-p10-glue.c | 0 arch/powerpc/{ => lib}/crypto/poly1305-p10le_64.S | 0 lib/crypto/Kconfig | 3 +++ 10 files changed, 27 insertions(+), 18 deletions(-) create mode 100644 arch/powerpc/lib/crypto/Kconfig create mode 100644 arch/powerpc/lib/crypto/Makefile rename arch/powerpc/{ => lib}/crypto/chacha-p10-glue.c (100%) rename arch/powerpc/{ => lib}/crypto/chacha-p10le-8x.S (100%) rename arch/powerpc/{ => lib}/crypto/poly1305-p10-glue.c (100%) rename arch/powerpc/{ => lib}/crypto/poly1305-p10le_64.S (100%) diff --git a/arch/powerpc/crypto/Kconfig b/arch/powerpc/crypto/Kconfig index 8bcc69013464..4bf7b01228e7 100644 --- a/arch/powerpc/crypto/Kconfig +++ b/arch/powerpc/crypto/Kconfig @@ -90,20 +90,6 @@ config CRYPTO_AES_GCM_P10 Support for cryptographic acceleration instructions on Power10 or later CPU. This module supports stitched acceleration for AES/GCM. -config CRYPTO_CHACHA20_P10 - tristate - depends on PPC64 && CPU_LITTLE_ENDIAN && VSX - select CRYPTO_LIB_CHACHA_GENERIC - select CRYPTO_ARCH_HAVE_LIB_CHACHA - default CRYPTO_LIB_CHACHA_INTERNAL - -config CRYPTO_POLY1305_P10 - tristate - depends on PPC64 && CPU_LITTLE_ENDIAN && VSX - select CRYPTO_ARCH_HAVE_LIB_POLY1305 - select CRYPTO_LIB_POLY1305_GENERIC - default CRYPTO_LIB_POLY1305_INTERNAL - config CRYPTO_DEV_VMX bool "Support for VMX cryptographic acceleration instructions" depends on PPC64 && VSX diff --git a/arch/powerpc/crypto/Makefile b/arch/powerpc/crypto/Makefile index 2f00b22b0823..f13aec8a1833 100644 --- a/arch/powerpc/crypto/Makefile +++ b/arch/powerpc/crypto/Makefile @@ -11,8 +11,6 @@ obj-$(CONFIG_CRYPTO_SHA1_PPC) += sha1-powerpc.o obj-$(CONFIG_CRYPTO_SHA1_PPC_SPE) += sha1-ppc-spe.o obj-$(CONFIG_CRYPTO_SHA256_PPC_SPE) += sha256-ppc-spe.o obj-$(CONFIG_CRYPTO_AES_GCM_P10) += aes-gcm-p10-crypto.o -obj-$(CONFIG_CRYPTO_CHACHA20_P10) += chacha-p10-crypto.o -obj-$(CONFIG_CRYPTO_POLY1305_P10) += poly1305-p10-crypto.o obj-$(CONFIG_CRYPTO_DEV_VMX_ENCRYPT) += vmx-crypto.o obj-$(CONFIG_CRYPTO_CURVE25519_PPC64) += curve25519-ppc64le.o @@ -22,8 +20,6 @@ sha1-powerpc-y := sha1-powerpc-asm.o sha1.o sha1-ppc-spe-y := sha1-spe-asm.o sha1-spe-glue.o sha256-ppc-spe-y := sha256-spe-asm.o sha256-spe-glue.o aes-gcm-p10-crypto-y := aes-gcm-p10-glue.o aes-gcm-p10.o ghashp10-ppc.o aesp10-ppc.o -chacha-p10-crypto-y := chacha-p10-glue.o chacha-p10le-8x.o -poly1305-p10-crypto-y := poly1305-p10-glue.o poly1305-p10le_64.o vmx-crypto-objs := vmx.o aesp8-ppc.o ghashp8-ppc.o aes.o aes_cbc.o aes_ctr.o aes_xts.o ghash.o curve25519-ppc64le-y := curve25519-ppc64le-core.o curve25519-ppc64le_asm.o diff --git a/arch/powerpc/lib/Makefile b/arch/powerpc/lib/Makefile index dd8a4b52a0cc..1cd74673cbf7 100644 --- a/arch/powerpc/lib/Makefile +++ b/arch/powerpc/lib/Makefile @@ -3,6 +3,8 @@ # Makefile for ppc-specific library files.. # +obj-y += crypto/ + CFLAGS_code-patching.o += -fno-stack-protector CFLAGS_feature-fixups.o += -fno-stack-protector diff --git a/arch/powerpc/lib/crypto/Kconfig b/arch/powerpc/lib/crypto/Kconfig new file mode 100644 index 000000000000..3f52610e45eb --- /dev/null +++ b/arch/powerpc/lib/crypto/Kconfig @@ -0,0 +1,15 @@ +# SPDX-License-Identifier: GPL-2.0-only + +config CRYPTO_CHACHA20_P10 + tristate + depends on PPC64 && CPU_LITTLE_ENDIAN && VSX + default CRYPTO_LIB_CHACHA_INTERNAL + select CRYPTO_LIB_CHACHA_GENERIC + select CRYPTO_ARCH_HAVE_LIB_CHACHA + +config CRYPTO_POLY1305_P10 + tristate + depends on PPC64 && CPU_LITTLE_ENDIAN && VSX + default CRYPTO_LIB_POLY1305_INTERNAL + select CRYPTO_ARCH_HAVE_LIB_POLY1305 + select CRYPTO_LIB_POLY1305_GENERIC diff --git a/arch/powerpc/lib/crypto/Makefile b/arch/powerpc/lib/crypto/Makefile new file mode 100644 index 000000000000..5709ae14258a --- /dev/null +++ b/arch/powerpc/lib/crypto/Makefile @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: GPL-2.0-only + +obj-$(CONFIG_CRYPTO_CHACHA20_P10) += chacha-p10-crypto.o +chacha-p10-crypto-y := chacha-p10-glue.o chacha-p10le-8x.o + +obj-$(CONFIG_CRYPTO_POLY1305_P10) += poly1305-p10-crypto.o +poly1305-p10-crypto-y := poly1305-p10-glue.o poly1305-p10le_64.o diff --git a/arch/powerpc/crypto/chacha-p10-glue.c b/arch/powerpc/lib/crypto/chacha-p10-glue.c similarity index 100% rename from arch/powerpc/crypto/chacha-p10-glue.c rename to arch/powerpc/lib/crypto/chacha-p10-glue.c diff --git a/arch/powerpc/crypto/chacha-p10le-8x.S b/arch/powerpc/lib/crypto/chacha-p10le-8x.S similarity index 100% rename from arch/powerpc/crypto/chacha-p10le-8x.S rename to arch/powerpc/lib/crypto/chacha-p10le-8x.S diff --git a/arch/powerpc/crypto/poly1305-p10-glue.c b/arch/powerpc/lib/crypto/poly1305-p10-glue.c similarity index 100% rename from arch/powerpc/crypto/poly1305-p10-glue.c rename to arch/powerpc/lib/crypto/poly1305-p10-glue.c diff --git a/arch/powerpc/crypto/poly1305-p10le_64.S b/arch/powerpc/lib/crypto/poly1305-p10le_64.S similarity index 100% rename from arch/powerpc/crypto/poly1305-p10le_64.S rename to arch/powerpc/lib/crypto/poly1305-p10le_64.S diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index c5c01bc3569d..4b3e94ed84bb 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -165,6 +165,9 @@ endif if MIPS source "arch/mips/lib/crypto/Kconfig" endif +if PPC +source "arch/powerpc/lib/crypto/Kconfig" +endif endif endmenu From d604877c2f9473848345db8e79341d4f6338dd78 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Tue, 22 Apr 2025 08:27:12 -0700 Subject: [PATCH 228/382] crypto: riscv - move library functions to arch/riscv/lib/crypto/ Continue disentangling the crypto library functions from the generic crypto infrastructure by moving the riscv ChaCha library functions into a new directory arch/riscv/lib/crypto/ that does not depend on CRYPTO. This mirrors the distinction between crypto/ and lib/crypto/. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/riscv/crypto/Kconfig | 7 ------- arch/riscv/crypto/Makefile | 3 --- arch/riscv/lib/Makefile | 1 + arch/riscv/lib/crypto/Kconfig | 8 ++++++++ arch/riscv/lib/crypto/Makefile | 4 ++++ arch/riscv/{ => lib}/crypto/chacha-riscv64-glue.c | 0 arch/riscv/{ => lib}/crypto/chacha-riscv64-zvkb.S | 0 lib/crypto/Kconfig | 3 +++ 8 files changed, 16 insertions(+), 10 deletions(-) create mode 100644 arch/riscv/lib/crypto/Kconfig create mode 100644 arch/riscv/lib/crypto/Makefile rename arch/riscv/{ => lib}/crypto/chacha-riscv64-glue.c (100%) rename arch/riscv/{ => lib}/crypto/chacha-riscv64-zvkb.S (100%) diff --git a/arch/riscv/crypto/Kconfig b/arch/riscv/crypto/Kconfig index 27a1f26d41bd..4863be2a4ec2 100644 --- a/arch/riscv/crypto/Kconfig +++ b/arch/riscv/crypto/Kconfig @@ -18,13 +18,6 @@ config CRYPTO_AES_RISCV64 - Zvkb vector crypto extension (CTR) - Zvkg vector crypto extension (XTS) -config CRYPTO_CHACHA_RISCV64 - tristate - depends on 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO - select CRYPTO_ARCH_HAVE_LIB_CHACHA - select CRYPTO_LIB_CHACHA_GENERIC - default CRYPTO_LIB_CHACHA_INTERNAL - config CRYPTO_GHASH_RISCV64 tristate "Hash functions: GHASH" depends on 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO diff --git a/arch/riscv/crypto/Makefile b/arch/riscv/crypto/Makefile index 247c7bc7288c..4ae9bf762e90 100644 --- a/arch/riscv/crypto/Makefile +++ b/arch/riscv/crypto/Makefile @@ -4,9 +4,6 @@ obj-$(CONFIG_CRYPTO_AES_RISCV64) += aes-riscv64.o aes-riscv64-y := aes-riscv64-glue.o aes-riscv64-zvkned.o \ aes-riscv64-zvkned-zvbb-zvkg.o aes-riscv64-zvkned-zvkb.o -obj-$(CONFIG_CRYPTO_CHACHA_RISCV64) += chacha-riscv64.o -chacha-riscv64-y := chacha-riscv64-glue.o chacha-riscv64-zvkb.o - obj-$(CONFIG_CRYPTO_GHASH_RISCV64) += ghash-riscv64.o ghash-riscv64-y := ghash-riscv64-glue.o ghash-riscv64-zvkg.o diff --git a/arch/riscv/lib/Makefile b/arch/riscv/lib/Makefile index b1c46153606a..0baec92d2f55 100644 --- a/arch/riscv/lib/Makefile +++ b/arch/riscv/lib/Makefile @@ -1,4 +1,5 @@ # SPDX-License-Identifier: GPL-2.0-only +obj-y += crypto/ lib-y += delay.o lib-y += memcpy.o lib-y += memset.o diff --git a/arch/riscv/lib/crypto/Kconfig b/arch/riscv/lib/crypto/Kconfig new file mode 100644 index 000000000000..46ce2a7ac1f2 --- /dev/null +++ b/arch/riscv/lib/crypto/Kconfig @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-2.0-only + +config CRYPTO_CHACHA_RISCV64 + tristate + depends on 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO + default CRYPTO_LIB_CHACHA_INTERNAL + select CRYPTO_ARCH_HAVE_LIB_CHACHA + select CRYPTO_LIB_CHACHA_GENERIC diff --git a/arch/riscv/lib/crypto/Makefile b/arch/riscv/lib/crypto/Makefile new file mode 100644 index 000000000000..e27b78f317fc --- /dev/null +++ b/arch/riscv/lib/crypto/Makefile @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0-only + +obj-$(CONFIG_CRYPTO_CHACHA_RISCV64) += chacha-riscv64.o +chacha-riscv64-y := chacha-riscv64-glue.o chacha-riscv64-zvkb.o diff --git a/arch/riscv/crypto/chacha-riscv64-glue.c b/arch/riscv/lib/crypto/chacha-riscv64-glue.c similarity index 100% rename from arch/riscv/crypto/chacha-riscv64-glue.c rename to arch/riscv/lib/crypto/chacha-riscv64-glue.c diff --git a/arch/riscv/crypto/chacha-riscv64-zvkb.S b/arch/riscv/lib/crypto/chacha-riscv64-zvkb.S similarity index 100% rename from arch/riscv/crypto/chacha-riscv64-zvkb.S rename to arch/riscv/lib/crypto/chacha-riscv64-zvkb.S diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index 4b3e94ed84bb..0b06c25eb38a 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -168,6 +168,9 @@ endif if PPC source "arch/powerpc/lib/crypto/Kconfig" endif +if RISCV +source "arch/riscv/lib/crypto/Kconfig" +endif endif endmenu From 3ea91323fe32011092fc75575d9a074ab374d673 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Tue, 22 Apr 2025 08:27:13 -0700 Subject: [PATCH 229/382] crypto: s390 - move library functions to arch/s390/lib/crypto/ Continue disentangling the crypto library functions from the generic crypto infrastructure by moving the s390 ChaCha library functions into a new directory arch/s390/lib/crypto/ that does not depend on CRYPTO. This mirrors the distinction between crypto/ and lib/crypto/. Acked-by: Heiko Carstens Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/s390/crypto/Kconfig | 6 ------ arch/s390/crypto/Makefile | 3 --- arch/s390/lib/Makefile | 1 + arch/s390/lib/crypto/Kconfig | 7 +++++++ arch/s390/lib/crypto/Makefile | 4 ++++ arch/s390/{ => lib}/crypto/chacha-glue.c | 0 arch/s390/{ => lib}/crypto/chacha-s390.S | 0 arch/s390/{ => lib}/crypto/chacha-s390.h | 0 lib/crypto/Kconfig | 3 +++ 9 files changed, 15 insertions(+), 9 deletions(-) create mode 100644 arch/s390/lib/crypto/Kconfig create mode 100644 arch/s390/lib/crypto/Makefile rename arch/s390/{ => lib}/crypto/chacha-glue.c (100%) rename arch/s390/{ => lib}/crypto/chacha-s390.S (100%) rename arch/s390/{ => lib}/crypto/chacha-s390.h (100%) diff --git a/arch/s390/crypto/Kconfig b/arch/s390/crypto/Kconfig index e88d9cd256ef..a2bfd6eef0ca 100644 --- a/arch/s390/crypto/Kconfig +++ b/arch/s390/crypto/Kconfig @@ -99,12 +99,6 @@ config CRYPTO_DES_S390 As of z990 the ECB and CBC mode are hardware accelerated. As of z196 the CTR mode is hardware accelerated. -config CRYPTO_CHACHA_S390 - tristate - select CRYPTO_LIB_CHACHA_GENERIC - select CRYPTO_ARCH_HAVE_LIB_CHACHA - default CRYPTO_LIB_CHACHA_INTERNAL - config CRYPTO_HMAC_S390 tristate "Keyed-hash message authentication code: HMAC" select CRYPTO_HASH diff --git a/arch/s390/crypto/Makefile b/arch/s390/crypto/Makefile index 14dafadbcbed..e3853774e1a3 100644 --- a/arch/s390/crypto/Makefile +++ b/arch/s390/crypto/Makefile @@ -11,10 +11,7 @@ obj-$(CONFIG_CRYPTO_SHA3_512_S390) += sha3_512_s390.o sha_common.o obj-$(CONFIG_CRYPTO_DES_S390) += des_s390.o obj-$(CONFIG_CRYPTO_AES_S390) += aes_s390.o obj-$(CONFIG_CRYPTO_PAES_S390) += paes_s390.o -obj-$(CONFIG_CRYPTO_CHACHA_S390) += chacha_s390.o obj-$(CONFIG_S390_PRNG) += prng.o obj-$(CONFIG_CRYPTO_GHASH_S390) += ghash_s390.o obj-$(CONFIG_CRYPTO_HMAC_S390) += hmac_s390.o obj-y += arch_random.o - -chacha_s390-y := chacha-glue.o chacha-s390.o diff --git a/arch/s390/lib/Makefile b/arch/s390/lib/Makefile index 14bbfe50033c..fce4edbe8a07 100644 --- a/arch/s390/lib/Makefile +++ b/arch/s390/lib/Makefile @@ -3,6 +3,7 @@ # Makefile for s390-specific library files.. # +obj-y += crypto/ lib-y += delay.o string.o uaccess.o find.o spinlock.o tishift.o lib-y += csum-partial.o obj-y += mem.o xor.o diff --git a/arch/s390/lib/crypto/Kconfig b/arch/s390/lib/crypto/Kconfig new file mode 100644 index 000000000000..b79fd91af9fe --- /dev/null +++ b/arch/s390/lib/crypto/Kconfig @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: GPL-2.0-only + +config CRYPTO_CHACHA_S390 + tristate + default CRYPTO_LIB_CHACHA_INTERNAL + select CRYPTO_LIB_CHACHA_GENERIC + select CRYPTO_ARCH_HAVE_LIB_CHACHA diff --git a/arch/s390/lib/crypto/Makefile b/arch/s390/lib/crypto/Makefile new file mode 100644 index 000000000000..06c2cf77178e --- /dev/null +++ b/arch/s390/lib/crypto/Makefile @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0-only + +obj-$(CONFIG_CRYPTO_CHACHA_S390) += chacha_s390.o +chacha_s390-y := chacha-glue.o chacha-s390.o diff --git a/arch/s390/crypto/chacha-glue.c b/arch/s390/lib/crypto/chacha-glue.c similarity index 100% rename from arch/s390/crypto/chacha-glue.c rename to arch/s390/lib/crypto/chacha-glue.c diff --git a/arch/s390/crypto/chacha-s390.S b/arch/s390/lib/crypto/chacha-s390.S similarity index 100% rename from arch/s390/crypto/chacha-s390.S rename to arch/s390/lib/crypto/chacha-s390.S diff --git a/arch/s390/crypto/chacha-s390.h b/arch/s390/lib/crypto/chacha-s390.h similarity index 100% rename from arch/s390/crypto/chacha-s390.h rename to arch/s390/lib/crypto/chacha-s390.h diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index 0b06c25eb38a..db19a7acc2fb 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -171,6 +171,9 @@ endif if RISCV source "arch/riscv/lib/crypto/Kconfig" endif +if S390 +source "arch/s390/lib/crypto/Kconfig" +endif endif endmenu From c7c18c94a6becc42c71a17fdc6a551aa8efb2999 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Tue, 22 Apr 2025 08:27:14 -0700 Subject: [PATCH 230/382] crypto: x86 - move library functions to arch/x86/lib/crypto/ Continue disentangling the crypto library functions from the generic crypto infrastructure by moving the x86 BLAKE2s, ChaCha, and Poly1305 library functions into a new directory arch/x86/lib/crypto/ that does not depend on CRYPTO. This mirrors the distinction between crypto/ and lib/crypto/. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/x86/crypto/Kconfig | 25 ------------------ arch/x86/crypto/Makefile | 15 ----------- arch/x86/lib/Makefile | 2 ++ arch/x86/lib/crypto/.gitignore | 2 ++ arch/x86/lib/crypto/Kconfig | 26 +++++++++++++++++++ arch/x86/lib/crypto/Makefile | 17 ++++++++++++ arch/x86/{ => lib}/crypto/blake2s-core.S | 0 arch/x86/{ => lib}/crypto/blake2s-glue.c | 0 .../x86/{ => lib}/crypto/chacha-avx2-x86_64.S | 0 .../{ => lib}/crypto/chacha-avx512vl-x86_64.S | 0 .../{ => lib}/crypto/chacha-ssse3-x86_64.S | 0 arch/x86/{ => lib}/crypto/chacha_glue.c | 0 .../crypto/poly1305-x86_64-cryptogams.pl | 0 arch/x86/{ => lib}/crypto/poly1305_glue.c | 0 lib/crypto/Kconfig | 3 +++ 15 files changed, 50 insertions(+), 40 deletions(-) create mode 100644 arch/x86/lib/crypto/.gitignore create mode 100644 arch/x86/lib/crypto/Kconfig create mode 100644 arch/x86/lib/crypto/Makefile rename arch/x86/{ => lib}/crypto/blake2s-core.S (100%) rename arch/x86/{ => lib}/crypto/blake2s-glue.c (100%) rename arch/x86/{ => lib}/crypto/chacha-avx2-x86_64.S (100%) rename arch/x86/{ => lib}/crypto/chacha-avx512vl-x86_64.S (100%) rename arch/x86/{ => lib}/crypto/chacha-ssse3-x86_64.S (100%) rename arch/x86/{ => lib}/crypto/chacha_glue.c (100%) rename arch/x86/{ => lib}/crypto/poly1305-x86_64-cryptogams.pl (100%) rename arch/x86/{ => lib}/crypto/poly1305_glue.c (100%) diff --git a/arch/x86/crypto/Kconfig b/arch/x86/crypto/Kconfig index b4df6cf37e0e..9e941362e4cd 100644 --- a/arch/x86/crypto/Kconfig +++ b/arch/x86/crypto/Kconfig @@ -335,13 +335,6 @@ config CRYPTO_ARIA_GFNI_AVX512_X86_64 Processes 64 blocks in parallel. -config CRYPTO_CHACHA20_X86_64 - tristate - depends on 64BIT - select CRYPTO_LIB_CHACHA_GENERIC - select CRYPTO_ARCH_HAVE_LIB_CHACHA - default CRYPTO_LIB_CHACHA_INTERNAL - config CRYPTO_AEGIS128_AESNI_SSE2 tristate "AEAD ciphers: AEGIS-128 (AES-NI/SSE4.1)" depends on 64BIT @@ -373,18 +366,6 @@ config CRYPTO_NHPOLY1305_AVX2 Architecture: x86_64 using: - AVX2 (Advanced Vector Extensions 2) -config CRYPTO_BLAKE2S_X86 - bool "Hash functions: BLAKE2s (SSSE3/AVX-512)" - depends on 64BIT - select CRYPTO_LIB_BLAKE2S_GENERIC - select CRYPTO_ARCH_HAVE_LIB_BLAKE2S - help - BLAKE2s cryptographic hash function (RFC 7693) - - Architecture: x86_64 using: - - SSSE3 (Supplemental SSE3) - - AVX-512 (Advanced Vector Extensions-512) - config CRYPTO_POLYVAL_CLMUL_NI tristate "Hash functions: POLYVAL (CLMUL-NI)" depends on 64BIT @@ -395,12 +376,6 @@ config CRYPTO_POLYVAL_CLMUL_NI Architecture: x86_64 using: - CLMUL-NI (carry-less multiplication new instructions) -config CRYPTO_POLY1305_X86_64 - tristate - depends on 64BIT - select CRYPTO_ARCH_HAVE_LIB_POLY1305 - default CRYPTO_LIB_POLY1305_INTERNAL - config CRYPTO_SHA1_SSSE3 tristate "Hash functions: SHA-1 (SSSE3/AVX/AVX2/SHA-NI)" depends on 64BIT diff --git a/arch/x86/crypto/Makefile b/arch/x86/crypto/Makefile index 2f22b381f244..fad59a6c6c26 100644 --- a/arch/x86/crypto/Makefile +++ b/arch/x86/crypto/Makefile @@ -42,9 +42,6 @@ cast6-avx-x86_64-y := cast6-avx-x86_64-asm_64.o cast6_avx_glue.o obj-$(CONFIG_CRYPTO_AEGIS128_AESNI_SSE2) += aegis128-aesni.o aegis128-aesni-y := aegis128-aesni-asm.o aegis128-aesni-glue.o -obj-$(CONFIG_CRYPTO_CHACHA20_X86_64) += chacha-x86_64.o -chacha-x86_64-y := chacha-avx2-x86_64.o chacha-ssse3-x86_64.o chacha-avx512vl-x86_64.o chacha_glue.o - obj-$(CONFIG_CRYPTO_AES_NI_INTEL) += aesni-intel.o aesni-intel-y := aesni-intel_asm.o aesni-intel_glue.o aesni-intel-$(CONFIG_64BIT) += aes-ctr-avx-x86_64.o \ @@ -63,19 +60,12 @@ sha256-ssse3-y := sha256-ssse3-asm.o sha256-avx-asm.o sha256-avx2-asm.o sha256_n obj-$(CONFIG_CRYPTO_SHA512_SSSE3) += sha512-ssse3.o sha512-ssse3-y := sha512-ssse3-asm.o sha512-avx-asm.o sha512-avx2-asm.o sha512_ssse3_glue.o -obj-$(CONFIG_CRYPTO_BLAKE2S_X86) += libblake2s-x86_64.o -libblake2s-x86_64-y := blake2s-core.o blake2s-glue.o - obj-$(CONFIG_CRYPTO_GHASH_CLMUL_NI_INTEL) += ghash-clmulni-intel.o ghash-clmulni-intel-y := ghash-clmulni-intel_asm.o ghash-clmulni-intel_glue.o obj-$(CONFIG_CRYPTO_POLYVAL_CLMUL_NI) += polyval-clmulni.o polyval-clmulni-y := polyval-clmulni_asm.o polyval-clmulni_glue.o -obj-$(CONFIG_CRYPTO_POLY1305_X86_64) += poly1305-x86_64.o -poly1305-x86_64-y := poly1305-x86_64-cryptogams.o poly1305_glue.o -targets += poly1305-x86_64-cryptogams.S - obj-$(CONFIG_CRYPTO_NHPOLY1305_SSE2) += nhpoly1305-sse2.o nhpoly1305-sse2-y := nh-sse2-x86_64.o nhpoly1305-sse2-glue.o obj-$(CONFIG_CRYPTO_NHPOLY1305_AVX2) += nhpoly1305-avx2.o @@ -101,10 +91,5 @@ aria-aesni-avx2-x86_64-y := aria-aesni-avx2-asm_64.o aria_aesni_avx2_glue.o obj-$(CONFIG_CRYPTO_ARIA_GFNI_AVX512_X86_64) += aria-gfni-avx512-x86_64.o aria-gfni-avx512-x86_64-y := aria-gfni-avx512-asm_64.o aria_gfni_avx512_glue.o -quiet_cmd_perlasm = PERLASM $@ - cmd_perlasm = $(PERL) $< > $@ -$(obj)/%.S: $(src)/%.pl FORCE - $(call if_changed,perlasm) - # Disable GCOV in odd or sensitive code GCOV_PROFILE_curve25519-x86_64.o := n diff --git a/arch/x86/lib/Makefile b/arch/x86/lib/Makefile index 1c50352eb49f..4f0d57a35426 100644 --- a/arch/x86/lib/Makefile +++ b/arch/x86/lib/Makefile @@ -3,6 +3,8 @@ # Makefile for x86 specific library files. # +obj-y += crypto/ + # Produces uninteresting flaky coverage. KCOV_INSTRUMENT_delay.o := n diff --git a/arch/x86/lib/crypto/.gitignore b/arch/x86/lib/crypto/.gitignore new file mode 100644 index 000000000000..580c839bb177 --- /dev/null +++ b/arch/x86/lib/crypto/.gitignore @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0-only +poly1305-x86_64-cryptogams.S diff --git a/arch/x86/lib/crypto/Kconfig b/arch/x86/lib/crypto/Kconfig new file mode 100644 index 000000000000..f83aa51dd912 --- /dev/null +++ b/arch/x86/lib/crypto/Kconfig @@ -0,0 +1,26 @@ +# SPDX-License-Identifier: GPL-2.0-only + +config CRYPTO_BLAKE2S_X86 + bool "Hash functions: BLAKE2s (SSSE3/AVX-512)" + depends on 64BIT + select CRYPTO_LIB_BLAKE2S_GENERIC + select CRYPTO_ARCH_HAVE_LIB_BLAKE2S + help + BLAKE2s cryptographic hash function (RFC 7693) + + Architecture: x86_64 using: + - SSSE3 (Supplemental SSE3) + - AVX-512 (Advanced Vector Extensions-512) + +config CRYPTO_CHACHA20_X86_64 + tristate + depends on 64BIT + default CRYPTO_LIB_CHACHA_INTERNAL + select CRYPTO_LIB_CHACHA_GENERIC + select CRYPTO_ARCH_HAVE_LIB_CHACHA + +config CRYPTO_POLY1305_X86_64 + tristate + depends on 64BIT + default CRYPTO_LIB_POLY1305_INTERNAL + select CRYPTO_ARCH_HAVE_LIB_POLY1305 diff --git a/arch/x86/lib/crypto/Makefile b/arch/x86/lib/crypto/Makefile new file mode 100644 index 000000000000..c2ff8c5f1046 --- /dev/null +++ b/arch/x86/lib/crypto/Makefile @@ -0,0 +1,17 @@ +# SPDX-License-Identifier: GPL-2.0-only + +obj-$(CONFIG_CRYPTO_BLAKE2S_X86) += libblake2s-x86_64.o +libblake2s-x86_64-y := blake2s-core.o blake2s-glue.o + +obj-$(CONFIG_CRYPTO_CHACHA20_X86_64) += chacha-x86_64.o +chacha-x86_64-y := chacha-avx2-x86_64.o chacha-ssse3-x86_64.o chacha-avx512vl-x86_64.o chacha_glue.o + +obj-$(CONFIG_CRYPTO_POLY1305_X86_64) += poly1305-x86_64.o +poly1305-x86_64-y := poly1305-x86_64-cryptogams.o poly1305_glue.o +targets += poly1305-x86_64-cryptogams.S + +quiet_cmd_perlasm = PERLASM $@ + cmd_perlasm = $(PERL) $< > $@ + +$(obj)/%.S: $(src)/%.pl FORCE + $(call if_changed,perlasm) diff --git a/arch/x86/crypto/blake2s-core.S b/arch/x86/lib/crypto/blake2s-core.S similarity index 100% rename from arch/x86/crypto/blake2s-core.S rename to arch/x86/lib/crypto/blake2s-core.S diff --git a/arch/x86/crypto/blake2s-glue.c b/arch/x86/lib/crypto/blake2s-glue.c similarity index 100% rename from arch/x86/crypto/blake2s-glue.c rename to arch/x86/lib/crypto/blake2s-glue.c diff --git a/arch/x86/crypto/chacha-avx2-x86_64.S b/arch/x86/lib/crypto/chacha-avx2-x86_64.S similarity index 100% rename from arch/x86/crypto/chacha-avx2-x86_64.S rename to arch/x86/lib/crypto/chacha-avx2-x86_64.S diff --git a/arch/x86/crypto/chacha-avx512vl-x86_64.S b/arch/x86/lib/crypto/chacha-avx512vl-x86_64.S similarity index 100% rename from arch/x86/crypto/chacha-avx512vl-x86_64.S rename to arch/x86/lib/crypto/chacha-avx512vl-x86_64.S diff --git a/arch/x86/crypto/chacha-ssse3-x86_64.S b/arch/x86/lib/crypto/chacha-ssse3-x86_64.S similarity index 100% rename from arch/x86/crypto/chacha-ssse3-x86_64.S rename to arch/x86/lib/crypto/chacha-ssse3-x86_64.S diff --git a/arch/x86/crypto/chacha_glue.c b/arch/x86/lib/crypto/chacha_glue.c similarity index 100% rename from arch/x86/crypto/chacha_glue.c rename to arch/x86/lib/crypto/chacha_glue.c diff --git a/arch/x86/crypto/poly1305-x86_64-cryptogams.pl b/arch/x86/lib/crypto/poly1305-x86_64-cryptogams.pl similarity index 100% rename from arch/x86/crypto/poly1305-x86_64-cryptogams.pl rename to arch/x86/lib/crypto/poly1305-x86_64-cryptogams.pl diff --git a/arch/x86/crypto/poly1305_glue.c b/arch/x86/lib/crypto/poly1305_glue.c similarity index 100% rename from arch/x86/crypto/poly1305_glue.c rename to arch/x86/lib/crypto/poly1305_glue.c diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index db19a7acc2fb..f321fe1a8681 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -174,6 +174,9 @@ endif if S390 source "arch/s390/lib/crypto/Kconfig" endif +if X86 +source "arch/x86/lib/crypto/Kconfig" +endif endif endmenu From 879f47548b8710f6cdfde38be7d5a6302e9e015d Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Tue, 22 Apr 2025 08:27:15 -0700 Subject: [PATCH 231/382] crypto: lib/chacha - remove INTERNAL symbol and selection of CRYPTO Now that the architecture-optimized ChaCha kconfig symbols are defined regardless of CRYPTO, there is no need for CRYPTO_LIB_CHACHA to select CRYPTO. So, remove that. This makes the indirection through the CRYPTO_LIB_CHACHA_INTERNAL symbol unnecessary, so get rid of that and just use CRYPTO_LIB_CHACHA directly. Finally, make the fallback to the generic implementation use a default value instead of a select; this makes it consistent with how the arch-optimized code gets enabled and also with how CRYPTO_LIB_BLAKE2S_GENERIC gets enabled. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/arm/lib/crypto/Kconfig | 2 +- arch/arm64/lib/crypto/Kconfig | 2 +- arch/mips/lib/crypto/Kconfig | 2 +- arch/powerpc/lib/crypto/Kconfig | 2 +- arch/riscv/lib/crypto/Kconfig | 2 +- arch/s390/lib/crypto/Kconfig | 2 +- arch/x86/lib/crypto/Kconfig | 2 +- crypto/Kconfig | 2 +- lib/crypto/Kconfig | 16 +++++----------- 9 files changed, 13 insertions(+), 19 deletions(-) diff --git a/arch/arm/lib/crypto/Kconfig b/arch/arm/lib/crypto/Kconfig index 181f138d563b..5d10bd13fc8d 100644 --- a/arch/arm/lib/crypto/Kconfig +++ b/arch/arm/lib/crypto/Kconfig @@ -15,7 +15,7 @@ config CRYPTO_BLAKE2S_ARM config CRYPTO_CHACHA20_NEON tristate - default CRYPTO_LIB_CHACHA_INTERNAL + default CRYPTO_LIB_CHACHA select CRYPTO_ARCH_HAVE_LIB_CHACHA config CRYPTO_POLY1305_ARM diff --git a/arch/arm64/lib/crypto/Kconfig b/arch/arm64/lib/crypto/Kconfig index 169311547efe..2a8ff7cfc08d 100644 --- a/arch/arm64/lib/crypto/Kconfig +++ b/arch/arm64/lib/crypto/Kconfig @@ -3,7 +3,7 @@ config CRYPTO_CHACHA20_NEON tristate depends on KERNEL_MODE_NEON - default CRYPTO_LIB_CHACHA_INTERNAL + default CRYPTO_LIB_CHACHA select CRYPTO_LIB_CHACHA_GENERIC select CRYPTO_ARCH_HAVE_LIB_CHACHA diff --git a/arch/mips/lib/crypto/Kconfig b/arch/mips/lib/crypto/Kconfig index 5b82ba753c55..454354e30d76 100644 --- a/arch/mips/lib/crypto/Kconfig +++ b/arch/mips/lib/crypto/Kconfig @@ -3,7 +3,7 @@ config CRYPTO_CHACHA_MIPS tristate depends on CPU_MIPS32_R2 - default CRYPTO_LIB_CHACHA_INTERNAL + default CRYPTO_LIB_CHACHA select CRYPTO_ARCH_HAVE_LIB_CHACHA config CRYPTO_POLY1305_MIPS diff --git a/arch/powerpc/lib/crypto/Kconfig b/arch/powerpc/lib/crypto/Kconfig index 3f52610e45eb..6627d28cd24e 100644 --- a/arch/powerpc/lib/crypto/Kconfig +++ b/arch/powerpc/lib/crypto/Kconfig @@ -3,7 +3,7 @@ config CRYPTO_CHACHA20_P10 tristate depends on PPC64 && CPU_LITTLE_ENDIAN && VSX - default CRYPTO_LIB_CHACHA_INTERNAL + default CRYPTO_LIB_CHACHA select CRYPTO_LIB_CHACHA_GENERIC select CRYPTO_ARCH_HAVE_LIB_CHACHA diff --git a/arch/riscv/lib/crypto/Kconfig b/arch/riscv/lib/crypto/Kconfig index 46ce2a7ac1f2..bc7a43f33eb3 100644 --- a/arch/riscv/lib/crypto/Kconfig +++ b/arch/riscv/lib/crypto/Kconfig @@ -3,6 +3,6 @@ config CRYPTO_CHACHA_RISCV64 tristate depends on 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO - default CRYPTO_LIB_CHACHA_INTERNAL + default CRYPTO_LIB_CHACHA select CRYPTO_ARCH_HAVE_LIB_CHACHA select CRYPTO_LIB_CHACHA_GENERIC diff --git a/arch/s390/lib/crypto/Kconfig b/arch/s390/lib/crypto/Kconfig index b79fd91af9fe..069b355fe51a 100644 --- a/arch/s390/lib/crypto/Kconfig +++ b/arch/s390/lib/crypto/Kconfig @@ -2,6 +2,6 @@ config CRYPTO_CHACHA_S390 tristate - default CRYPTO_LIB_CHACHA_INTERNAL + default CRYPTO_LIB_CHACHA select CRYPTO_LIB_CHACHA_GENERIC select CRYPTO_ARCH_HAVE_LIB_CHACHA diff --git a/arch/x86/lib/crypto/Kconfig b/arch/x86/lib/crypto/Kconfig index f83aa51dd912..e44403d9677f 100644 --- a/arch/x86/lib/crypto/Kconfig +++ b/arch/x86/lib/crypto/Kconfig @@ -15,7 +15,7 @@ config CRYPTO_BLAKE2S_X86 config CRYPTO_CHACHA20_X86_64 tristate depends on 64BIT - default CRYPTO_LIB_CHACHA_INTERNAL + default CRYPTO_LIB_CHACHA select CRYPTO_LIB_CHACHA_GENERIC select CRYPTO_ARCH_HAVE_LIB_CHACHA diff --git a/crypto/Kconfig b/crypto/Kconfig index 9322e42e562d..832af6363951 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -634,8 +634,8 @@ config CRYPTO_ARC4 config CRYPTO_CHACHA20 tristate "ChaCha" + select CRYPTO_LIB_CHACHA select CRYPTO_LIB_CHACHA_GENERIC - select CRYPTO_LIB_CHACHA_INTERNAL select CRYPTO_SKCIPHER help The ChaCha20, XChaCha20, and XChaCha12 stream cipher algorithms diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index f321fe1a8681..c6ab724c1dbd 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -50,22 +50,16 @@ config CRYPTO_ARCH_HAVE_LIB_CHACHA config CRYPTO_LIB_CHACHA_GENERIC tristate + default CRYPTO_LIB_CHACHA if !CRYPTO_ARCH_HAVE_LIB_CHACHA select CRYPTO_LIB_UTILS help - This symbol can be depended upon by arch implementations of the - ChaCha library interface that require the generic code as a - fallback, e.g., for SIMD implementations. If no arch specific - implementation is enabled, this implementation serves the users - of CRYPTO_LIB_CHACHA. - -config CRYPTO_LIB_CHACHA_INTERNAL - tristate - select CRYPTO_LIB_CHACHA_GENERIC if CRYPTO_ARCH_HAVE_LIB_CHACHA=n + This symbol can be selected by arch implementations of the ChaCha + library interface that require the generic code as a fallback, e.g., + for SIMD implementations. If no arch specific implementation is + enabled, this implementation serves the users of CRYPTO_LIB_CHACHA. config CRYPTO_LIB_CHACHA tristate - select CRYPTO - select CRYPTO_LIB_CHACHA_INTERNAL help Enable the ChaCha library interface. This interface may be fulfilled by either the generic implementation or an arch-specific one, if one From af9ce62783dd6acd595491badec08f1235c84739 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Tue, 22 Apr 2025 08:27:16 -0700 Subject: [PATCH 232/382] crypto: lib/poly1305 - remove INTERNAL symbol and selection of CRYPTO Now that the architecture-optimized Poly1305 kconfig symbols are defined regardless of CRYPTO, there is no need for CRYPTO_LIB_POLY1305 to select CRYPTO. So, remove that. This makes the indirection through the CRYPTO_LIB_POLY1305_INTERNAL symbol unnecessary, so get rid of that and just use CRYPTO_LIB_POLY1305 directly. Finally, make the fallback to the generic implementation use a default value instead of a select; this makes it consistent with how the arch-optimized code gets enabled and also with how CRYPTO_LIB_BLAKE2S_GENERIC gets enabled. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/arm/lib/crypto/Kconfig | 2 +- arch/arm64/lib/crypto/Kconfig | 2 +- arch/mips/lib/crypto/Kconfig | 2 +- arch/powerpc/lib/crypto/Kconfig | 2 +- arch/x86/lib/crypto/Kconfig | 2 +- crypto/Kconfig | 2 +- lib/crypto/Kconfig | 16 +++++----------- 7 files changed, 11 insertions(+), 17 deletions(-) diff --git a/arch/arm/lib/crypto/Kconfig b/arch/arm/lib/crypto/Kconfig index 5d10bd13fc8d..e8444fd0aae3 100644 --- a/arch/arm/lib/crypto/Kconfig +++ b/arch/arm/lib/crypto/Kconfig @@ -20,5 +20,5 @@ config CRYPTO_CHACHA20_NEON config CRYPTO_POLY1305_ARM tristate - default CRYPTO_LIB_POLY1305_INTERNAL + default CRYPTO_LIB_POLY1305 select CRYPTO_ARCH_HAVE_LIB_POLY1305 diff --git a/arch/arm64/lib/crypto/Kconfig b/arch/arm64/lib/crypto/Kconfig index 2a8ff7cfc08d..0b903ef524d8 100644 --- a/arch/arm64/lib/crypto/Kconfig +++ b/arch/arm64/lib/crypto/Kconfig @@ -10,5 +10,5 @@ config CRYPTO_CHACHA20_NEON config CRYPTO_POLY1305_NEON tristate depends on KERNEL_MODE_NEON - default CRYPTO_LIB_POLY1305_INTERNAL + default CRYPTO_LIB_POLY1305 select CRYPTO_ARCH_HAVE_LIB_POLY1305 diff --git a/arch/mips/lib/crypto/Kconfig b/arch/mips/lib/crypto/Kconfig index 454354e30d76..0670a170c1be 100644 --- a/arch/mips/lib/crypto/Kconfig +++ b/arch/mips/lib/crypto/Kconfig @@ -8,5 +8,5 @@ config CRYPTO_CHACHA_MIPS config CRYPTO_POLY1305_MIPS tristate - default CRYPTO_LIB_POLY1305_INTERNAL + default CRYPTO_LIB_POLY1305 select CRYPTO_ARCH_HAVE_LIB_POLY1305 diff --git a/arch/powerpc/lib/crypto/Kconfig b/arch/powerpc/lib/crypto/Kconfig index 6627d28cd24e..bf6d0ab22c27 100644 --- a/arch/powerpc/lib/crypto/Kconfig +++ b/arch/powerpc/lib/crypto/Kconfig @@ -10,6 +10,6 @@ config CRYPTO_CHACHA20_P10 config CRYPTO_POLY1305_P10 tristate depends on PPC64 && CPU_LITTLE_ENDIAN && VSX - default CRYPTO_LIB_POLY1305_INTERNAL + default CRYPTO_LIB_POLY1305 select CRYPTO_ARCH_HAVE_LIB_POLY1305 select CRYPTO_LIB_POLY1305_GENERIC diff --git a/arch/x86/lib/crypto/Kconfig b/arch/x86/lib/crypto/Kconfig index e44403d9677f..546fe2afe0b5 100644 --- a/arch/x86/lib/crypto/Kconfig +++ b/arch/x86/lib/crypto/Kconfig @@ -22,5 +22,5 @@ config CRYPTO_CHACHA20_X86_64 config CRYPTO_POLY1305_X86_64 tristate depends on 64BIT - default CRYPTO_LIB_POLY1305_INTERNAL + default CRYPTO_LIB_POLY1305 select CRYPTO_ARCH_HAVE_LIB_POLY1305 diff --git a/crypto/Kconfig b/crypto/Kconfig index 832af6363951..9878286d1d68 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -956,8 +956,8 @@ config CRYPTO_POLYVAL config CRYPTO_POLY1305 tristate "Poly1305" select CRYPTO_HASH + select CRYPTO_LIB_POLY1305 select CRYPTO_LIB_POLY1305_GENERIC - select CRYPTO_LIB_POLY1305_INTERNAL help Poly1305 authenticator algorithm (RFC7539) diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index c6ab724c1dbd..af2368799579 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -114,21 +114,15 @@ config CRYPTO_ARCH_HAVE_LIB_POLY1305 config CRYPTO_LIB_POLY1305_GENERIC tristate + default CRYPTO_LIB_POLY1305 if !CRYPTO_ARCH_HAVE_LIB_POLY1305 help - This symbol can be depended upon by arch implementations of the - Poly1305 library interface that require the generic code as a - fallback, e.g., for SIMD implementations. If no arch specific - implementation is enabled, this implementation serves the users - of CRYPTO_LIB_POLY1305. - -config CRYPTO_LIB_POLY1305_INTERNAL - tristate - select CRYPTO_LIB_POLY1305_GENERIC if CRYPTO_ARCH_HAVE_LIB_POLY1305=n + This symbol can be selected by arch implementations of the Poly1305 + library interface that require the generic code as a fallback, e.g., + for SIMD implementations. If no arch specific implementation is + enabled, this implementation serves the users of CRYPTO_LIB_POLY1305. config CRYPTO_LIB_POLY1305 tristate - select CRYPTO - select CRYPTO_LIB_POLY1305_INTERNAL help Enable the Poly1305 library interface. This interface may be fulfilled by either the generic implementation or an arch-specific one, if one From 19da081a28c95fe9b03ce952a2bf4a6f6bf5112c Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Wed, 23 Apr 2025 17:22:28 +0800 Subject: [PATCH 233/382] crypto: api - Add crypto_request_clone and fb Add a helper to clone crypto requests and eliminate code duplication. Use kmemdup in the helper. Also add an fb field to crypto_tfm. This also happens to fix the existing implementations which were buggy. Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202504230118.1CxUaUoX-lkp@intel.com/ Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202504230004.c7mrY0C6-lkp@intel.com/ Signed-off-by: Herbert Xu --- crypto/acompress.c | 29 +++----------------------- crypto/ahash.c | 32 +++++------------------------ crypto/api.c | 18 ++++++++++++++++ include/crypto/acompress.h | 9 +++++--- include/crypto/hash.h | 9 +++++--- include/crypto/internal/acompress.h | 7 ++++++- include/crypto/internal/hash.h | 7 ++++++- include/linux/crypto.h | 11 +++++++--- 8 files changed, 58 insertions(+), 64 deletions(-) diff --git a/crypto/acompress.c b/crypto/acompress.c index 4c665c6fb5d6..9dea76ed4513 100644 --- a/crypto/acompress.c +++ b/crypto/acompress.c @@ -11,15 +11,13 @@ #include #include #include -#include +#include #include #include -#include #include #include #include #include -#include #include #include #include @@ -79,7 +77,7 @@ static void crypto_acomp_exit_tfm(struct crypto_tfm *tfm) alg->exit(acomp); if (acomp_is_async(acomp)) - crypto_free_acomp(acomp->fb); + crypto_free_acomp(crypto_acomp_fb(acomp)); } static int crypto_acomp_init_tfm(struct crypto_tfm *tfm) @@ -89,8 +87,6 @@ static int crypto_acomp_init_tfm(struct crypto_tfm *tfm) struct crypto_acomp *fb = NULL; int err; - acomp->fb = acomp; - if (tfm->__crt_alg->cra_type != &crypto_acomp_type) return crypto_init_scomp_ops_async(tfm); @@ -104,7 +100,7 @@ static int crypto_acomp_init_tfm(struct crypto_tfm *tfm) if (crypto_acomp_reqsize(fb) > MAX_SYNC_COMP_REQSIZE) goto out_free_fb; - acomp->fb = fb; + tfm->fb = crypto_acomp_tfm(fb); } acomp->compress = alg->compress; @@ -570,24 +566,5 @@ int acomp_walk_virt(struct acomp_walk *__restrict walk, } EXPORT_SYMBOL_GPL(acomp_walk_virt); -struct acomp_req *acomp_request_clone(struct acomp_req *req, - size_t total, gfp_t gfp) -{ - struct crypto_acomp *tfm = crypto_acomp_reqtfm(req); - struct acomp_req *nreq; - - nreq = kmalloc(total, gfp); - if (!nreq) { - acomp_request_set_tfm(req, tfm->fb); - req->base.flags = CRYPTO_TFM_REQ_ON_STACK; - return req; - } - - memcpy(nreq, req, total); - acomp_request_set_tfm(req, tfm); - return req; -} -EXPORT_SYMBOL_GPL(acomp_request_clone); - MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Asynchronous compression type"); diff --git a/crypto/ahash.c b/crypto/ahash.c index 7a74092323b9..9b813f7b9177 100644 --- a/crypto/ahash.c +++ b/crypto/ahash.c @@ -12,13 +12,11 @@ * Copyright (c) 2008 Loc Ho */ -#include #include #include #include #include #include -#include #include #include #include @@ -301,7 +299,8 @@ int crypto_ahash_setkey(struct crypto_ahash *tfm, const u8 *key, err = alg->setkey(tfm, key, keylen); if (!err && ahash_is_async(tfm)) - err = crypto_ahash_setkey(tfm->fb, key, keylen); + err = crypto_ahash_setkey(crypto_ahash_fb(tfm), + key, keylen); if (unlikely(err)) { ahash_set_needkey(tfm, alg); return err; @@ -732,7 +731,7 @@ static void crypto_ahash_exit_tfm(struct crypto_tfm *tfm) tfm->__crt_alg->cra_exit(tfm); if (ahash_is_async(hash)) - crypto_free_ahash(hash->fb); + crypto_free_ahash(crypto_ahash_fb(hash)); } static int crypto_ahash_init_tfm(struct crypto_tfm *tfm) @@ -745,8 +744,6 @@ static int crypto_ahash_init_tfm(struct crypto_tfm *tfm) crypto_ahash_set_statesize(hash, alg->halg.statesize); crypto_ahash_set_reqsize(hash, crypto_tfm_alg_reqsize(tfm)); - hash->fb = hash; - if (tfm->__crt_alg->cra_type == &crypto_shash_type) return crypto_init_ahash_using_shash(tfm); @@ -756,7 +753,7 @@ static int crypto_ahash_init_tfm(struct crypto_tfm *tfm) if (IS_ERR(fb)) return PTR_ERR(fb); - hash->fb = fb; + tfm->fb = crypto_ahash_tfm(fb); } ahash_set_needkey(hash, alg); @@ -1036,7 +1033,7 @@ EXPORT_SYMBOL_GPL(ahash_request_free); int crypto_hash_digest(struct crypto_ahash *tfm, const u8 *data, unsigned int len, u8 *out) { - HASH_REQUEST_ON_STACK(req, tfm->fb); + HASH_REQUEST_ON_STACK(req, crypto_ahash_fb(tfm)); int err; ahash_request_set_callback(req, 0, NULL, NULL); @@ -1049,24 +1046,5 @@ int crypto_hash_digest(struct crypto_ahash *tfm, const u8 *data, } EXPORT_SYMBOL_GPL(crypto_hash_digest); -struct ahash_request *ahash_request_clone(struct ahash_request *req, - size_t total, gfp_t gfp) -{ - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - struct ahash_request *nreq; - - nreq = kmalloc(total, gfp); - if (!nreq) { - ahash_request_set_tfm(req, tfm->fb); - req->base.flags = CRYPTO_TFM_REQ_ON_STACK; - return req; - } - - memcpy(nreq, req, total); - ahash_request_set_tfm(req, tfm); - return req; -} -EXPORT_SYMBOL_GPL(ahash_request_clone); - MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Asynchronous cryptographic hash type"); diff --git a/crypto/api.c b/crypto/api.c index e427cc5662b5..172e82f79c69 100644 --- a/crypto/api.c +++ b/crypto/api.c @@ -528,6 +528,7 @@ void *crypto_create_tfm_node(struct crypto_alg *alg, goto out; tfm = (struct crypto_tfm *)(mem + frontend->tfmsize); + tfm->fb = tfm; err = frontend->init_tfm(tfm); if (err) @@ -712,5 +713,22 @@ void crypto_destroy_alg(struct crypto_alg *alg) } EXPORT_SYMBOL_GPL(crypto_destroy_alg); +struct crypto_async_request *crypto_request_clone( + struct crypto_async_request *req, size_t total, gfp_t gfp) +{ + struct crypto_tfm *tfm = req->tfm; + struct crypto_async_request *nreq; + + nreq = kmemdup(req, total, gfp); + if (!nreq) { + req->tfm = tfm->fb; + return req; + } + + nreq->flags &= ~CRYPTO_TFM_REQ_ON_STACK; + return nreq; +} +EXPORT_SYMBOL_GPL(crypto_request_clone); + MODULE_DESCRIPTION("Cryptographic core API"); MODULE_LICENSE("GPL"); diff --git a/include/crypto/acompress.h b/include/crypto/acompress.h index 1b30290d6380..933c48a4855b 100644 --- a/include/crypto/acompress.h +++ b/include/crypto/acompress.h @@ -114,7 +114,6 @@ struct crypto_acomp { int (*compress)(struct acomp_req *req); int (*decompress)(struct acomp_req *req); unsigned int reqsize; - struct crypto_acomp *fb; struct crypto_tfm base; }; @@ -553,7 +552,11 @@ static inline struct acomp_req *acomp_request_on_stack_init( return req; } -struct acomp_req *acomp_request_clone(struct acomp_req *req, - size_t total, gfp_t gfp); +static inline struct acomp_req *acomp_request_clone(struct acomp_req *req, + size_t total, gfp_t gfp) +{ + return container_of(crypto_request_clone(&req->base, total, gfp), + struct acomp_req, base); +} #endif diff --git a/include/crypto/hash.h b/include/crypto/hash.h index 5f87d1040a7c..68813a83443b 100644 --- a/include/crypto/hash.h +++ b/include/crypto/hash.h @@ -246,7 +246,6 @@ struct crypto_ahash { bool using_shash; /* Underlying algorithm is shash, not ahash */ unsigned int statesize; unsigned int reqsize; - struct crypto_ahash *fb; struct crypto_tfm base; }; @@ -1035,7 +1034,11 @@ static inline struct ahash_request *ahash_request_on_stack_init( return req; } -struct ahash_request *ahash_request_clone(struct ahash_request *req, - size_t total, gfp_t gfp); +static inline struct ahash_request *ahash_request_clone( + struct ahash_request *req, size_t total, gfp_t gfp) +{ + return container_of(crypto_request_clone(&req->base, total, gfp), + struct ahash_request, base); +} #endif /* _CRYPTO_HASH_H */ diff --git a/include/crypto/internal/acompress.h b/include/crypto/internal/acompress.h index 7eda32619024..6550dad18e0f 100644 --- a/include/crypto/internal/acompress.h +++ b/include/crypto/internal/acompress.h @@ -220,13 +220,18 @@ static inline u32 acomp_request_flags(struct acomp_req *req) return crypto_request_flags(&req->base) & ~CRYPTO_ACOMP_REQ_PRIVATE; } +static inline struct crypto_acomp *crypto_acomp_fb(struct crypto_acomp *tfm) +{ + return __crypto_acomp_tfm(crypto_acomp_tfm(tfm)->fb); +} + static inline struct acomp_req *acomp_fbreq_on_stack_init( char *buf, struct acomp_req *old) { struct crypto_acomp *tfm = crypto_acomp_reqtfm(old); struct acomp_req *req = (void *)buf; - acomp_request_set_tfm(req, tfm->fb); + acomp_request_set_tfm(req, crypto_acomp_fb(tfm)); req->base.flags = CRYPTO_TFM_REQ_ON_STACK; acomp_request_set_callback(req, acomp_request_flags(old), NULL, NULL); req->base.flags &= ~CRYPTO_ACOMP_REQ_PRIVATE; diff --git a/include/crypto/internal/hash.h b/include/crypto/internal/hash.h index 1e80dd084a23..0bc0fefc9b3c 100644 --- a/include/crypto/internal/hash.h +++ b/include/crypto/internal/hash.h @@ -272,13 +272,18 @@ static inline bool crypto_ahash_req_chain(struct crypto_ahash *tfm) return crypto_tfm_req_chain(&tfm->base); } +static inline struct crypto_ahash *crypto_ahash_fb(struct crypto_ahash *tfm) +{ + return __crypto_ahash_cast(crypto_ahash_tfm(tfm)->fb); +} + static inline struct ahash_request *ahash_fbreq_on_stack_init( char *buf, struct ahash_request *old) { struct crypto_ahash *tfm = crypto_ahash_reqtfm(old); struct ahash_request *req = (void *)buf; - ahash_request_set_tfm(req, tfm->fb); + ahash_request_set_tfm(req, crypto_ahash_fb(tfm)); req->base.flags = CRYPTO_TFM_REQ_ON_STACK; ahash_request_set_callback(req, ahash_request_flags(old), NULL, NULL); req->base.flags &= ~CRYPTO_AHASH_REQ_PRIVATE; diff --git a/include/linux/crypto.h b/include/linux/crypto.h index f691ce01745e..fe75320ff9a3 100644 --- a/include/linux/crypto.h +++ b/include/linux/crypto.h @@ -14,7 +14,7 @@ #include #include -#include +#include #include #include @@ -411,9 +411,11 @@ struct crypto_tfm { u32 crt_flags; int node; - + + struct crypto_tfm *fb; + void (*exit)(struct crypto_tfm *tfm); - + struct crypto_alg *__crt_alg; void *__crt_ctx[] CRYPTO_MINALIGN_ATTR; @@ -509,5 +511,8 @@ static inline void crypto_request_set_tfm(struct crypto_async_request *req, req->flags &= ~CRYPTO_TFM_REQ_ON_STACK; } +struct crypto_async_request *crypto_request_clone( + struct crypto_async_request *req, size_t total, gfp_t gfp); + #endif /* _LINUX_CRYPTO_H */ From 2cfe41630a1a4f24d46825aa9656a51a38fb7f7d Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Wed, 23 Apr 2025 17:22:31 +0800 Subject: [PATCH 234/382] crypto: hash - Fix clone error handling Do not copy the exit function in crypto_clone_tfm as it should only be set after init_tfm or clone_tfm has succeeded. Move the setting into crypto_clone_ahash and crypto_clone_shash instead. Also clone the fb if necessary. Signed-off-by: Herbert Xu --- crypto/ahash.c | 19 +++++++++++++++++-- crypto/api.c | 2 +- crypto/shash.c | 3 +++ 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/crypto/ahash.c b/crypto/ahash.c index 9b813f7b9177..cc9885d5cfd2 100644 --- a/crypto/ahash.c +++ b/crypto/ahash.c @@ -877,6 +877,7 @@ struct crypto_ahash *crypto_clone_ahash(struct crypto_ahash *hash) { struct hash_alg_common *halg = crypto_hash_alg_common(hash); struct crypto_tfm *tfm = crypto_ahash_tfm(hash); + struct crypto_ahash *fb = NULL; struct crypto_ahash *nhash; struct ahash_alg *alg; int err; @@ -906,22 +907,36 @@ struct crypto_ahash *crypto_clone_ahash(struct crypto_ahash *hash) err = PTR_ERR(shash); goto out_free_nhash; } + crypto_ahash_tfm(nhash)->exit = crypto_exit_ahash_using_shash; nhash->using_shash = true; *nctx = shash; return nhash; } + if (ahash_is_async(hash)) { + fb = crypto_clone_ahash(crypto_ahash_fb(hash)); + err = PTR_ERR(fb); + if (IS_ERR(fb)) + goto out_free_nhash; + + crypto_ahash_tfm(nhash)->fb = crypto_ahash_tfm(fb); + } + err = -ENOSYS; alg = crypto_ahash_alg(hash); if (!alg->clone_tfm) - goto out_free_nhash; + goto out_free_fb; err = alg->clone_tfm(nhash, hash); if (err) - goto out_free_nhash; + goto out_free_fb; + + crypto_ahash_tfm(nhash)->exit = crypto_ahash_exit_tfm; return nhash; +out_free_fb: + crypto_free_ahash(fb); out_free_nhash: crypto_free_ahash(nhash); return ERR_PTR(err); diff --git a/crypto/api.c b/crypto/api.c index 172e82f79c69..5cd5ec105bb1 100644 --- a/crypto/api.c +++ b/crypto/api.c @@ -570,7 +570,7 @@ void *crypto_clone_tfm(const struct crypto_type *frontend, tfm = (struct crypto_tfm *)(mem + frontend->tfmsize); tfm->crt_flags = otfm->crt_flags; - tfm->exit = otfm->exit; + tfm->fb = tfm; out: return mem; diff --git a/crypto/shash.c b/crypto/shash.c index b6c79a4a044a..c4a724e55d7a 100644 --- a/crypto/shash.c +++ b/crypto/shash.c @@ -413,6 +413,9 @@ struct crypto_shash *crypto_clone_shash(struct crypto_shash *hash) } } + if (alg->exit_tfm) + crypto_shash_tfm(nhash)->exit = crypto_shash_exit_tfm; + return nhash; } EXPORT_SYMBOL_GPL(crypto_clone_shash); From d5a582a782f31c8712d4bfa07b6dc6d64f4ae330 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Thu, 24 Apr 2025 18:47:26 +0800 Subject: [PATCH 235/382] crypto: arm64/polyval - Use API partial block handling Use the Crypto API partial block handling. Also remove the unnecessary SIMD fallback path. Signed-off-by: Herbert Xu --- arch/arm64/crypto/polyval-ce-glue.c | 73 ++++++++--------------------- 1 file changed, 20 insertions(+), 53 deletions(-) diff --git a/arch/arm64/crypto/polyval-ce-glue.c b/arch/arm64/crypto/polyval-ce-glue.c index 0a3b5718df85..c4e653688ea0 100644 --- a/arch/arm64/crypto/polyval-ce-glue.c +++ b/arch/arm64/crypto/polyval-ce-glue.c @@ -15,17 +15,15 @@ * ARMv8 Crypto Extensions instructions to implement the finite field operations. */ -#include +#include #include -#include #include -#include -#include +#include +#include +#include #include #include -#include -#include -#include +#include #define NUM_KEY_POWERS 8 @@ -38,7 +36,6 @@ struct polyval_tfm_ctx { struct polyval_desc_ctx { u8 buffer[POLYVAL_BLOCK_SIZE]; - u32 bytes; }; asmlinkage void pmull_polyval_update(const struct polyval_tfm_ctx *keys, @@ -48,25 +45,16 @@ asmlinkage void pmull_polyval_mul(u8 *op1, const u8 *op2); static void internal_polyval_update(const struct polyval_tfm_ctx *keys, const u8 *in, size_t nblocks, u8 *accumulator) { - if (likely(crypto_simd_usable())) { - kernel_neon_begin(); - pmull_polyval_update(keys, in, nblocks, accumulator); - kernel_neon_end(); - } else { - polyval_update_non4k(keys->key_powers[NUM_KEY_POWERS-1], in, - nblocks, accumulator); - } + kernel_neon_begin(); + pmull_polyval_update(keys, in, nblocks, accumulator); + kernel_neon_end(); } static void internal_polyval_mul(u8 *op1, const u8 *op2) { - if (likely(crypto_simd_usable())) { - kernel_neon_begin(); - pmull_polyval_mul(op1, op2); - kernel_neon_end(); - } else { - polyval_mul_non4k(op1, op2); - } + kernel_neon_begin(); + pmull_polyval_mul(op1, op2); + kernel_neon_end(); } static int polyval_arm64_setkey(struct crypto_shash *tfm, @@ -103,49 +91,27 @@ static int polyval_arm64_update(struct shash_desc *desc, { struct polyval_desc_ctx *dctx = shash_desc_ctx(desc); const struct polyval_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); - u8 *pos; unsigned int nblocks; - unsigned int n; - if (dctx->bytes) { - n = min(srclen, dctx->bytes); - pos = dctx->buffer + POLYVAL_BLOCK_SIZE - dctx->bytes; - - dctx->bytes -= n; - srclen -= n; - - while (n--) - *pos++ ^= *src++; - - if (!dctx->bytes) - internal_polyval_mul(dctx->buffer, - tctx->key_powers[NUM_KEY_POWERS-1]); - } - - while (srclen >= POLYVAL_BLOCK_SIZE) { + do { /* allow rescheduling every 4K bytes */ nblocks = min(srclen, 4096U) / POLYVAL_BLOCK_SIZE; internal_polyval_update(tctx, src, nblocks, dctx->buffer); srclen -= nblocks * POLYVAL_BLOCK_SIZE; src += nblocks * POLYVAL_BLOCK_SIZE; - } + } while (srclen >= POLYVAL_BLOCK_SIZE); - if (srclen) { - dctx->bytes = POLYVAL_BLOCK_SIZE - srclen; - pos = dctx->buffer; - while (srclen--) - *pos++ ^= *src++; - } - - return 0; + return srclen; } -static int polyval_arm64_final(struct shash_desc *desc, u8 *dst) +static int polyval_arm64_finup(struct shash_desc *desc, const u8 *src, + unsigned int len, u8 *dst) { struct polyval_desc_ctx *dctx = shash_desc_ctx(desc); const struct polyval_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); - if (dctx->bytes) { + if (len) { + crypto_xor(dctx->buffer, src, len); internal_polyval_mul(dctx->buffer, tctx->key_powers[NUM_KEY_POWERS-1]); } @@ -159,13 +125,14 @@ static struct shash_alg polyval_alg = { .digestsize = POLYVAL_DIGEST_SIZE, .init = polyval_arm64_init, .update = polyval_arm64_update, - .final = polyval_arm64_final, + .finup = polyval_arm64_finup, .setkey = polyval_arm64_setkey, .descsize = sizeof(struct polyval_desc_ctx), .base = { .cra_name = "polyval", .cra_driver_name = "polyval-ce", .cra_priority = 200, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_blocksize = POLYVAL_BLOCK_SIZE, .cra_ctxsize = sizeof(struct polyval_tfm_ctx), .cra_module = THIS_MODULE, From 74df89ff7676479651059f82af35a3c9bb41327c Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Thu, 24 Apr 2025 18:47:28 +0800 Subject: [PATCH 236/382] crypto: x86/polyval - Use API partial block handling Use the Crypto API partial block handling. Also remove the unnecessary SIMD fallback path. Signed-off-by: Herbert Xu --- arch/x86/crypto/polyval-clmulni_glue.c | 72 +++++++------------------- 1 file changed, 20 insertions(+), 52 deletions(-) diff --git a/arch/x86/crypto/polyval-clmulni_glue.c b/arch/x86/crypto/polyval-clmulni_glue.c index 8fa58b0f3cb3..6b466867f91a 100644 --- a/arch/x86/crypto/polyval-clmulni_glue.c +++ b/arch/x86/crypto/polyval-clmulni_glue.c @@ -16,16 +16,15 @@ * operations. */ -#include +#include +#include #include -#include #include -#include -#include +#include +#include #include #include -#include -#include +#include #define POLYVAL_ALIGN 16 #define POLYVAL_ALIGN_ATTR __aligned(POLYVAL_ALIGN) @@ -42,7 +41,6 @@ struct polyval_tfm_ctx { struct polyval_desc_ctx { u8 buffer[POLYVAL_BLOCK_SIZE]; - u32 bytes; }; asmlinkage void clmul_polyval_update(const struct polyval_tfm_ctx *keys, @@ -57,25 +55,16 @@ static inline struct polyval_tfm_ctx *polyval_tfm_ctx(struct crypto_shash *tfm) static void internal_polyval_update(const struct polyval_tfm_ctx *keys, const u8 *in, size_t nblocks, u8 *accumulator) { - if (likely(crypto_simd_usable())) { - kernel_fpu_begin(); - clmul_polyval_update(keys, in, nblocks, accumulator); - kernel_fpu_end(); - } else { - polyval_update_non4k(keys->key_powers[NUM_KEY_POWERS-1], in, - nblocks, accumulator); - } + kernel_fpu_begin(); + clmul_polyval_update(keys, in, nblocks, accumulator); + kernel_fpu_end(); } static void internal_polyval_mul(u8 *op1, const u8 *op2) { - if (likely(crypto_simd_usable())) { - kernel_fpu_begin(); - clmul_polyval_mul(op1, op2); - kernel_fpu_end(); - } else { - polyval_mul_non4k(op1, op2); - } + kernel_fpu_begin(); + clmul_polyval_mul(op1, op2); + kernel_fpu_end(); } static int polyval_x86_setkey(struct crypto_shash *tfm, @@ -112,49 +101,27 @@ static int polyval_x86_update(struct shash_desc *desc, { struct polyval_desc_ctx *dctx = shash_desc_ctx(desc); const struct polyval_tfm_ctx *tctx = polyval_tfm_ctx(desc->tfm); - u8 *pos; unsigned int nblocks; - unsigned int n; - if (dctx->bytes) { - n = min(srclen, dctx->bytes); - pos = dctx->buffer + POLYVAL_BLOCK_SIZE - dctx->bytes; - - dctx->bytes -= n; - srclen -= n; - - while (n--) - *pos++ ^= *src++; - - if (!dctx->bytes) - internal_polyval_mul(dctx->buffer, - tctx->key_powers[NUM_KEY_POWERS-1]); - } - - while (srclen >= POLYVAL_BLOCK_SIZE) { + do { /* Allow rescheduling every 4K bytes. */ nblocks = min(srclen, 4096U) / POLYVAL_BLOCK_SIZE; internal_polyval_update(tctx, src, nblocks, dctx->buffer); srclen -= nblocks * POLYVAL_BLOCK_SIZE; src += nblocks * POLYVAL_BLOCK_SIZE; - } + } while (srclen >= POLYVAL_BLOCK_SIZE); - if (srclen) { - dctx->bytes = POLYVAL_BLOCK_SIZE - srclen; - pos = dctx->buffer; - while (srclen--) - *pos++ ^= *src++; - } - - return 0; + return srclen; } -static int polyval_x86_final(struct shash_desc *desc, u8 *dst) +static int polyval_x86_finup(struct shash_desc *desc, const u8 *src, + unsigned int len, u8 *dst) { struct polyval_desc_ctx *dctx = shash_desc_ctx(desc); const struct polyval_tfm_ctx *tctx = polyval_tfm_ctx(desc->tfm); - if (dctx->bytes) { + if (len) { + crypto_xor(dctx->buffer, src, len); internal_polyval_mul(dctx->buffer, tctx->key_powers[NUM_KEY_POWERS-1]); } @@ -168,13 +135,14 @@ static struct shash_alg polyval_alg = { .digestsize = POLYVAL_DIGEST_SIZE, .init = polyval_x86_init, .update = polyval_x86_update, - .final = polyval_x86_final, + .finup = polyval_x86_finup, .setkey = polyval_x86_setkey, .descsize = sizeof(struct polyval_desc_ctx), .base = { .cra_name = "polyval", .cra_driver_name = "polyval-clmulni", .cra_priority = 200, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_blocksize = POLYVAL_BLOCK_SIZE, .cra_ctxsize = POLYVAL_CTX_SIZE, .cra_module = THIS_MODULE, From e3b49657228c114099b4591821465dc6f9884c61 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Thu, 24 Apr 2025 18:47:30 +0800 Subject: [PATCH 237/382] crypto: polyval-generic - Use API partial block handling Use the Crypto API partial block handling. The accelerated export format on x86/arm64 is easier to use so switch the generic polyval algorithm to use that format instead. Signed-off-by: Herbert Xu --- crypto/polyval-generic.c | 120 +++++++++++++-------------------------- include/crypto/polyval.h | 8 --- 2 files changed, 40 insertions(+), 88 deletions(-) diff --git a/crypto/polyval-generic.c b/crypto/polyval-generic.c index 4f98910bcdb5..ffd174e75420 100644 --- a/crypto/polyval-generic.c +++ b/crypto/polyval-generic.c @@ -44,15 +44,15 @@ * */ -#include -#include #include -#include #include -#include -#include +#include +#include +#include #include #include +#include +#include struct polyval_tfm_ctx { struct gf128mul_4k *gf128; @@ -63,7 +63,6 @@ struct polyval_desc_ctx { u8 buffer[POLYVAL_BLOCK_SIZE]; be128 buffer128; }; - u32 bytes; }; static void copy_and_reverse(u8 dst[POLYVAL_BLOCK_SIZE], @@ -76,46 +75,6 @@ static void copy_and_reverse(u8 dst[POLYVAL_BLOCK_SIZE], put_unaligned(swab64(b), (u64 *)&dst[0]); } -/* - * Performs multiplication in the POLYVAL field using the GHASH field as a - * subroutine. This function is used as a fallback for hardware accelerated - * implementations when simd registers are unavailable. - * - * Note: This function is not used for polyval-generic, instead we use the 4k - * lookup table implementation for finite field multiplication. - */ -void polyval_mul_non4k(u8 *op1, const u8 *op2) -{ - be128 a, b; - - // Assume one argument is in Montgomery form and one is not. - copy_and_reverse((u8 *)&a, op1); - copy_and_reverse((u8 *)&b, op2); - gf128mul_x_lle(&a, &a); - gf128mul_lle(&a, &b); - copy_and_reverse(op1, (u8 *)&a); -} -EXPORT_SYMBOL_GPL(polyval_mul_non4k); - -/* - * Perform a POLYVAL update using non4k multiplication. This function is used - * as a fallback for hardware accelerated implementations when simd registers - * are unavailable. - * - * Note: This function is not used for polyval-generic, instead we use the 4k - * lookup table implementation of finite field multiplication. - */ -void polyval_update_non4k(const u8 *key, const u8 *in, - size_t nblocks, u8 *accumulator) -{ - while (nblocks--) { - crypto_xor(accumulator, in, POLYVAL_BLOCK_SIZE); - polyval_mul_non4k(accumulator, key); - in += POLYVAL_BLOCK_SIZE; - } -} -EXPORT_SYMBOL_GPL(polyval_update_non4k); - static int polyval_setkey(struct crypto_shash *tfm, const u8 *key, unsigned int keylen) { @@ -154,56 +113,53 @@ static int polyval_update(struct shash_desc *desc, { struct polyval_desc_ctx *dctx = shash_desc_ctx(desc); const struct polyval_tfm_ctx *ctx = crypto_shash_ctx(desc->tfm); - u8 *pos; u8 tmp[POLYVAL_BLOCK_SIZE]; - int n; - if (dctx->bytes) { - n = min(srclen, dctx->bytes); - pos = dctx->buffer + dctx->bytes - 1; - - dctx->bytes -= n; - srclen -= n; - - while (n--) - *pos-- ^= *src++; - - if (!dctx->bytes) - gf128mul_4k_lle(&dctx->buffer128, ctx->gf128); - } - - while (srclen >= POLYVAL_BLOCK_SIZE) { + do { copy_and_reverse(tmp, src); crypto_xor(dctx->buffer, tmp, POLYVAL_BLOCK_SIZE); gf128mul_4k_lle(&dctx->buffer128, ctx->gf128); src += POLYVAL_BLOCK_SIZE; srclen -= POLYVAL_BLOCK_SIZE; - } + } while (srclen >= POLYVAL_BLOCK_SIZE); - if (srclen) { - dctx->bytes = POLYVAL_BLOCK_SIZE - srclen; - pos = dctx->buffer + POLYVAL_BLOCK_SIZE - 1; - while (srclen--) - *pos-- ^= *src++; - } - - return 0; + return srclen; } -static int polyval_final(struct shash_desc *desc, u8 *dst) +static int polyval_finup(struct shash_desc *desc, const u8 *src, + unsigned int len, u8 *dst) { struct polyval_desc_ctx *dctx = shash_desc_ctx(desc); - const struct polyval_tfm_ctx *ctx = crypto_shash_ctx(desc->tfm); - if (dctx->bytes) - gf128mul_4k_lle(&dctx->buffer128, ctx->gf128); + if (len) { + u8 tmp[POLYVAL_BLOCK_SIZE] = {}; + + memcpy(tmp, src, len); + polyval_update(desc, tmp, POLYVAL_BLOCK_SIZE); + } copy_and_reverse(dst, dctx->buffer); return 0; } -static void polyval_exit_tfm(struct crypto_tfm *tfm) +static int polyval_export(struct shash_desc *desc, void *out) { - struct polyval_tfm_ctx *ctx = crypto_tfm_ctx(tfm); + struct polyval_desc_ctx *dctx = shash_desc_ctx(desc); + + copy_and_reverse(out, dctx->buffer); + return 0; +} + +static int polyval_import(struct shash_desc *desc, const void *in) +{ + struct polyval_desc_ctx *dctx = shash_desc_ctx(desc); + + copy_and_reverse(dctx->buffer, in); + return 0; +} + +static void polyval_exit_tfm(struct crypto_shash *tfm) +{ + struct polyval_tfm_ctx *ctx = crypto_shash_ctx(tfm); gf128mul_free_4k(ctx->gf128); } @@ -212,17 +168,21 @@ static struct shash_alg polyval_alg = { .digestsize = POLYVAL_DIGEST_SIZE, .init = polyval_init, .update = polyval_update, - .final = polyval_final, + .finup = polyval_finup, .setkey = polyval_setkey, + .export = polyval_export, + .import = polyval_import, + .exit_tfm = polyval_exit_tfm, + .statesize = sizeof(struct polyval_desc_ctx), .descsize = sizeof(struct polyval_desc_ctx), .base = { .cra_name = "polyval", .cra_driver_name = "polyval-generic", .cra_priority = 100, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_blocksize = POLYVAL_BLOCK_SIZE, .cra_ctxsize = sizeof(struct polyval_tfm_ctx), .cra_module = THIS_MODULE, - .cra_exit = polyval_exit_tfm, }, }; diff --git a/include/crypto/polyval.h b/include/crypto/polyval.h index 1d630f371f77..d2e63743e592 100644 --- a/include/crypto/polyval.h +++ b/include/crypto/polyval.h @@ -8,15 +8,7 @@ #ifndef _CRYPTO_POLYVAL_H #define _CRYPTO_POLYVAL_H -#include -#include - #define POLYVAL_BLOCK_SIZE 16 #define POLYVAL_DIGEST_SIZE 16 -void polyval_mul_non4k(u8 *op1, const u8 *op2); - -void polyval_update_non4k(const u8 *key, const u8 *in, - size_t nblocks, u8 *accumulator); - #endif From b75fa20c127eb736b0ac9b30be051f526a2316a9 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 25 Apr 2025 11:05:29 +0800 Subject: [PATCH 238/382] crypto: api - Add crypto_stack_request_init and initialise flags fully Add a helper to initialise crypto stack requests and use it for ahash and acomp. Make sure that the flags field is initialised fully in the helper to silence false-positive warnings from the compiler. Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202504250751.mdy28Ibr-lkp@intel.com/ Signed-off-by: Herbert Xu --- include/crypto/acompress.h | 3 +-- include/crypto/hash.h | 3 +-- include/crypto/internal/acompress.h | 4 ++-- include/crypto/internal/hash.h | 4 ++-- include/linux/crypto.h | 8 ++++++++ 5 files changed, 14 insertions(+), 8 deletions(-) diff --git a/include/crypto/acompress.h b/include/crypto/acompress.h index 933c48a4855b..f1812e92d3e3 100644 --- a/include/crypto/acompress.h +++ b/include/crypto/acompress.h @@ -547,8 +547,7 @@ static inline struct acomp_req *acomp_request_on_stack_init( { struct acomp_req *req = (void *)buf; - acomp_request_set_tfm(req, tfm); - req->base.flags = CRYPTO_TFM_REQ_ON_STACK; + crypto_stack_request_init(&req->base, crypto_acomp_tfm(tfm)); return req; } diff --git a/include/crypto/hash.h b/include/crypto/hash.h index 68813a83443b..c2497c300a28 100644 --- a/include/crypto/hash.h +++ b/include/crypto/hash.h @@ -1029,8 +1029,7 @@ static inline struct ahash_request *ahash_request_on_stack_init( { struct ahash_request *req = (void *)buf; - ahash_request_set_tfm(req, tfm); - req->base.flags = CRYPTO_TFM_REQ_ON_STACK; + crypto_stack_request_init(&req->base, crypto_ahash_tfm(tfm)); return req; } diff --git a/include/crypto/internal/acompress.h b/include/crypto/internal/acompress.h index 6550dad18e0f..b72bb7a6a2b2 100644 --- a/include/crypto/internal/acompress.h +++ b/include/crypto/internal/acompress.h @@ -231,8 +231,8 @@ static inline struct acomp_req *acomp_fbreq_on_stack_init( struct crypto_acomp *tfm = crypto_acomp_reqtfm(old); struct acomp_req *req = (void *)buf; - acomp_request_set_tfm(req, crypto_acomp_fb(tfm)); - req->base.flags = CRYPTO_TFM_REQ_ON_STACK; + crypto_stack_request_init(&req->base, + crypto_acomp_tfm(crypto_acomp_fb(tfm))); acomp_request_set_callback(req, acomp_request_flags(old), NULL, NULL); req->base.flags &= ~CRYPTO_ACOMP_REQ_PRIVATE; req->base.flags |= old->base.flags & CRYPTO_ACOMP_REQ_PRIVATE; diff --git a/include/crypto/internal/hash.h b/include/crypto/internal/hash.h index 0bc0fefc9b3c..33d45275f5bd 100644 --- a/include/crypto/internal/hash.h +++ b/include/crypto/internal/hash.h @@ -283,8 +283,8 @@ static inline struct ahash_request *ahash_fbreq_on_stack_init( struct crypto_ahash *tfm = crypto_ahash_reqtfm(old); struct ahash_request *req = (void *)buf; - ahash_request_set_tfm(req, crypto_ahash_fb(tfm)); - req->base.flags = CRYPTO_TFM_REQ_ON_STACK; + crypto_stack_request_init(&req->base, + crypto_ahash_tfm(crypto_ahash_fb(tfm))); ahash_request_set_callback(req, ahash_request_flags(old), NULL, NULL); req->base.flags &= ~CRYPTO_AHASH_REQ_PRIVATE; req->base.flags |= old->base.flags & CRYPTO_AHASH_REQ_PRIVATE; diff --git a/include/linux/crypto.h b/include/linux/crypto.h index fe75320ff9a3..b8d875b11193 100644 --- a/include/linux/crypto.h +++ b/include/linux/crypto.h @@ -514,5 +514,13 @@ static inline void crypto_request_set_tfm(struct crypto_async_request *req, struct crypto_async_request *crypto_request_clone( struct crypto_async_request *req, size_t total, gfp_t gfp); +static inline void crypto_stack_request_init(struct crypto_async_request *req, + struct crypto_tfm *tfm) +{ + req->flags = 0; + crypto_request_set_tfm(req, tfm); + req->flags |= CRYPTO_TFM_REQ_ON_STACK; +} + #endif /* _LINUX_CRYPTO_H */ From f31adc3e356f7350d4a4d68c98d3f60f2f6e26b3 Mon Sep 17 00:00:00 2001 From: Ovidiu Panait Date: Fri, 25 Apr 2025 15:45:14 +0300 Subject: [PATCH 239/382] crypto: sun8i-ce-cipher - fix error handling in sun8i_ce_cipher_prepare() Fix two DMA cleanup issues on the error path in sun8i_ce_cipher_prepare(): 1] If dma_map_sg() fails for areq->dst, the device driver would try to free DMA memory it has not allocated in the first place. To fix this, on the "theend_sgs" error path, call dma unmap only if the corresponding dma map was successful. 2] If the dma_map_single() call for the IV fails, the device driver would try to free an invalid DMA memory address on the "theend_iv" path: ------------[ cut here ]------------ DMA-API: sun8i-ce 1904000.crypto: device driver tries to free an invalid DMA memory address WARNING: CPU: 2 PID: 69 at kernel/dma/debug.c:968 check_unmap+0x123c/0x1b90 Modules linked in: skcipher_example(O+) CPU: 2 UID: 0 PID: 69 Comm: 1904000.crypto- Tainted: G O 6.15.0-rc3+ #24 PREEMPT Tainted: [O]=OOT_MODULE Hardware name: OrangePi Zero2 (DT) pc : check_unmap+0x123c/0x1b90 lr : check_unmap+0x123c/0x1b90 ... Call trace: check_unmap+0x123c/0x1b90 (P) debug_dma_unmap_page+0xac/0xc0 dma_unmap_page_attrs+0x1f4/0x5fc sun8i_ce_cipher_do_one+0x1bd4/0x1f40 crypto_pump_work+0x334/0x6e0 kthread_worker_fn+0x21c/0x438 kthread+0x374/0x664 ret_from_fork+0x10/0x20 ---[ end trace 0000000000000000 ]--- To fix this, check for !dma_mapping_error() before calling dma_unmap_single() on the "theend_iv" path. Fixes: 06f751b61329 ("crypto: allwinner - Add sun8i-ce Crypto Engine") Signed-off-by: Ovidiu Panait Signed-off-by: Herbert Xu --- drivers/crypto/allwinner/sun8i-ce/sun8i-ce-cipher.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-cipher.c b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-cipher.c index 19b7fb4a93e8..05f67661553c 100644 --- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-cipher.c +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-cipher.c @@ -275,13 +275,16 @@ theend_sgs: } else { if (nr_sgs > 0) dma_unmap_sg(ce->dev, areq->src, ns, DMA_TO_DEVICE); - dma_unmap_sg(ce->dev, areq->dst, nd, DMA_FROM_DEVICE); + + if (nr_sgd > 0) + dma_unmap_sg(ce->dev, areq->dst, nd, DMA_FROM_DEVICE); } theend_iv: if (areq->iv && ivsize > 0) { - if (rctx->addr_iv) + if (!dma_mapping_error(ce->dev, rctx->addr_iv)) dma_unmap_single(ce->dev, rctx->addr_iv, rctx->ivlen, DMA_TO_DEVICE); + offset = areq->cryptlen - ivsize; if (rctx->op_dir & CE_DECRYPTION) { memcpy(areq->iv, chan->backup_iv, ivsize); From 8e379e8fb83f426a60a3ae0827743ebf881f6fde Mon Sep 17 00:00:00 2001 From: Ovidiu Panait Date: Fri, 25 Apr 2025 15:45:15 +0300 Subject: [PATCH 240/382] crypto: sun8i-ce-cipher - use crypto_skcipher_driver_name() Use crypto_skcipher_driver_name() helper from , instead of accessing struct crypto_alg directly. Signed-off-by: Ovidiu Panait Acked-by: Corentin LABBE Tested-by: Corentin LABBE Signed-off-by: Herbert Xu --- drivers/crypto/allwinner/sun8i-ce/sun8i-ce-cipher.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-cipher.c b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-cipher.c index 05f67661553c..ba50f98fdf09 100644 --- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-cipher.c +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-cipher.c @@ -438,7 +438,7 @@ int sun8i_ce_cipher_init(struct crypto_tfm *tfm) crypto_skcipher_reqsize(op->fallback_tfm)); memcpy(algt->fbname, - crypto_tfm_alg_driver_name(crypto_skcipher_tfm(op->fallback_tfm)), + crypto_skcipher_driver_name(op->fallback_tfm), CRYPTO_MAX_ALG_NAME); err = pm_runtime_get_sync(op->ce->dev); From 031bc3e8b9e230df0c5380eb4a8767558c4c1a25 Mon Sep 17 00:00:00 2001 From: Ovidiu Panait Date: Fri, 25 Apr 2025 15:45:16 +0300 Subject: [PATCH 241/382] crypto: sun8i-ce-cipher - use IS_ENABLED() checks for debugfs stats Add IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG) checks before the fallback counter updates to make sure the code is not included when debugfs statistics support is not enabled. Also, drop the existing ifdef guards, since 'struct sun8i_ce_alg_template' is always defined, even with CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG disabled. Signed-off-by: Ovidiu Panait Tested-by: Corentin LABBE Acked-by: Corentin LABBE Signed-off-by: Herbert Xu --- .../allwinner/sun8i-ce/sun8i-ce-cipher.c | 46 ++++++++++++------- 1 file changed, 30 insertions(+), 16 deletions(-) diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-cipher.c b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-cipher.c index ba50f98fdf09..91bc3c2c488f 100644 --- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-cipher.c +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-cipher.c @@ -33,22 +33,30 @@ static int sun8i_ce_cipher_need_fallback(struct skcipher_request *areq) if (sg_nents_for_len(areq->src, areq->cryptlen) > MAX_SG || sg_nents_for_len(areq->dst, areq->cryptlen) > MAX_SG) { - algt->stat_fb_maxsg++; + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) + algt->stat_fb_maxsg++; + return true; } if (areq->cryptlen < crypto_skcipher_ivsize(tfm)) { - algt->stat_fb_leniv++; + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) + algt->stat_fb_leniv++; + return true; } if (areq->cryptlen == 0) { - algt->stat_fb_len0++; + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) + algt->stat_fb_len0++; + return true; } if (areq->cryptlen % 16) { - algt->stat_fb_mod16++; + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) + algt->stat_fb_mod16++; + return true; } @@ -56,12 +64,16 @@ static int sun8i_ce_cipher_need_fallback(struct skcipher_request *areq) sg = areq->src; while (sg) { if (!IS_ALIGNED(sg->offset, sizeof(u32))) { - algt->stat_fb_srcali++; + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) + algt->stat_fb_srcali++; + return true; } todo = min(len, sg->length); if (todo % 4) { - algt->stat_fb_srclen++; + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) + algt->stat_fb_srclen++; + return true; } len -= todo; @@ -72,12 +84,16 @@ static int sun8i_ce_cipher_need_fallback(struct skcipher_request *areq) sg = areq->dst; while (sg) { if (!IS_ALIGNED(sg->offset, sizeof(u32))) { - algt->stat_fb_dstali++; + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) + algt->stat_fb_dstali++; + return true; } todo = min(len, sg->length); if (todo % 4) { - algt->stat_fb_dstlen++; + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) + algt->stat_fb_dstlen++; + return true; } len -= todo; @@ -100,9 +116,7 @@ static int sun8i_ce_cipher_fallback(struct skcipher_request *areq) algt = container_of(alg, struct sun8i_ce_alg_template, alg.skcipher.base); -#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG algt->stat_fb++; -#endif } skcipher_request_set_tfm(&rctx->fallback_req, op->fallback_tfm); @@ -146,9 +160,8 @@ static int sun8i_ce_cipher_prepare(struct crypto_engine *engine, void *async_req rctx->op_dir, areq->iv, crypto_skcipher_ivsize(tfm), op->keylen); -#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG - algt->stat_req++; -#endif + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) + algt->stat_req++; flow = rctx->flow; @@ -437,9 +450,10 @@ int sun8i_ce_cipher_init(struct crypto_tfm *tfm) crypto_skcipher_set_reqsize(sktfm, sizeof(struct sun8i_cipher_req_ctx) + crypto_skcipher_reqsize(op->fallback_tfm)); - memcpy(algt->fbname, - crypto_skcipher_driver_name(op->fallback_tfm), - CRYPTO_MAX_ALG_NAME); + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) + memcpy(algt->fbname, + crypto_skcipher_driver_name(op->fallback_tfm), + CRYPTO_MAX_ALG_NAME); err = pm_runtime_get_sync(op->ce->dev); if (err < 0) From b9580cf5f9ace0d64903164f553cbef674cefb46 Mon Sep 17 00:00:00 2001 From: Ovidiu Panait Date: Fri, 25 Apr 2025 15:45:17 +0300 Subject: [PATCH 242/382] crypto: sun8i-ce-cipher - use pm_runtime_resume_and_get() Replace pm_runtime_get_sync() usage with pm_runtime_resume_and_get() to simplify error handling. This is recommended in the documentation of pm_runtime_get_sync(): /** * pm_runtime_get_sync - Bump up usage counter of a device and resume it. ... * Consider using pm_runtime_resume_and_get() instead of it, especially * if its return value is checked by the caller, as this is likely to result * in cleaner code. ... */ Signed-off-by: Ovidiu Panait Tested-by: Corentin LABBE Acked-by: Corentin LABBE Signed-off-by: Herbert Xu --- drivers/crypto/allwinner/sun8i-ce/sun8i-ce-cipher.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-cipher.c b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-cipher.c index 91bc3c2c488f..f9cf00d690e2 100644 --- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-cipher.c +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-cipher.c @@ -455,13 +455,12 @@ int sun8i_ce_cipher_init(struct crypto_tfm *tfm) crypto_skcipher_driver_name(op->fallback_tfm), CRYPTO_MAX_ALG_NAME); - err = pm_runtime_get_sync(op->ce->dev); + err = pm_runtime_resume_and_get(op->ce->dev); if (err < 0) goto error_pm; return 0; error_pm: - pm_runtime_put_noidle(op->ce->dev); crypto_free_skcipher(op->fallback_tfm); return err; } From db4978d2f0559288a1595f82bda419fad216beed Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Fri, 25 Apr 2025 23:11:31 -0700 Subject: [PATCH 243/382] crypto: iaa - Adjust workqueue allocation type In preparation for making the kmalloc family of allocators type aware, we need to make sure that the returned type from the allocation matches the type of the variable being assigned. (Before, the allocator would always return "void *", which can be implicitly cast to any pointer type.) The assigned type is "struct idxd_wq **", but the returned type will be "struct wq **". These are the same size allocation (pointer sized), but the types don't match. Adjust the allocation type to match the assignment. Signed-off-by: Kees Cook Signed-off-by: Herbert Xu --- drivers/crypto/intel/iaa/iaa_crypto_main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/intel/iaa/iaa_crypto_main.c b/drivers/crypto/intel/iaa/iaa_crypto_main.c index be3b899c6977..e9c762124f11 100644 --- a/drivers/crypto/intel/iaa/iaa_crypto_main.c +++ b/drivers/crypto/intel/iaa/iaa_crypto_main.c @@ -725,7 +725,7 @@ static int alloc_wq_table(int max_wqs) for (cpu = 0; cpu < nr_cpus; cpu++) { entry = per_cpu_ptr(wq_table, cpu); - entry->wqs = kcalloc(max_wqs, sizeof(struct wq *), GFP_KERNEL); + entry->wqs = kcalloc(max_wqs, sizeof(*entry->wqs), GFP_KERNEL); if (!entry->wqs) { free_wq_table(); return -ENOMEM; From 9a7b87412390b008c923828913e7d296d3ae5afd Mon Sep 17 00:00:00 2001 From: Thorsten Blum Date: Sun, 27 Apr 2025 00:26:19 +0200 Subject: [PATCH 244/382] crypto: octeontx2 - Simplify multiple return statements Simplify multiple return statements by directly returning the boolean expressions. Signed-off-by: Thorsten Blum Signed-off-by: Herbert Xu --- .../marvell/octeontx2/otx2_cpt_common.h | 34 ++++++------------- 1 file changed, 10 insertions(+), 24 deletions(-) diff --git a/drivers/crypto/marvell/octeontx2/otx2_cpt_common.h b/drivers/crypto/marvell/octeontx2/otx2_cpt_common.h index c5b7c57574ef..84f2bb0ca11c 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cpt_common.h +++ b/drivers/crypto/marvell/octeontx2/otx2_cpt_common.h @@ -145,11 +145,8 @@ static inline u64 otx2_cpt_read64(void __iomem *reg_base, u64 blk, u64 slot, static inline bool is_dev_otx2(struct pci_dev *pdev) { - if (pdev->device == OTX2_CPT_PCI_PF_DEVICE_ID || - pdev->device == OTX2_CPT_PCI_VF_DEVICE_ID) - return true; - - return false; + return pdev->device == OTX2_CPT_PCI_PF_DEVICE_ID || + pdev->device == OTX2_CPT_PCI_VF_DEVICE_ID; } static inline bool is_dev_cn10ka(struct pci_dev *pdev) @@ -159,12 +156,10 @@ static inline bool is_dev_cn10ka(struct pci_dev *pdev) static inline bool is_dev_cn10ka_ax(struct pci_dev *pdev) { - if (pdev->subsystem_device == CPT_PCI_SUBSYS_DEVID_CN10K_A && - ((pdev->revision & 0xFF) == 4 || (pdev->revision & 0xFF) == 0x50 || - (pdev->revision & 0xff) == 0x51)) - return true; - - return false; + return pdev->subsystem_device == CPT_PCI_SUBSYS_DEVID_CN10K_A && + ((pdev->revision & 0xFF) == 4 || + (pdev->revision & 0xFF) == 0x50 || + (pdev->revision & 0xFF) == 0x51); } static inline bool is_dev_cn10kb(struct pci_dev *pdev) @@ -174,11 +169,8 @@ static inline bool is_dev_cn10kb(struct pci_dev *pdev) static inline bool is_dev_cn10ka_b0(struct pci_dev *pdev) { - if (pdev->subsystem_device == CPT_PCI_SUBSYS_DEVID_CN10K_A && - (pdev->revision & 0xFF) == 0x54) - return true; - - return false; + return pdev->subsystem_device == CPT_PCI_SUBSYS_DEVID_CN10K_A && + (pdev->revision & 0xFF) == 0x54; } static inline void otx2_cpt_set_hw_caps(struct pci_dev *pdev, @@ -192,18 +184,12 @@ static inline void otx2_cpt_set_hw_caps(struct pci_dev *pdev, static inline bool cpt_is_errata_38550_exists(struct pci_dev *pdev) { - if (is_dev_otx2(pdev) || is_dev_cn10ka_ax(pdev)) - return true; - - return false; + return is_dev_otx2(pdev) || is_dev_cn10ka_ax(pdev); } static inline bool cpt_feature_sgv2(struct pci_dev *pdev) { - if (!is_dev_otx2(pdev) && !is_dev_cn10ka_ax(pdev)) - return true; - - return false; + return !is_dev_otx2(pdev) && !is_dev_cn10ka_ax(pdev); } int otx2_cpt_send_ready_msg(struct otx2_mbox *mbox, struct pci_dev *pdev); From 1dea68318aad0899011a49748634aff54ddc327a Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sun, 27 Apr 2025 08:42:51 +0800 Subject: [PATCH 245/382] crypto: ccp - Include crypto/utils.h for crypto_memneq Include crypto/utils.h to get crypto_memneq rather than relying on random inclusions. Signed-off-by: Herbert Xu --- drivers/crypto/ccp/ccp-ops.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/drivers/crypto/ccp/ccp-ops.c b/drivers/crypto/ccp/ccp-ops.c index cb8e99936abb..109b5aef4034 100644 --- a/drivers/crypto/ccp/ccp-ops.c +++ b/drivers/crypto/ccp/ccp-ops.c @@ -8,13 +8,14 @@ * Author: Gary R Hook */ -#include -#include -#include -#include -#include #include +#include +#include #include +#include +#include +#include +#include #include "ccp-dev.h" From 0f8d42bf128d349ad490e87d5574d211245e40f1 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sun, 27 Apr 2025 08:42:54 +0800 Subject: [PATCH 246/382] crypto: scatterwalk - Move skcipher walk and use it for memcpy_sglist Move the generic part of skcipher walk into scatterwalk, and use it to implement memcpy_sglist. This makes memcpy_sglist do the right thing when two distinct SG lists contain identical subsets (e.g., the AD part of AEAD). Signed-off-by: Herbert Xu --- crypto/scatterwalk.c | 274 +++++++++++++++++++++++++++-- crypto/skcipher.c | 261 +-------------------------- include/crypto/algapi.h | 12 -- include/crypto/internal/skcipher.h | 48 +---- include/crypto/scatterwalk.h | 65 ++++++- 5 files changed, 329 insertions(+), 331 deletions(-) diff --git a/crypto/scatterwalk.c b/crypto/scatterwalk.c index 8225801488d5..1d010e2a1b1a 100644 --- a/crypto/scatterwalk.c +++ b/crypto/scatterwalk.c @@ -10,10 +10,25 @@ */ #include +#include +#include #include #include #include #include +#include + +enum { + SKCIPHER_WALK_SLOW = 1 << 0, + SKCIPHER_WALK_COPY = 1 << 1, + SKCIPHER_WALK_DIFF = 1 << 2, + SKCIPHER_WALK_SLEEP = 1 << 3, +}; + +static inline gfp_t skcipher_walk_gfp(struct skcipher_walk *walk) +{ + return walk->flags & SKCIPHER_WALK_SLEEP ? GFP_KERNEL : GFP_ATOMIC; +} void scatterwalk_skip(struct scatter_walk *walk, unsigned int nbytes) { @@ -89,27 +104,23 @@ EXPORT_SYMBOL_GPL(memcpy_to_sglist); void memcpy_sglist(struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { - struct scatter_walk swalk; - struct scatter_walk dwalk; + struct skcipher_walk walk = {}; if (unlikely(nbytes == 0)) /* in case sg == NULL */ return; - scatterwalk_start(&swalk, src); - scatterwalk_start(&dwalk, dst); + walk.total = nbytes; + scatterwalk_start(&walk.in, src); + scatterwalk_start(&walk.out, dst); + + skcipher_walk_first(&walk, true); do { - unsigned int slen, dlen; - unsigned int len; - - slen = scatterwalk_next(&swalk, nbytes); - dlen = scatterwalk_next(&dwalk, nbytes); - len = min(slen, dlen); - memcpy(dwalk.addr, swalk.addr, len); - scatterwalk_done_dst(&dwalk, len); - scatterwalk_done_src(&swalk, len); - nbytes -= len; - } while (nbytes); + if (walk.src.virt.addr != walk.dst.virt.addr) + memcpy(walk.dst.virt.addr, walk.src.virt.addr, + walk.nbytes); + skcipher_walk_done(&walk, 0); + } while (walk.nbytes); } EXPORT_SYMBOL_GPL(memcpy_sglist); @@ -135,3 +146,236 @@ struct scatterlist *scatterwalk_ffwd(struct scatterlist dst[2], return dst; } EXPORT_SYMBOL_GPL(scatterwalk_ffwd); + +static int skcipher_next_slow(struct skcipher_walk *walk, unsigned int bsize) +{ + unsigned alignmask = walk->alignmask; + unsigned n; + void *buffer; + + if (!walk->buffer) + walk->buffer = walk->page; + buffer = walk->buffer; + if (!buffer) { + /* Min size for a buffer of bsize bytes aligned to alignmask */ + n = bsize + (alignmask & ~(crypto_tfm_ctx_alignment() - 1)); + + buffer = kzalloc(n, skcipher_walk_gfp(walk)); + if (!buffer) + return skcipher_walk_done(walk, -ENOMEM); + walk->buffer = buffer; + } + + buffer = PTR_ALIGN(buffer, alignmask + 1); + memcpy_from_scatterwalk(buffer, &walk->in, bsize); + walk->out.__addr = buffer; + walk->in.__addr = walk->out.addr; + + walk->nbytes = bsize; + walk->flags |= SKCIPHER_WALK_SLOW; + + return 0; +} + +static int skcipher_next_copy(struct skcipher_walk *walk) +{ + void *tmp = walk->page; + + scatterwalk_map(&walk->in); + memcpy(tmp, walk->in.addr, walk->nbytes); + scatterwalk_unmap(&walk->in); + /* + * walk->in is advanced later when the number of bytes actually + * processed (which might be less than walk->nbytes) is known. + */ + + walk->in.__addr = tmp; + walk->out.__addr = tmp; + return 0; +} + +static int skcipher_next_fast(struct skcipher_walk *walk) +{ + unsigned long diff; + + diff = offset_in_page(walk->in.offset) - + offset_in_page(walk->out.offset); + diff |= (u8 *)(sg_page(walk->in.sg) + (walk->in.offset >> PAGE_SHIFT)) - + (u8 *)(sg_page(walk->out.sg) + (walk->out.offset >> PAGE_SHIFT)); + + scatterwalk_map(&walk->out); + walk->in.__addr = walk->out.__addr; + + if (diff) { + walk->flags |= SKCIPHER_WALK_DIFF; + scatterwalk_map(&walk->in); + } + + return 0; +} + +static int skcipher_walk_next(struct skcipher_walk *walk) +{ + unsigned int bsize; + unsigned int n; + + n = walk->total; + bsize = min(walk->stride, max(n, walk->blocksize)); + n = scatterwalk_clamp(&walk->in, n); + n = scatterwalk_clamp(&walk->out, n); + + if (unlikely(n < bsize)) { + if (unlikely(walk->total < walk->blocksize)) + return skcipher_walk_done(walk, -EINVAL); + +slow_path: + return skcipher_next_slow(walk, bsize); + } + walk->nbytes = n; + + if (unlikely((walk->in.offset | walk->out.offset) & walk->alignmask)) { + if (!walk->page) { + gfp_t gfp = skcipher_walk_gfp(walk); + + walk->page = (void *)__get_free_page(gfp); + if (!walk->page) + goto slow_path; + } + walk->flags |= SKCIPHER_WALK_COPY; + return skcipher_next_copy(walk); + } + + return skcipher_next_fast(walk); +} + +static int skcipher_copy_iv(struct skcipher_walk *walk) +{ + unsigned alignmask = walk->alignmask; + unsigned ivsize = walk->ivsize; + unsigned aligned_stride = ALIGN(walk->stride, alignmask + 1); + unsigned size; + u8 *iv; + + /* Min size for a buffer of stride + ivsize, aligned to alignmask */ + size = aligned_stride + ivsize + + (alignmask & ~(crypto_tfm_ctx_alignment() - 1)); + + walk->buffer = kmalloc(size, skcipher_walk_gfp(walk)); + if (!walk->buffer) + return -ENOMEM; + + iv = PTR_ALIGN(walk->buffer, alignmask + 1) + aligned_stride; + + walk->iv = memcpy(iv, walk->iv, walk->ivsize); + return 0; +} + +int skcipher_walk_first(struct skcipher_walk *walk, bool atomic) +{ + if (WARN_ON_ONCE(in_hardirq())) + return -EDEADLK; + + walk->flags = atomic ? 0 : SKCIPHER_WALK_SLEEP; + + walk->buffer = NULL; + if (unlikely(((unsigned long)walk->iv & walk->alignmask))) { + int err = skcipher_copy_iv(walk); + if (err) + return err; + } + + walk->page = NULL; + + return skcipher_walk_next(walk); +} +EXPORT_SYMBOL_GPL(skcipher_walk_first); + +/** + * skcipher_walk_done() - finish one step of a skcipher_walk + * @walk: the skcipher_walk + * @res: number of bytes *not* processed (>= 0) from walk->nbytes, + * or a -errno value to terminate the walk due to an error + * + * This function cleans up after one step of walking through the source and + * destination scatterlists, and advances to the next step if applicable. + * walk->nbytes is set to the number of bytes available in the next step, + * walk->total is set to the new total number of bytes remaining, and + * walk->{src,dst}.virt.addr is set to the next pair of data pointers. If there + * is no more data, or if an error occurred (i.e. -errno return), then + * walk->nbytes and walk->total are set to 0 and all resources owned by the + * skcipher_walk are freed. + * + * Return: 0 or a -errno value. If @res was a -errno value then it will be + * returned, but other errors may occur too. + */ +int skcipher_walk_done(struct skcipher_walk *walk, int res) +{ + unsigned int n = walk->nbytes; /* num bytes processed this step */ + unsigned int total = 0; /* new total remaining */ + + if (!n) + goto finish; + + if (likely(res >= 0)) { + n -= res; /* subtract num bytes *not* processed */ + total = walk->total - n; + } + + if (likely(!(walk->flags & (SKCIPHER_WALK_SLOW | + SKCIPHER_WALK_COPY | + SKCIPHER_WALK_DIFF)))) { + scatterwalk_advance(&walk->in, n); + } else if (walk->flags & SKCIPHER_WALK_DIFF) { + scatterwalk_done_src(&walk->in, n); + } else if (walk->flags & SKCIPHER_WALK_COPY) { + scatterwalk_advance(&walk->in, n); + scatterwalk_map(&walk->out); + memcpy(walk->out.addr, walk->page, n); + } else { /* SKCIPHER_WALK_SLOW */ + if (res > 0) { + /* + * Didn't process all bytes. Either the algorithm is + * broken, or this was the last step and it turned out + * the message wasn't evenly divisible into blocks but + * the algorithm requires it. + */ + res = -EINVAL; + total = 0; + } else + memcpy_to_scatterwalk(&walk->out, walk->out.addr, n); + goto dst_done; + } + + scatterwalk_done_dst(&walk->out, n); +dst_done: + + if (res > 0) + res = 0; + + walk->total = total; + walk->nbytes = 0; + + if (total) { + if (walk->flags & SKCIPHER_WALK_SLEEP) + cond_resched(); + walk->flags &= ~(SKCIPHER_WALK_SLOW | SKCIPHER_WALK_COPY | + SKCIPHER_WALK_DIFF); + return skcipher_walk_next(walk); + } + +finish: + /* Short-circuit for the common/fast path. */ + if (!((unsigned long)walk->buffer | (unsigned long)walk->page)) + goto out; + + if (walk->iv != walk->oiv) + memcpy(walk->oiv, walk->iv, walk->ivsize); + if (walk->buffer != walk->page) + kfree(walk->buffer); + if (walk->page) + free_page((unsigned long)walk->page); + +out: + return res; +} +EXPORT_SYMBOL_GPL(skcipher_walk_done); diff --git a/crypto/skcipher.c b/crypto/skcipher.c index 319215cfded5..de5fc91bba26 100644 --- a/crypto/skcipher.c +++ b/crypto/skcipher.c @@ -17,7 +17,6 @@ #include #include #include -#include #include #include #include @@ -28,258 +27,14 @@ #define CRYPTO_ALG_TYPE_SKCIPHER_MASK 0x0000000e -enum { - SKCIPHER_WALK_SLOW = 1 << 0, - SKCIPHER_WALK_COPY = 1 << 1, - SKCIPHER_WALK_DIFF = 1 << 2, - SKCIPHER_WALK_SLEEP = 1 << 3, -}; - static const struct crypto_type crypto_skcipher_type; -static int skcipher_walk_next(struct skcipher_walk *walk); - -static inline gfp_t skcipher_walk_gfp(struct skcipher_walk *walk) -{ - return walk->flags & SKCIPHER_WALK_SLEEP ? GFP_KERNEL : GFP_ATOMIC; -} - static inline struct skcipher_alg *__crypto_skcipher_alg( struct crypto_alg *alg) { return container_of(alg, struct skcipher_alg, base); } -/** - * skcipher_walk_done() - finish one step of a skcipher_walk - * @walk: the skcipher_walk - * @res: number of bytes *not* processed (>= 0) from walk->nbytes, - * or a -errno value to terminate the walk due to an error - * - * This function cleans up after one step of walking through the source and - * destination scatterlists, and advances to the next step if applicable. - * walk->nbytes is set to the number of bytes available in the next step, - * walk->total is set to the new total number of bytes remaining, and - * walk->{src,dst}.virt.addr is set to the next pair of data pointers. If there - * is no more data, or if an error occurred (i.e. -errno return), then - * walk->nbytes and walk->total are set to 0 and all resources owned by the - * skcipher_walk are freed. - * - * Return: 0 or a -errno value. If @res was a -errno value then it will be - * returned, but other errors may occur too. - */ -int skcipher_walk_done(struct skcipher_walk *walk, int res) -{ - unsigned int n = walk->nbytes; /* num bytes processed this step */ - unsigned int total = 0; /* new total remaining */ - - if (!n) - goto finish; - - if (likely(res >= 0)) { - n -= res; /* subtract num bytes *not* processed */ - total = walk->total - n; - } - - if (likely(!(walk->flags & (SKCIPHER_WALK_SLOW | - SKCIPHER_WALK_COPY | - SKCIPHER_WALK_DIFF)))) { - scatterwalk_advance(&walk->in, n); - } else if (walk->flags & SKCIPHER_WALK_DIFF) { - scatterwalk_done_src(&walk->in, n); - } else if (walk->flags & SKCIPHER_WALK_COPY) { - scatterwalk_advance(&walk->in, n); - scatterwalk_map(&walk->out); - memcpy(walk->out.addr, walk->page, n); - } else { /* SKCIPHER_WALK_SLOW */ - if (res > 0) { - /* - * Didn't process all bytes. Either the algorithm is - * broken, or this was the last step and it turned out - * the message wasn't evenly divisible into blocks but - * the algorithm requires it. - */ - res = -EINVAL; - total = 0; - } else - memcpy_to_scatterwalk(&walk->out, walk->out.addr, n); - goto dst_done; - } - - scatterwalk_done_dst(&walk->out, n); -dst_done: - - if (res > 0) - res = 0; - - walk->total = total; - walk->nbytes = 0; - - if (total) { - if (walk->flags & SKCIPHER_WALK_SLEEP) - cond_resched(); - walk->flags &= ~(SKCIPHER_WALK_SLOW | SKCIPHER_WALK_COPY | - SKCIPHER_WALK_DIFF); - return skcipher_walk_next(walk); - } - -finish: - /* Short-circuit for the common/fast path. */ - if (!((unsigned long)walk->buffer | (unsigned long)walk->page)) - goto out; - - if (walk->iv != walk->oiv) - memcpy(walk->oiv, walk->iv, walk->ivsize); - if (walk->buffer != walk->page) - kfree(walk->buffer); - if (walk->page) - free_page((unsigned long)walk->page); - -out: - return res; -} -EXPORT_SYMBOL_GPL(skcipher_walk_done); - -static int skcipher_next_slow(struct skcipher_walk *walk, unsigned int bsize) -{ - unsigned alignmask = walk->alignmask; - unsigned n; - void *buffer; - - if (!walk->buffer) - walk->buffer = walk->page; - buffer = walk->buffer; - if (!buffer) { - /* Min size for a buffer of bsize bytes aligned to alignmask */ - n = bsize + (alignmask & ~(crypto_tfm_ctx_alignment() - 1)); - - buffer = kzalloc(n, skcipher_walk_gfp(walk)); - if (!buffer) - return skcipher_walk_done(walk, -ENOMEM); - walk->buffer = buffer; - } - - buffer = PTR_ALIGN(buffer, alignmask + 1); - memcpy_from_scatterwalk(buffer, &walk->in, bsize); - walk->out.__addr = buffer; - walk->in.__addr = walk->out.addr; - - walk->nbytes = bsize; - walk->flags |= SKCIPHER_WALK_SLOW; - - return 0; -} - -static int skcipher_next_copy(struct skcipher_walk *walk) -{ - void *tmp = walk->page; - - scatterwalk_map(&walk->in); - memcpy(tmp, walk->in.addr, walk->nbytes); - scatterwalk_unmap(&walk->in); - /* - * walk->in is advanced later when the number of bytes actually - * processed (which might be less than walk->nbytes) is known. - */ - - walk->in.__addr = tmp; - walk->out.__addr = tmp; - return 0; -} - -static int skcipher_next_fast(struct skcipher_walk *walk) -{ - unsigned long diff; - - diff = offset_in_page(walk->in.offset) - - offset_in_page(walk->out.offset); - diff |= (u8 *)(sg_page(walk->in.sg) + (walk->in.offset >> PAGE_SHIFT)) - - (u8 *)(sg_page(walk->out.sg) + (walk->out.offset >> PAGE_SHIFT)); - - scatterwalk_map(&walk->out); - walk->in.__addr = walk->out.__addr; - - if (diff) { - walk->flags |= SKCIPHER_WALK_DIFF; - scatterwalk_map(&walk->in); - } - - return 0; -} - -static int skcipher_walk_next(struct skcipher_walk *walk) -{ - unsigned int bsize; - unsigned int n; - - n = walk->total; - bsize = min(walk->stride, max(n, walk->blocksize)); - n = scatterwalk_clamp(&walk->in, n); - n = scatterwalk_clamp(&walk->out, n); - - if (unlikely(n < bsize)) { - if (unlikely(walk->total < walk->blocksize)) - return skcipher_walk_done(walk, -EINVAL); - -slow_path: - return skcipher_next_slow(walk, bsize); - } - walk->nbytes = n; - - if (unlikely((walk->in.offset | walk->out.offset) & walk->alignmask)) { - if (!walk->page) { - gfp_t gfp = skcipher_walk_gfp(walk); - - walk->page = (void *)__get_free_page(gfp); - if (!walk->page) - goto slow_path; - } - walk->flags |= SKCIPHER_WALK_COPY; - return skcipher_next_copy(walk); - } - - return skcipher_next_fast(walk); -} - -static int skcipher_copy_iv(struct skcipher_walk *walk) -{ - unsigned alignmask = walk->alignmask; - unsigned ivsize = walk->ivsize; - unsigned aligned_stride = ALIGN(walk->stride, alignmask + 1); - unsigned size; - u8 *iv; - - /* Min size for a buffer of stride + ivsize, aligned to alignmask */ - size = aligned_stride + ivsize + - (alignmask & ~(crypto_tfm_ctx_alignment() - 1)); - - walk->buffer = kmalloc(size, skcipher_walk_gfp(walk)); - if (!walk->buffer) - return -ENOMEM; - - iv = PTR_ALIGN(walk->buffer, alignmask + 1) + aligned_stride; - - walk->iv = memcpy(iv, walk->iv, walk->ivsize); - return 0; -} - -static int skcipher_walk_first(struct skcipher_walk *walk) -{ - if (WARN_ON_ONCE(in_hardirq())) - return -EDEADLK; - - walk->buffer = NULL; - if (unlikely(((unsigned long)walk->iv & walk->alignmask))) { - int err = skcipher_copy_iv(walk); - if (err) - return err; - } - - walk->page = NULL; - - return skcipher_walk_next(walk); -} - int skcipher_walk_virt(struct skcipher_walk *__restrict walk, struct skcipher_request *__restrict req, bool atomic) { @@ -294,10 +49,8 @@ int skcipher_walk_virt(struct skcipher_walk *__restrict walk, walk->nbytes = 0; walk->iv = req->iv; walk->oiv = req->iv; - if ((req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) && !atomic) - walk->flags = SKCIPHER_WALK_SLEEP; - else - walk->flags = 0; + if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP)) + atomic = true; if (unlikely(!walk->total)) return 0; @@ -314,7 +67,7 @@ int skcipher_walk_virt(struct skcipher_walk *__restrict walk, else walk->stride = alg->walksize; - return skcipher_walk_first(walk); + return skcipher_walk_first(walk, atomic); } EXPORT_SYMBOL_GPL(skcipher_walk_virt); @@ -327,10 +80,8 @@ static int skcipher_walk_aead_common(struct skcipher_walk *__restrict walk, walk->nbytes = 0; walk->iv = req->iv; walk->oiv = req->iv; - if ((req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) && !atomic) - walk->flags = SKCIPHER_WALK_SLEEP; - else - walk->flags = 0; + if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP)) + atomic = true; if (unlikely(!walk->total)) return 0; @@ -343,7 +94,7 @@ static int skcipher_walk_aead_common(struct skcipher_walk *__restrict walk, walk->ivsize = crypto_aead_ivsize(tfm); walk->alignmask = crypto_aead_alignmask(tfm); - return skcipher_walk_first(walk); + return skcipher_walk_first(walk, atomic); } int skcipher_walk_aead_encrypt(struct skcipher_walk *__restrict walk, diff --git a/include/crypto/algapi.h b/include/crypto/algapi.h index 6999e10ea09e..f5f730969d72 100644 --- a/include/crypto/algapi.h +++ b/include/crypto/algapi.h @@ -107,18 +107,6 @@ struct crypto_queue { unsigned int max_qlen; }; -struct scatter_walk { - /* Must be the first member, see struct skcipher_walk. */ - union { - void *const addr; - - /* Private API field, do not touch. */ - union crypto_no_such_thing *__addr; - }; - struct scatterlist *sg; - unsigned int offset; -}; - struct crypto_attr_alg { char name[CRYPTO_MAX_ALG_NAME]; }; diff --git a/include/crypto/internal/skcipher.h b/include/crypto/internal/skcipher.h index 0cad8e7364c8..d5aa535263f6 100644 --- a/include/crypto/internal/skcipher.h +++ b/include/crypto/internal/skcipher.h @@ -10,6 +10,7 @@ #include #include +#include #include #include @@ -54,47 +55,6 @@ struct crypto_lskcipher_spawn { struct crypto_spawn base; }; -struct skcipher_walk { - union { - /* Virtual address of the source. */ - struct { - struct { - const void *const addr; - } virt; - } src; - - /* Private field for the API, do not use. */ - struct scatter_walk in; - }; - - union { - /* Virtual address of the destination. */ - struct { - struct { - void *const addr; - } virt; - } dst; - - /* Private field for the API, do not use. */ - struct scatter_walk out; - }; - - unsigned int nbytes; - unsigned int total; - - u8 *page; - u8 *buffer; - u8 *oiv; - void *iv; - - unsigned int ivsize; - - int flags; - unsigned int blocksize; - unsigned int stride; - unsigned int alignmask; -}; - static inline struct crypto_instance *skcipher_crypto_instance( struct skcipher_instance *inst) { @@ -211,7 +171,6 @@ void crypto_unregister_lskciphers(struct lskcipher_alg *algs, int count); int lskcipher_register_instance(struct crypto_template *tmpl, struct lskcipher_instance *inst); -int skcipher_walk_done(struct skcipher_walk *walk, int res); int skcipher_walk_virt(struct skcipher_walk *__restrict walk, struct skcipher_request *__restrict req, bool atomic); @@ -222,11 +181,6 @@ int skcipher_walk_aead_decrypt(struct skcipher_walk *__restrict walk, struct aead_request *__restrict req, bool atomic); -static inline void skcipher_walk_abort(struct skcipher_walk *walk) -{ - skcipher_walk_done(walk, -ECANCELED); -} - static inline void *crypto_skcipher_ctx(struct crypto_skcipher *tfm) { return crypto_tfm_ctx(&tfm->base); diff --git a/include/crypto/scatterwalk.h b/include/crypto/scatterwalk.h index 94a8585f26b2..15ab743f68c8 100644 --- a/include/crypto/scatterwalk.h +++ b/include/crypto/scatterwalk.h @@ -11,11 +11,64 @@ #ifndef _CRYPTO_SCATTERWALK_H #define _CRYPTO_SCATTERWALK_H -#include - +#include #include #include #include +#include + +struct scatter_walk { + /* Must be the first member, see struct skcipher_walk. */ + union { + void *const addr; + + /* Private API field, do not touch. */ + union crypto_no_such_thing *__addr; + }; + struct scatterlist *sg; + unsigned int offset; +}; + +struct skcipher_walk { + union { + /* Virtual address of the source. */ + struct { + struct { + const void *const addr; + } virt; + } src; + + /* Private field for the API, do not use. */ + struct scatter_walk in; + }; + + union { + /* Virtual address of the destination. */ + struct { + struct { + void *const addr; + } virt; + } dst; + + /* Private field for the API, do not use. */ + struct scatter_walk out; + }; + + unsigned int nbytes; + unsigned int total; + + u8 *page; + u8 *buffer; + u8 *oiv; + void *iv; + + unsigned int ivsize; + + int flags; + unsigned int blocksize; + unsigned int stride; + unsigned int alignmask; +}; static inline void scatterwalk_crypto_chain(struct scatterlist *head, struct scatterlist *sg, int num) @@ -243,4 +296,12 @@ struct scatterlist *scatterwalk_ffwd(struct scatterlist dst[2], struct scatterlist *src, unsigned int len); +int skcipher_walk_first(struct skcipher_walk *walk, bool atomic); +int skcipher_walk_done(struct skcipher_walk *walk, int res); + +static inline void skcipher_walk_abort(struct skcipher_walk *walk) +{ + skcipher_walk_done(walk, -ECANCELED); +} + #endif /* _CRYPTO_SCATTERWALK_H */ From 2dfc7cd74a5e062a5405560447517e7aab1c7341 Mon Sep 17 00:00:00 2001 From: Corentin Labbe Date: Sun, 27 Apr 2025 13:12:36 +0200 Subject: [PATCH 247/382] crypto: sun8i-ss - do not use sg_dma_len before calling DMA functions When testing sun8i-ss with multi_v7_defconfig, all CBC algorithm fail crypto selftests. This is strange since on sunxi_defconfig, everything was ok. The problem was in the IV setup loop which never run because sg_dma_len was 0. Fixes: 359e893e8af4 ("crypto: sun8i-ss - rework handling of IV") Signed-off-by: Corentin Labbe Signed-off-by: Herbert Xu --- drivers/crypto/allwinner/sun8i-ss/sun8i-ss-cipher.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-cipher.c b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-cipher.c index 9b9605ce8ee6..8831bcb230c2 100644 --- a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-cipher.c +++ b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-cipher.c @@ -141,7 +141,7 @@ static int sun8i_ss_setup_ivs(struct skcipher_request *areq) /* we need to copy all IVs from source in case DMA is bi-directionnal */ while (sg && len) { - if (sg_dma_len(sg) == 0) { + if (sg->length == 0) { sg = sg_next(sg); continue; } From 5b39aa368bcfe193666ad4a68ef55f41dfd74028 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Tue, 29 Apr 2025 17:12:30 +0800 Subject: [PATCH 248/382] crypto: s390/sha512 - Fix sha512 state size The sha512 state size in s390_sha_ctx is out by a factor of 8, fix it so that it stays below HASH_MAX_DESCSIZE. Also fix the state init function which was writing garbage to the state. Luckily this is not actually used for anything other than export. Reported-by: Harald Freudenberger Fixes: 572b5c4682c7 ("crypto: s390/sha512 - Use API partial block handling") Signed-off-by: Herbert Xu --- arch/s390/crypto/sha.h | 2 +- arch/s390/crypto/sha512_s390.c | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/arch/s390/crypto/sha.h b/arch/s390/crypto/sha.h index 0a3cc1739144..d757ccbce2b4 100644 --- a/arch/s390/crypto/sha.h +++ b/arch/s390/crypto/sha.h @@ -24,7 +24,7 @@ struct s390_sha_ctx { union { u32 state[CPACF_MAX_PARMBLOCK_SIZE / sizeof(u32)]; struct { - u64 state[SHA512_DIGEST_SIZE]; + u64 state[SHA512_DIGEST_SIZE / sizeof(u64)]; u64 count_hi; } sha512; }; diff --git a/arch/s390/crypto/sha512_s390.c b/arch/s390/crypto/sha512_s390.c index 14818fcc9cd4..3c5175e6dda6 100644 --- a/arch/s390/crypto/sha512_s390.c +++ b/arch/s390/crypto/sha512_s390.c @@ -22,13 +22,13 @@ static int sha512_init(struct shash_desc *desc) struct s390_sha_ctx *ctx = shash_desc_ctx(desc); ctx->sha512.state[0] = SHA512_H0; - ctx->sha512.state[2] = SHA512_H1; - ctx->sha512.state[4] = SHA512_H2; - ctx->sha512.state[6] = SHA512_H3; - ctx->sha512.state[8] = SHA512_H4; - ctx->sha512.state[10] = SHA512_H5; - ctx->sha512.state[12] = SHA512_H6; - ctx->sha512.state[14] = SHA512_H7; + ctx->sha512.state[1] = SHA512_H1; + ctx->sha512.state[2] = SHA512_H2; + ctx->sha512.state[3] = SHA512_H3; + ctx->sha512.state[4] = SHA512_H4; + ctx->sha512.state[5] = SHA512_H5; + ctx->sha512.state[6] = SHA512_H6; + ctx->sha512.state[7] = SHA512_H7; ctx->count = 0; ctx->sha512.count_hi = 0; ctx->func = CPACF_KIMD_SHA_512; From 74a43a2cf5e8a3eeab3b55a2e64b33281f5ac554 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Mon, 28 Apr 2025 12:56:05 +0800 Subject: [PATCH 249/382] crypto: lib/sha256 - Move partial block handling out Extract the common partial block handling into a helper macro that can be reused by other library code. Also delete the unused sha256_base_do_finalize function. Signed-off-by: Herbert Xu --- include/crypto/internal/blockhash.h | 52 +++++++++++++++++++++++++++++ include/crypto/sha2.h | 9 +++-- include/crypto/sha256_base.h | 38 ++------------------- 3 files changed, 62 insertions(+), 37 deletions(-) create mode 100644 include/crypto/internal/blockhash.h diff --git a/include/crypto/internal/blockhash.h b/include/crypto/internal/blockhash.h new file mode 100644 index 000000000000..52d9d4c82493 --- /dev/null +++ b/include/crypto/internal/blockhash.h @@ -0,0 +1,52 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Handle partial blocks for block hash. + * + * Copyright (c) 2015 Linaro Ltd + * Copyright (c) 2025 Herbert Xu + */ + +#ifndef _CRYPTO_INTERNAL_BLOCKHASH_H +#define _CRYPTO_INTERNAL_BLOCKHASH_H + +#include +#include + +#define BLOCK_HASH_UPDATE_BASE(block_fn, state, src, nbytes, bs, dv, \ + buf, buflen) \ + ({ \ + typeof(block_fn) *_block_fn = &(block_fn); \ + typeof(state + 0) _state = (state); \ + unsigned int _buflen = (buflen); \ + size_t _nbytes = (nbytes); \ + unsigned int _bs = (bs); \ + const u8 *_src = (src); \ + u8 *_buf = (buf); \ + while ((_buflen + _nbytes) >= _bs) { \ + const u8 *data = _src; \ + size_t len = _nbytes; \ + size_t blocks; \ + int remain; \ + if (_buflen) { \ + remain = _bs - _buflen; \ + memcpy(_buf + _buflen, _src, remain); \ + data = _buf; \ + len = _bs; \ + } \ + remain = len % bs; \ + blocks = (len - remain) / (dv); \ + (*_block_fn)(_state, data, blocks); \ + _src += len - remain - _buflen; \ + _nbytes -= len - remain - _buflen; \ + _buflen = 0; \ + } \ + memcpy(_buf + _buflen, _src, _nbytes); \ + _buflen += _nbytes; \ + }) + +#define BLOCK_HASH_UPDATE(block, state, src, nbytes, bs, buf, buflen) \ + BLOCK_HASH_UPDATE_BASE(block, state, src, nbytes, bs, 1, buf, buflen) +#define BLOCK_HASH_UPDATE_BLOCKS(block, state, src, nbytes, bs, buf, buflen) \ + BLOCK_HASH_UPDATE_BASE(block, state, src, nbytes, bs, bs, buf, buflen) + +#endif /* _CRYPTO_INTERNAL_BLOCKHASH_H */ diff --git a/include/crypto/sha2.h b/include/crypto/sha2.h index abbd882f7849..f873c2207b1e 100644 --- a/include/crypto/sha2.h +++ b/include/crypto/sha2.h @@ -71,8 +71,13 @@ struct crypto_sha256_state { }; struct sha256_state { - u32 state[SHA256_DIGEST_SIZE / 4]; - u64 count; + union { + struct crypto_sha256_state ctx; + struct { + u32 state[SHA256_DIGEST_SIZE / 4]; + u64 count; + }; + }; u8 buf[SHA256_BLOCK_SIZE]; }; diff --git a/include/crypto/sha256_base.h b/include/crypto/sha256_base.h index 08cd5e41d4fd..9f284bed5a51 100644 --- a/include/crypto/sha256_base.h +++ b/include/crypto/sha256_base.h @@ -8,6 +8,7 @@ #ifndef _CRYPTO_SHA256_BASE_H #define _CRYPTO_SHA256_BASE_H +#include #include #include #include @@ -40,35 +41,10 @@ static inline int lib_sha256_base_do_update(struct sha256_state *sctx, sha256_block_fn *block_fn) { unsigned int partial = sctx->count % SHA256_BLOCK_SIZE; - struct crypto_sha256_state *state = (void *)sctx; sctx->count += len; - - if (unlikely((partial + len) >= SHA256_BLOCK_SIZE)) { - int blocks; - - if (partial) { - int p = SHA256_BLOCK_SIZE - partial; - - memcpy(sctx->buf + partial, data, p); - data += p; - len -= p; - - block_fn(state, sctx->buf, 1); - } - - blocks = len / SHA256_BLOCK_SIZE; - len %= SHA256_BLOCK_SIZE; - - if (blocks) { - block_fn(state, data, blocks); - data += blocks * SHA256_BLOCK_SIZE; - } - partial = 0; - } - if (len) - memcpy(sctx->buf + partial, data, len); - + BLOCK_HASH_UPDATE_BLOCKS(block_fn, &sctx->ctx, data, len, + SHA256_BLOCK_SIZE, sctx->buf, partial); return 0; } @@ -140,14 +116,6 @@ static inline int lib_sha256_base_do_finalize(struct sha256_state *sctx, return lib_sha256_base_do_finup(state, sctx->buf, partial, block_fn); } -static inline int sha256_base_do_finalize(struct shash_desc *desc, - sha256_block_fn *block_fn) -{ - struct sha256_state *sctx = shash_desc_ctx(desc); - - return lib_sha256_base_do_finalize(sctx, block_fn); -} - static inline int __sha256_base_finish(u32 state[SHA256_DIGEST_SIZE / 4], u8 *out, unsigned int digest_size) { From 9b84cb897803c484e15eb1885cd45a895ce1e436 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Mon, 28 Apr 2025 12:56:07 +0800 Subject: [PATCH 250/382] crypto: lib/poly1305 - Add block-only interface Add a block-only interface for poly1305. Implement the generic code first. Also use the generic partial block helper. Signed-off-by: Herbert Xu --- include/crypto/internal/poly1305.h | 28 ++++++++++++++-- include/crypto/poly1305.h | 25 ++++++++++---- lib/crypto/poly1305.c | 54 +++++++++++++----------------- 3 files changed, 68 insertions(+), 39 deletions(-) diff --git a/include/crypto/internal/poly1305.h b/include/crypto/internal/poly1305.h index e614594f88c1..c60315f47562 100644 --- a/include/crypto/internal/poly1305.h +++ b/include/crypto/internal/poly1305.h @@ -6,9 +6,8 @@ #ifndef _CRYPTO_INTERNAL_POLY1305_H #define _CRYPTO_INTERNAL_POLY1305_H -#include -#include #include +#include /* * Poly1305 core functions. These only accept whole blocks; the caller must @@ -31,4 +30,29 @@ void poly1305_core_blocks(struct poly1305_state *state, void poly1305_core_emit(const struct poly1305_state *state, const u32 nonce[4], void *dst); +void poly1305_block_init_arch(struct poly1305_block_state *state, + const u8 raw_key[POLY1305_BLOCK_SIZE]); +void poly1305_block_init_generic(struct poly1305_block_state *state, + const u8 raw_key[POLY1305_BLOCK_SIZE]); +void poly1305_blocks_arch(struct poly1305_block_state *state, const u8 *src, + unsigned int len, u32 padbit); + +static inline void poly1305_blocks_generic(struct poly1305_block_state *state, + const u8 *src, unsigned int len, + u32 padbit) +{ + poly1305_core_blocks(&state->h, &state->core_r, src, + len / POLY1305_BLOCK_SIZE, padbit); +} + +void poly1305_emit_arch(const struct poly1305_state *state, + u8 digest[POLY1305_DIGEST_SIZE], const u32 nonce[4]); + +static inline void poly1305_emit_generic(const struct poly1305_state *state, + u8 digest[POLY1305_DIGEST_SIZE], + const u32 nonce[4]) +{ + poly1305_core_emit(state, nonce, digest); +} + #endif diff --git a/include/crypto/poly1305.h b/include/crypto/poly1305.h index 6e21ec2d1dc2..027d74842cd5 100644 --- a/include/crypto/poly1305.h +++ b/include/crypto/poly1305.h @@ -7,7 +7,6 @@ #define _CRYPTO_POLY1305_H #include -#include #define POLY1305_BLOCK_SIZE 16 #define POLY1305_KEY_SIZE 32 @@ -38,6 +37,17 @@ struct poly1305_state { }; }; +/* Combined state for block function. */ +struct poly1305_block_state { + /* accumulator */ + struct poly1305_state h; + /* key */ + union { + struct poly1305_key opaque_r[CONFIG_CRYPTO_LIB_POLY1305_RSIZE]; + struct poly1305_core_key core_r; + }; +}; + struct poly1305_desc_ctx { /* partial buffer */ u8 buf[POLY1305_BLOCK_SIZE]; @@ -45,12 +55,15 @@ struct poly1305_desc_ctx { unsigned int buflen; /* finalize key */ u32 s[4]; - /* accumulator */ - struct poly1305_state h; - /* key */ union { - struct poly1305_key opaque_r[CONFIG_CRYPTO_LIB_POLY1305_RSIZE]; - struct poly1305_core_key core_r; + struct { + struct poly1305_state h; + union { + struct poly1305_key opaque_r[CONFIG_CRYPTO_LIB_POLY1305_RSIZE]; + struct poly1305_core_key core_r; + }; + }; + struct poly1305_block_state state; }; }; diff --git a/lib/crypto/poly1305.c b/lib/crypto/poly1305.c index b633b043f0f6..9fec64a599c1 100644 --- a/lib/crypto/poly1305.c +++ b/lib/crypto/poly1305.c @@ -7,54 +7,45 @@ * Based on public domain code by Andrew Moon and Daniel J. Bernstein. */ +#include #include #include #include +#include #include +void poly1305_block_init_generic(struct poly1305_block_state *desc, + const u8 raw_key[POLY1305_BLOCK_SIZE]) +{ + poly1305_core_init(&desc->h); + poly1305_core_setkey(&desc->core_r, raw_key); +} +EXPORT_SYMBOL_GPL(poly1305_block_init_generic); + void poly1305_init_generic(struct poly1305_desc_ctx *desc, const u8 key[POLY1305_KEY_SIZE]) { - poly1305_core_setkey(&desc->core_r, key); desc->s[0] = get_unaligned_le32(key + 16); desc->s[1] = get_unaligned_le32(key + 20); desc->s[2] = get_unaligned_le32(key + 24); desc->s[3] = get_unaligned_le32(key + 28); - poly1305_core_init(&desc->h); desc->buflen = 0; + poly1305_block_init_generic(&desc->state, key); } EXPORT_SYMBOL_GPL(poly1305_init_generic); +static inline void poly1305_blocks(struct poly1305_block_state *state, + const u8 *src, unsigned int len) +{ + poly1305_blocks_generic(state, src, len, 1); +} + void poly1305_update_generic(struct poly1305_desc_ctx *desc, const u8 *src, unsigned int nbytes) { - unsigned int bytes; - - if (unlikely(desc->buflen)) { - bytes = min(nbytes, POLY1305_BLOCK_SIZE - desc->buflen); - memcpy(desc->buf + desc->buflen, src, bytes); - src += bytes; - nbytes -= bytes; - desc->buflen += bytes; - - if (desc->buflen == POLY1305_BLOCK_SIZE) { - poly1305_core_blocks(&desc->h, &desc->core_r, desc->buf, - 1, 1); - desc->buflen = 0; - } - } - - if (likely(nbytes >= POLY1305_BLOCK_SIZE)) { - poly1305_core_blocks(&desc->h, &desc->core_r, src, - nbytes / POLY1305_BLOCK_SIZE, 1); - src += nbytes - (nbytes % POLY1305_BLOCK_SIZE); - nbytes %= POLY1305_BLOCK_SIZE; - } - - if (unlikely(nbytes)) { - desc->buflen = nbytes; - memcpy(desc->buf, src, nbytes); - } + desc->buflen = BLOCK_HASH_UPDATE(poly1305_blocks, &desc->state, + src, nbytes, POLY1305_BLOCK_SIZE, + desc->buf, desc->buflen); } EXPORT_SYMBOL_GPL(poly1305_update_generic); @@ -64,10 +55,11 @@ void poly1305_final_generic(struct poly1305_desc_ctx *desc, u8 *dst) desc->buf[desc->buflen++] = 1; memset(desc->buf + desc->buflen, 0, POLY1305_BLOCK_SIZE - desc->buflen); - poly1305_core_blocks(&desc->h, &desc->core_r, desc->buf, 1, 0); + poly1305_blocks_generic(&desc->state, desc->buf, + POLY1305_BLOCK_SIZE, 0); } - poly1305_core_emit(&desc->h, desc->s, dst); + poly1305_emit_generic(&desc->h, dst, desc->s); *desc = (struct poly1305_desc_ctx){}; } EXPORT_SYMBOL_GPL(poly1305_final_generic); From 773426f4771bdd82ac5c834bf4c1775315c73a46 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Mon, 28 Apr 2025 12:56:09 +0800 Subject: [PATCH 251/382] crypto: arm/poly1305 - Add block-only interface Add block-only interface. Also remove the unnecessary SIMD fallback path. Signed-off-by: Herbert Xu --- arch/arm/lib/crypto/poly1305-armv4.pl | 4 +- arch/arm/lib/crypto/poly1305-glue.c | 78 +++++++++++++++------------ 2 files changed, 47 insertions(+), 35 deletions(-) diff --git a/arch/arm/lib/crypto/poly1305-armv4.pl b/arch/arm/lib/crypto/poly1305-armv4.pl index 6d79498d3115..d57c6e2fc84a 100644 --- a/arch/arm/lib/crypto/poly1305-armv4.pl +++ b/arch/arm/lib/crypto/poly1305-armv4.pl @@ -43,9 +43,9 @@ $code.=<<___; #else # define __ARM_ARCH__ __LINUX_ARM_ARCH__ # define __ARM_MAX_ARCH__ __LINUX_ARM_ARCH__ -# define poly1305_init poly1305_init_arm +# define poly1305_init poly1305_block_init_arch # define poly1305_blocks poly1305_blocks_arm -# define poly1305_emit poly1305_emit_arm +# define poly1305_emit poly1305_emit_arch .globl poly1305_blocks_neon #endif diff --git a/arch/arm/lib/crypto/poly1305-glue.c b/arch/arm/lib/crypto/poly1305-glue.c index 42d0ebde1ae1..3ee16048ec7c 100644 --- a/arch/arm/lib/crypto/poly1305-glue.c +++ b/arch/arm/lib/crypto/poly1305-glue.c @@ -7,20 +7,29 @@ #include #include -#include -#include -#include +#include #include #include +#include #include +#include #include -void poly1305_init_arm(void *state, const u8 *key); -void poly1305_blocks_arm(void *state, const u8 *src, u32 len, u32 hibit); -void poly1305_blocks_neon(void *state, const u8 *src, u32 len, u32 hibit); -void poly1305_emit_arm(void *state, u8 *digest, const u32 *nonce); +asmlinkage void poly1305_block_init_arch( + struct poly1305_block_state *state, + const u8 raw_key[POLY1305_BLOCK_SIZE]); +EXPORT_SYMBOL_GPL(poly1305_block_init_arch); +asmlinkage void poly1305_blocks_arm(struct poly1305_block_state *state, + const u8 *src, u32 len, u32 hibit); +asmlinkage void poly1305_blocks_neon(struct poly1305_block_state *state, + const u8 *src, u32 len, u32 hibit); +asmlinkage void poly1305_emit_arch(const struct poly1305_state *state, + u8 digest[POLY1305_DIGEST_SIZE], + const u32 nonce[4]); +EXPORT_SYMBOL_GPL(poly1305_emit_arch); -void __weak poly1305_blocks_neon(void *state, const u8 *src, u32 len, u32 hibit) +void __weak poly1305_blocks_neon(struct poly1305_block_state *state, + const u8 *src, u32 len, u32 hibit) { } @@ -28,21 +37,39 @@ static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon); void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 key[POLY1305_KEY_SIZE]) { - poly1305_init_arm(&dctx->h, key); dctx->s[0] = get_unaligned_le32(key + 16); dctx->s[1] = get_unaligned_le32(key + 20); dctx->s[2] = get_unaligned_le32(key + 24); dctx->s[3] = get_unaligned_le32(key + 28); dctx->buflen = 0; + poly1305_block_init_arch(&dctx->state, key); } EXPORT_SYMBOL(poly1305_init_arch); +void poly1305_blocks_arch(struct poly1305_block_state *state, const u8 *src, + unsigned int len, u32 padbit) +{ + len = round_down(len, POLY1305_BLOCK_SIZE); + if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && + static_branch_likely(&have_neon)) { + do { + unsigned int todo = min_t(unsigned int, len, SZ_4K); + + kernel_neon_begin(); + poly1305_blocks_neon(state, src, todo, padbit); + kernel_neon_end(); + + len -= todo; + src += todo; + } while (len); + } else + poly1305_blocks_arm(state, src, len, padbit); +} +EXPORT_SYMBOL_GPL(poly1305_blocks_arch); + void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src, unsigned int nbytes) { - bool do_neon = IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && - crypto_simd_usable(); - if (unlikely(dctx->buflen)) { u32 bytes = min(nbytes, POLY1305_BLOCK_SIZE - dctx->buflen); @@ -52,30 +79,15 @@ void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src, dctx->buflen += bytes; if (dctx->buflen == POLY1305_BLOCK_SIZE) { - poly1305_blocks_arm(&dctx->h, dctx->buf, - POLY1305_BLOCK_SIZE, 1); + poly1305_blocks_arch(&dctx->state, dctx->buf, + POLY1305_BLOCK_SIZE, 1); dctx->buflen = 0; } } if (likely(nbytes >= POLY1305_BLOCK_SIZE)) { - unsigned int len = round_down(nbytes, POLY1305_BLOCK_SIZE); - - if (static_branch_likely(&have_neon) && do_neon) { - do { - unsigned int todo = min_t(unsigned int, len, SZ_4K); - - kernel_neon_begin(); - poly1305_blocks_neon(&dctx->h, src, todo, 1); - kernel_neon_end(); - - len -= todo; - src += todo; - } while (len); - } else { - poly1305_blocks_arm(&dctx->h, src, len, 1); - src += len; - } + poly1305_blocks_arch(&dctx->state, src, nbytes, 1); + src += round_down(nbytes, POLY1305_BLOCK_SIZE); nbytes %= POLY1305_BLOCK_SIZE; } @@ -92,10 +104,10 @@ void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst) dctx->buf[dctx->buflen++] = 1; memset(dctx->buf + dctx->buflen, 0, POLY1305_BLOCK_SIZE - dctx->buflen); - poly1305_blocks_arm(&dctx->h, dctx->buf, POLY1305_BLOCK_SIZE, 0); + poly1305_blocks_arch(&dctx->state, dctx->buf, POLY1305_BLOCK_SIZE, 0); } - poly1305_emit_arm(&dctx->h, dst, dctx->s); + poly1305_emit_arch(&dctx->h, dst, dctx->s); *dctx = (struct poly1305_desc_ctx){}; } EXPORT_SYMBOL(poly1305_final_arch); From a59e5468a921937cb7317892779c67046ad9f5cc Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Mon, 28 Apr 2025 12:56:11 +0800 Subject: [PATCH 252/382] crypto: arm64/poly1305 - Add block-only interface Add block-only interface. Also remove the unnecessary SIMD fallback path. Signed-off-by: Herbert Xu --- arch/arm64/lib/crypto/Makefile | 3 +- arch/arm64/lib/crypto/poly1305-glue.c | 71 ++++++++++++++++----------- 2 files changed, 45 insertions(+), 29 deletions(-) diff --git a/arch/arm64/lib/crypto/Makefile b/arch/arm64/lib/crypto/Makefile index ac624c3effda..6207088397a7 100644 --- a/arch/arm64/lib/crypto/Makefile +++ b/arch/arm64/lib/crypto/Makefile @@ -5,7 +5,8 @@ chacha-neon-y := chacha-neon-core.o chacha-neon-glue.o obj-$(CONFIG_CRYPTO_POLY1305_NEON) += poly1305-neon.o poly1305-neon-y := poly1305-core.o poly1305-glue.o -AFLAGS_poly1305-core.o += -Dpoly1305_init=poly1305_init_arm64 +AFLAGS_poly1305-core.o += -Dpoly1305_init=poly1305_block_init_arch +AFLAGS_poly1305-core.o += -Dpoly1305_emit=poly1305_emit_arch quiet_cmd_perlasm = PERLASM $@ cmd_perlasm = $(PERL) $(<) void $(@) diff --git a/arch/arm64/lib/crypto/poly1305-glue.c b/arch/arm64/lib/crypto/poly1305-glue.c index 906970dd5373..d66a820e32d5 100644 --- a/arch/arm64/lib/crypto/poly1305-glue.c +++ b/arch/arm64/lib/crypto/poly1305-glue.c @@ -7,32 +7,60 @@ #include #include -#include -#include -#include +#include #include #include +#include #include +#include #include -asmlinkage void poly1305_init_arm64(void *state, const u8 *key); -asmlinkage void poly1305_blocks(void *state, const u8 *src, u32 len, u32 hibit); -asmlinkage void poly1305_blocks_neon(void *state, const u8 *src, u32 len, u32 hibit); -asmlinkage void poly1305_emit(void *state, u8 *digest, const u32 *nonce); +asmlinkage void poly1305_block_init_arch( + struct poly1305_block_state *state, + const u8 raw_key[POLY1305_BLOCK_SIZE]); +EXPORT_SYMBOL_GPL(poly1305_block_init_arch); +asmlinkage void poly1305_blocks(struct poly1305_block_state *state, + const u8 *src, u32 len, u32 hibit); +asmlinkage void poly1305_blocks_neon(struct poly1305_block_state *state, + const u8 *src, u32 len, u32 hibit); +asmlinkage void poly1305_emit_arch(const struct poly1305_state *state, + u8 digest[POLY1305_DIGEST_SIZE], + const u32 nonce[4]); +EXPORT_SYMBOL_GPL(poly1305_emit_arch); static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon); void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 key[POLY1305_KEY_SIZE]) { - poly1305_init_arm64(&dctx->h, key); dctx->s[0] = get_unaligned_le32(key + 16); dctx->s[1] = get_unaligned_le32(key + 20); dctx->s[2] = get_unaligned_le32(key + 24); dctx->s[3] = get_unaligned_le32(key + 28); dctx->buflen = 0; + poly1305_block_init_arch(&dctx->state, key); } EXPORT_SYMBOL(poly1305_init_arch); +void poly1305_blocks_arch(struct poly1305_block_state *state, const u8 *src, + unsigned int len, u32 padbit) +{ + len = round_down(len, POLY1305_BLOCK_SIZE); + if (static_branch_likely(&have_neon)) { + do { + unsigned int todo = min_t(unsigned int, len, SZ_4K); + + kernel_neon_begin(); + poly1305_blocks_neon(state, src, todo, 1); + kernel_neon_end(); + + len -= todo; + src += todo; + } while (len); + } else + poly1305_blocks(state, src, len, 1); +} +EXPORT_SYMBOL_GPL(poly1305_blocks_arch); + void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src, unsigned int nbytes) { @@ -45,29 +73,15 @@ void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src, dctx->buflen += bytes; if (dctx->buflen == POLY1305_BLOCK_SIZE) { - poly1305_blocks(&dctx->h, dctx->buf, POLY1305_BLOCK_SIZE, 1); + poly1305_blocks_arch(&dctx->state, dctx->buf, + POLY1305_BLOCK_SIZE, 1); dctx->buflen = 0; } } if (likely(nbytes >= POLY1305_BLOCK_SIZE)) { - unsigned int len = round_down(nbytes, POLY1305_BLOCK_SIZE); - - if (static_branch_likely(&have_neon) && crypto_simd_usable()) { - do { - unsigned int todo = min_t(unsigned int, len, SZ_4K); - - kernel_neon_begin(); - poly1305_blocks_neon(&dctx->h, src, todo, 1); - kernel_neon_end(); - - len -= todo; - src += todo; - } while (len); - } else { - poly1305_blocks(&dctx->h, src, len, 1); - src += len; - } + poly1305_blocks_arch(&dctx->state, src, nbytes, 1); + src += round_down(nbytes, POLY1305_BLOCK_SIZE); nbytes %= POLY1305_BLOCK_SIZE; } @@ -84,10 +98,11 @@ void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst) dctx->buf[dctx->buflen++] = 1; memset(dctx->buf + dctx->buflen, 0, POLY1305_BLOCK_SIZE - dctx->buflen); - poly1305_blocks(&dctx->h, dctx->buf, POLY1305_BLOCK_SIZE, 0); + poly1305_blocks_arch(&dctx->state, dctx->buf, + POLY1305_BLOCK_SIZE, 0); } - poly1305_emit(&dctx->h, dst, dctx->s); + poly1305_emit_arch(&dctx->h, dst, dctx->s); memzero_explicit(dctx, sizeof(*dctx)); } EXPORT_SYMBOL(poly1305_final_arch); From ffe5ca295d757d912b226239df17396c3cd8dbca Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Mon, 28 Apr 2025 12:56:14 +0800 Subject: [PATCH 253/382] crypto: mips/poly1305 - Add block-only interface Add block-only interface. Signed-off-by: Herbert Xu --- arch/mips/lib/crypto/poly1305-glue.c | 29 ++++++++++++++++++--------- arch/mips/lib/crypto/poly1305-mips.pl | 12 +++++------ 2 files changed, 26 insertions(+), 15 deletions(-) diff --git a/arch/mips/lib/crypto/poly1305-glue.c b/arch/mips/lib/crypto/poly1305-glue.c index 576e7a58e0b1..2fea4cacfe27 100644 --- a/arch/mips/lib/crypto/poly1305-glue.c +++ b/arch/mips/lib/crypto/poly1305-glue.c @@ -5,23 +5,33 @@ * Copyright (C) 2019 Linaro Ltd. */ -#include +#include #include +#include #include +#include #include -asmlinkage void poly1305_init_mips(void *state, const u8 *key); -asmlinkage void poly1305_blocks_mips(void *state, const u8 *src, u32 len, u32 hibit); -asmlinkage void poly1305_emit_mips(void *state, u8 *digest, const u32 *nonce); +asmlinkage void poly1305_block_init_arch( + struct poly1305_block_state *state, + const u8 raw_key[POLY1305_BLOCK_SIZE]); +EXPORT_SYMBOL_GPL(poly1305_block_init_arch); +asmlinkage void poly1305_blocks_arch(struct poly1305_block_state *state, + const u8 *src, u32 len, u32 hibit); +EXPORT_SYMBOL_GPL(poly1305_blocks_arch); +asmlinkage void poly1305_emit_arch(const struct poly1305_state *state, + u8 digest[POLY1305_DIGEST_SIZE], + const u32 nonce[4]); +EXPORT_SYMBOL_GPL(poly1305_emit_arch); void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 key[POLY1305_KEY_SIZE]) { - poly1305_init_mips(&dctx->h, key); dctx->s[0] = get_unaligned_le32(key + 16); dctx->s[1] = get_unaligned_le32(key + 20); dctx->s[2] = get_unaligned_le32(key + 24); dctx->s[3] = get_unaligned_le32(key + 28); dctx->buflen = 0; + poly1305_block_init_arch(&dctx->state, key); } EXPORT_SYMBOL(poly1305_init_arch); @@ -37,7 +47,7 @@ void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src, dctx->buflen += bytes; if (dctx->buflen == POLY1305_BLOCK_SIZE) { - poly1305_blocks_mips(&dctx->h, dctx->buf, + poly1305_blocks_arch(&dctx->state, dctx->buf, POLY1305_BLOCK_SIZE, 1); dctx->buflen = 0; } @@ -46,7 +56,7 @@ void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src, if (likely(nbytes >= POLY1305_BLOCK_SIZE)) { unsigned int len = round_down(nbytes, POLY1305_BLOCK_SIZE); - poly1305_blocks_mips(&dctx->h, src, len, 1); + poly1305_blocks_arch(&dctx->state, src, len, 1); src += len; nbytes %= POLY1305_BLOCK_SIZE; } @@ -64,10 +74,11 @@ void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst) dctx->buf[dctx->buflen++] = 1; memset(dctx->buf + dctx->buflen, 0, POLY1305_BLOCK_SIZE - dctx->buflen); - poly1305_blocks_mips(&dctx->h, dctx->buf, POLY1305_BLOCK_SIZE, 0); + poly1305_blocks_arch(&dctx->state, dctx->buf, + POLY1305_BLOCK_SIZE, 0); } - poly1305_emit_mips(&dctx->h, dst, dctx->s); + poly1305_emit_arch(&dctx->h, dst, dctx->s); *dctx = (struct poly1305_desc_ctx){}; } EXPORT_SYMBOL(poly1305_final_arch); diff --git a/arch/mips/lib/crypto/poly1305-mips.pl b/arch/mips/lib/crypto/poly1305-mips.pl index b05bab884ed2..399f10c3e385 100644 --- a/arch/mips/lib/crypto/poly1305-mips.pl +++ b/arch/mips/lib/crypto/poly1305-mips.pl @@ -93,9 +93,9 @@ $code.=<<___; #endif #ifdef __KERNEL__ -# define poly1305_init poly1305_init_mips -# define poly1305_blocks poly1305_blocks_mips -# define poly1305_emit poly1305_emit_mips +# define poly1305_init poly1305_block_init_arch +# define poly1305_blocks poly1305_blocks_arch +# define poly1305_emit poly1305_emit_arch #endif #if defined(__MIPSEB__) && !defined(MIPSEB) @@ -565,9 +565,9 @@ $code.=<<___; #endif #ifdef __KERNEL__ -# define poly1305_init poly1305_init_mips -# define poly1305_blocks poly1305_blocks_mips -# define poly1305_emit poly1305_emit_mips +# define poly1305_init poly1305_block_init_arch +# define poly1305_blocks poly1305_blocks_arch +# define poly1305_emit poly1305_emit_arch #endif #if defined(__MIPSEB__) && !defined(MIPSEB) From 14d31979145dbafaeb28a5bc4c90c4db918bb772 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Mon, 28 Apr 2025 12:56:16 +0800 Subject: [PATCH 254/382] crypto: powerpc/poly1305 - Add block-only interface Add block-only interface. Also remove the unnecessary SIMD fallback path. Signed-off-by: Herbert Xu --- arch/powerpc/lib/crypto/poly1305-p10-glue.c | 84 ++++++++++++--------- 1 file changed, 49 insertions(+), 35 deletions(-) diff --git a/arch/powerpc/lib/crypto/poly1305-p10-glue.c b/arch/powerpc/lib/crypto/poly1305-p10-glue.c index 00617f4c58e6..708435beaba6 100644 --- a/arch/powerpc/lib/crypto/poly1305-p10-glue.c +++ b/arch/powerpc/lib/crypto/poly1305-p10-glue.c @@ -4,19 +4,20 @@ * * Copyright 2023- IBM Corp. All rights reserved. */ +#include +#include +#include +#include #include #include -#include -#include -#include -#include +#include #include -#include -#include -asmlinkage void poly1305_p10le_4blocks(void *h, const u8 *m, u32 mlen); -asmlinkage void poly1305_64s(void *h, const u8 *m, u32 mlen, int highbit); -asmlinkage void poly1305_emit_64(void *h, void *s, u8 *dst); +asmlinkage void poly1305_p10le_4blocks(struct poly1305_block_state *state, const u8 *m, u32 mlen); +asmlinkage void poly1305_64s(struct poly1305_block_state *state, const u8 *m, u32 mlen, int highbit); +asmlinkage void poly1305_emit_arch(const struct poly1305_state *state, + u8 digest[POLY1305_DIGEST_SIZE], + const u32 nonce[4]); static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_p10); @@ -32,22 +33,49 @@ static void vsx_end(void) preempt_enable(); } -void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 key[POLY1305_KEY_SIZE]) +void poly1305_block_init_arch(struct poly1305_block_state *dctx, + const u8 raw_key[POLY1305_BLOCK_SIZE]) { if (!static_key_enabled(&have_p10)) - return poly1305_init_generic(dctx, key); + return poly1305_block_init_generic(dctx, raw_key); dctx->h = (struct poly1305_state){}; - dctx->core_r.key.r64[0] = get_unaligned_le64(key + 0); - dctx->core_r.key.r64[1] = get_unaligned_le64(key + 8); + dctx->core_r.key.r64[0] = get_unaligned_le64(raw_key + 0); + dctx->core_r.key.r64[1] = get_unaligned_le64(raw_key + 8); +} +EXPORT_SYMBOL_GPL(poly1305_block_init_arch); + +void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 key[POLY1305_KEY_SIZE]) +{ dctx->s[0] = get_unaligned_le32(key + 16); dctx->s[1] = get_unaligned_le32(key + 20); dctx->s[2] = get_unaligned_le32(key + 24); dctx->s[3] = get_unaligned_le32(key + 28); dctx->buflen = 0; + poly1305_block_init_arch(&dctx->state, key); } EXPORT_SYMBOL(poly1305_init_arch); +void poly1305_blocks_arch(struct poly1305_block_state *state, const u8 *src, + unsigned int len, u32 padbit) +{ + if (!static_key_enabled(&have_p10)) + return poly1305_blocks_generic(state, src, len, padbit); + vsx_begin(); + if (len >= POLY1305_BLOCK_SIZE * 4) { + poly1305_p10le_4blocks(state, src, len); + src += len - (len % (POLY1305_BLOCK_SIZE * 4)); + len %= POLY1305_BLOCK_SIZE * 4; + } + while (len >= POLY1305_BLOCK_SIZE) { + poly1305_64s(state, src, POLY1305_BLOCK_SIZE, padbit); + len -= POLY1305_BLOCK_SIZE; + src += POLY1305_BLOCK_SIZE; + } + vsx_end(); +} +EXPORT_SYMBOL_GPL(poly1305_blocks_arch); + void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src, unsigned int srclen) { @@ -64,28 +92,15 @@ void poly1305_update_arch(struct poly1305_desc_ctx *dctx, dctx->buflen += bytes; if (dctx->buflen < POLY1305_BLOCK_SIZE) return; - vsx_begin(); - poly1305_64s(&dctx->h, dctx->buf, POLY1305_BLOCK_SIZE, 1); - vsx_end(); + poly1305_blocks_arch(&dctx->state, dctx->buf, + POLY1305_BLOCK_SIZE, 1); dctx->buflen = 0; } if (likely(srclen >= POLY1305_BLOCK_SIZE)) { - bytes = round_down(srclen, POLY1305_BLOCK_SIZE); - if (crypto_simd_usable() && (srclen >= POLY1305_BLOCK_SIZE*4)) { - vsx_begin(); - poly1305_p10le_4blocks(&dctx->h, src, srclen); - vsx_end(); - src += srclen - (srclen % (POLY1305_BLOCK_SIZE * 4)); - srclen %= POLY1305_BLOCK_SIZE * 4; - } - while (srclen >= POLY1305_BLOCK_SIZE) { - vsx_begin(); - poly1305_64s(&dctx->h, src, POLY1305_BLOCK_SIZE, 1); - vsx_end(); - srclen -= POLY1305_BLOCK_SIZE; - src += POLY1305_BLOCK_SIZE; - } + poly1305_blocks_arch(&dctx->state, src, srclen, 1); + src += srclen - (srclen % POLY1305_BLOCK_SIZE); + srclen %= POLY1305_BLOCK_SIZE; } if (unlikely(srclen)) { @@ -104,12 +119,11 @@ void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst) dctx->buf[dctx->buflen++] = 1; memset(dctx->buf + dctx->buflen, 0, POLY1305_BLOCK_SIZE - dctx->buflen); - vsx_begin(); - poly1305_64s(&dctx->h, dctx->buf, POLY1305_BLOCK_SIZE, 0); - vsx_end(); + poly1305_blocks_arch(&dctx->state, dctx->buf, + POLY1305_BLOCK_SIZE, 0); } - poly1305_emit_64(&dctx->h, &dctx->s, dst); + poly1305_emit_arch(&dctx->h, dst, dctx->s); } EXPORT_SYMBOL(poly1305_final_arch); From 318c53ae02f2abf1542062543741068278780d09 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Mon, 28 Apr 2025 12:56:18 +0800 Subject: [PATCH 255/382] crypto: x86/poly1305 - Add block-only interface Add block-only interface. Also remove the unnecessary SIMD fallback path. Signed-off-by: Herbert Xu --- .../lib/crypto/poly1305-x86_64-cryptogams.pl | 33 +++-- arch/x86/lib/crypto/poly1305_glue.c | 125 +++++++----------- 2 files changed, 71 insertions(+), 87 deletions(-) diff --git a/arch/x86/lib/crypto/poly1305-x86_64-cryptogams.pl b/arch/x86/lib/crypto/poly1305-x86_64-cryptogams.pl index 409ec6955733..501827254fed 100644 --- a/arch/x86/lib/crypto/poly1305-x86_64-cryptogams.pl +++ b/arch/x86/lib/crypto/poly1305-x86_64-cryptogams.pl @@ -118,6 +118,19 @@ sub declare_function() { } } +sub declare_typed_function() { + my ($name, $align, $nargs) = @_; + if($kernel) { + $code .= "SYM_TYPED_FUNC_START($name)\n"; + $code .= ".L$name:\n"; + } else { + $code .= ".globl $name\n"; + $code .= ".type $name,\@function,$nargs\n"; + $code .= ".align $align\n"; + $code .= "$name:\n"; + } +} + sub end_function() { my ($name) = @_; if($kernel) { @@ -128,7 +141,7 @@ sub end_function() { } $code.=<<___ if $kernel; -#include +#include ___ if ($avx) { @@ -236,14 +249,14 @@ ___ $code.=<<___ if (!$kernel); .extern OPENSSL_ia32cap_P -.globl poly1305_init_x86_64 -.hidden poly1305_init_x86_64 +.globl poly1305_block_init_arch +.hidden poly1305_block_init_arch .globl poly1305_blocks_x86_64 .hidden poly1305_blocks_x86_64 .globl poly1305_emit_x86_64 .hidden poly1305_emit_x86_64 ___ -&declare_function("poly1305_init_x86_64", 32, 3); +&declare_typed_function("poly1305_block_init_arch", 32, 3); $code.=<<___; xor %eax,%eax mov %rax,0($ctx) # initialize hash value @@ -298,7 +311,7 @@ $code.=<<___; .Lno_key: RET ___ -&end_function("poly1305_init_x86_64"); +&end_function("poly1305_block_init_arch"); &declare_function("poly1305_blocks_x86_64", 32, 4); $code.=<<___; @@ -4105,9 +4118,9 @@ avx_handler: .section .pdata .align 4 - .rva .LSEH_begin_poly1305_init_x86_64 - .rva .LSEH_end_poly1305_init_x86_64 - .rva .LSEH_info_poly1305_init_x86_64 + .rva .LSEH_begin_poly1305_block_init_arch + .rva .LSEH_end_poly1305_block_init_arch + .rva .LSEH_info_poly1305_block_init_arch .rva .LSEH_begin_poly1305_blocks_x86_64 .rva .LSEH_end_poly1305_blocks_x86_64 @@ -4155,10 +4168,10 @@ ___ $code.=<<___; .section .xdata .align 8 -.LSEH_info_poly1305_init_x86_64: +.LSEH_info_poly1305_block_init_arch: .byte 9,0,0,0 .rva se_handler - .rva .LSEH_begin_poly1305_init_x86_64,.LSEH_begin_poly1305_init_x86_64 + .rva .LSEH_begin_poly1305_block_init_arch,.LSEH_begin_poly1305_block_init_arch .LSEH_info_poly1305_blocks_x86_64: .byte 9,0,0,0 diff --git a/arch/x86/lib/crypto/poly1305_glue.c b/arch/x86/lib/crypto/poly1305_glue.c index cff35ca5822a..d98764ec3b47 100644 --- a/arch/x86/lib/crypto/poly1305_glue.c +++ b/arch/x86/lib/crypto/poly1305_glue.c @@ -3,34 +3,15 @@ * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. */ -#include -#include +#include +#include +#include #include #include #include #include +#include #include -#include -#include - -asmlinkage void poly1305_init_x86_64(void *ctx, - const u8 key[POLY1305_BLOCK_SIZE]); -asmlinkage void poly1305_blocks_x86_64(void *ctx, const u8 *inp, - const size_t len, const u32 padbit); -asmlinkage void poly1305_emit_x86_64(void *ctx, u8 mac[POLY1305_DIGEST_SIZE], - const u32 nonce[4]); -asmlinkage void poly1305_emit_avx(void *ctx, u8 mac[POLY1305_DIGEST_SIZE], - const u32 nonce[4]); -asmlinkage void poly1305_blocks_avx(void *ctx, const u8 *inp, const size_t len, - const u32 padbit); -asmlinkage void poly1305_blocks_avx2(void *ctx, const u8 *inp, const size_t len, - const u32 padbit); -asmlinkage void poly1305_blocks_avx512(void *ctx, const u8 *inp, - const size_t len, const u32 padbit); - -static __ro_after_init DEFINE_STATIC_KEY_FALSE(poly1305_use_avx); -static __ro_after_init DEFINE_STATIC_KEY_FALSE(poly1305_use_avx2); -static __ro_after_init DEFINE_STATIC_KEY_FALSE(poly1305_use_avx512); struct poly1305_arch_internal { union { @@ -45,64 +26,50 @@ struct poly1305_arch_internal { struct { u32 r2, r1, r4, r3; } rn[9]; }; -/* The AVX code uses base 2^26, while the scalar code uses base 2^64. If we hit - * the unfortunate situation of using AVX and then having to go back to scalar - * -- because the user is silly and has called the update function from two - * separate contexts -- then we need to convert back to the original base before - * proceeding. It is possible to reason that the initial reduction below is - * sufficient given the implementation invariants. However, for an avoidance of - * doubt and because this is not performance critical, we do the full reduction - * anyway. Z3 proof of below function: https://xn--4db.cc/ltPtHCKN/py - */ -static void convert_to_base2_64(void *ctx) +asmlinkage void poly1305_block_init_arch( + struct poly1305_block_state *state, + const u8 raw_key[POLY1305_BLOCK_SIZE]); +EXPORT_SYMBOL_GPL(poly1305_block_init_arch); +asmlinkage void poly1305_blocks_x86_64(struct poly1305_arch_internal *ctx, + const u8 *inp, + const size_t len, const u32 padbit); +asmlinkage void poly1305_emit_x86_64(const struct poly1305_state *ctx, + u8 mac[POLY1305_DIGEST_SIZE], + const u32 nonce[4]); +asmlinkage void poly1305_emit_avx(const struct poly1305_state *ctx, + u8 mac[POLY1305_DIGEST_SIZE], + const u32 nonce[4]); +asmlinkage void poly1305_blocks_avx(struct poly1305_arch_internal *ctx, + const u8 *inp, const size_t len, + const u32 padbit); +asmlinkage void poly1305_blocks_avx2(struct poly1305_arch_internal *ctx, + const u8 *inp, const size_t len, + const u32 padbit); +asmlinkage void poly1305_blocks_avx512(struct poly1305_arch_internal *ctx, + const u8 *inp, + const size_t len, const u32 padbit); + +static __ro_after_init DEFINE_STATIC_KEY_FALSE(poly1305_use_avx); +static __ro_after_init DEFINE_STATIC_KEY_FALSE(poly1305_use_avx2); +static __ro_after_init DEFINE_STATIC_KEY_FALSE(poly1305_use_avx512); + +void poly1305_blocks_arch(struct poly1305_block_state *state, const u8 *inp, + unsigned int len, u32 padbit) { - struct poly1305_arch_internal *state = ctx; - u32 cy; - - if (!state->is_base2_26) - return; - - cy = state->h[0] >> 26; state->h[0] &= 0x3ffffff; state->h[1] += cy; - cy = state->h[1] >> 26; state->h[1] &= 0x3ffffff; state->h[2] += cy; - cy = state->h[2] >> 26; state->h[2] &= 0x3ffffff; state->h[3] += cy; - cy = state->h[3] >> 26; state->h[3] &= 0x3ffffff; state->h[4] += cy; - state->hs[0] = ((u64)state->h[2] << 52) | ((u64)state->h[1] << 26) | state->h[0]; - state->hs[1] = ((u64)state->h[4] << 40) | ((u64)state->h[3] << 14) | (state->h[2] >> 12); - state->hs[2] = state->h[4] >> 24; -#define ULT(a, b) ((a ^ ((a ^ b) | ((a - b) ^ b))) >> (sizeof(a) * 8 - 1)) - cy = (state->hs[2] >> 2) + (state->hs[2] & ~3ULL); - state->hs[2] &= 3; - state->hs[0] += cy; - state->hs[1] += (cy = ULT(state->hs[0], cy)); - state->hs[2] += ULT(state->hs[1], cy); -#undef ULT - state->is_base2_26 = 0; -} - -static void poly1305_simd_init(void *ctx, const u8 key[POLY1305_BLOCK_SIZE]) -{ - poly1305_init_x86_64(ctx, key); -} - -static void poly1305_simd_blocks(void *ctx, const u8 *inp, size_t len, - const u32 padbit) -{ - struct poly1305_arch_internal *state = ctx; + struct poly1305_arch_internal *ctx = + container_of(&state->h.h, struct poly1305_arch_internal, h); /* SIMD disables preemption, so relax after processing each page. */ BUILD_BUG_ON(SZ_4K < POLY1305_BLOCK_SIZE || SZ_4K % POLY1305_BLOCK_SIZE); - if (!static_branch_likely(&poly1305_use_avx) || - (len < (POLY1305_BLOCK_SIZE * 18) && !state->is_base2_26) || - !crypto_simd_usable()) { - convert_to_base2_64(ctx); + if (!static_branch_likely(&poly1305_use_avx)) { poly1305_blocks_x86_64(ctx, inp, len, padbit); return; } do { - const size_t bytes = min_t(size_t, len, SZ_4K); + const unsigned int bytes = min(len, SZ_4K); kernel_fpu_begin(); if (static_branch_likely(&poly1305_use_avx512)) @@ -117,24 +84,26 @@ static void poly1305_simd_blocks(void *ctx, const u8 *inp, size_t len, inp += bytes; } while (len); } +EXPORT_SYMBOL_GPL(poly1305_blocks_arch); -static void poly1305_simd_emit(void *ctx, u8 mac[POLY1305_DIGEST_SIZE], - const u32 nonce[4]) +void poly1305_emit_arch(const struct poly1305_state *ctx, + u8 mac[POLY1305_DIGEST_SIZE], const u32 nonce[4]) { if (!static_branch_likely(&poly1305_use_avx)) poly1305_emit_x86_64(ctx, mac, nonce); else poly1305_emit_avx(ctx, mac, nonce); } +EXPORT_SYMBOL_GPL(poly1305_emit_arch); void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 key[POLY1305_KEY_SIZE]) { - poly1305_simd_init(&dctx->h, key); dctx->s[0] = get_unaligned_le32(&key[16]); dctx->s[1] = get_unaligned_le32(&key[20]); dctx->s[2] = get_unaligned_le32(&key[24]); dctx->s[3] = get_unaligned_le32(&key[28]); dctx->buflen = 0; + poly1305_block_init_arch(&dctx->state, key); } EXPORT_SYMBOL(poly1305_init_arch); @@ -151,14 +120,15 @@ void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src, dctx->buflen += bytes; if (dctx->buflen == POLY1305_BLOCK_SIZE) { - poly1305_simd_blocks(&dctx->h, dctx->buf, POLY1305_BLOCK_SIZE, 1); + poly1305_blocks_arch(&dctx->state, dctx->buf, + POLY1305_BLOCK_SIZE, 1); dctx->buflen = 0; } } if (likely(srclen >= POLY1305_BLOCK_SIZE)) { bytes = round_down(srclen, POLY1305_BLOCK_SIZE); - poly1305_simd_blocks(&dctx->h, src, bytes, 1); + poly1305_blocks_arch(&dctx->state, src, bytes, 1); src += bytes; srclen -= bytes; } @@ -176,10 +146,11 @@ void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst) dctx->buf[dctx->buflen++] = 1; memset(dctx->buf + dctx->buflen, 0, POLY1305_BLOCK_SIZE - dctx->buflen); - poly1305_simd_blocks(&dctx->h, dctx->buf, POLY1305_BLOCK_SIZE, 0); + poly1305_blocks_arch(&dctx->state, dctx->buf, + POLY1305_BLOCK_SIZE, 0); } - poly1305_simd_emit(&dctx->h, dst, dctx->s); + poly1305_emit_arch(&dctx->h, dst, dctx->s); memzero_explicit(dctx, sizeof(*dctx)); } EXPORT_SYMBOL(poly1305_final_arch); From a298765e28adaea199f722142c10dae7e24dedf8 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Mon, 28 Apr 2025 12:56:21 +0800 Subject: [PATCH 256/382] crypto: chacha20poly1305 - Use lib/crypto poly1305 Since the poly1305 algorithm is fixed, there is no point in going through the Crypto API for it. Use the lib/crypto poly1305 interface instead. For compatiblity keep the poly1305 parameter in the algorithm name. Signed-off-by: Herbert Xu --- crypto/Kconfig | 2 +- crypto/chacha20poly1305.c | 323 ++++++++------------------------------ 2 files changed, 67 insertions(+), 258 deletions(-) diff --git a/crypto/Kconfig b/crypto/Kconfig index 9878286d1d68..f87e2a26d2dd 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -784,8 +784,8 @@ config CRYPTO_AEGIS128_SIMD config CRYPTO_CHACHA20POLY1305 tristate "ChaCha20-Poly1305" select CRYPTO_CHACHA20 - select CRYPTO_POLY1305 select CRYPTO_AEAD + select CRYPTO_LIB_POLY1305 select CRYPTO_MANAGER help ChaCha20 stream cipher and Poly1305 authenticator combined diff --git a/crypto/chacha20poly1305.c b/crypto/chacha20poly1305.c index d740849f1c19..b29f66ba1e2f 100644 --- a/crypto/chacha20poly1305.c +++ b/crypto/chacha20poly1305.c @@ -12,36 +12,23 @@ #include #include #include -#include #include +#include #include +#include struct chachapoly_instance_ctx { struct crypto_skcipher_spawn chacha; - struct crypto_ahash_spawn poly; unsigned int saltlen; }; struct chachapoly_ctx { struct crypto_skcipher *chacha; - struct crypto_ahash *poly; /* key bytes we use for the ChaCha20 IV */ unsigned int saltlen; u8 salt[] __counted_by(saltlen); }; -struct poly_req { - /* zero byte padding for AD/ciphertext, as needed */ - u8 pad[POLY1305_BLOCK_SIZE]; - /* tail data with AD/ciphertext lengths */ - struct { - __le64 assoclen; - __le64 cryptlen; - } tail; - struct scatterlist src[1]; - struct ahash_request req; /* must be last member */ -}; - struct chacha_req { u8 iv[CHACHA_IV_SIZE]; struct scatterlist src[1]; @@ -62,7 +49,6 @@ struct chachapoly_req_ctx { /* request flags, with MAY_SLEEP cleared if needed */ u32 flags; union { - struct poly_req poly; struct chacha_req chacha; } u; }; @@ -105,16 +91,6 @@ static int poly_verify_tag(struct aead_request *req) return 0; } -static int poly_copy_tag(struct aead_request *req) -{ - struct chachapoly_req_ctx *rctx = aead_request_ctx(req); - - scatterwalk_map_and_copy(rctx->tag, req->dst, - req->assoclen + rctx->cryptlen, - sizeof(rctx->tag), 1); - return 0; -} - static void chacha_decrypt_done(void *data, int err) { async_done_continue(data, err, poly_verify_tag); @@ -151,210 +127,76 @@ skip: return poly_verify_tag(req); } -static int poly_tail_continue(struct aead_request *req) +static int poly_hash(struct aead_request *req) { struct chachapoly_req_ctx *rctx = aead_request_ctx(req); + const void *zp = page_address(ZERO_PAGE(0)); + struct scatterlist *sg = req->src; + struct poly1305_desc_ctx desc; + struct scatter_walk walk; + struct { + union { + struct { + __le64 assoclen; + __le64 cryptlen; + }; + u8 u8[16]; + }; + } tail; + unsigned int padlen; + unsigned int total; + + if (sg != req->dst) + memcpy_sglist(req->dst, sg, req->assoclen); if (rctx->cryptlen == req->cryptlen) /* encrypting */ - return poly_copy_tag(req); + sg = req->dst; - return chacha_decrypt(req); -} + poly1305_init(&desc, rctx->key); + scatterwalk_start(&walk, sg); -static void poly_tail_done(void *data, int err) -{ - async_done_continue(data, err, poly_tail_continue); -} + total = rctx->assoclen; + while (total) { + unsigned int n = scatterwalk_next(&walk, total); -static int poly_tail(struct aead_request *req) -{ - struct crypto_aead *tfm = crypto_aead_reqtfm(req); - struct chachapoly_ctx *ctx = crypto_aead_ctx(tfm); - struct chachapoly_req_ctx *rctx = aead_request_ctx(req); - struct poly_req *preq = &rctx->u.poly; - int err; - - preq->tail.assoclen = cpu_to_le64(rctx->assoclen); - preq->tail.cryptlen = cpu_to_le64(rctx->cryptlen); - sg_init_one(preq->src, &preq->tail, sizeof(preq->tail)); - - ahash_request_set_callback(&preq->req, rctx->flags, - poly_tail_done, req); - ahash_request_set_tfm(&preq->req, ctx->poly); - ahash_request_set_crypt(&preq->req, preq->src, - rctx->tag, sizeof(preq->tail)); - - err = crypto_ahash_finup(&preq->req); - if (err) - return err; - - return poly_tail_continue(req); -} - -static void poly_cipherpad_done(void *data, int err) -{ - async_done_continue(data, err, poly_tail); -} - -static int poly_cipherpad(struct aead_request *req) -{ - struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); - struct chachapoly_req_ctx *rctx = aead_request_ctx(req); - struct poly_req *preq = &rctx->u.poly; - unsigned int padlen; - int err; - - padlen = -rctx->cryptlen % POLY1305_BLOCK_SIZE; - memset(preq->pad, 0, sizeof(preq->pad)); - sg_init_one(preq->src, preq->pad, padlen); - - ahash_request_set_callback(&preq->req, rctx->flags, - poly_cipherpad_done, req); - ahash_request_set_tfm(&preq->req, ctx->poly); - ahash_request_set_crypt(&preq->req, preq->src, NULL, padlen); - - err = crypto_ahash_update(&preq->req); - if (err) - return err; - - return poly_tail(req); -} - -static void poly_cipher_done(void *data, int err) -{ - async_done_continue(data, err, poly_cipherpad); -} - -static int poly_cipher(struct aead_request *req) -{ - struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); - struct chachapoly_req_ctx *rctx = aead_request_ctx(req); - struct poly_req *preq = &rctx->u.poly; - struct scatterlist *crypt = req->src; - int err; - - if (rctx->cryptlen == req->cryptlen) /* encrypting */ - crypt = req->dst; - - crypt = scatterwalk_ffwd(rctx->src, crypt, req->assoclen); - - ahash_request_set_callback(&preq->req, rctx->flags, - poly_cipher_done, req); - ahash_request_set_tfm(&preq->req, ctx->poly); - ahash_request_set_crypt(&preq->req, crypt, NULL, rctx->cryptlen); - - err = crypto_ahash_update(&preq->req); - if (err) - return err; - - return poly_cipherpad(req); -} - -static void poly_adpad_done(void *data, int err) -{ - async_done_continue(data, err, poly_cipher); -} - -static int poly_adpad(struct aead_request *req) -{ - struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); - struct chachapoly_req_ctx *rctx = aead_request_ctx(req); - struct poly_req *preq = &rctx->u.poly; - unsigned int padlen; - int err; + poly1305_update(&desc, walk.addr, n); + scatterwalk_done_src(&walk, n); + total -= n; + } padlen = -rctx->assoclen % POLY1305_BLOCK_SIZE; - memset(preq->pad, 0, sizeof(preq->pad)); - sg_init_one(preq->src, preq->pad, padlen); + poly1305_update(&desc, zp, padlen); - ahash_request_set_callback(&preq->req, rctx->flags, - poly_adpad_done, req); - ahash_request_set_tfm(&preq->req, ctx->poly); - ahash_request_set_crypt(&preq->req, preq->src, NULL, padlen); + scatterwalk_skip(&walk, req->assoclen - rctx->assoclen); - err = crypto_ahash_update(&preq->req); - if (err) - return err; + total = rctx->cryptlen; + while (total) { + unsigned int n = scatterwalk_next(&walk, total); - return poly_cipher(req); -} + poly1305_update(&desc, walk.addr, n); + scatterwalk_done_src(&walk, n); + total -= n; + } -static void poly_ad_done(void *data, int err) -{ - async_done_continue(data, err, poly_adpad); -} + padlen = -rctx->cryptlen % POLY1305_BLOCK_SIZE; + poly1305_update(&desc, zp, padlen); -static int poly_ad(struct aead_request *req) -{ - struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); - struct chachapoly_req_ctx *rctx = aead_request_ctx(req); - struct poly_req *preq = &rctx->u.poly; - int err; + tail.assoclen = cpu_to_le64(rctx->assoclen); + tail.cryptlen = cpu_to_le64(rctx->cryptlen); + poly1305_update(&desc, tail.u8, sizeof(tail)); + memzero_explicit(&tail, sizeof(tail)); + poly1305_final(&desc, rctx->tag); - ahash_request_set_callback(&preq->req, rctx->flags, - poly_ad_done, req); - ahash_request_set_tfm(&preq->req, ctx->poly); - ahash_request_set_crypt(&preq->req, req->src, NULL, rctx->assoclen); + if (rctx->cryptlen != req->cryptlen) + return chacha_decrypt(req); - err = crypto_ahash_update(&preq->req); - if (err) - return err; - - return poly_adpad(req); -} - -static void poly_setkey_done(void *data, int err) -{ - async_done_continue(data, err, poly_ad); -} - -static int poly_setkey(struct aead_request *req) -{ - struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); - struct chachapoly_req_ctx *rctx = aead_request_ctx(req); - struct poly_req *preq = &rctx->u.poly; - int err; - - sg_init_one(preq->src, rctx->key, sizeof(rctx->key)); - - ahash_request_set_callback(&preq->req, rctx->flags, - poly_setkey_done, req); - ahash_request_set_tfm(&preq->req, ctx->poly); - ahash_request_set_crypt(&preq->req, preq->src, NULL, sizeof(rctx->key)); - - err = crypto_ahash_update(&preq->req); - if (err) - return err; - - return poly_ad(req); -} - -static void poly_init_done(void *data, int err) -{ - async_done_continue(data, err, poly_setkey); -} - -static int poly_init(struct aead_request *req) -{ - struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); - struct chachapoly_req_ctx *rctx = aead_request_ctx(req); - struct poly_req *preq = &rctx->u.poly; - int err; - - ahash_request_set_callback(&preq->req, rctx->flags, - poly_init_done, req); - ahash_request_set_tfm(&preq->req, ctx->poly); - - err = crypto_ahash_init(&preq->req); - if (err) - return err; - - return poly_setkey(req); + memcpy_to_scatterwalk(&walk, rctx->tag, sizeof(rctx->tag)); + return 0; } static void poly_genkey_done(void *data, int err) { - async_done_continue(data, err, poly_init); + async_done_continue(data, err, poly_hash); } static int poly_genkey(struct aead_request *req) @@ -388,7 +230,7 @@ static int poly_genkey(struct aead_request *req) if (err) return err; - return poly_init(req); + return poly_hash(req); } static void chacha_encrypt_done(void *data, int err) @@ -437,14 +279,7 @@ static int chachapoly_encrypt(struct aead_request *req) /* encrypt call chain: * - chacha_encrypt/done() * - poly_genkey/done() - * - poly_init/done() - * - poly_setkey/done() - * - poly_ad/done() - * - poly_adpad/done() - * - poly_cipher/done() - * - poly_cipherpad/done() - * - poly_tail/done/continue() - * - poly_copy_tag() + * - poly_hash() */ return chacha_encrypt(req); } @@ -458,13 +293,7 @@ static int chachapoly_decrypt(struct aead_request *req) /* decrypt call chain: * - poly_genkey/done() - * - poly_init/done() - * - poly_setkey/done() - * - poly_ad/done() - * - poly_adpad/done() - * - poly_cipher/done() - * - poly_cipherpad/done() - * - poly_tail/done/continue() + * - poly_hash() * - chacha_decrypt/done() * - poly_verify_tag() */ @@ -503,21 +332,13 @@ static int chachapoly_init(struct crypto_aead *tfm) struct chachapoly_instance_ctx *ictx = aead_instance_ctx(inst); struct chachapoly_ctx *ctx = crypto_aead_ctx(tfm); struct crypto_skcipher *chacha; - struct crypto_ahash *poly; unsigned long align; - poly = crypto_spawn_ahash(&ictx->poly); - if (IS_ERR(poly)) - return PTR_ERR(poly); - chacha = crypto_spawn_skcipher(&ictx->chacha); - if (IS_ERR(chacha)) { - crypto_free_ahash(poly); + if (IS_ERR(chacha)) return PTR_ERR(chacha); - } ctx->chacha = chacha; - ctx->poly = poly; ctx->saltlen = ictx->saltlen; align = crypto_aead_alignmask(tfm); @@ -525,12 +346,9 @@ static int chachapoly_init(struct crypto_aead *tfm) crypto_aead_set_reqsize( tfm, align + offsetof(struct chachapoly_req_ctx, u) + - max(offsetof(struct chacha_req, req) + - sizeof(struct skcipher_request) + - crypto_skcipher_reqsize(chacha), - offsetof(struct poly_req, req) + - sizeof(struct ahash_request) + - crypto_ahash_reqsize(poly))); + offsetof(struct chacha_req, req) + + sizeof(struct skcipher_request) + + crypto_skcipher_reqsize(chacha)); return 0; } @@ -539,7 +357,6 @@ static void chachapoly_exit(struct crypto_aead *tfm) { struct chachapoly_ctx *ctx = crypto_aead_ctx(tfm); - crypto_free_ahash(ctx->poly); crypto_free_skcipher(ctx->chacha); } @@ -548,7 +365,6 @@ static void chachapoly_free(struct aead_instance *inst) struct chachapoly_instance_ctx *ctx = aead_instance_ctx(inst); crypto_drop_skcipher(&ctx->chacha); - crypto_drop_ahash(&ctx->poly); kfree(inst); } @@ -559,7 +375,6 @@ static int chachapoly_create(struct crypto_template *tmpl, struct rtattr **tb, struct aead_instance *inst; struct chachapoly_instance_ctx *ctx; struct skcipher_alg_common *chacha; - struct hash_alg_common *poly; int err; if (ivsize > CHACHAPOLY_IV_SIZE) @@ -581,14 +396,9 @@ static int chachapoly_create(struct crypto_template *tmpl, struct rtattr **tb, goto err_free_inst; chacha = crypto_spawn_skcipher_alg_common(&ctx->chacha); - err = crypto_grab_ahash(&ctx->poly, aead_crypto_instance(inst), - crypto_attr_alg_name(tb[2]), 0, mask); - if (err) - goto err_free_inst; - poly = crypto_spawn_ahash_alg(&ctx->poly); - err = -EINVAL; - if (poly->digestsize != POLY1305_DIGEST_SIZE) + if (strcmp(crypto_attr_alg_name(tb[2]), "poly1305") && + strcmp(crypto_attr_alg_name(tb[2]), "poly1305-generic")) goto err_free_inst; /* Need 16-byte IV size, including Initial Block Counter value */ if (chacha->ivsize != CHACHA_IV_SIZE) @@ -599,16 +409,15 @@ static int chachapoly_create(struct crypto_template *tmpl, struct rtattr **tb, err = -ENAMETOOLONG; if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME, - "%s(%s,%s)", name, chacha->base.cra_name, - poly->base.cra_name) >= CRYPTO_MAX_ALG_NAME) + "%s(%s,poly1305)", name, + chacha->base.cra_name) >= CRYPTO_MAX_ALG_NAME) goto err_free_inst; if (snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME, - "%s(%s,%s)", name, chacha->base.cra_driver_name, - poly->base.cra_driver_name) >= CRYPTO_MAX_ALG_NAME) + "%s(%s,poly1305-generic)", name, + chacha->base.cra_driver_name) >= CRYPTO_MAX_ALG_NAME) goto err_free_inst; - inst->alg.base.cra_priority = (chacha->base.cra_priority + - poly->base.cra_priority) / 2; + inst->alg.base.cra_priority = chacha->base.cra_priority; inst->alg.base.cra_blocksize = 1; inst->alg.base.cra_alignmask = chacha->base.cra_alignmask; inst->alg.base.cra_ctxsize = sizeof(struct chachapoly_ctx) + From 34c418b742a64f4add67ad7975b3fa0e20d6cd92 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Mon, 28 Apr 2025 12:56:23 +0800 Subject: [PATCH 257/382] crypto: testmgr - Remove poly1305 As poly1305 no longer has any in-kernel users, remove its tests. Signed-off-by: Herbert Xu --- crypto/testmgr.c | 6 - crypto/testmgr.h | 288 ----------------------------------------------- 2 files changed, 294 deletions(-) diff --git a/crypto/testmgr.c b/crypto/testmgr.c index 82977ea25db3..f100be516f52 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -5406,12 +5406,6 @@ static const struct alg_test_desc alg_test_descs[] = { .alg = "pkcs1pad(rsa)", .test = alg_test_null, .fips_allowed = 1, - }, { - .alg = "poly1305", - .test = alg_test_hash, - .suite = { - .hash = __VECS(poly1305_tv_template) - } }, { .alg = "polyval", .test = alg_test_hash, diff --git a/crypto/testmgr.h b/crypto/testmgr.h index afc10af59b0a..32d099ac9e73 100644 --- a/crypto/testmgr.h +++ b/crypto/testmgr.h @@ -8836,294 +8836,6 @@ static const struct hash_testvec hmac_sha3_512_tv_template[] = { }, }; -/* - * Poly1305 test vectors from RFC7539 A.3. - */ - -static const struct hash_testvec poly1305_tv_template[] = { - { /* Test Vector #1 */ - .plaintext = "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00", - .psize = 96, - .digest = "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00", - }, { /* Test Vector #2 */ - .plaintext = "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x36\xe5\xf6\xb5\xc5\xe0\x60\x70" - "\xf0\xef\xca\x96\x22\x7a\x86\x3e" - "\x41\x6e\x79\x20\x73\x75\x62\x6d" - "\x69\x73\x73\x69\x6f\x6e\x20\x74" - "\x6f\x20\x74\x68\x65\x20\x49\x45" - "\x54\x46\x20\x69\x6e\x74\x65\x6e" - "\x64\x65\x64\x20\x62\x79\x20\x74" - "\x68\x65\x20\x43\x6f\x6e\x74\x72" - "\x69\x62\x75\x74\x6f\x72\x20\x66" - "\x6f\x72\x20\x70\x75\x62\x6c\x69" - "\x63\x61\x74\x69\x6f\x6e\x20\x61" - "\x73\x20\x61\x6c\x6c\x20\x6f\x72" - "\x20\x70\x61\x72\x74\x20\x6f\x66" - "\x20\x61\x6e\x20\x49\x45\x54\x46" - "\x20\x49\x6e\x74\x65\x72\x6e\x65" - "\x74\x2d\x44\x72\x61\x66\x74\x20" - "\x6f\x72\x20\x52\x46\x43\x20\x61" - "\x6e\x64\x20\x61\x6e\x79\x20\x73" - "\x74\x61\x74\x65\x6d\x65\x6e\x74" - "\x20\x6d\x61\x64\x65\x20\x77\x69" - "\x74\x68\x69\x6e\x20\x74\x68\x65" - "\x20\x63\x6f\x6e\x74\x65\x78\x74" - "\x20\x6f\x66\x20\x61\x6e\x20\x49" - "\x45\x54\x46\x20\x61\x63\x74\x69" - "\x76\x69\x74\x79\x20\x69\x73\x20" - "\x63\x6f\x6e\x73\x69\x64\x65\x72" - "\x65\x64\x20\x61\x6e\x20\x22\x49" - "\x45\x54\x46\x20\x43\x6f\x6e\x74" - "\x72\x69\x62\x75\x74\x69\x6f\x6e" - "\x22\x2e\x20\x53\x75\x63\x68\x20" - "\x73\x74\x61\x74\x65\x6d\x65\x6e" - "\x74\x73\x20\x69\x6e\x63\x6c\x75" - "\x64\x65\x20\x6f\x72\x61\x6c\x20" - "\x73\x74\x61\x74\x65\x6d\x65\x6e" - "\x74\x73\x20\x69\x6e\x20\x49\x45" - "\x54\x46\x20\x73\x65\x73\x73\x69" - "\x6f\x6e\x73\x2c\x20\x61\x73\x20" - "\x77\x65\x6c\x6c\x20\x61\x73\x20" - "\x77\x72\x69\x74\x74\x65\x6e\x20" - "\x61\x6e\x64\x20\x65\x6c\x65\x63" - "\x74\x72\x6f\x6e\x69\x63\x20\x63" - "\x6f\x6d\x6d\x75\x6e\x69\x63\x61" - "\x74\x69\x6f\x6e\x73\x20\x6d\x61" - "\x64\x65\x20\x61\x74\x20\x61\x6e" - "\x79\x20\x74\x69\x6d\x65\x20\x6f" - "\x72\x20\x70\x6c\x61\x63\x65\x2c" - "\x20\x77\x68\x69\x63\x68\x20\x61" - "\x72\x65\x20\x61\x64\x64\x72\x65" - "\x73\x73\x65\x64\x20\x74\x6f", - .psize = 407, - .digest = "\x36\xe5\xf6\xb5\xc5\xe0\x60\x70" - "\xf0\xef\xca\x96\x22\x7a\x86\x3e", - }, { /* Test Vector #3 */ - .plaintext = "\x36\xe5\xf6\xb5\xc5\xe0\x60\x70" - "\xf0\xef\xca\x96\x22\x7a\x86\x3e" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x41\x6e\x79\x20\x73\x75\x62\x6d" - "\x69\x73\x73\x69\x6f\x6e\x20\x74" - "\x6f\x20\x74\x68\x65\x20\x49\x45" - "\x54\x46\x20\x69\x6e\x74\x65\x6e" - "\x64\x65\x64\x20\x62\x79\x20\x74" - "\x68\x65\x20\x43\x6f\x6e\x74\x72" - "\x69\x62\x75\x74\x6f\x72\x20\x66" - "\x6f\x72\x20\x70\x75\x62\x6c\x69" - "\x63\x61\x74\x69\x6f\x6e\x20\x61" - "\x73\x20\x61\x6c\x6c\x20\x6f\x72" - "\x20\x70\x61\x72\x74\x20\x6f\x66" - "\x20\x61\x6e\x20\x49\x45\x54\x46" - "\x20\x49\x6e\x74\x65\x72\x6e\x65" - "\x74\x2d\x44\x72\x61\x66\x74\x20" - "\x6f\x72\x20\x52\x46\x43\x20\x61" - "\x6e\x64\x20\x61\x6e\x79\x20\x73" - "\x74\x61\x74\x65\x6d\x65\x6e\x74" - "\x20\x6d\x61\x64\x65\x20\x77\x69" - "\x74\x68\x69\x6e\x20\x74\x68\x65" - "\x20\x63\x6f\x6e\x74\x65\x78\x74" - "\x20\x6f\x66\x20\x61\x6e\x20\x49" - "\x45\x54\x46\x20\x61\x63\x74\x69" - "\x76\x69\x74\x79\x20\x69\x73\x20" - "\x63\x6f\x6e\x73\x69\x64\x65\x72" - "\x65\x64\x20\x61\x6e\x20\x22\x49" - "\x45\x54\x46\x20\x43\x6f\x6e\x74" - "\x72\x69\x62\x75\x74\x69\x6f\x6e" - "\x22\x2e\x20\x53\x75\x63\x68\x20" - "\x73\x74\x61\x74\x65\x6d\x65\x6e" - "\x74\x73\x20\x69\x6e\x63\x6c\x75" - "\x64\x65\x20\x6f\x72\x61\x6c\x20" - "\x73\x74\x61\x74\x65\x6d\x65\x6e" - "\x74\x73\x20\x69\x6e\x20\x49\x45" - "\x54\x46\x20\x73\x65\x73\x73\x69" - "\x6f\x6e\x73\x2c\x20\x61\x73\x20" - "\x77\x65\x6c\x6c\x20\x61\x73\x20" - "\x77\x72\x69\x74\x74\x65\x6e\x20" - "\x61\x6e\x64\x20\x65\x6c\x65\x63" - "\x74\x72\x6f\x6e\x69\x63\x20\x63" - "\x6f\x6d\x6d\x75\x6e\x69\x63\x61" - "\x74\x69\x6f\x6e\x73\x20\x6d\x61" - "\x64\x65\x20\x61\x74\x20\x61\x6e" - "\x79\x20\x74\x69\x6d\x65\x20\x6f" - "\x72\x20\x70\x6c\x61\x63\x65\x2c" - "\x20\x77\x68\x69\x63\x68\x20\x61" - "\x72\x65\x20\x61\x64\x64\x72\x65" - "\x73\x73\x65\x64\x20\x74\x6f", - .psize = 407, - .digest = "\xf3\x47\x7e\x7c\xd9\x54\x17\xaf" - "\x89\xa6\xb8\x79\x4c\x31\x0c\xf0", - }, { /* Test Vector #4 */ - .plaintext = "\x1c\x92\x40\xa5\xeb\x55\xd3\x8a" - "\xf3\x33\x88\x86\x04\xf6\xb5\xf0" - "\x47\x39\x17\xc1\x40\x2b\x80\x09" - "\x9d\xca\x5c\xbc\x20\x70\x75\xc0" - "\x27\x54\x77\x61\x73\x20\x62\x72" - "\x69\x6c\x6c\x69\x67\x2c\x20\x61" - "\x6e\x64\x20\x74\x68\x65\x20\x73" - "\x6c\x69\x74\x68\x79\x20\x74\x6f" - "\x76\x65\x73\x0a\x44\x69\x64\x20" - "\x67\x79\x72\x65\x20\x61\x6e\x64" - "\x20\x67\x69\x6d\x62\x6c\x65\x20" - "\x69\x6e\x20\x74\x68\x65\x20\x77" - "\x61\x62\x65\x3a\x0a\x41\x6c\x6c" - "\x20\x6d\x69\x6d\x73\x79\x20\x77" - "\x65\x72\x65\x20\x74\x68\x65\x20" - "\x62\x6f\x72\x6f\x67\x6f\x76\x65" - "\x73\x2c\x0a\x41\x6e\x64\x20\x74" - "\x68\x65\x20\x6d\x6f\x6d\x65\x20" - "\x72\x61\x74\x68\x73\x20\x6f\x75" - "\x74\x67\x72\x61\x62\x65\x2e", - .psize = 159, - .digest = "\x45\x41\x66\x9a\x7e\xaa\xee\x61" - "\xe7\x08\xdc\x7c\xbc\xc5\xeb\x62", - }, { /* Test Vector #5 */ - .plaintext = "\x02\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff", - .psize = 48, - .digest = "\x03\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00", - }, { /* Test Vector #6 */ - .plaintext = "\x02\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\x02\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00", - .psize = 48, - .digest = "\x03\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00", - }, { /* Test Vector #7 */ - .plaintext = "\x01\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xf0\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\x11\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00", - .psize = 80, - .digest = "\x05\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00", - }, { /* Test Vector #8 */ - .plaintext = "\x01\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xfb\xfe\xfe\xfe\xfe\xfe\xfe\xfe" - "\xfe\xfe\xfe\xfe\xfe\xfe\xfe\xfe" - "\x01\x01\x01\x01\x01\x01\x01\x01" - "\x01\x01\x01\x01\x01\x01\x01\x01", - .psize = 80, - .digest = "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00", - }, { /* Test Vector #9 */ - .plaintext = "\x02\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\xfd\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff", - .psize = 48, - .digest = "\xfa\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff", - }, { /* Test Vector #10 */ - .plaintext = "\x01\x00\x00\x00\x00\x00\x00\x00" - "\x04\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\xe3\x35\x94\xd7\x50\x5e\x43\xb9" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x33\x94\xd7\x50\x5e\x43\x79\xcd" - "\x01\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x01\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00", - .psize = 96, - .digest = "\x14\x00\x00\x00\x00\x00\x00\x00" - "\x55\x00\x00\x00\x00\x00\x00\x00", - }, { /* Test Vector #11 */ - .plaintext = "\x01\x00\x00\x00\x00\x00\x00\x00" - "\x04\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\xe3\x35\x94\xd7\x50\x5e\x43\xb9" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x33\x94\xd7\x50\x5e\x43\x79\xcd" - "\x01\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00", - .psize = 80, - .digest = "\x13\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00", - }, { /* Regression test for overflow in AVX2 implementation */ - .plaintext = "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff", - .psize = 300, - .digest = "\xfb\x5e\x96\xd8\x61\xd5\xc7\xc8" - "\x78\xe5\x87\xcc\x2d\x5a\x22\xe1", - } -}; - /* NHPoly1305 test vectors from https://github.com/google/adiantum */ static const struct hash_testvec nhpoly1305_tv_template[] = { { From ceef731b0e22df80a13d67773ae9afd55a971f9e Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Mon, 28 Apr 2025 12:56:25 +0800 Subject: [PATCH 258/382] crypto: poly1305 - Remove algorithm As there are no in-kernel users of the Crypto API poly1305 left, remove it. Signed-off-by: Herbert Xu --- crypto/Kconfig | 12 ---- crypto/Makefile | 2 - crypto/poly1305.c | 152 ---------------------------------------------- 3 files changed, 166 deletions(-) delete mode 100644 crypto/poly1305.c diff --git a/crypto/Kconfig b/crypto/Kconfig index f87e2a26d2dd..3cb5563dc4ab 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -953,18 +953,6 @@ config CRYPTO_POLYVAL This is used in HCTR2. It is not a general-purpose cryptographic hash function. -config CRYPTO_POLY1305 - tristate "Poly1305" - select CRYPTO_HASH - select CRYPTO_LIB_POLY1305 - select CRYPTO_LIB_POLY1305_GENERIC - help - Poly1305 authenticator algorithm (RFC7539) - - Poly1305 is an authenticator algorithm designed by Daniel J. Bernstein. - It is used for the ChaCha20-Poly1305 AEAD, specified in RFC7539 for use - in IETF protocols. This is the portable C implementation of Poly1305. - config CRYPTO_RMD160 tristate "RIPEMD-160" select CRYPTO_HASH diff --git a/crypto/Makefile b/crypto/Makefile index 5d2f2a28d8a0..587bc74b6d74 100644 --- a/crypto/Makefile +++ b/crypto/Makefile @@ -149,8 +149,6 @@ obj-$(CONFIG_CRYPTO_SEED) += seed.o obj-$(CONFIG_CRYPTO_ARIA) += aria_generic.o obj-$(CONFIG_CRYPTO_CHACHA20) += chacha.o CFLAGS_chacha.o += -DARCH=$(ARCH) -obj-$(CONFIG_CRYPTO_POLY1305) += poly1305.o -CFLAGS_poly1305.o += -DARCH=$(ARCH) obj-$(CONFIG_CRYPTO_DEFLATE) += deflate.o obj-$(CONFIG_CRYPTO_MICHAEL_MIC) += michael_mic.o obj-$(CONFIG_CRYPTO_CRC32C) += crc32c_generic.o diff --git a/crypto/poly1305.c b/crypto/poly1305.c deleted file mode 100644 index e0436bdc462b..000000000000 --- a/crypto/poly1305.c +++ /dev/null @@ -1,152 +0,0 @@ -/* - * Crypto API wrapper for the Poly1305 library functions - * - * Copyright (C) 2015 Martin Willi - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - */ - -#include -#include -#include -#include -#include -#include - -struct crypto_poly1305_desc_ctx { - struct poly1305_desc_ctx base; - u8 key[POLY1305_KEY_SIZE]; - unsigned int keysize; -}; - -static int crypto_poly1305_init(struct shash_desc *desc) -{ - struct crypto_poly1305_desc_ctx *dctx = shash_desc_ctx(desc); - - dctx->keysize = 0; - return 0; -} - -static int crypto_poly1305_update(struct shash_desc *desc, - const u8 *src, unsigned int srclen, bool arch) -{ - struct crypto_poly1305_desc_ctx *dctx = shash_desc_ctx(desc); - unsigned int bytes; - - /* - * The key is passed as the first 32 "data" bytes. The actual - * poly1305_init() can be called only once the full key is available. - */ - if (dctx->keysize < POLY1305_KEY_SIZE) { - bytes = min(srclen, POLY1305_KEY_SIZE - dctx->keysize); - memcpy(&dctx->key[dctx->keysize], src, bytes); - dctx->keysize += bytes; - if (dctx->keysize < POLY1305_KEY_SIZE) - return 0; - if (arch) - poly1305_init(&dctx->base, dctx->key); - else - poly1305_init_generic(&dctx->base, dctx->key); - src += bytes; - srclen -= bytes; - } - - if (arch) - poly1305_update(&dctx->base, src, srclen); - else - poly1305_update_generic(&dctx->base, src, srclen); - - return 0; -} - -static int crypto_poly1305_update_generic(struct shash_desc *desc, - const u8 *src, unsigned int srclen) -{ - return crypto_poly1305_update(desc, src, srclen, false); -} - -static int crypto_poly1305_update_arch(struct shash_desc *desc, - const u8 *src, unsigned int srclen) -{ - return crypto_poly1305_update(desc, src, srclen, true); -} - -static int crypto_poly1305_final(struct shash_desc *desc, u8 *dst, bool arch) -{ - struct crypto_poly1305_desc_ctx *dctx = shash_desc_ctx(desc); - - if (unlikely(dctx->keysize != POLY1305_KEY_SIZE)) - return -ENOKEY; - - if (arch) - poly1305_final(&dctx->base, dst); - else - poly1305_final_generic(&dctx->base, dst); - memzero_explicit(&dctx->key, sizeof(dctx->key)); - return 0; -} - -static int crypto_poly1305_final_generic(struct shash_desc *desc, u8 *dst) -{ - return crypto_poly1305_final(desc, dst, false); -} - -static int crypto_poly1305_final_arch(struct shash_desc *desc, u8 *dst) -{ - return crypto_poly1305_final(desc, dst, true); -} - -static struct shash_alg poly1305_algs[] = { - { - .base.cra_name = "poly1305", - .base.cra_driver_name = "poly1305-generic", - .base.cra_priority = 100, - .base.cra_blocksize = POLY1305_BLOCK_SIZE, - .base.cra_module = THIS_MODULE, - .digestsize = POLY1305_DIGEST_SIZE, - .init = crypto_poly1305_init, - .update = crypto_poly1305_update_generic, - .final = crypto_poly1305_final_generic, - .descsize = sizeof(struct crypto_poly1305_desc_ctx), - }, - { - .base.cra_name = "poly1305", - .base.cra_driver_name = "poly1305-" __stringify(ARCH), - .base.cra_priority = 300, - .base.cra_blocksize = POLY1305_BLOCK_SIZE, - .base.cra_module = THIS_MODULE, - .digestsize = POLY1305_DIGEST_SIZE, - .init = crypto_poly1305_init, - .update = crypto_poly1305_update_arch, - .final = crypto_poly1305_final_arch, - .descsize = sizeof(struct crypto_poly1305_desc_ctx), - }, -}; - -static int num_algs; - -static int __init poly1305_mod_init(void) -{ - /* register the arch flavours only if they differ from generic */ - num_algs = poly1305_is_arch_optimized() ? 2 : 1; - - return crypto_register_shashes(poly1305_algs, num_algs); -} - -static void __exit poly1305_mod_exit(void) -{ - crypto_unregister_shashes(poly1305_algs, num_algs); -} - -subsys_initcall(poly1305_mod_init); -module_exit(poly1305_mod_exit); - -MODULE_LICENSE("GPL"); -MODULE_AUTHOR("Martin Willi "); -MODULE_DESCRIPTION("Crypto API wrapper for the Poly1305 library functions"); -MODULE_ALIAS_CRYPTO("poly1305"); -MODULE_ALIAS_CRYPTO("poly1305-generic"); -MODULE_ALIAS_CRYPTO("poly1305-" __stringify(ARCH)); From 10a6d72ea355b730aa9702da0fd36aef0898a80e Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Mon, 28 Apr 2025 12:56:28 +0800 Subject: [PATCH 259/382] crypto: lib/poly1305 - Use block-only interface Now that every architecture provides a block function, use that to implement the lib/poly1305 and remove the old per-arch code. Signed-off-by: Herbert Xu --- arch/arm/lib/crypto/poly1305-glue.c | 57 ------------------- arch/arm64/lib/crypto/poly1305-glue.c | 58 ------------------- arch/mips/lib/crypto/poly1305-glue.c | 60 -------------------- arch/powerpc/lib/crypto/poly1305-p10-glue.c | 63 --------------------- arch/x86/lib/crypto/poly1305_glue.c | 60 -------------------- include/crypto/poly1305.h | 53 ++--------------- lib/crypto/poly1305.c | 39 ++++++++----- 7 files changed, 32 insertions(+), 358 deletions(-) diff --git a/arch/arm/lib/crypto/poly1305-glue.c b/arch/arm/lib/crypto/poly1305-glue.c index 3ee16048ec7c..91da42b26d9c 100644 --- a/arch/arm/lib/crypto/poly1305-glue.c +++ b/arch/arm/lib/crypto/poly1305-glue.c @@ -12,7 +12,6 @@ #include #include #include -#include #include asmlinkage void poly1305_block_init_arch( @@ -35,17 +34,6 @@ void __weak poly1305_blocks_neon(struct poly1305_block_state *state, static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon); -void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 key[POLY1305_KEY_SIZE]) -{ - dctx->s[0] = get_unaligned_le32(key + 16); - dctx->s[1] = get_unaligned_le32(key + 20); - dctx->s[2] = get_unaligned_le32(key + 24); - dctx->s[3] = get_unaligned_le32(key + 28); - dctx->buflen = 0; - poly1305_block_init_arch(&dctx->state, key); -} -EXPORT_SYMBOL(poly1305_init_arch); - void poly1305_blocks_arch(struct poly1305_block_state *state, const u8 *src, unsigned int len, u32 padbit) { @@ -67,51 +55,6 @@ void poly1305_blocks_arch(struct poly1305_block_state *state, const u8 *src, } EXPORT_SYMBOL_GPL(poly1305_blocks_arch); -void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src, - unsigned int nbytes) -{ - if (unlikely(dctx->buflen)) { - u32 bytes = min(nbytes, POLY1305_BLOCK_SIZE - dctx->buflen); - - memcpy(dctx->buf + dctx->buflen, src, bytes); - src += bytes; - nbytes -= bytes; - dctx->buflen += bytes; - - if (dctx->buflen == POLY1305_BLOCK_SIZE) { - poly1305_blocks_arch(&dctx->state, dctx->buf, - POLY1305_BLOCK_SIZE, 1); - dctx->buflen = 0; - } - } - - if (likely(nbytes >= POLY1305_BLOCK_SIZE)) { - poly1305_blocks_arch(&dctx->state, src, nbytes, 1); - src += round_down(nbytes, POLY1305_BLOCK_SIZE); - nbytes %= POLY1305_BLOCK_SIZE; - } - - if (unlikely(nbytes)) { - dctx->buflen = nbytes; - memcpy(dctx->buf, src, nbytes); - } -} -EXPORT_SYMBOL(poly1305_update_arch); - -void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst) -{ - if (unlikely(dctx->buflen)) { - dctx->buf[dctx->buflen++] = 1; - memset(dctx->buf + dctx->buflen, 0, - POLY1305_BLOCK_SIZE - dctx->buflen); - poly1305_blocks_arch(&dctx->state, dctx->buf, POLY1305_BLOCK_SIZE, 0); - } - - poly1305_emit_arch(&dctx->h, dst, dctx->s); - *dctx = (struct poly1305_desc_ctx){}; -} -EXPORT_SYMBOL(poly1305_final_arch); - bool poly1305_is_arch_optimized(void) { /* We always can use at least the ARM scalar implementation. */ diff --git a/arch/arm64/lib/crypto/poly1305-glue.c b/arch/arm64/lib/crypto/poly1305-glue.c index d66a820e32d5..681c26557336 100644 --- a/arch/arm64/lib/crypto/poly1305-glue.c +++ b/arch/arm64/lib/crypto/poly1305-glue.c @@ -12,7 +12,6 @@ #include #include #include -#include #include asmlinkage void poly1305_block_init_arch( @@ -30,17 +29,6 @@ EXPORT_SYMBOL_GPL(poly1305_emit_arch); static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon); -void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 key[POLY1305_KEY_SIZE]) -{ - dctx->s[0] = get_unaligned_le32(key + 16); - dctx->s[1] = get_unaligned_le32(key + 20); - dctx->s[2] = get_unaligned_le32(key + 24); - dctx->s[3] = get_unaligned_le32(key + 28); - dctx->buflen = 0; - poly1305_block_init_arch(&dctx->state, key); -} -EXPORT_SYMBOL(poly1305_init_arch); - void poly1305_blocks_arch(struct poly1305_block_state *state, const u8 *src, unsigned int len, u32 padbit) { @@ -61,52 +49,6 @@ void poly1305_blocks_arch(struct poly1305_block_state *state, const u8 *src, } EXPORT_SYMBOL_GPL(poly1305_blocks_arch); -void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src, - unsigned int nbytes) -{ - if (unlikely(dctx->buflen)) { - u32 bytes = min(nbytes, POLY1305_BLOCK_SIZE - dctx->buflen); - - memcpy(dctx->buf + dctx->buflen, src, bytes); - src += bytes; - nbytes -= bytes; - dctx->buflen += bytes; - - if (dctx->buflen == POLY1305_BLOCK_SIZE) { - poly1305_blocks_arch(&dctx->state, dctx->buf, - POLY1305_BLOCK_SIZE, 1); - dctx->buflen = 0; - } - } - - if (likely(nbytes >= POLY1305_BLOCK_SIZE)) { - poly1305_blocks_arch(&dctx->state, src, nbytes, 1); - src += round_down(nbytes, POLY1305_BLOCK_SIZE); - nbytes %= POLY1305_BLOCK_SIZE; - } - - if (unlikely(nbytes)) { - dctx->buflen = nbytes; - memcpy(dctx->buf, src, nbytes); - } -} -EXPORT_SYMBOL(poly1305_update_arch); - -void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst) -{ - if (unlikely(dctx->buflen)) { - dctx->buf[dctx->buflen++] = 1; - memset(dctx->buf + dctx->buflen, 0, - POLY1305_BLOCK_SIZE - dctx->buflen); - poly1305_blocks_arch(&dctx->state, dctx->buf, - POLY1305_BLOCK_SIZE, 0); - } - - poly1305_emit_arch(&dctx->h, dst, dctx->s); - memzero_explicit(dctx, sizeof(*dctx)); -} -EXPORT_SYMBOL(poly1305_final_arch); - bool poly1305_is_arch_optimized(void) { /* We always can use at least the ARM64 scalar implementation. */ diff --git a/arch/mips/lib/crypto/poly1305-glue.c b/arch/mips/lib/crypto/poly1305-glue.c index 2fea4cacfe27..764a38a65200 100644 --- a/arch/mips/lib/crypto/poly1305-glue.c +++ b/arch/mips/lib/crypto/poly1305-glue.c @@ -9,7 +9,6 @@ #include #include #include -#include #include asmlinkage void poly1305_block_init_arch( @@ -24,65 +23,6 @@ asmlinkage void poly1305_emit_arch(const struct poly1305_state *state, const u32 nonce[4]); EXPORT_SYMBOL_GPL(poly1305_emit_arch); -void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 key[POLY1305_KEY_SIZE]) -{ - dctx->s[0] = get_unaligned_le32(key + 16); - dctx->s[1] = get_unaligned_le32(key + 20); - dctx->s[2] = get_unaligned_le32(key + 24); - dctx->s[3] = get_unaligned_le32(key + 28); - dctx->buflen = 0; - poly1305_block_init_arch(&dctx->state, key); -} -EXPORT_SYMBOL(poly1305_init_arch); - -void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src, - unsigned int nbytes) -{ - if (unlikely(dctx->buflen)) { - u32 bytes = min(nbytes, POLY1305_BLOCK_SIZE - dctx->buflen); - - memcpy(dctx->buf + dctx->buflen, src, bytes); - src += bytes; - nbytes -= bytes; - dctx->buflen += bytes; - - if (dctx->buflen == POLY1305_BLOCK_SIZE) { - poly1305_blocks_arch(&dctx->state, dctx->buf, - POLY1305_BLOCK_SIZE, 1); - dctx->buflen = 0; - } - } - - if (likely(nbytes >= POLY1305_BLOCK_SIZE)) { - unsigned int len = round_down(nbytes, POLY1305_BLOCK_SIZE); - - poly1305_blocks_arch(&dctx->state, src, len, 1); - src += len; - nbytes %= POLY1305_BLOCK_SIZE; - } - - if (unlikely(nbytes)) { - dctx->buflen = nbytes; - memcpy(dctx->buf, src, nbytes); - } -} -EXPORT_SYMBOL(poly1305_update_arch); - -void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst) -{ - if (unlikely(dctx->buflen)) { - dctx->buf[dctx->buflen++] = 1; - memset(dctx->buf + dctx->buflen, 0, - POLY1305_BLOCK_SIZE - dctx->buflen); - poly1305_blocks_arch(&dctx->state, dctx->buf, - POLY1305_BLOCK_SIZE, 0); - } - - poly1305_emit_arch(&dctx->h, dst, dctx->s); - *dctx = (struct poly1305_desc_ctx){}; -} -EXPORT_SYMBOL(poly1305_final_arch); - bool poly1305_is_arch_optimized(void) { return true; diff --git a/arch/powerpc/lib/crypto/poly1305-p10-glue.c b/arch/powerpc/lib/crypto/poly1305-p10-glue.c index 708435beaba6..50ac802220e0 100644 --- a/arch/powerpc/lib/crypto/poly1305-p10-glue.c +++ b/arch/powerpc/lib/crypto/poly1305-p10-glue.c @@ -10,7 +10,6 @@ #include #include #include -#include #include asmlinkage void poly1305_p10le_4blocks(struct poly1305_block_state *state, const u8 *m, u32 mlen); @@ -45,17 +44,6 @@ void poly1305_block_init_arch(struct poly1305_block_state *dctx, } EXPORT_SYMBOL_GPL(poly1305_block_init_arch); -void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 key[POLY1305_KEY_SIZE]) -{ - dctx->s[0] = get_unaligned_le32(key + 16); - dctx->s[1] = get_unaligned_le32(key + 20); - dctx->s[2] = get_unaligned_le32(key + 24); - dctx->s[3] = get_unaligned_le32(key + 28); - dctx->buflen = 0; - poly1305_block_init_arch(&dctx->state, key); -} -EXPORT_SYMBOL(poly1305_init_arch); - void poly1305_blocks_arch(struct poly1305_block_state *state, const u8 *src, unsigned int len, u32 padbit) { @@ -76,57 +64,6 @@ void poly1305_blocks_arch(struct poly1305_block_state *state, const u8 *src, } EXPORT_SYMBOL_GPL(poly1305_blocks_arch); -void poly1305_update_arch(struct poly1305_desc_ctx *dctx, - const u8 *src, unsigned int srclen) -{ - unsigned int bytes; - - if (!static_key_enabled(&have_p10)) - return poly1305_update_generic(dctx, src, srclen); - - if (unlikely(dctx->buflen)) { - bytes = min(srclen, POLY1305_BLOCK_SIZE - dctx->buflen); - memcpy(dctx->buf + dctx->buflen, src, bytes); - src += bytes; - srclen -= bytes; - dctx->buflen += bytes; - if (dctx->buflen < POLY1305_BLOCK_SIZE) - return; - poly1305_blocks_arch(&dctx->state, dctx->buf, - POLY1305_BLOCK_SIZE, 1); - dctx->buflen = 0; - } - - if (likely(srclen >= POLY1305_BLOCK_SIZE)) { - poly1305_blocks_arch(&dctx->state, src, srclen, 1); - src += srclen - (srclen % POLY1305_BLOCK_SIZE); - srclen %= POLY1305_BLOCK_SIZE; - } - - if (unlikely(srclen)) { - dctx->buflen = srclen; - memcpy(dctx->buf, src, srclen); - } -} -EXPORT_SYMBOL(poly1305_update_arch); - -void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst) -{ - if (!static_key_enabled(&have_p10)) - return poly1305_final_generic(dctx, dst); - - if (dctx->buflen) { - dctx->buf[dctx->buflen++] = 1; - memset(dctx->buf + dctx->buflen, 0, - POLY1305_BLOCK_SIZE - dctx->buflen); - poly1305_blocks_arch(&dctx->state, dctx->buf, - POLY1305_BLOCK_SIZE, 0); - } - - poly1305_emit_arch(&dctx->h, dst, dctx->s); -} -EXPORT_SYMBOL(poly1305_final_arch); - bool poly1305_is_arch_optimized(void) { return static_key_enabled(&have_p10); diff --git a/arch/x86/lib/crypto/poly1305_glue.c b/arch/x86/lib/crypto/poly1305_glue.c index d98764ec3b47..f799828c5809 100644 --- a/arch/x86/lib/crypto/poly1305_glue.c +++ b/arch/x86/lib/crypto/poly1305_glue.c @@ -10,7 +10,6 @@ #include #include #include -#include #include struct poly1305_arch_internal { @@ -96,65 +95,6 @@ void poly1305_emit_arch(const struct poly1305_state *ctx, } EXPORT_SYMBOL_GPL(poly1305_emit_arch); -void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 key[POLY1305_KEY_SIZE]) -{ - dctx->s[0] = get_unaligned_le32(&key[16]); - dctx->s[1] = get_unaligned_le32(&key[20]); - dctx->s[2] = get_unaligned_le32(&key[24]); - dctx->s[3] = get_unaligned_le32(&key[28]); - dctx->buflen = 0; - poly1305_block_init_arch(&dctx->state, key); -} -EXPORT_SYMBOL(poly1305_init_arch); - -void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src, - unsigned int srclen) -{ - unsigned int bytes; - - if (unlikely(dctx->buflen)) { - bytes = min(srclen, POLY1305_BLOCK_SIZE - dctx->buflen); - memcpy(dctx->buf + dctx->buflen, src, bytes); - src += bytes; - srclen -= bytes; - dctx->buflen += bytes; - - if (dctx->buflen == POLY1305_BLOCK_SIZE) { - poly1305_blocks_arch(&dctx->state, dctx->buf, - POLY1305_BLOCK_SIZE, 1); - dctx->buflen = 0; - } - } - - if (likely(srclen >= POLY1305_BLOCK_SIZE)) { - bytes = round_down(srclen, POLY1305_BLOCK_SIZE); - poly1305_blocks_arch(&dctx->state, src, bytes, 1); - src += bytes; - srclen -= bytes; - } - - if (unlikely(srclen)) { - dctx->buflen = srclen; - memcpy(dctx->buf, src, srclen); - } -} -EXPORT_SYMBOL(poly1305_update_arch); - -void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst) -{ - if (unlikely(dctx->buflen)) { - dctx->buf[dctx->buflen++] = 1; - memset(dctx->buf + dctx->buflen, 0, - POLY1305_BLOCK_SIZE - dctx->buflen); - poly1305_blocks_arch(&dctx->state, dctx->buf, - POLY1305_BLOCK_SIZE, 0); - } - - poly1305_emit_arch(&dctx->h, dst, dctx->s); - memzero_explicit(dctx, sizeof(*dctx)); -} -EXPORT_SYMBOL(poly1305_final_arch); - bool poly1305_is_arch_optimized(void) { return static_key_enabled(&poly1305_use_avx); diff --git a/include/crypto/poly1305.h b/include/crypto/poly1305.h index 027d74842cd5..e54abda8cfe9 100644 --- a/include/crypto/poly1305.h +++ b/include/crypto/poly1305.h @@ -55,55 +55,14 @@ struct poly1305_desc_ctx { unsigned int buflen; /* finalize key */ u32 s[4]; - union { - struct { - struct poly1305_state h; - union { - struct poly1305_key opaque_r[CONFIG_CRYPTO_LIB_POLY1305_RSIZE]; - struct poly1305_core_key core_r; - }; - }; - struct poly1305_block_state state; - }; + struct poly1305_block_state state; }; -void poly1305_init_arch(struct poly1305_desc_ctx *desc, - const u8 key[POLY1305_KEY_SIZE]); -void poly1305_init_generic(struct poly1305_desc_ctx *desc, - const u8 key[POLY1305_KEY_SIZE]); - -static inline void poly1305_init(struct poly1305_desc_ctx *desc, const u8 *key) -{ - if (IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_POLY1305)) - poly1305_init_arch(desc, key); - else - poly1305_init_generic(desc, key); -} - -void poly1305_update_arch(struct poly1305_desc_ctx *desc, const u8 *src, - unsigned int nbytes); -void poly1305_update_generic(struct poly1305_desc_ctx *desc, const u8 *src, - unsigned int nbytes); - -static inline void poly1305_update(struct poly1305_desc_ctx *desc, - const u8 *src, unsigned int nbytes) -{ - if (IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_POLY1305)) - poly1305_update_arch(desc, src, nbytes); - else - poly1305_update_generic(desc, src, nbytes); -} - -void poly1305_final_arch(struct poly1305_desc_ctx *desc, u8 *digest); -void poly1305_final_generic(struct poly1305_desc_ctx *desc, u8 *digest); - -static inline void poly1305_final(struct poly1305_desc_ctx *desc, u8 *digest) -{ - if (IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_POLY1305)) - poly1305_final_arch(desc, digest); - else - poly1305_final_generic(desc, digest); -} +void poly1305_init(struct poly1305_desc_ctx *desc, + const u8 key[POLY1305_KEY_SIZE]); +void poly1305_update(struct poly1305_desc_ctx *desc, + const u8 *src, unsigned int nbytes); +void poly1305_final(struct poly1305_desc_ctx *desc, u8 *digest); #if IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_POLY1305) bool poly1305_is_arch_optimized(void); diff --git a/lib/crypto/poly1305.c b/lib/crypto/poly1305.c index 9fec64a599c1..4c9996864090 100644 --- a/lib/crypto/poly1305.c +++ b/lib/crypto/poly1305.c @@ -22,47 +22,60 @@ void poly1305_block_init_generic(struct poly1305_block_state *desc, } EXPORT_SYMBOL_GPL(poly1305_block_init_generic); -void poly1305_init_generic(struct poly1305_desc_ctx *desc, - const u8 key[POLY1305_KEY_SIZE]) +void poly1305_init(struct poly1305_desc_ctx *desc, + const u8 key[POLY1305_KEY_SIZE]) { desc->s[0] = get_unaligned_le32(key + 16); desc->s[1] = get_unaligned_le32(key + 20); desc->s[2] = get_unaligned_le32(key + 24); desc->s[3] = get_unaligned_le32(key + 28); desc->buflen = 0; - poly1305_block_init_generic(&desc->state, key); + if (IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_POLY1305)) + poly1305_block_init_arch(&desc->state, key); + else + poly1305_block_init_generic(&desc->state, key); } -EXPORT_SYMBOL_GPL(poly1305_init_generic); +EXPORT_SYMBOL(poly1305_init); static inline void poly1305_blocks(struct poly1305_block_state *state, const u8 *src, unsigned int len) { - poly1305_blocks_generic(state, src, len, 1); + if (IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_POLY1305)) + poly1305_blocks_arch(state, src, len, 1); + else + poly1305_blocks_generic(state, src, len, 1); } -void poly1305_update_generic(struct poly1305_desc_ctx *desc, const u8 *src, - unsigned int nbytes) +void poly1305_update(struct poly1305_desc_ctx *desc, + const u8 *src, unsigned int nbytes) { desc->buflen = BLOCK_HASH_UPDATE(poly1305_blocks, &desc->state, src, nbytes, POLY1305_BLOCK_SIZE, desc->buf, desc->buflen); } -EXPORT_SYMBOL_GPL(poly1305_update_generic); +EXPORT_SYMBOL(poly1305_update); -void poly1305_final_generic(struct poly1305_desc_ctx *desc, u8 *dst) +void poly1305_final(struct poly1305_desc_ctx *desc, u8 *dst) { if (unlikely(desc->buflen)) { desc->buf[desc->buflen++] = 1; memset(desc->buf + desc->buflen, 0, POLY1305_BLOCK_SIZE - desc->buflen); - poly1305_blocks_generic(&desc->state, desc->buf, - POLY1305_BLOCK_SIZE, 0); + if (IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_POLY1305)) + poly1305_blocks_arch(&desc->state, desc->buf, + POLY1305_BLOCK_SIZE, 0); + else + poly1305_blocks_generic(&desc->state, desc->buf, + POLY1305_BLOCK_SIZE, 0); } - poly1305_emit_generic(&desc->h, dst, desc->s); + if (IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_POLY1305)) + poly1305_emit_arch(&desc->state.h, dst, desc->s); + else + poly1305_emit_generic(&desc->state.h, dst, desc->s); *desc = (struct poly1305_desc_ctx){}; } -EXPORT_SYMBOL_GPL(poly1305_final_generic); +EXPORT_SYMBOL(poly1305_final); MODULE_LICENSE("GPL"); MODULE_AUTHOR("Martin Willi "); From 950e5c84118c9e5b06bb9a9b64edf989ee4034df Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 28 Apr 2025 10:00:26 -0700 Subject: [PATCH 260/382] crypto: sha256 - support arch-optimized lib and expose through shash As has been done for various other algorithms, rework the design of the SHA-256 library to support arch-optimized implementations, and make crypto/sha256.c expose both generic and arch-optimized shash algorithms that wrap the library functions. This allows users of the SHA-256 library functions to take advantage of the arch-optimized code, and this makes it much simpler to integrate SHA-256 for each architecture. Note that sha256_base.h is not used in the new design. It will be removed once all the architecture-specific code has been updated. Move the generic block function into its own module to avoid a circular dependency from libsha256.ko => sha256-$ARCH.ko => libsha256.ko. Signed-off-by: Eric Biggers Add export and import functions to maintain existing export format. Signed-off-by: Herbert Xu --- crypto/Kconfig | 1 + crypto/Makefile | 3 +- crypto/sha256.c | 243 +++++++++++++++++++++++++++++++++ crypto/sha256_generic.c | 102 -------------- include/crypto/internal/sha2.h | 28 ++++ include/crypto/sha2.h | 15 +- include/crypto/sha256_base.h | 9 +- lib/crypto/Kconfig | 19 +++ lib/crypto/Makefile | 3 + lib/crypto/sha256-generic.c | 137 +++++++++++++++++++ lib/crypto/sha256.c | 196 ++++++++++++-------------- 11 files changed, 529 insertions(+), 227 deletions(-) create mode 100644 crypto/sha256.c delete mode 100644 crypto/sha256_generic.c create mode 100644 include/crypto/internal/sha2.h create mode 100644 lib/crypto/sha256-generic.c diff --git a/crypto/Kconfig b/crypto/Kconfig index 3cb5563dc4ab..7bfad077f308 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -982,6 +982,7 @@ config CRYPTO_SHA256 tristate "SHA-224 and SHA-256" select CRYPTO_HASH select CRYPTO_LIB_SHA256 + select CRYPTO_LIB_SHA256_GENERIC help SHA-224 and SHA-256 secure hash algorithms (FIPS 180, ISO/IEC 10118-3) diff --git a/crypto/Makefile b/crypto/Makefile index 587bc74b6d74..84f6911dc9ba 100644 --- a/crypto/Makefile +++ b/crypto/Makefile @@ -76,7 +76,8 @@ obj-$(CONFIG_CRYPTO_MD4) += md4.o obj-$(CONFIG_CRYPTO_MD5) += md5.o obj-$(CONFIG_CRYPTO_RMD160) += rmd160.o obj-$(CONFIG_CRYPTO_SHA1) += sha1_generic.o -obj-$(CONFIG_CRYPTO_SHA256) += sha256_generic.o +obj-$(CONFIG_CRYPTO_SHA256) += sha256.o +CFLAGS_sha256.o += -DARCH=$(ARCH) obj-$(CONFIG_CRYPTO_SHA512) += sha512_generic.o obj-$(CONFIG_CRYPTO_SHA3) += sha3_generic.o obj-$(CONFIG_CRYPTO_SM3_GENERIC) += sm3_generic.o diff --git a/crypto/sha256.c b/crypto/sha256.c new file mode 100644 index 000000000000..c2588d08ee3e --- /dev/null +++ b/crypto/sha256.c @@ -0,0 +1,243 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Crypto API wrapper for the SHA-256 and SHA-224 library functions + * + * Copyright (c) Jean-Luc Cooke + * Copyright (c) Andrew McDonald + * Copyright (c) 2002 James Morris + * SHA224 Support Copyright 2007 Intel Corporation + */ +#include +#include +#include +#include + +const u8 sha224_zero_message_hash[SHA224_DIGEST_SIZE] = { + 0xd1, 0x4a, 0x02, 0x8c, 0x2a, 0x3a, 0x2b, 0xc9, 0x47, + 0x61, 0x02, 0xbb, 0x28, 0x82, 0x34, 0xc4, 0x15, 0xa2, + 0xb0, 0x1f, 0x82, 0x8e, 0xa6, 0x2a, 0xc5, 0xb3, 0xe4, + 0x2f +}; +EXPORT_SYMBOL_GPL(sha224_zero_message_hash); + +const u8 sha256_zero_message_hash[SHA256_DIGEST_SIZE] = { + 0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, + 0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, + 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, + 0xa4, 0x95, 0x99, 0x1b, 0x78, 0x52, 0xb8, 0x55 +}; +EXPORT_SYMBOL_GPL(sha256_zero_message_hash); + +static int crypto_sha256_init(struct shash_desc *desc) +{ + sha256_init(shash_desc_ctx(desc)); + return 0; +} + +static int crypto_sha256_update_generic(struct shash_desc *desc, const u8 *data, + unsigned int len) +{ + sha256_update_generic(shash_desc_ctx(desc), data, len); + return 0; +} + +static int crypto_sha256_update_arch(struct shash_desc *desc, const u8 *data, + unsigned int len) +{ + sha256_update(shash_desc_ctx(desc), data, len); + return 0; +} + +static int crypto_sha256_final_generic(struct shash_desc *desc, u8 *out) +{ + sha256_final_generic(shash_desc_ctx(desc), out); + return 0; +} + +static int crypto_sha256_final_arch(struct shash_desc *desc, u8 *out) +{ + sha256_final(shash_desc_ctx(desc), out); + return 0; +} + +static int crypto_sha256_finup_generic(struct shash_desc *desc, const u8 *data, + unsigned int len, u8 *out) +{ + struct sha256_state *sctx = shash_desc_ctx(desc); + + sha256_update_generic(sctx, data, len); + sha256_final_generic(sctx, out); + return 0; +} + +static int crypto_sha256_finup_arch(struct shash_desc *desc, const u8 *data, + unsigned int len, u8 *out) +{ + struct sha256_state *sctx = shash_desc_ctx(desc); + + sha256_update(sctx, data, len); + sha256_final(sctx, out); + return 0; +} + +static int crypto_sha256_digest_generic(struct shash_desc *desc, const u8 *data, + unsigned int len, u8 *out) +{ + struct sha256_state *sctx = shash_desc_ctx(desc); + + sha256_init(sctx); + sha256_update_generic(sctx, data, len); + sha256_final_generic(sctx, out); + return 0; +} + +static int crypto_sha256_digest_arch(struct shash_desc *desc, const u8 *data, + unsigned int len, u8 *out) +{ + sha256(data, len, out); + return 0; +} + +static int crypto_sha224_init(struct shash_desc *desc) +{ + sha224_init(shash_desc_ctx(desc)); + return 0; +} + +static int crypto_sha224_final_generic(struct shash_desc *desc, u8 *out) +{ + sha224_final_generic(shash_desc_ctx(desc), out); + return 0; +} + +static int crypto_sha224_final_arch(struct shash_desc *desc, u8 *out) +{ + sha224_final(shash_desc_ctx(desc), out); + return 0; +} + +static int crypto_sha256_import_lib(struct shash_desc *desc, const void *in) +{ + struct sha256_state *sctx = shash_desc_ctx(desc); + const u8 *p = in; + + memcpy(sctx, p, sizeof(*sctx)); + p += sizeof(*sctx); + sctx->count += *p; + return 0; +} + +static int crypto_sha256_export_lib(struct shash_desc *desc, void *out) +{ + struct sha256_state *sctx0 = shash_desc_ctx(desc); + struct sha256_state sctx = *sctx0; + unsigned int partial; + u8 *p = out; + + partial = sctx.count % SHA256_BLOCK_SIZE; + sctx.count -= partial; + memcpy(p, &sctx, sizeof(sctx)); + p += sizeof(sctx); + *p = partial; + return 0; +} + +static struct shash_alg algs[] = { + { + .base.cra_name = "sha256", + .base.cra_driver_name = "sha256-generic", + .base.cra_priority = 100, + .base.cra_blocksize = SHA256_BLOCK_SIZE, + .base.cra_module = THIS_MODULE, + .digestsize = SHA256_DIGEST_SIZE, + .init = crypto_sha256_init, + .update = crypto_sha256_update_generic, + .final = crypto_sha256_final_generic, + .finup = crypto_sha256_finup_generic, + .digest = crypto_sha256_digest_generic, + .descsize = sizeof(struct sha256_state), + .statesize = sizeof(struct crypto_sha256_state) + + SHA256_BLOCK_SIZE + 1, + .import = crypto_sha256_import_lib, + .export = crypto_sha256_export_lib, + }, + { + .base.cra_name = "sha224", + .base.cra_driver_name = "sha224-generic", + .base.cra_priority = 100, + .base.cra_blocksize = SHA224_BLOCK_SIZE, + .base.cra_module = THIS_MODULE, + .digestsize = SHA224_DIGEST_SIZE, + .init = crypto_sha224_init, + .update = crypto_sha256_update_generic, + .final = crypto_sha224_final_generic, + .descsize = sizeof(struct sha256_state), + .statesize = sizeof(struct crypto_sha256_state) + + SHA256_BLOCK_SIZE + 1, + .import = crypto_sha256_import_lib, + .export = crypto_sha256_export_lib, + }, + { + .base.cra_name = "sha256", + .base.cra_driver_name = "sha256-" __stringify(ARCH), + .base.cra_priority = 300, + .base.cra_blocksize = SHA256_BLOCK_SIZE, + .base.cra_module = THIS_MODULE, + .digestsize = SHA256_DIGEST_SIZE, + .init = crypto_sha256_init, + .update = crypto_sha256_update_arch, + .final = crypto_sha256_final_arch, + .finup = crypto_sha256_finup_arch, + .digest = crypto_sha256_digest_arch, + .descsize = sizeof(struct sha256_state), + .statesize = sizeof(struct crypto_sha256_state) + + SHA256_BLOCK_SIZE + 1, + .import = crypto_sha256_import_lib, + .export = crypto_sha256_export_lib, + }, + { + .base.cra_name = "sha224", + .base.cra_driver_name = "sha224-" __stringify(ARCH), + .base.cra_priority = 300, + .base.cra_blocksize = SHA224_BLOCK_SIZE, + .base.cra_module = THIS_MODULE, + .digestsize = SHA224_DIGEST_SIZE, + .init = crypto_sha224_init, + .update = crypto_sha256_update_arch, + .final = crypto_sha224_final_arch, + .descsize = sizeof(struct sha256_state), + .statesize = sizeof(struct crypto_sha256_state) + + SHA256_BLOCK_SIZE + 1, + .import = crypto_sha256_import_lib, + .export = crypto_sha256_export_lib, + }, +}; + +static unsigned int num_algs; + +static int __init crypto_sha256_mod_init(void) +{ + /* register the arch flavours only if they differ from generic */ + num_algs = ARRAY_SIZE(algs); + BUILD_BUG_ON(ARRAY_SIZE(algs) % 2 != 0); + if (!sha256_is_arch_optimized()) + num_algs /= 2; + return crypto_register_shashes(algs, ARRAY_SIZE(algs)); +} +subsys_initcall(crypto_sha256_mod_init); + +static void __exit crypto_sha256_mod_exit(void) +{ + crypto_unregister_shashes(algs, num_algs); +} +module_exit(crypto_sha256_mod_exit); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Crypto API wrapper for the SHA-256 and SHA-224 library functions"); + +MODULE_ALIAS_CRYPTO("sha256"); +MODULE_ALIAS_CRYPTO("sha256-generic"); +MODULE_ALIAS_CRYPTO("sha256-" __stringify(ARCH)); +MODULE_ALIAS_CRYPTO("sha224"); +MODULE_ALIAS_CRYPTO("sha224-generic"); +MODULE_ALIAS_CRYPTO("sha224-" __stringify(ARCH)); diff --git a/crypto/sha256_generic.c b/crypto/sha256_generic.c deleted file mode 100644 index 05084e5bbaec..000000000000 --- a/crypto/sha256_generic.c +++ /dev/null @@ -1,102 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Crypto API wrapper for the generic SHA256 code from lib/crypto/sha256.c - * - * Copyright (c) Jean-Luc Cooke - * Copyright (c) Andrew McDonald - * Copyright (c) 2002 James Morris - * SHA224 Support Copyright 2007 Intel Corporation - */ -#include -#include -#include -#include -#include - -const u8 sha224_zero_message_hash[SHA224_DIGEST_SIZE] = { - 0xd1, 0x4a, 0x02, 0x8c, 0x2a, 0x3a, 0x2b, 0xc9, 0x47, - 0x61, 0x02, 0xbb, 0x28, 0x82, 0x34, 0xc4, 0x15, 0xa2, - 0xb0, 0x1f, 0x82, 0x8e, 0xa6, 0x2a, 0xc5, 0xb3, 0xe4, - 0x2f -}; -EXPORT_SYMBOL_GPL(sha224_zero_message_hash); - -const u8 sha256_zero_message_hash[SHA256_DIGEST_SIZE] = { - 0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, - 0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, - 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, - 0xa4, 0x95, 0x99, 0x1b, 0x78, 0x52, 0xb8, 0x55 -}; -EXPORT_SYMBOL_GPL(sha256_zero_message_hash); - -static void sha256_block(struct crypto_sha256_state *sctx, const u8 *input, - int blocks) -{ - sha256_transform_blocks(sctx, input, blocks); -} - -static int crypto_sha256_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - return sha256_base_do_update_blocks(desc, data, len, sha256_block); -} - -static int crypto_sha256_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *hash) -{ - sha256_base_do_finup(desc, data, len, sha256_block); - return sha256_base_finish(desc, hash); -} - -static struct shash_alg sha256_algs[2] = { { - .digestsize = SHA256_DIGEST_SIZE, - .init = sha256_base_init, - .update = crypto_sha256_update, - .finup = crypto_sha256_finup, - .descsize = sizeof(struct crypto_sha256_state), - .base = { - .cra_name = "sha256", - .cra_driver_name= "sha256-generic", - .cra_priority = 100, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_blocksize = SHA256_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}, { - .digestsize = SHA224_DIGEST_SIZE, - .init = sha224_base_init, - .update = crypto_sha256_update, - .finup = crypto_sha256_finup, - .descsize = sizeof(struct crypto_sha256_state), - .base = { - .cra_name = "sha224", - .cra_driver_name= "sha224-generic", - .cra_priority = 100, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_blocksize = SHA224_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -} }; - -static int __init sha256_generic_mod_init(void) -{ - return crypto_register_shashes(sha256_algs, ARRAY_SIZE(sha256_algs)); -} - -static void __exit sha256_generic_mod_fini(void) -{ - crypto_unregister_shashes(sha256_algs, ARRAY_SIZE(sha256_algs)); -} - -subsys_initcall(sha256_generic_mod_init); -module_exit(sha256_generic_mod_fini); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("SHA-224 and SHA-256 Secure Hash Algorithm"); - -MODULE_ALIAS_CRYPTO("sha224"); -MODULE_ALIAS_CRYPTO("sha224-generic"); -MODULE_ALIAS_CRYPTO("sha256"); -MODULE_ALIAS_CRYPTO("sha256-generic"); diff --git a/include/crypto/internal/sha2.h b/include/crypto/internal/sha2.h new file mode 100644 index 000000000000..d641c67abcbc --- /dev/null +++ b/include/crypto/internal/sha2.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#ifndef _CRYPTO_INTERNAL_SHA2_H +#define _CRYPTO_INTERNAL_SHA2_H + +#include + +void sha256_update_generic(struct sha256_state *sctx, + const u8 *data, size_t len); +void sha256_final_generic(struct sha256_state *sctx, + u8 out[SHA256_DIGEST_SIZE]); +void sha224_final_generic(struct sha256_state *sctx, + u8 out[SHA224_DIGEST_SIZE]); + +#if IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_SHA256) +bool sha256_is_arch_optimized(void); +#else +static inline bool sha256_is_arch_optimized(void) +{ + return false; +} +#endif +void sha256_blocks_generic(u32 state[SHA256_STATE_WORDS], + const u8 *data, size_t nblocks); +void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS], + const u8 *data, size_t nblocks); + +#endif /* _CRYPTO_INTERNAL_SHA2_H */ diff --git a/include/crypto/sha2.h b/include/crypto/sha2.h index f873c2207b1e..9a56286d736d 100644 --- a/include/crypto/sha2.h +++ b/include/crypto/sha2.h @@ -13,6 +13,7 @@ #define SHA256_DIGEST_SIZE 32 #define SHA256_BLOCK_SIZE 64 +#define SHA256_STATE_WORDS 8 #define SHA384_DIGEST_SIZE 48 #define SHA384_BLOCK_SIZE 128 @@ -66,7 +67,7 @@ extern const u8 sha384_zero_message_hash[SHA384_DIGEST_SIZE]; extern const u8 sha512_zero_message_hash[SHA512_DIGEST_SIZE]; struct crypto_sha256_state { - u32 state[SHA256_DIGEST_SIZE / 4]; + u32 state[SHA256_STATE_WORDS]; u64 count; }; @@ -74,7 +75,7 @@ struct sha256_state { union { struct crypto_sha256_state ctx; struct { - u32 state[SHA256_DIGEST_SIZE / 4]; + u32 state[SHA256_STATE_WORDS]; u64 count; }; }; @@ -87,16 +88,6 @@ struct sha512_state { u8 buf[SHA512_BLOCK_SIZE]; }; -/* - * Stand-alone implementation of the SHA256 algorithm. It is designed to - * have as little dependencies as possible so it can be used in the - * kexec_file purgatory. In other cases you should generally use the - * hash APIs from include/crypto/hash.h. Especially when hashing large - * amounts of data as those APIs may be hw-accelerated. - * - * For details see lib/crypto/sha256.c - */ - static inline void sha256_init(struct sha256_state *sctx) { sctx->state[0] = SHA256_H0; diff --git a/include/crypto/sha256_base.h b/include/crypto/sha256_base.h index 9f284bed5a51..804361731a7a 100644 --- a/include/crypto/sha256_base.h +++ b/include/crypto/sha256_base.h @@ -10,7 +10,7 @@ #include #include -#include +#include #include #include #include @@ -142,7 +142,10 @@ static inline int sha256_base_finish(struct shash_desc *desc, u8 *out) return __sha256_base_finish(sctx->state, out, digest_size); } -void sha256_transform_blocks(struct crypto_sha256_state *sst, - const u8 *input, int blocks); +static inline void sha256_transform_blocks(struct crypto_sha256_state *sst, + const u8 *input, int blocks) +{ + sha256_blocks_generic(sst->state, input, blocks); +} #endif /* _CRYPTO_SHA256_BASE_H */ diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index af2368799579..7fe678047939 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -139,6 +139,25 @@ config CRYPTO_LIB_SHA1 config CRYPTO_LIB_SHA256 tristate + help + Enable the SHA-256 library interface. This interface may be fulfilled + by either the generic implementation or an arch-specific one, if one + is available and enabled. + +config CRYPTO_ARCH_HAVE_LIB_SHA256 + bool + help + Declares whether the architecture provides an arch-specific + accelerated implementation of the SHA-256 library interface. + +config CRYPTO_LIB_SHA256_GENERIC + tristate + default CRYPTO_LIB_SHA256 if !CRYPTO_ARCH_HAVE_LIB_SHA256 + help + This symbol can be selected by arch implementations of the SHA-256 + library interface that require the generic code as a fallback, e.g., + for SIMD implementations. If no arch specific implementation is + enabled, this implementation serves the users of CRYPTO_LIB_SHA256. config CRYPTO_LIB_SM3 tristate diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index 4dd62bc5bee3..71d3d05d666a 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -51,6 +51,9 @@ libsha1-y := sha1.o obj-$(CONFIG_CRYPTO_LIB_SHA256) += libsha256.o libsha256-y := sha256.o +obj-$(CONFIG_CRYPTO_LIB_SHA256_GENERIC) += libsha256-generic.o +libsha256-generic-y := sha256-generic.o + ifneq ($(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS),y) libblake2s-y += blake2s-selftest.o libchacha20poly1305-y += chacha20poly1305-selftest.o diff --git a/lib/crypto/sha256-generic.c b/lib/crypto/sha256-generic.c new file mode 100644 index 000000000000..a16ad4f25ebb --- /dev/null +++ b/lib/crypto/sha256-generic.c @@ -0,0 +1,137 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * SHA-256, as specified in + * http://csrc.nist.gov/groups/STM/cavp/documents/shs/sha256-384-512.pdf + * + * SHA-256 code by Jean-Luc Cooke . + * + * Copyright (c) Jean-Luc Cooke + * Copyright (c) Andrew McDonald + * Copyright (c) 2002 James Morris + * Copyright (c) 2014 Red Hat Inc. + */ + +#include +#include +#include +#include +#include + +static const u32 SHA256_K[] = { + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, + 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, + 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, + 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, + 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, + 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, + 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, + 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, + 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, + 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, + 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, + 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, + 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, + 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2, +}; + +static inline u32 Ch(u32 x, u32 y, u32 z) +{ + return z ^ (x & (y ^ z)); +} + +static inline u32 Maj(u32 x, u32 y, u32 z) +{ + return (x & y) | (z & (x | y)); +} + +#define e0(x) (ror32(x, 2) ^ ror32(x, 13) ^ ror32(x, 22)) +#define e1(x) (ror32(x, 6) ^ ror32(x, 11) ^ ror32(x, 25)) +#define s0(x) (ror32(x, 7) ^ ror32(x, 18) ^ (x >> 3)) +#define s1(x) (ror32(x, 17) ^ ror32(x, 19) ^ (x >> 10)) + +static inline void LOAD_OP(int I, u32 *W, const u8 *input) +{ + W[I] = get_unaligned_be32((__u32 *)input + I); +} + +static inline void BLEND_OP(int I, u32 *W) +{ + W[I] = s1(W[I-2]) + W[I-7] + s0(W[I-15]) + W[I-16]; +} + +#define SHA256_ROUND(i, a, b, c, d, e, f, g, h) do { \ + u32 t1, t2; \ + t1 = h + e1(e) + Ch(e, f, g) + SHA256_K[i] + W[i]; \ + t2 = e0(a) + Maj(a, b, c); \ + d += t1; \ + h = t1 + t2; \ +} while (0) + +static void sha256_block_generic(u32 state[SHA256_STATE_WORDS], + const u8 *input, u32 W[64]) +{ + u32 a, b, c, d, e, f, g, h; + int i; + + /* load the input */ + for (i = 0; i < 16; i += 8) { + LOAD_OP(i + 0, W, input); + LOAD_OP(i + 1, W, input); + LOAD_OP(i + 2, W, input); + LOAD_OP(i + 3, W, input); + LOAD_OP(i + 4, W, input); + LOAD_OP(i + 5, W, input); + LOAD_OP(i + 6, W, input); + LOAD_OP(i + 7, W, input); + } + + /* now blend */ + for (i = 16; i < 64; i += 8) { + BLEND_OP(i + 0, W); + BLEND_OP(i + 1, W); + BLEND_OP(i + 2, W); + BLEND_OP(i + 3, W); + BLEND_OP(i + 4, W); + BLEND_OP(i + 5, W); + BLEND_OP(i + 6, W); + BLEND_OP(i + 7, W); + } + + /* load the state into our registers */ + a = state[0]; b = state[1]; c = state[2]; d = state[3]; + e = state[4]; f = state[5]; g = state[6]; h = state[7]; + + /* now iterate */ + for (i = 0; i < 64; i += 8) { + SHA256_ROUND(i + 0, a, b, c, d, e, f, g, h); + SHA256_ROUND(i + 1, h, a, b, c, d, e, f, g); + SHA256_ROUND(i + 2, g, h, a, b, c, d, e, f); + SHA256_ROUND(i + 3, f, g, h, a, b, c, d, e); + SHA256_ROUND(i + 4, e, f, g, h, a, b, c, d); + SHA256_ROUND(i + 5, d, e, f, g, h, a, b, c); + SHA256_ROUND(i + 6, c, d, e, f, g, h, a, b); + SHA256_ROUND(i + 7, b, c, d, e, f, g, h, a); + } + + state[0] += a; state[1] += b; state[2] += c; state[3] += d; + state[4] += e; state[5] += f; state[6] += g; state[7] += h; +} + +void sha256_blocks_generic(u32 state[SHA256_STATE_WORDS], + const u8 *data, size_t nblocks) +{ + u32 W[64]; + + do { + sha256_block_generic(state, data, W); + data += SHA256_BLOCK_SIZE; + } while (--nblocks); + + memzero_explicit(W, sizeof(W)); +} +EXPORT_SYMBOL_GPL(sha256_blocks_generic); + +MODULE_DESCRIPTION("SHA-256 Algorithm (generic implementation)"); +MODULE_LICENSE("GPL"); diff --git a/lib/crypto/sha256.c b/lib/crypto/sha256.c index a89bab377de1..4b19cf977ef1 100644 --- a/lib/crypto/sha256.c +++ b/lib/crypto/sha256.c @@ -11,148 +11,105 @@ * Copyright (c) 2014 Red Hat Inc. */ -#include -#include +#include #include #include #include +#include -static const u32 SHA256_K[] = { - 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, - 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, - 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, - 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, - 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, - 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, - 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, - 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, - 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, - 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, - 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, - 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, - 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, - 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, - 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, - 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2, -}; +/* + * If __DISABLE_EXPORTS is defined, then this file is being compiled for a + * pre-boot environment. In that case, ignore the kconfig options, pull the + * generic code into the same translation unit, and use that only. + */ +#ifdef __DISABLE_EXPORTS +#include "sha256-generic.c" +#endif -static inline u32 Ch(u32 x, u32 y, u32 z) +static inline void sha256_blocks(u32 state[SHA256_STATE_WORDS], const u8 *data, + size_t nblocks, bool force_generic) { - return z ^ (x & (y ^ z)); +#if IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_SHA256) && !defined(__DISABLE_EXPORTS) + if (!force_generic) + return sha256_blocks_arch(state, data, nblocks); +#endif + sha256_blocks_generic(state, data, nblocks); } -static inline u32 Maj(u32 x, u32 y, u32 z) +static inline void __sha256_update(struct sha256_state *sctx, const u8 *data, + size_t len, bool force_generic) { - return (x & y) | (z & (x | y)); -} + size_t partial = sctx->count % SHA256_BLOCK_SIZE; -#define e0(x) (ror32(x, 2) ^ ror32(x, 13) ^ ror32(x, 22)) -#define e1(x) (ror32(x, 6) ^ ror32(x, 11) ^ ror32(x, 25)) -#define s0(x) (ror32(x, 7) ^ ror32(x, 18) ^ (x >> 3)) -#define s1(x) (ror32(x, 17) ^ ror32(x, 19) ^ (x >> 10)) + sctx->count += len; -static inline void LOAD_OP(int I, u32 *W, const u8 *input) -{ - W[I] = get_unaligned_be32((__u32 *)input + I); -} + if (partial + len >= SHA256_BLOCK_SIZE) { + size_t nblocks; -static inline void BLEND_OP(int I, u32 *W) -{ - W[I] = s1(W[I-2]) + W[I-7] + s0(W[I-15]) + W[I-16]; -} + if (partial) { + size_t l = SHA256_BLOCK_SIZE - partial; -#define SHA256_ROUND(i, a, b, c, d, e, f, g, h) do { \ - u32 t1, t2; \ - t1 = h + e1(e) + Ch(e, f, g) + SHA256_K[i] + W[i]; \ - t2 = e0(a) + Maj(a, b, c); \ - d += t1; \ - h = t1 + t2; \ -} while (0) + memcpy(&sctx->buf[partial], data, l); + data += l; + len -= l; -static void sha256_transform(u32 *state, const u8 *input, u32 *W) -{ - u32 a, b, c, d, e, f, g, h; - int i; + sha256_blocks(sctx->state, sctx->buf, 1, force_generic); + } - /* load the input */ - for (i = 0; i < 16; i += 8) { - LOAD_OP(i + 0, W, input); - LOAD_OP(i + 1, W, input); - LOAD_OP(i + 2, W, input); - LOAD_OP(i + 3, W, input); - LOAD_OP(i + 4, W, input); - LOAD_OP(i + 5, W, input); - LOAD_OP(i + 6, W, input); - LOAD_OP(i + 7, W, input); + nblocks = len / SHA256_BLOCK_SIZE; + len %= SHA256_BLOCK_SIZE; + + if (nblocks) { + sha256_blocks(sctx->state, data, nblocks, + force_generic); + data += nblocks * SHA256_BLOCK_SIZE; + } + partial = 0; } - - /* now blend */ - for (i = 16; i < 64; i += 8) { - BLEND_OP(i + 0, W); - BLEND_OP(i + 1, W); - BLEND_OP(i + 2, W); - BLEND_OP(i + 3, W); - BLEND_OP(i + 4, W); - BLEND_OP(i + 5, W); - BLEND_OP(i + 6, W); - BLEND_OP(i + 7, W); - } - - /* load the state into our registers */ - a = state[0]; b = state[1]; c = state[2]; d = state[3]; - e = state[4]; f = state[5]; g = state[6]; h = state[7]; - - /* now iterate */ - for (i = 0; i < 64; i += 8) { - SHA256_ROUND(i + 0, a, b, c, d, e, f, g, h); - SHA256_ROUND(i + 1, h, a, b, c, d, e, f, g); - SHA256_ROUND(i + 2, g, h, a, b, c, d, e, f); - SHA256_ROUND(i + 3, f, g, h, a, b, c, d, e); - SHA256_ROUND(i + 4, e, f, g, h, a, b, c, d); - SHA256_ROUND(i + 5, d, e, f, g, h, a, b, c); - SHA256_ROUND(i + 6, c, d, e, f, g, h, a, b); - SHA256_ROUND(i + 7, b, c, d, e, f, g, h, a); - } - - state[0] += a; state[1] += b; state[2] += c; state[3] += d; - state[4] += e; state[5] += f; state[6] += g; state[7] += h; + if (len) + memcpy(&sctx->buf[partial], data, len); } -void sha256_transform_blocks(struct crypto_sha256_state *sst, - const u8 *input, int blocks) -{ - u32 W[64]; - - do { - sha256_transform(sst->state, input, W); - input += SHA256_BLOCK_SIZE; - } while (--blocks); - - memzero_explicit(W, sizeof(W)); -} -EXPORT_SYMBOL_GPL(sha256_transform_blocks); - void sha256_update(struct sha256_state *sctx, const u8 *data, unsigned int len) { - lib_sha256_base_do_update(sctx, data, len, sha256_transform_blocks); + __sha256_update(sctx, data, len, false); } EXPORT_SYMBOL(sha256_update); -static void __sha256_final(struct sha256_state *sctx, u8 *out, int digest_size) +static inline void __sha256_final(struct sha256_state *sctx, u8 *out, + size_t digest_size, bool force_generic) { - lib_sha256_base_do_finalize(sctx, sha256_transform_blocks); - lib_sha256_base_finish(sctx, out, digest_size); + const size_t bit_offset = SHA256_BLOCK_SIZE - sizeof(__be64); + __be64 *bits = (__be64 *)&sctx->buf[bit_offset]; + size_t partial = sctx->count % SHA256_BLOCK_SIZE; + size_t i; + + sctx->buf[partial++] = 0x80; + if (partial > bit_offset) { + memset(&sctx->buf[partial], 0, SHA256_BLOCK_SIZE - partial); + sha256_blocks(sctx->state, sctx->buf, 1, force_generic); + partial = 0; + } + + memset(&sctx->buf[partial], 0, bit_offset - partial); + *bits = cpu_to_be64(sctx->count << 3); + sha256_blocks(sctx->state, sctx->buf, 1, force_generic); + + for (i = 0; i < digest_size; i += 4) + put_unaligned_be32(sctx->state[i / 4], out + i); + + memzero_explicit(sctx, sizeof(*sctx)); } void sha256_final(struct sha256_state *sctx, u8 *out) { - __sha256_final(sctx, out, 32); + __sha256_final(sctx, out, SHA256_DIGEST_SIZE, false); } EXPORT_SYMBOL(sha256_final); void sha224_final(struct sha256_state *sctx, u8 *out) { - __sha256_final(sctx, out, 28); + __sha256_final(sctx, out, SHA224_DIGEST_SIZE, false); } EXPORT_SYMBOL(sha224_final); @@ -166,5 +123,26 @@ void sha256(const u8 *data, unsigned int len, u8 *out) } EXPORT_SYMBOL(sha256); +#if IS_ENABLED(CONFIG_CRYPTO_SHA256) && !defined(__DISABLE_EXPORTS) +void sha256_update_generic(struct sha256_state *sctx, + const u8 *data, size_t len) +{ + __sha256_update(sctx, data, len, true); +} +EXPORT_SYMBOL(sha256_update_generic); + +void sha256_final_generic(struct sha256_state *sctx, u8 out[SHA256_DIGEST_SIZE]) +{ + __sha256_final(sctx, out, SHA256_DIGEST_SIZE, true); +} +EXPORT_SYMBOL(sha256_final_generic); + +void sha224_final_generic(struct sha256_state *sctx, u8 out[SHA224_DIGEST_SIZE]) +{ + __sha256_final(sctx, out, SHA224_DIGEST_SIZE, true); +} +EXPORT_SYMBOL(sha224_final_generic); +#endif + MODULE_DESCRIPTION("SHA-256 Algorithm"); MODULE_LICENSE("GPL"); From ca4477e41c68b58043e67bc78074cd6fcc59ee5e Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 28 Apr 2025 10:00:27 -0700 Subject: [PATCH 261/382] crypto: arm/sha256 - implement library instead of shash Instead of providing crypto_shash algorithms for the arch-optimized SHA-256 code, instead implement the SHA-256 library. This is much simpler, it makes the SHA-256 library functions be arch-optimized, and it fixes the longstanding issue where the arch-optimized SHA-256 was disabled by default. SHA-256 still remains available through crypto_shash, but individual architectures no longer need to handle it. To merge the scalar, NEON, and CE code all into one module cleanly, add !CPU_V7M as a direct dependency of the CE code. Previously, !CPU_V7M was only a direct dependency of the scalar and NEON code. The result is still the same because CPU_V7M implies !KERNEL_MODE_NEON, so !CPU_V7M was already an indirect dependency of the CE code. To match sha256_blocks_arch(), change the type of the nblocks parameter of the assembly functions from int to size_t. The assembly functions actually already treated it as size_t. While renaming the assembly files, also fix the naming quirk where "sha2" meant sha256. (SHA-512 is also part of SHA-2.) Reviewed-by: Ard Biesheuvel Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/arm/configs/exynos_defconfig | 1 - arch/arm/configs/milbeaut_m10v_defconfig | 1 - arch/arm/configs/multi_v7_defconfig | 1 - arch/arm/configs/omap2plus_defconfig | 1 - arch/arm/configs/pxa_defconfig | 1 - arch/arm/crypto/Kconfig | 21 ---- arch/arm/crypto/Makefile | 8 +- arch/arm/crypto/sha2-ce-glue.c | 87 -------------- arch/arm/crypto/sha256_glue.c | 107 ------------------ arch/arm/crypto/sha256_glue.h | 9 -- arch/arm/crypto/sha256_neon_glue.c | 75 ------------ arch/arm/lib/crypto/.gitignore | 1 + arch/arm/lib/crypto/Kconfig | 6 + arch/arm/lib/crypto/Makefile | 8 +- arch/arm/{ => lib}/crypto/sha256-armv4.pl | 0 .../sha2-ce-core.S => lib/crypto/sha256-ce.S} | 10 +- arch/arm/lib/crypto/sha256.c | 64 +++++++++++ 17 files changed, 84 insertions(+), 317 deletions(-) delete mode 100644 arch/arm/crypto/sha2-ce-glue.c delete mode 100644 arch/arm/crypto/sha256_glue.c delete mode 100644 arch/arm/crypto/sha256_glue.h delete mode 100644 arch/arm/crypto/sha256_neon_glue.c rename arch/arm/{ => lib}/crypto/sha256-armv4.pl (100%) rename arch/arm/{crypto/sha2-ce-core.S => lib/crypto/sha256-ce.S} (91%) create mode 100644 arch/arm/lib/crypto/sha256.c diff --git a/arch/arm/configs/exynos_defconfig b/arch/arm/configs/exynos_defconfig index e81a5d6c1c20..c6792c0256a6 100644 --- a/arch/arm/configs/exynos_defconfig +++ b/arch/arm/configs/exynos_defconfig @@ -364,7 +364,6 @@ CONFIG_CRYPTO_USER_API_SKCIPHER=m CONFIG_CRYPTO_USER_API_RNG=m CONFIG_CRYPTO_USER_API_AEAD=m CONFIG_CRYPTO_SHA1_ARM_NEON=m -CONFIG_CRYPTO_SHA256_ARM=m CONFIG_CRYPTO_SHA512_ARM=m CONFIG_CRYPTO_AES_ARM_BS=m CONFIG_CRYPTO_CHACHA20_NEON=m diff --git a/arch/arm/configs/milbeaut_m10v_defconfig b/arch/arm/configs/milbeaut_m10v_defconfig index 275ddf7a3a14..4ec21f477c63 100644 --- a/arch/arm/configs/milbeaut_m10v_defconfig +++ b/arch/arm/configs/milbeaut_m10v_defconfig @@ -101,7 +101,6 @@ CONFIG_CRYPTO_SEQIV=m CONFIG_CRYPTO_GHASH_ARM_CE=m CONFIG_CRYPTO_SHA1_ARM_NEON=m CONFIG_CRYPTO_SHA1_ARM_CE=m -CONFIG_CRYPTO_SHA2_ARM_CE=m CONFIG_CRYPTO_SHA512_ARM=m CONFIG_CRYPTO_AES_ARM=m CONFIG_CRYPTO_AES_ARM_BS=m diff --git a/arch/arm/configs/multi_v7_defconfig b/arch/arm/configs/multi_v7_defconfig index ad037c175fdb..96178acedad0 100644 --- a/arch/arm/configs/multi_v7_defconfig +++ b/arch/arm/configs/multi_v7_defconfig @@ -1301,7 +1301,6 @@ CONFIG_CRYPTO_USER_API_AEAD=m CONFIG_CRYPTO_GHASH_ARM_CE=m CONFIG_CRYPTO_SHA1_ARM_NEON=m CONFIG_CRYPTO_SHA1_ARM_CE=m -CONFIG_CRYPTO_SHA2_ARM_CE=m CONFIG_CRYPTO_SHA512_ARM=m CONFIG_CRYPTO_AES_ARM=m CONFIG_CRYPTO_AES_ARM_BS=m diff --git a/arch/arm/configs/omap2plus_defconfig b/arch/arm/configs/omap2plus_defconfig index 75b326bc7830..317f977e509e 100644 --- a/arch/arm/configs/omap2plus_defconfig +++ b/arch/arm/configs/omap2plus_defconfig @@ -697,7 +697,6 @@ CONFIG_SECURITY=y CONFIG_CRYPTO_MICHAEL_MIC=y CONFIG_CRYPTO_GHASH_ARM_CE=m CONFIG_CRYPTO_SHA1_ARM_NEON=m -CONFIG_CRYPTO_SHA256_ARM=m CONFIG_CRYPTO_SHA512_ARM=m CONFIG_CRYPTO_AES_ARM=m CONFIG_CRYPTO_AES_ARM_BS=m diff --git a/arch/arm/configs/pxa_defconfig b/arch/arm/configs/pxa_defconfig index 24fca8608554..56be85752909 100644 --- a/arch/arm/configs/pxa_defconfig +++ b/arch/arm/configs/pxa_defconfig @@ -660,7 +660,6 @@ CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_DEFLATE=y CONFIG_CRYPTO_LZO=y CONFIG_CRYPTO_SHA1_ARM=m -CONFIG_CRYPTO_SHA256_ARM=m CONFIG_CRYPTO_SHA512_ARM=m CONFIG_CRYPTO_AES_ARM=m CONFIG_FONTS=y diff --git a/arch/arm/crypto/Kconfig b/arch/arm/crypto/Kconfig index 1f889d6bab77..7efb9a8596e4 100644 --- a/arch/arm/crypto/Kconfig +++ b/arch/arm/crypto/Kconfig @@ -93,27 +93,6 @@ config CRYPTO_SHA1_ARM_CE Architecture: arm using ARMv8 Crypto Extensions -config CRYPTO_SHA2_ARM_CE - tristate "Hash functions: SHA-224 and SHA-256 (ARMv8 Crypto Extensions)" - depends on KERNEL_MODE_NEON - select CRYPTO_SHA256_ARM - select CRYPTO_HASH - help - SHA-224 and SHA-256 secure hash algorithms (FIPS 180) - - Architecture: arm using - - ARMv8 Crypto Extensions - -config CRYPTO_SHA256_ARM - tristate "Hash functions: SHA-224 and SHA-256 (NEON)" - select CRYPTO_HASH - depends on !CPU_V7M - help - SHA-224 and SHA-256 secure hash algorithms (FIPS 180) - - Architecture: arm using - - NEON (Advanced SIMD) extensions - config CRYPTO_SHA512_ARM tristate "Hash functions: SHA-384 and SHA-512 (NEON)" select CRYPTO_HASH diff --git a/arch/arm/crypto/Makefile b/arch/arm/crypto/Makefile index ecabe6603e08..8479137c6e80 100644 --- a/arch/arm/crypto/Makefile +++ b/arch/arm/crypto/Makefile @@ -7,7 +7,6 @@ obj-$(CONFIG_CRYPTO_AES_ARM) += aes-arm.o obj-$(CONFIG_CRYPTO_AES_ARM_BS) += aes-arm-bs.o obj-$(CONFIG_CRYPTO_SHA1_ARM) += sha1-arm.o obj-$(CONFIG_CRYPTO_SHA1_ARM_NEON) += sha1-arm-neon.o -obj-$(CONFIG_CRYPTO_SHA256_ARM) += sha256-arm.o obj-$(CONFIG_CRYPTO_SHA512_ARM) += sha512-arm.o obj-$(CONFIG_CRYPTO_BLAKE2B_NEON) += blake2b-neon.o obj-$(CONFIG_CRYPTO_NHPOLY1305_NEON) += nhpoly1305-neon.o @@ -15,20 +14,16 @@ obj-$(CONFIG_CRYPTO_CURVE25519_NEON) += curve25519-neon.o obj-$(CONFIG_CRYPTO_AES_ARM_CE) += aes-arm-ce.o obj-$(CONFIG_CRYPTO_SHA1_ARM_CE) += sha1-arm-ce.o -obj-$(CONFIG_CRYPTO_SHA2_ARM_CE) += sha2-arm-ce.o obj-$(CONFIG_CRYPTO_GHASH_ARM_CE) += ghash-arm-ce.o aes-arm-y := aes-cipher-core.o aes-cipher-glue.o aes-arm-bs-y := aes-neonbs-core.o aes-neonbs-glue.o sha1-arm-y := sha1-armv4-large.o sha1_glue.o sha1-arm-neon-y := sha1-armv7-neon.o sha1_neon_glue.o -sha256-arm-neon-$(CONFIG_KERNEL_MODE_NEON) := sha256_neon_glue.o -sha256-arm-y := sha256-core.o sha256_glue.o $(sha256-arm-neon-y) sha512-arm-neon-$(CONFIG_KERNEL_MODE_NEON) := sha512-neon-glue.o sha512-arm-y := sha512-core.o sha512-glue.o $(sha512-arm-neon-y) blake2b-neon-y := blake2b-neon-core.o blake2b-neon-glue.o sha1-arm-ce-y := sha1-ce-core.o sha1-ce-glue.o -sha2-arm-ce-y := sha2-ce-core.o sha2-ce-glue.o aes-arm-ce-y := aes-ce-core.o aes-ce-glue.o ghash-arm-ce-y := ghash-ce-core.o ghash-ce-glue.o nhpoly1305-neon-y := nh-neon-core.o nhpoly1305-neon-glue.o @@ -40,9 +35,8 @@ quiet_cmd_perl = PERL $@ $(obj)/%-core.S: $(src)/%-armv4.pl $(call cmd,perl) -clean-files += sha256-core.S sha512-core.S +clean-files += sha512-core.S aflags-thumb2-$(CONFIG_THUMB2_KERNEL) := -U__thumb2__ -D__thumb2__=1 -AFLAGS_sha256-core.o += $(aflags-thumb2-y) AFLAGS_sha512-core.o += $(aflags-thumb2-y) diff --git a/arch/arm/crypto/sha2-ce-glue.c b/arch/arm/crypto/sha2-ce-glue.c deleted file mode 100644 index 1e9d16f79678..000000000000 --- a/arch/arm/crypto/sha2-ce-glue.c +++ /dev/null @@ -1,87 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * sha2-ce-glue.c - SHA-224/SHA-256 using ARMv8 Crypto Extensions - * - * Copyright (C) 2015 Linaro Ltd - */ - -#include -#include -#include -#include -#include -#include -#include - -MODULE_DESCRIPTION("SHA-224/SHA-256 secure hash using ARMv8 Crypto Extensions"); -MODULE_AUTHOR("Ard Biesheuvel "); -MODULE_LICENSE("GPL v2"); - -asmlinkage void sha2_ce_transform(struct crypto_sha256_state *sst, - u8 const *src, int blocks); - -static int sha2_ce_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - int remain; - - kernel_neon_begin(); - remain = sha256_base_do_update_blocks(desc, data, len, - sha2_ce_transform); - kernel_neon_end(); - return remain; -} - -static int sha2_ce_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - kernel_neon_begin(); - sha256_base_do_finup(desc, data, len, sha2_ce_transform); - kernel_neon_end(); - return sha256_base_finish(desc, out); -} - -static struct shash_alg algs[] = { { - .init = sha224_base_init, - .update = sha2_ce_update, - .finup = sha2_ce_finup, - .descsize = sizeof(struct crypto_sha256_state), - .digestsize = SHA224_DIGEST_SIZE, - .base = { - .cra_name = "sha224", - .cra_driver_name = "sha224-ce", - .cra_priority = 300, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_blocksize = SHA256_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}, { - .init = sha256_base_init, - .update = sha2_ce_update, - .finup = sha2_ce_finup, - .descsize = sizeof(struct crypto_sha256_state), - .digestsize = SHA256_DIGEST_SIZE, - .base = { - .cra_name = "sha256", - .cra_driver_name = "sha256-ce", - .cra_priority = 300, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_blocksize = SHA256_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -} }; - -static int __init sha2_ce_mod_init(void) -{ - return crypto_register_shashes(algs, ARRAY_SIZE(algs)); -} - -static void __exit sha2_ce_mod_fini(void) -{ - crypto_unregister_shashes(algs, ARRAY_SIZE(algs)); -} - -module_cpu_feature_match(SHA2, sha2_ce_mod_init); -module_exit(sha2_ce_mod_fini); diff --git a/arch/arm/crypto/sha256_glue.c b/arch/arm/crypto/sha256_glue.c deleted file mode 100644 index d04c4e6bae6d..000000000000 --- a/arch/arm/crypto/sha256_glue.c +++ /dev/null @@ -1,107 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Glue code for the SHA256 Secure Hash Algorithm assembly implementation - * using optimized ARM assembler and NEON instructions. - * - * Copyright © 2015 Google Inc. - * - * This file is based on sha256_ssse3_glue.c: - * Copyright (C) 2013 Intel Corporation - * Author: Tim Chen - */ - -#include -#include -#include -#include -#include -#include - -#include "sha256_glue.h" - -asmlinkage void sha256_block_data_order(struct crypto_sha256_state *state, - const u8 *data, int num_blks); - -static int crypto_sha256_arm_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - /* make sure casting to sha256_block_fn() is safe */ - BUILD_BUG_ON(offsetof(struct crypto_sha256_state, state) != 0); - - return sha256_base_do_update_blocks(desc, data, len, - sha256_block_data_order); -} - -static int crypto_sha256_arm_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - sha256_base_do_finup(desc, data, len, sha256_block_data_order); - return sha256_base_finish(desc, out); -} - -static struct shash_alg algs[] = { { - .digestsize = SHA256_DIGEST_SIZE, - .init = sha256_base_init, - .update = crypto_sha256_arm_update, - .finup = crypto_sha256_arm_finup, - .descsize = sizeof(struct crypto_sha256_state), - .base = { - .cra_name = "sha256", - .cra_driver_name = "sha256-asm", - .cra_priority = 150, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_blocksize = SHA256_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}, { - .digestsize = SHA224_DIGEST_SIZE, - .init = sha224_base_init, - .update = crypto_sha256_arm_update, - .finup = crypto_sha256_arm_finup, - .descsize = sizeof(struct crypto_sha256_state), - .base = { - .cra_name = "sha224", - .cra_driver_name = "sha224-asm", - .cra_priority = 150, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_blocksize = SHA224_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -} }; - -static int __init sha256_mod_init(void) -{ - int res = crypto_register_shashes(algs, ARRAY_SIZE(algs)); - - if (res < 0) - return res; - - if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && cpu_has_neon()) { - res = crypto_register_shashes(sha256_neon_algs, - ARRAY_SIZE(sha256_neon_algs)); - - if (res < 0) - crypto_unregister_shashes(algs, ARRAY_SIZE(algs)); - } - - return res; -} - -static void __exit sha256_mod_fini(void) -{ - crypto_unregister_shashes(algs, ARRAY_SIZE(algs)); - - if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && cpu_has_neon()) - crypto_unregister_shashes(sha256_neon_algs, - ARRAY_SIZE(sha256_neon_algs)); -} - -module_init(sha256_mod_init); -module_exit(sha256_mod_fini); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("SHA256 Secure Hash Algorithm (ARM), including NEON"); - -MODULE_ALIAS_CRYPTO("sha256"); diff --git a/arch/arm/crypto/sha256_glue.h b/arch/arm/crypto/sha256_glue.h deleted file mode 100644 index 9881c9a115d1..000000000000 --- a/arch/arm/crypto/sha256_glue.h +++ /dev/null @@ -1,9 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _CRYPTO_SHA256_GLUE_H -#define _CRYPTO_SHA256_GLUE_H - -#include - -extern struct shash_alg sha256_neon_algs[2]; - -#endif /* _CRYPTO_SHA256_GLUE_H */ diff --git a/arch/arm/crypto/sha256_neon_glue.c b/arch/arm/crypto/sha256_neon_glue.c deleted file mode 100644 index 76eb3cdc21c9..000000000000 --- a/arch/arm/crypto/sha256_neon_glue.c +++ /dev/null @@ -1,75 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Glue code for the SHA256 Secure Hash Algorithm assembly implementation - * using NEON instructions. - * - * Copyright © 2015 Google Inc. - * - * This file is based on sha512_neon_glue.c: - * Copyright © 2014 Jussi Kivilinna - */ - -#include -#include -#include -#include -#include -#include - -#include "sha256_glue.h" - -asmlinkage void sha256_block_data_order_neon( - struct crypto_sha256_state *digest, const u8 *data, int num_blks); - -static int crypto_sha256_neon_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - int remain; - - kernel_neon_begin(); - remain = sha256_base_do_update_blocks(desc, data, len, - sha256_block_data_order_neon); - kernel_neon_end(); - return remain; -} - -static int crypto_sha256_neon_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - kernel_neon_begin(); - sha256_base_do_finup(desc, data, len, sha256_block_data_order_neon); - kernel_neon_end(); - return sha256_base_finish(desc, out); -} - -struct shash_alg sha256_neon_algs[] = { { - .digestsize = SHA256_DIGEST_SIZE, - .init = sha256_base_init, - .update = crypto_sha256_neon_update, - .finup = crypto_sha256_neon_finup, - .descsize = sizeof(struct crypto_sha256_state), - .base = { - .cra_name = "sha256", - .cra_driver_name = "sha256-neon", - .cra_priority = 250, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_blocksize = SHA256_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}, { - .digestsize = SHA224_DIGEST_SIZE, - .init = sha224_base_init, - .update = crypto_sha256_neon_update, - .finup = crypto_sha256_neon_finup, - .descsize = sizeof(struct crypto_sha256_state), - .base = { - .cra_name = "sha224", - .cra_driver_name = "sha224-neon", - .cra_priority = 250, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_blocksize = SHA224_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -} }; diff --git a/arch/arm/lib/crypto/.gitignore b/arch/arm/lib/crypto/.gitignore index 0d47d4f21c6d..12d74d8b03d0 100644 --- a/arch/arm/lib/crypto/.gitignore +++ b/arch/arm/lib/crypto/.gitignore @@ -1,2 +1,3 @@ # SPDX-License-Identifier: GPL-2.0-only poly1305-core.S +sha256-core.S diff --git a/arch/arm/lib/crypto/Kconfig b/arch/arm/lib/crypto/Kconfig index e8444fd0aae3..9f3ff30f4032 100644 --- a/arch/arm/lib/crypto/Kconfig +++ b/arch/arm/lib/crypto/Kconfig @@ -22,3 +22,9 @@ config CRYPTO_POLY1305_ARM tristate default CRYPTO_LIB_POLY1305 select CRYPTO_ARCH_HAVE_LIB_POLY1305 + +config CRYPTO_SHA256_ARM + tristate + depends on !CPU_V7M + default CRYPTO_LIB_SHA256 + select CRYPTO_ARCH_HAVE_LIB_SHA256 diff --git a/arch/arm/lib/crypto/Makefile b/arch/arm/lib/crypto/Makefile index 4c042a4c77ed..431f77c3ff6f 100644 --- a/arch/arm/lib/crypto/Makefile +++ b/arch/arm/lib/crypto/Makefile @@ -10,13 +10,17 @@ chacha-neon-$(CONFIG_KERNEL_MODE_NEON) += chacha-neon-core.o obj-$(CONFIG_CRYPTO_POLY1305_ARM) += poly1305-arm.o poly1305-arm-y := poly1305-core.o poly1305-glue.o +obj-$(CONFIG_CRYPTO_SHA256_ARM) += sha256-arm.o +sha256-arm-y := sha256.o sha256-core.o +sha256-arm-$(CONFIG_KERNEL_MODE_NEON) += sha256-ce.o + quiet_cmd_perl = PERL $@ cmd_perl = $(PERL) $(<) > $(@) $(obj)/%-core.S: $(src)/%-armv4.pl $(call cmd,perl) -clean-files += poly1305-core.S +clean-files += poly1305-core.S sha256-core.S aflags-thumb2-$(CONFIG_THUMB2_KERNEL) := -U__thumb2__ -D__thumb2__=1 @@ -24,3 +28,5 @@ aflags-thumb2-$(CONFIG_THUMB2_KERNEL) := -U__thumb2__ -D__thumb2__=1 poly1305-aflags-$(CONFIG_CPU_V7) := -U__LINUX_ARM_ARCH__ -D__LINUX_ARM_ARCH__=5 poly1305-aflags-$(CONFIG_KERNEL_MODE_NEON) := -U__LINUX_ARM_ARCH__ -D__LINUX_ARM_ARCH__=7 AFLAGS_poly1305-core.o += $(poly1305-aflags-y) $(aflags-thumb2-y) + +AFLAGS_sha256-core.o += $(aflags-thumb2-y) diff --git a/arch/arm/crypto/sha256-armv4.pl b/arch/arm/lib/crypto/sha256-armv4.pl similarity index 100% rename from arch/arm/crypto/sha256-armv4.pl rename to arch/arm/lib/crypto/sha256-armv4.pl diff --git a/arch/arm/crypto/sha2-ce-core.S b/arch/arm/lib/crypto/sha256-ce.S similarity index 91% rename from arch/arm/crypto/sha2-ce-core.S rename to arch/arm/lib/crypto/sha256-ce.S index b6369d2440a1..ac2c9b01b22d 100644 --- a/arch/arm/crypto/sha2-ce-core.S +++ b/arch/arm/lib/crypto/sha256-ce.S @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* - * sha2-ce-core.S - SHA-224/256 secure hash using ARMv8 Crypto Extensions + * sha256-ce.S - SHA-224/256 secure hash using ARMv8 Crypto Extensions * * Copyright (C) 2015 Linaro Ltd. * Author: Ard Biesheuvel @@ -67,10 +67,10 @@ .word 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2 /* - * void sha2_ce_transform(struct sha256_state *sst, u8 const *src, - int blocks); + * void sha256_ce_transform(u32 state[SHA256_STATE_WORDS], + * const u8 *data, size_t nblocks); */ -ENTRY(sha2_ce_transform) +ENTRY(sha256_ce_transform) /* load state */ vld1.32 {dga-dgb}, [r0] @@ -120,4 +120,4 @@ ENTRY(sha2_ce_transform) /* store new state */ vst1.32 {dga-dgb}, [r0] bx lr -ENDPROC(sha2_ce_transform) +ENDPROC(sha256_ce_transform) diff --git a/arch/arm/lib/crypto/sha256.c b/arch/arm/lib/crypto/sha256.c new file mode 100644 index 000000000000..3a8dfc304807 --- /dev/null +++ b/arch/arm/lib/crypto/sha256.c @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * SHA-256 optimized for ARM + * + * Copyright 2025 Google LLC + */ +#include +#include +#include +#include +#include + +asmlinkage void sha256_block_data_order(u32 state[SHA256_STATE_WORDS], + const u8 *data, size_t nblocks); +asmlinkage void sha256_block_data_order_neon(u32 state[SHA256_STATE_WORDS], + const u8 *data, size_t nblocks); +asmlinkage void sha256_ce_transform(u32 state[SHA256_STATE_WORDS], + const u8 *data, size_t nblocks); + +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon); +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_ce); + +void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS], + const u8 *data, size_t nblocks) +{ + if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && + static_branch_likely(&have_neon) && crypto_simd_usable()) { + kernel_neon_begin(); + if (static_branch_likely(&have_ce)) + sha256_ce_transform(state, data, nblocks); + else + sha256_block_data_order_neon(state, data, nblocks); + kernel_neon_end(); + } else { + sha256_block_data_order(state, data, nblocks); + } +} +EXPORT_SYMBOL(sha256_blocks_arch); + +bool sha256_is_arch_optimized(void) +{ + /* We always can use at least the ARM scalar implementation. */ + return true; +} +EXPORT_SYMBOL(sha256_is_arch_optimized); + +static int __init sha256_arm_mod_init(void) +{ + if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && (elf_hwcap & HWCAP_NEON)) { + static_branch_enable(&have_neon); + if (elf_hwcap2 & HWCAP2_SHA2) + static_branch_enable(&have_ce); + } + return 0; +} +arch_initcall(sha256_arm_mod_init); + +static void __exit sha256_arm_mod_exit(void) +{ +} +module_exit(sha256_arm_mod_exit); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("SHA-256 optimized for ARM"); From 642cfc0680ff9aae73cd87d6fffcc84d9434938b Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 28 Apr 2025 10:00:28 -0700 Subject: [PATCH 262/382] crypto: arm64/sha256 - remove obsolete chunking logic Since kernel-mode NEON sections are now preemptible on arm64, there is no longer any need to limit the length of them. Reviewed-by: Ard Biesheuvel Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/arm64/crypto/sha256-glue.c | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/arch/arm64/crypto/sha256-glue.c b/arch/arm64/crypto/sha256-glue.c index 26f9fdfae87b..d63ea82e1374 100644 --- a/arch/arm64/crypto/sha256-glue.c +++ b/arch/arm64/crypto/sha256-glue.c @@ -86,23 +86,8 @@ static struct shash_alg algs[] = { { static int sha256_update_neon(struct shash_desc *desc, const u8 *data, unsigned int len) { - do { - unsigned int chunk = len; - - /* - * Don't hog the CPU for the entire time it takes to process all - * input when running on a preemptible kernel, but process the - * data block by block instead. - */ - if (IS_ENABLED(CONFIG_PREEMPTION)) - chunk = SHA256_BLOCK_SIZE; - - chunk -= sha256_base_do_update_blocks(desc, data, chunk, - sha256_neon_transform); - data += chunk; - len -= chunk; - } while (len >= SHA256_BLOCK_SIZE); - return len; + return sha256_base_do_update_blocks(desc, data, len, + sha256_neon_transform); } static int sha256_finup_neon(struct shash_desc *desc, const u8 *data, From 6e36be511d2846f40c0095c408797ceef17db4f5 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 28 Apr 2025 10:00:29 -0700 Subject: [PATCH 263/382] crypto: arm64/sha256 - implement library instead of shash Instead of providing crypto_shash algorithms for the arch-optimized SHA-256 code, instead implement the SHA-256 library. This is much simpler, it makes the SHA-256 library functions be arch-optimized, and it fixes the longstanding issue where the arch-optimized SHA-256 was disabled by default. SHA-256 still remains available through crypto_shash, but individual architectures no longer need to handle it. Remove support for SHA-256 finalization from the ARMv8 CE assembly code, since the library does not yet support architecture-specific overrides of the finalization. (Support for that has been omitted for now, for simplicity and because usually it isn't performance-critical.) To match sha256_blocks_arch(), change the type of the nblocks parameter of the assembly functions from int or 'unsigned int' to size_t. Update the ARMv8 CE assembly function accordingly. The scalar and NEON assembly functions actually already treated it as size_t. While renaming the assembly files, also fix the naming quirks where "sha2" meant sha256, and "sha512" meant both sha256 and sha512. Reviewed-by: Ard Biesheuvel Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/arm64/configs/defconfig | 1 - arch/arm64/crypto/Kconfig | 19 --- arch/arm64/crypto/Makefile | 13 +- arch/arm64/crypto/sha2-ce-glue.c | 138 ---------------- arch/arm64/crypto/sha256-glue.c | 156 ------------------ arch/arm64/lib/crypto/.gitignore | 1 + arch/arm64/lib/crypto/Kconfig | 5 + arch/arm64/lib/crypto/Makefile | 9 +- .../crypto/sha2-armv8.pl} | 0 .../sha2-ce-core.S => lib/crypto/sha256-ce.S} | 36 +--- arch/arm64/lib/crypto/sha256.c | 75 +++++++++ 11 files changed, 98 insertions(+), 355 deletions(-) delete mode 100644 arch/arm64/crypto/sha2-ce-glue.c delete mode 100644 arch/arm64/crypto/sha256-glue.c rename arch/arm64/{crypto/sha512-armv8.pl => lib/crypto/sha2-armv8.pl} (100%) rename arch/arm64/{crypto/sha2-ce-core.S => lib/crypto/sha256-ce.S} (80%) create mode 100644 arch/arm64/lib/crypto/sha256.c diff --git a/arch/arm64/configs/defconfig b/arch/arm64/configs/defconfig index 5bb8f09422a2..b0d4c7d173ea 100644 --- a/arch/arm64/configs/defconfig +++ b/arch/arm64/configs/defconfig @@ -1737,7 +1737,6 @@ CONFIG_CRYPTO_USER_API_RNG=m CONFIG_CRYPTO_CHACHA20_NEON=m CONFIG_CRYPTO_GHASH_ARM64_CE=y CONFIG_CRYPTO_SHA1_ARM64_CE=y -CONFIG_CRYPTO_SHA2_ARM64_CE=y CONFIG_CRYPTO_SHA512_ARM64_CE=m CONFIG_CRYPTO_SHA3_ARM64=m CONFIG_CRYPTO_SM3_ARM64_CE=m diff --git a/arch/arm64/crypto/Kconfig b/arch/arm64/crypto/Kconfig index 55a7d87a6769..c44b0f202a1f 100644 --- a/arch/arm64/crypto/Kconfig +++ b/arch/arm64/crypto/Kconfig @@ -36,25 +36,6 @@ config CRYPTO_SHA1_ARM64_CE Architecture: arm64 using: - ARMv8 Crypto Extensions -config CRYPTO_SHA256_ARM64 - tristate "Hash functions: SHA-224 and SHA-256" - select CRYPTO_HASH - help - SHA-224 and SHA-256 secure hash algorithms (FIPS 180) - - Architecture: arm64 - -config CRYPTO_SHA2_ARM64_CE - tristate "Hash functions: SHA-224 and SHA-256 (ARMv8 Crypto Extensions)" - depends on KERNEL_MODE_NEON - select CRYPTO_HASH - select CRYPTO_SHA256_ARM64 - help - SHA-224 and SHA-256 secure hash algorithms (FIPS 180) - - Architecture: arm64 using: - - ARMv8 Crypto Extensions - config CRYPTO_SHA512_ARM64 tristate "Hash functions: SHA-384 and SHA-512" select CRYPTO_HASH diff --git a/arch/arm64/crypto/Makefile b/arch/arm64/crypto/Makefile index 089ae3ddde81..c231c980c514 100644 --- a/arch/arm64/crypto/Makefile +++ b/arch/arm64/crypto/Makefile @@ -8,9 +8,6 @@ obj-$(CONFIG_CRYPTO_SHA1_ARM64_CE) += sha1-ce.o sha1-ce-y := sha1-ce-glue.o sha1-ce-core.o -obj-$(CONFIG_CRYPTO_SHA2_ARM64_CE) += sha2-ce.o -sha2-ce-y := sha2-ce-glue.o sha2-ce-core.o - obj-$(CONFIG_CRYPTO_SHA512_ARM64_CE) += sha512-ce.o sha512-ce-y := sha512-ce-glue.o sha512-ce-core.o @@ -56,9 +53,6 @@ aes-ce-blk-y := aes-glue-ce.o aes-ce.o obj-$(CONFIG_CRYPTO_AES_ARM64_NEON_BLK) += aes-neon-blk.o aes-neon-blk-y := aes-glue-neon.o aes-neon.o -obj-$(CONFIG_CRYPTO_SHA256_ARM64) += sha256-arm64.o -sha256-arm64-y := sha256-glue.o sha256-core.o - obj-$(CONFIG_CRYPTO_SHA512_ARM64) += sha512-arm64.o sha512-arm64-y := sha512-glue.o sha512-core.o @@ -74,10 +68,7 @@ aes-neon-bs-y := aes-neonbs-core.o aes-neonbs-glue.o quiet_cmd_perlasm = PERLASM $@ cmd_perlasm = $(PERL) $(<) void $(@) -$(obj)/%-core.S: $(src)/%-armv8.pl +$(obj)/sha512-core.S: $(src)/../lib/crypto/sha2-armv8.pl $(call cmd,perlasm) -$(obj)/sha256-core.S: $(src)/sha512-armv8.pl - $(call cmd,perlasm) - -clean-files += sha256-core.S sha512-core.S +clean-files += sha512-core.S diff --git a/arch/arm64/crypto/sha2-ce-glue.c b/arch/arm64/crypto/sha2-ce-glue.c deleted file mode 100644 index 912c215101eb..000000000000 --- a/arch/arm64/crypto/sha2-ce-glue.c +++ /dev/null @@ -1,138 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * sha2-ce-glue.c - SHA-224/SHA-256 using ARMv8 Crypto Extensions - * - * Copyright (C) 2014 - 2017 Linaro Ltd - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -MODULE_DESCRIPTION("SHA-224/SHA-256 secure hash using ARMv8 Crypto Extensions"); -MODULE_AUTHOR("Ard Biesheuvel "); -MODULE_LICENSE("GPL v2"); -MODULE_ALIAS_CRYPTO("sha224"); -MODULE_ALIAS_CRYPTO("sha256"); - -struct sha256_ce_state { - struct crypto_sha256_state sst; - u32 finalize; -}; - -extern const u32 sha256_ce_offsetof_count; -extern const u32 sha256_ce_offsetof_finalize; - -asmlinkage int __sha256_ce_transform(struct sha256_ce_state *sst, u8 const *src, - int blocks); - -static void sha256_ce_transform(struct crypto_sha256_state *sst, u8 const *src, - int blocks) -{ - while (blocks) { - int rem; - - kernel_neon_begin(); - rem = __sha256_ce_transform(container_of(sst, - struct sha256_ce_state, - sst), src, blocks); - kernel_neon_end(); - src += (blocks - rem) * SHA256_BLOCK_SIZE; - blocks = rem; - } -} - -const u32 sha256_ce_offsetof_count = offsetof(struct sha256_ce_state, - sst.count); -const u32 sha256_ce_offsetof_finalize = offsetof(struct sha256_ce_state, - finalize); - -static int sha256_ce_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - struct sha256_ce_state *sctx = shash_desc_ctx(desc); - - sctx->finalize = 0; - return sha256_base_do_update_blocks(desc, data, len, - sha256_ce_transform); -} - -static int sha256_ce_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - struct sha256_ce_state *sctx = shash_desc_ctx(desc); - bool finalize = !(len % SHA256_BLOCK_SIZE) && len; - - /* - * Allow the asm code to perform the finalization if there is no - * partial data and the input is a round multiple of the block size. - */ - sctx->finalize = finalize; - - if (finalize) - sha256_base_do_update_blocks(desc, data, len, - sha256_ce_transform); - else - sha256_base_do_finup(desc, data, len, sha256_ce_transform); - return sha256_base_finish(desc, out); -} - -static int sha256_ce_digest(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - sha256_base_init(desc); - return sha256_ce_finup(desc, data, len, out); -} - -static struct shash_alg algs[] = { { - .init = sha224_base_init, - .update = sha256_ce_update, - .finup = sha256_ce_finup, - .descsize = sizeof(struct sha256_ce_state), - .statesize = sizeof(struct crypto_sha256_state), - .digestsize = SHA224_DIGEST_SIZE, - .base = { - .cra_name = "sha224", - .cra_driver_name = "sha224-ce", - .cra_priority = 200, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_blocksize = SHA256_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}, { - .init = sha256_base_init, - .update = sha256_ce_update, - .finup = sha256_ce_finup, - .digest = sha256_ce_digest, - .descsize = sizeof(struct sha256_ce_state), - .statesize = sizeof(struct crypto_sha256_state), - .digestsize = SHA256_DIGEST_SIZE, - .base = { - .cra_name = "sha256", - .cra_driver_name = "sha256-ce", - .cra_priority = 200, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_blocksize = SHA256_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -} }; - -static int __init sha2_ce_mod_init(void) -{ - return crypto_register_shashes(algs, ARRAY_SIZE(algs)); -} - -static void __exit sha2_ce_mod_fini(void) -{ - crypto_unregister_shashes(algs, ARRAY_SIZE(algs)); -} - -module_cpu_feature_match(SHA2, sha2_ce_mod_init); -module_exit(sha2_ce_mod_fini); diff --git a/arch/arm64/crypto/sha256-glue.c b/arch/arm64/crypto/sha256-glue.c deleted file mode 100644 index d63ea82e1374..000000000000 --- a/arch/arm64/crypto/sha256-glue.c +++ /dev/null @@ -1,156 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Linux/arm64 port of the OpenSSL SHA256 implementation for AArch64 - * - * Copyright (c) 2016 Linaro Ltd. - */ - -#include -#include -#include -#include -#include -#include -#include - -MODULE_DESCRIPTION("SHA-224/SHA-256 secure hash for arm64"); -MODULE_AUTHOR("Andy Polyakov "); -MODULE_AUTHOR("Ard Biesheuvel "); -MODULE_LICENSE("GPL v2"); -MODULE_ALIAS_CRYPTO("sha224"); -MODULE_ALIAS_CRYPTO("sha256"); - -asmlinkage void sha256_block_data_order(u32 *digest, const void *data, - unsigned int num_blks); -EXPORT_SYMBOL(sha256_block_data_order); - -static void sha256_arm64_transform(struct crypto_sha256_state *sst, - u8 const *src, int blocks) -{ - sha256_block_data_order(sst->state, src, blocks); -} - -asmlinkage void sha256_block_neon(u32 *digest, const void *data, - unsigned int num_blks); - -static void sha256_neon_transform(struct crypto_sha256_state *sst, - u8 const *src, int blocks) -{ - kernel_neon_begin(); - sha256_block_neon(sst->state, src, blocks); - kernel_neon_end(); -} - -static int crypto_sha256_arm64_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - return sha256_base_do_update_blocks(desc, data, len, - sha256_arm64_transform); -} - -static int crypto_sha256_arm64_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - sha256_base_do_finup(desc, data, len, sha256_arm64_transform); - return sha256_base_finish(desc, out); -} - -static struct shash_alg algs[] = { { - .digestsize = SHA256_DIGEST_SIZE, - .init = sha256_base_init, - .update = crypto_sha256_arm64_update, - .finup = crypto_sha256_arm64_finup, - .descsize = sizeof(struct crypto_sha256_state), - .base.cra_name = "sha256", - .base.cra_driver_name = "sha256-arm64", - .base.cra_priority = 125, - .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .base.cra_blocksize = SHA256_BLOCK_SIZE, - .base.cra_module = THIS_MODULE, -}, { - .digestsize = SHA224_DIGEST_SIZE, - .init = sha224_base_init, - .update = crypto_sha256_arm64_update, - .finup = crypto_sha256_arm64_finup, - .descsize = sizeof(struct crypto_sha256_state), - .base.cra_name = "sha224", - .base.cra_driver_name = "sha224-arm64", - .base.cra_priority = 125, - .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .base.cra_blocksize = SHA224_BLOCK_SIZE, - .base.cra_module = THIS_MODULE, -} }; - -static int sha256_update_neon(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - return sha256_base_do_update_blocks(desc, data, len, - sha256_neon_transform); -} - -static int sha256_finup_neon(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - if (len >= SHA256_BLOCK_SIZE) { - int remain = sha256_update_neon(desc, data, len); - - data += len - remain; - len = remain; - } - sha256_base_do_finup(desc, data, len, sha256_neon_transform); - return sha256_base_finish(desc, out); -} - -static struct shash_alg neon_algs[] = { { - .digestsize = SHA256_DIGEST_SIZE, - .init = sha256_base_init, - .update = sha256_update_neon, - .finup = sha256_finup_neon, - .descsize = sizeof(struct crypto_sha256_state), - .base.cra_name = "sha256", - .base.cra_driver_name = "sha256-arm64-neon", - .base.cra_priority = 150, - .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .base.cra_blocksize = SHA256_BLOCK_SIZE, - .base.cra_module = THIS_MODULE, -}, { - .digestsize = SHA224_DIGEST_SIZE, - .init = sha224_base_init, - .update = sha256_update_neon, - .finup = sha256_finup_neon, - .descsize = sizeof(struct crypto_sha256_state), - .base.cra_name = "sha224", - .base.cra_driver_name = "sha224-arm64-neon", - .base.cra_priority = 150, - .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .base.cra_blocksize = SHA224_BLOCK_SIZE, - .base.cra_module = THIS_MODULE, -} }; - -static int __init sha256_mod_init(void) -{ - int ret = crypto_register_shashes(algs, ARRAY_SIZE(algs)); - if (ret) - return ret; - - if (cpu_have_named_feature(ASIMD)) { - ret = crypto_register_shashes(neon_algs, ARRAY_SIZE(neon_algs)); - if (ret) - crypto_unregister_shashes(algs, ARRAY_SIZE(algs)); - } - return ret; -} - -static void __exit sha256_mod_fini(void) -{ - if (cpu_have_named_feature(ASIMD)) - crypto_unregister_shashes(neon_algs, ARRAY_SIZE(neon_algs)); - crypto_unregister_shashes(algs, ARRAY_SIZE(algs)); -} - -module_init(sha256_mod_init); -module_exit(sha256_mod_fini); diff --git a/arch/arm64/lib/crypto/.gitignore b/arch/arm64/lib/crypto/.gitignore index 0d47d4f21c6d..12d74d8b03d0 100644 --- a/arch/arm64/lib/crypto/.gitignore +++ b/arch/arm64/lib/crypto/.gitignore @@ -1,2 +1,3 @@ # SPDX-License-Identifier: GPL-2.0-only poly1305-core.S +sha256-core.S diff --git a/arch/arm64/lib/crypto/Kconfig b/arch/arm64/lib/crypto/Kconfig index 0b903ef524d8..49e57bfdb5b5 100644 --- a/arch/arm64/lib/crypto/Kconfig +++ b/arch/arm64/lib/crypto/Kconfig @@ -12,3 +12,8 @@ config CRYPTO_POLY1305_NEON depends on KERNEL_MODE_NEON default CRYPTO_LIB_POLY1305 select CRYPTO_ARCH_HAVE_LIB_POLY1305 + +config CRYPTO_SHA256_ARM64 + tristate + default CRYPTO_LIB_SHA256 + select CRYPTO_ARCH_HAVE_LIB_SHA256 diff --git a/arch/arm64/lib/crypto/Makefile b/arch/arm64/lib/crypto/Makefile index 6207088397a7..946c09903711 100644 --- a/arch/arm64/lib/crypto/Makefile +++ b/arch/arm64/lib/crypto/Makefile @@ -8,10 +8,17 @@ poly1305-neon-y := poly1305-core.o poly1305-glue.o AFLAGS_poly1305-core.o += -Dpoly1305_init=poly1305_block_init_arch AFLAGS_poly1305-core.o += -Dpoly1305_emit=poly1305_emit_arch +obj-$(CONFIG_CRYPTO_SHA256_ARM64) += sha256-arm64.o +sha256-arm64-y := sha256.o sha256-core.o +sha256-arm64-$(CONFIG_KERNEL_MODE_NEON) += sha256-ce.o + quiet_cmd_perlasm = PERLASM $@ cmd_perlasm = $(PERL) $(<) void $(@) $(obj)/%-core.S: $(src)/%-armv8.pl $(call cmd,perlasm) -clean-files += poly1305-core.S +$(obj)/sha256-core.S: $(src)/sha2-armv8.pl + $(call cmd,perlasm) + +clean-files += poly1305-core.S sha256-core.S diff --git a/arch/arm64/crypto/sha512-armv8.pl b/arch/arm64/lib/crypto/sha2-armv8.pl similarity index 100% rename from arch/arm64/crypto/sha512-armv8.pl rename to arch/arm64/lib/crypto/sha2-armv8.pl diff --git a/arch/arm64/crypto/sha2-ce-core.S b/arch/arm64/lib/crypto/sha256-ce.S similarity index 80% rename from arch/arm64/crypto/sha2-ce-core.S rename to arch/arm64/lib/crypto/sha256-ce.S index fce84d88ddb2..a8461d6dad63 100644 --- a/arch/arm64/crypto/sha2-ce-core.S +++ b/arch/arm64/lib/crypto/sha256-ce.S @@ -71,8 +71,8 @@ .word 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2 /* - * int __sha256_ce_transform(struct sha256_ce_state *sst, u8 const *src, - * int blocks) + * size_t __sha256_ce_transform(u32 state[SHA256_STATE_WORDS], + * const u8 *data, size_t nblocks); */ .text SYM_FUNC_START(__sha256_ce_transform) @@ -86,20 +86,16 @@ SYM_FUNC_START(__sha256_ce_transform) /* load state */ ld1 {dgav.4s, dgbv.4s}, [x0] - /* load sha256_ce_state::finalize */ - ldr_l w4, sha256_ce_offsetof_finalize, x4 - ldr w4, [x0, x4] - /* load input */ 0: ld1 {v16.4s-v19.4s}, [x1], #64 - sub w2, w2, #1 + sub x2, x2, #1 CPU_LE( rev32 v16.16b, v16.16b ) CPU_LE( rev32 v17.16b, v17.16b ) CPU_LE( rev32 v18.16b, v18.16b ) CPU_LE( rev32 v19.16b, v19.16b ) -1: add t0.4s, v16.4s, v0.4s + add t0.4s, v16.4s, v0.4s mov dg0v.16b, dgav.16b mov dg1v.16b, dgbv.16b @@ -128,30 +124,12 @@ CPU_LE( rev32 v19.16b, v19.16b ) add dgbv.4s, dgbv.4s, dg1v.4s /* handled all input blocks? */ - cbz w2, 2f + cbz x2, 1f cond_yield 3f, x5, x6 b 0b - /* - * Final block: add padding and total bit count. - * Skip if the input size was not a round multiple of the block size, - * the padding is handled by the C code in that case. - */ -2: cbz x4, 3f - ldr_l w4, sha256_ce_offsetof_count, x4 - ldr x4, [x0, x4] - movi v17.2d, #0 - mov x8, #0x80000000 - movi v18.2d, #0 - ror x7, x4, #29 // ror(lsl(x4, 3), 32) - fmov d16, x8 - mov x4, #0 - mov v19.d[0], xzr - mov v19.d[1], x7 - b 1b - /* store new state */ -3: st1 {dgav.4s, dgbv.4s}, [x0] - mov w0, w2 +1: st1 {dgav.4s, dgbv.4s}, [x0] + mov x0, x2 ret SYM_FUNC_END(__sha256_ce_transform) diff --git a/arch/arm64/lib/crypto/sha256.c b/arch/arm64/lib/crypto/sha256.c new file mode 100644 index 000000000000..2bd413c586d2 --- /dev/null +++ b/arch/arm64/lib/crypto/sha256.c @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * SHA-256 optimized for ARM64 + * + * Copyright 2025 Google LLC + */ +#include +#include +#include +#include +#include + +asmlinkage void sha256_block_data_order(u32 state[SHA256_STATE_WORDS], + const u8 *data, size_t nblocks); +asmlinkage void sha256_block_neon(u32 state[SHA256_STATE_WORDS], + const u8 *data, size_t nblocks); +asmlinkage size_t __sha256_ce_transform(u32 state[SHA256_STATE_WORDS], + const u8 *data, size_t nblocks); + +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon); +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_ce); + +void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS], + const u8 *data, size_t nblocks) +{ + if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && + static_branch_likely(&have_neon) && crypto_simd_usable()) { + if (static_branch_likely(&have_ce)) { + do { + size_t rem; + + kernel_neon_begin(); + rem = __sha256_ce_transform(state, + data, nblocks); + kernel_neon_end(); + data += (nblocks - rem) * SHA256_BLOCK_SIZE; + nblocks = rem; + } while (nblocks); + } else { + kernel_neon_begin(); + sha256_block_neon(state, data, nblocks); + kernel_neon_end(); + } + } else { + sha256_block_data_order(state, data, nblocks); + } +} +EXPORT_SYMBOL(sha256_blocks_arch); + +bool sha256_is_arch_optimized(void) +{ + /* We always can use at least the ARM64 scalar implementation. */ + return true; +} +EXPORT_SYMBOL(sha256_is_arch_optimized); + +static int __init sha256_arm64_mod_init(void) +{ + if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && + cpu_have_named_feature(ASIMD)) { + static_branch_enable(&have_neon); + if (cpu_have_named_feature(SHA2)) + static_branch_enable(&have_ce); + } + return 0; +} +arch_initcall(sha256_arm64_mod_init); + +static void __exit sha256_arm64_mod_exit(void) +{ +} +module_exit(sha256_arm64_mod_exit); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("SHA-256 optimized for ARM64"); From b67b6f9adb326f4a816b11bc3dabf9190070f536 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 28 Apr 2025 10:00:30 -0700 Subject: [PATCH 264/382] crypto: mips/sha256 - implement library instead of shash Instead of providing crypto_shash algorithms for the arch-optimized SHA-256 code, instead implement the SHA-256 library. This is much simpler, it makes the SHA-256 library functions be arch-optimized, and it fixes the longstanding issue where the arch-optimized SHA-256 was disabled by default. SHA-256 still remains available through crypto_shash, but individual architectures no longer need to handle it. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/mips/cavium-octeon/Kconfig | 6 + .../mips/cavium-octeon/crypto/octeon-sha256.c | 139 ++++-------------- arch/mips/configs/cavium_octeon_defconfig | 1 - arch/mips/crypto/Kconfig | 10 -- 4 files changed, 35 insertions(+), 121 deletions(-) diff --git a/arch/mips/cavium-octeon/Kconfig b/arch/mips/cavium-octeon/Kconfig index 450e979ef5d9..11f4aa6e80e9 100644 --- a/arch/mips/cavium-octeon/Kconfig +++ b/arch/mips/cavium-octeon/Kconfig @@ -23,6 +23,12 @@ config CAVIUM_OCTEON_CVMSEG_SIZE legally range is from zero to 54 cache blocks (i.e. CVMSEG LM is between zero and 6192 bytes). +config CRYPTO_SHA256_OCTEON + tristate + default CRYPTO_LIB_SHA256 + select CRYPTO_ARCH_HAVE_LIB_SHA256 + select CRYPTO_LIB_SHA256_GENERIC + endif # CPU_CAVIUM_OCTEON if CAVIUM_OCTEON_SOC diff --git a/arch/mips/cavium-octeon/crypto/octeon-sha256.c b/arch/mips/cavium-octeon/crypto/octeon-sha256.c index 8e85ea65387c..f169054852bc 100644 --- a/arch/mips/cavium-octeon/crypto/octeon-sha256.c +++ b/arch/mips/cavium-octeon/crypto/octeon-sha256.c @@ -1,8 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-or-later /* - * Cryptographic API. - * - * SHA-224 and SHA-256 Secure Hash Algorithm. + * SHA-256 Secure Hash Algorithm. * * Adapted for OCTEON by Aaro Koskinen . * @@ -15,9 +13,7 @@ */ #include -#include -#include -#include +#include #include #include @@ -27,31 +23,24 @@ * We pass everything as 64-bit. OCTEON can handle misaligned data. */ -static void octeon_sha256_store_hash(struct crypto_sha256_state *sctx) +void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS], + const u8 *data, size_t nblocks) { - u64 *hash = (u64 *)sctx->state; + struct octeon_cop2_state cop2_state; + u64 *state64 = (u64 *)state; + unsigned long flags; - write_octeon_64bit_hash_dword(hash[0], 0); - write_octeon_64bit_hash_dword(hash[1], 1); - write_octeon_64bit_hash_dword(hash[2], 2); - write_octeon_64bit_hash_dword(hash[3], 3); -} + if (!octeon_has_crypto()) + return sha256_blocks_generic(state, data, nblocks); -static void octeon_sha256_read_hash(struct crypto_sha256_state *sctx) -{ - u64 *hash = (u64 *)sctx->state; + flags = octeon_crypto_enable(&cop2_state); + write_octeon_64bit_hash_dword(state64[0], 0); + write_octeon_64bit_hash_dword(state64[1], 1); + write_octeon_64bit_hash_dword(state64[2], 2); + write_octeon_64bit_hash_dword(state64[3], 3); - hash[0] = read_octeon_64bit_hash_dword(0); - hash[1] = read_octeon_64bit_hash_dword(1); - hash[2] = read_octeon_64bit_hash_dword(2); - hash[3] = read_octeon_64bit_hash_dword(3); -} - -static void octeon_sha256_transform(struct crypto_sha256_state *sctx, - const u8 *src, int blocks) -{ do { - const u64 *block = (const u64 *)src; + const u64 *block = (const u64 *)data; write_octeon_64bit_block_dword(block[0], 0); write_octeon_64bit_block_dword(block[1], 1); @@ -62,93 +51,23 @@ static void octeon_sha256_transform(struct crypto_sha256_state *sctx, write_octeon_64bit_block_dword(block[6], 6); octeon_sha256_start(block[7]); - src += SHA256_BLOCK_SIZE; - } while (--blocks); -} + data += SHA256_BLOCK_SIZE; + } while (--nblocks); -static int octeon_sha256_update(struct shash_desc *desc, const u8 *data, - unsigned int len) + state64[0] = read_octeon_64bit_hash_dword(0); + state64[1] = read_octeon_64bit_hash_dword(1); + state64[2] = read_octeon_64bit_hash_dword(2); + state64[3] = read_octeon_64bit_hash_dword(3); + octeon_crypto_disable(&cop2_state, flags); +} +EXPORT_SYMBOL(sha256_blocks_arch); + +bool sha256_is_arch_optimized(void) { - struct crypto_sha256_state *sctx = shash_desc_ctx(desc); - struct octeon_cop2_state state; - unsigned long flags; - int remain; - - flags = octeon_crypto_enable(&state); - octeon_sha256_store_hash(sctx); - - remain = sha256_base_do_update_blocks(desc, data, len, - octeon_sha256_transform); - - octeon_sha256_read_hash(sctx); - octeon_crypto_disable(&state, flags); - return remain; + return octeon_has_crypto(); } - -static int octeon_sha256_finup(struct shash_desc *desc, const u8 *src, - unsigned int len, u8 *out) -{ - struct crypto_sha256_state *sctx = shash_desc_ctx(desc); - struct octeon_cop2_state state; - unsigned long flags; - - flags = octeon_crypto_enable(&state); - octeon_sha256_store_hash(sctx); - - sha256_base_do_finup(desc, src, len, octeon_sha256_transform); - - octeon_sha256_read_hash(sctx); - octeon_crypto_disable(&state, flags); - return sha256_base_finish(desc, out); -} - -static struct shash_alg octeon_sha256_algs[2] = { { - .digestsize = SHA256_DIGEST_SIZE, - .init = sha256_base_init, - .update = octeon_sha256_update, - .finup = octeon_sha256_finup, - .descsize = sizeof(struct crypto_sha256_state), - .base = { - .cra_name = "sha256", - .cra_driver_name= "octeon-sha256", - .cra_priority = OCTEON_CR_OPCODE_PRIORITY, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, - .cra_blocksize = SHA256_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}, { - .digestsize = SHA224_DIGEST_SIZE, - .init = sha224_base_init, - .update = octeon_sha256_update, - .finup = octeon_sha256_finup, - .descsize = sizeof(struct crypto_sha256_state), - .base = { - .cra_name = "sha224", - .cra_driver_name= "octeon-sha224", - .cra_priority = OCTEON_CR_OPCODE_PRIORITY, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, - .cra_blocksize = SHA224_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -} }; - -static int __init octeon_sha256_mod_init(void) -{ - if (!octeon_has_crypto()) - return -ENOTSUPP; - return crypto_register_shashes(octeon_sha256_algs, - ARRAY_SIZE(octeon_sha256_algs)); -} - -static void __exit octeon_sha256_mod_fini(void) -{ - crypto_unregister_shashes(octeon_sha256_algs, - ARRAY_SIZE(octeon_sha256_algs)); -} - -module_init(octeon_sha256_mod_init); -module_exit(octeon_sha256_mod_fini); +EXPORT_SYMBOL(sha256_is_arch_optimized); MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("SHA-224 and SHA-256 Secure Hash Algorithm (OCTEON)"); +MODULE_DESCRIPTION("SHA-256 Secure Hash Algorithm (OCTEON)"); MODULE_AUTHOR("Aaro Koskinen "); diff --git a/arch/mips/configs/cavium_octeon_defconfig b/arch/mips/configs/cavium_octeon_defconfig index f523ee6f25bf..88ae0aa85364 100644 --- a/arch/mips/configs/cavium_octeon_defconfig +++ b/arch/mips/configs/cavium_octeon_defconfig @@ -157,7 +157,6 @@ CONFIG_CRYPTO_CBC=y CONFIG_CRYPTO_HMAC=y CONFIG_CRYPTO_MD5_OCTEON=y CONFIG_CRYPTO_SHA1_OCTEON=m -CONFIG_CRYPTO_SHA256_OCTEON=m CONFIG_CRYPTO_SHA512_OCTEON=m CONFIG_CRYPTO_DES=y CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y diff --git a/arch/mips/crypto/Kconfig b/arch/mips/crypto/Kconfig index 9db1fd6d9f0e..6bf073ae7613 100644 --- a/arch/mips/crypto/Kconfig +++ b/arch/mips/crypto/Kconfig @@ -22,16 +22,6 @@ config CRYPTO_SHA1_OCTEON Architecture: mips OCTEON -config CRYPTO_SHA256_OCTEON - tristate "Hash functions: SHA-224 and SHA-256 (OCTEON)" - depends on CPU_CAVIUM_OCTEON - select CRYPTO_SHA256 - select CRYPTO_HASH - help - SHA-224 and SHA-256 secure hash algorithms (FIPS 180) - - Architecture: mips OCTEON using crypto instructions, when available - config CRYPTO_SHA512_OCTEON tristate "Hash functions: SHA-384 and SHA-512 (OCTEON)" depends on CPU_CAVIUM_OCTEON From 1a49c573bf886349c7bc958b331c700cf18601d8 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 28 Apr 2025 10:00:31 -0700 Subject: [PATCH 265/382] crypto: powerpc/sha256 - implement library instead of shash Instead of providing crypto_shash algorithms for the arch-optimized SHA-256 code, instead implement the SHA-256 library. This is much simpler, it makes the SHA-256 library functions be arch-optimized, and it fixes the longstanding issue where the arch-optimized SHA-256 was disabled by default. SHA-256 still remains available through crypto_shash, but individual architectures no longer need to handle it. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/powerpc/crypto/Kconfig | 11 -- arch/powerpc/crypto/Makefile | 2 - arch/powerpc/crypto/sha256-spe-glue.c | 128 ------------------ arch/powerpc/lib/crypto/Kconfig | 6 + arch/powerpc/lib/crypto/Makefile | 3 + .../powerpc/{ => lib}/crypto/sha256-spe-asm.S | 0 arch/powerpc/lib/crypto/sha256.c | 70 ++++++++++ 7 files changed, 79 insertions(+), 141 deletions(-) delete mode 100644 arch/powerpc/crypto/sha256-spe-glue.c rename arch/powerpc/{ => lib}/crypto/sha256-spe-asm.S (100%) create mode 100644 arch/powerpc/lib/crypto/sha256.c diff --git a/arch/powerpc/crypto/Kconfig b/arch/powerpc/crypto/Kconfig index 4bf7b01228e7..caaa359f4742 100644 --- a/arch/powerpc/crypto/Kconfig +++ b/arch/powerpc/crypto/Kconfig @@ -39,17 +39,6 @@ config CRYPTO_SHA1_PPC_SPE Architecture: powerpc using - SPE (Signal Processing Engine) extensions -config CRYPTO_SHA256_PPC_SPE - tristate "Hash functions: SHA-224 and SHA-256 (SPE)" - depends on SPE - select CRYPTO_SHA256 - select CRYPTO_HASH - help - SHA-224 and SHA-256 secure hash algorithms (FIPS 180) - - Architecture: powerpc using - - SPE (Signal Processing Engine) extensions - config CRYPTO_AES_PPC_SPE tristate "Ciphers: AES, modes: ECB/CBC/CTR/XTS (SPE)" depends on SPE diff --git a/arch/powerpc/crypto/Makefile b/arch/powerpc/crypto/Makefile index f13aec8a1833..8c2936ae466f 100644 --- a/arch/powerpc/crypto/Makefile +++ b/arch/powerpc/crypto/Makefile @@ -9,7 +9,6 @@ obj-$(CONFIG_CRYPTO_AES_PPC_SPE) += aes-ppc-spe.o obj-$(CONFIG_CRYPTO_MD5_PPC) += md5-ppc.o obj-$(CONFIG_CRYPTO_SHA1_PPC) += sha1-powerpc.o obj-$(CONFIG_CRYPTO_SHA1_PPC_SPE) += sha1-ppc-spe.o -obj-$(CONFIG_CRYPTO_SHA256_PPC_SPE) += sha256-ppc-spe.o obj-$(CONFIG_CRYPTO_AES_GCM_P10) += aes-gcm-p10-crypto.o obj-$(CONFIG_CRYPTO_DEV_VMX_ENCRYPT) += vmx-crypto.o obj-$(CONFIG_CRYPTO_CURVE25519_PPC64) += curve25519-ppc64le.o @@ -18,7 +17,6 @@ aes-ppc-spe-y := aes-spe-core.o aes-spe-keys.o aes-tab-4k.o aes-spe-modes.o aes- md5-ppc-y := md5-asm.o md5-glue.o sha1-powerpc-y := sha1-powerpc-asm.o sha1.o sha1-ppc-spe-y := sha1-spe-asm.o sha1-spe-glue.o -sha256-ppc-spe-y := sha256-spe-asm.o sha256-spe-glue.o aes-gcm-p10-crypto-y := aes-gcm-p10-glue.o aes-gcm-p10.o ghashp10-ppc.o aesp10-ppc.o vmx-crypto-objs := vmx.o aesp8-ppc.o ghashp8-ppc.o aes.o aes_cbc.o aes_ctr.o aes_xts.o ghash.o curve25519-ppc64le-y := curve25519-ppc64le-core.o curve25519-ppc64le_asm.o diff --git a/arch/powerpc/crypto/sha256-spe-glue.c b/arch/powerpc/crypto/sha256-spe-glue.c deleted file mode 100644 index 42c76bf8062d..000000000000 --- a/arch/powerpc/crypto/sha256-spe-glue.c +++ /dev/null @@ -1,128 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Glue code for SHA-256 implementation for SPE instructions (PPC) - * - * Based on generic implementation. The assembler module takes care - * about the SPE registers so it can run from interrupt context. - * - * Copyright (c) 2015 Markus Stockhausen - */ - -#include -#include -#include -#include -#include -#include -#include - -/* - * MAX_BYTES defines the number of bytes that are allowed to be processed - * between preempt_disable() and preempt_enable(). SHA256 takes ~2,000 - * operations per 64 bytes. e500 cores can issue two arithmetic instructions - * per clock cycle using one 32/64 bit unit (SU1) and one 32 bit unit (SU2). - * Thus 1KB of input data will need an estimated maximum of 18,000 cycles. - * Headroom for cache misses included. Even with the low end model clocked - * at 667 MHz this equals to a critical time window of less than 27us. - * - */ -#define MAX_BYTES 1024 - -extern void ppc_spe_sha256_transform(u32 *state, const u8 *src, u32 blocks); - -static void spe_begin(void) -{ - /* We just start SPE operations and will save SPE registers later. */ - preempt_disable(); - enable_kernel_spe(); -} - -static void spe_end(void) -{ - disable_kernel_spe(); - /* reenable preemption */ - preempt_enable(); -} - -static void ppc_spe_sha256_block(struct crypto_sha256_state *sctx, - const u8 *src, int blocks) -{ - do { - /* cut input data into smaller blocks */ - int unit = min(blocks, MAX_BYTES / SHA256_BLOCK_SIZE); - - spe_begin(); - ppc_spe_sha256_transform(sctx->state, src, unit); - spe_end(); - - src += unit * SHA256_BLOCK_SIZE; - blocks -= unit; - } while (blocks); -} - -static int ppc_spe_sha256_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - return sha256_base_do_update_blocks(desc, data, len, - ppc_spe_sha256_block); -} - -static int ppc_spe_sha256_finup(struct shash_desc *desc, const u8 *src, - unsigned int len, u8 *out) -{ - sha256_base_do_finup(desc, src, len, ppc_spe_sha256_block); - return sha256_base_finish(desc, out); -} - -static struct shash_alg algs[2] = { { - .digestsize = SHA256_DIGEST_SIZE, - .init = sha256_base_init, - .update = ppc_spe_sha256_update, - .finup = ppc_spe_sha256_finup, - .descsize = sizeof(struct crypto_sha256_state), - .base = { - .cra_name = "sha256", - .cra_driver_name= "sha256-ppc-spe", - .cra_priority = 300, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_blocksize = SHA256_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}, { - .digestsize = SHA224_DIGEST_SIZE, - .init = sha224_base_init, - .update = ppc_spe_sha256_update, - .finup = ppc_spe_sha256_finup, - .descsize = sizeof(struct crypto_sha256_state), - .base = { - .cra_name = "sha224", - .cra_driver_name= "sha224-ppc-spe", - .cra_priority = 300, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_blocksize = SHA224_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -} }; - -static int __init ppc_spe_sha256_mod_init(void) -{ - return crypto_register_shashes(algs, ARRAY_SIZE(algs)); -} - -static void __exit ppc_spe_sha256_mod_fini(void) -{ - crypto_unregister_shashes(algs, ARRAY_SIZE(algs)); -} - -module_init(ppc_spe_sha256_mod_init); -module_exit(ppc_spe_sha256_mod_fini); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("SHA-224 and SHA-256 Secure Hash Algorithm, SPE optimized"); - -MODULE_ALIAS_CRYPTO("sha224"); -MODULE_ALIAS_CRYPTO("sha224-ppc-spe"); -MODULE_ALIAS_CRYPTO("sha256"); -MODULE_ALIAS_CRYPTO("sha256-ppc-spe"); diff --git a/arch/powerpc/lib/crypto/Kconfig b/arch/powerpc/lib/crypto/Kconfig index bf6d0ab22c27..ffa541ad6d5d 100644 --- a/arch/powerpc/lib/crypto/Kconfig +++ b/arch/powerpc/lib/crypto/Kconfig @@ -13,3 +13,9 @@ config CRYPTO_POLY1305_P10 default CRYPTO_LIB_POLY1305 select CRYPTO_ARCH_HAVE_LIB_POLY1305 select CRYPTO_LIB_POLY1305_GENERIC + +config CRYPTO_SHA256_PPC_SPE + tristate + depends on SPE + default CRYPTO_LIB_SHA256 + select CRYPTO_ARCH_HAVE_LIB_SHA256 diff --git a/arch/powerpc/lib/crypto/Makefile b/arch/powerpc/lib/crypto/Makefile index 5709ae14258a..27f231f8e334 100644 --- a/arch/powerpc/lib/crypto/Makefile +++ b/arch/powerpc/lib/crypto/Makefile @@ -5,3 +5,6 @@ chacha-p10-crypto-y := chacha-p10-glue.o chacha-p10le-8x.o obj-$(CONFIG_CRYPTO_POLY1305_P10) += poly1305-p10-crypto.o poly1305-p10-crypto-y := poly1305-p10-glue.o poly1305-p10le_64.o + +obj-$(CONFIG_CRYPTO_SHA256_PPC_SPE) += sha256-ppc-spe.o +sha256-ppc-spe-y := sha256.o sha256-spe-asm.o diff --git a/arch/powerpc/crypto/sha256-spe-asm.S b/arch/powerpc/lib/crypto/sha256-spe-asm.S similarity index 100% rename from arch/powerpc/crypto/sha256-spe-asm.S rename to arch/powerpc/lib/crypto/sha256-spe-asm.S diff --git a/arch/powerpc/lib/crypto/sha256.c b/arch/powerpc/lib/crypto/sha256.c new file mode 100644 index 000000000000..c05023c5acdd --- /dev/null +++ b/arch/powerpc/lib/crypto/sha256.c @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * SHA-256 Secure Hash Algorithm, SPE optimized + * + * Based on generic implementation. The assembler module takes care + * about the SPE registers so it can run from interrupt context. + * + * Copyright (c) 2015 Markus Stockhausen + */ + +#include +#include +#include +#include +#include + +/* + * MAX_BYTES defines the number of bytes that are allowed to be processed + * between preempt_disable() and preempt_enable(). SHA256 takes ~2,000 + * operations per 64 bytes. e500 cores can issue two arithmetic instructions + * per clock cycle using one 32/64 bit unit (SU1) and one 32 bit unit (SU2). + * Thus 1KB of input data will need an estimated maximum of 18,000 cycles. + * Headroom for cache misses included. Even with the low end model clocked + * at 667 MHz this equals to a critical time window of less than 27us. + * + */ +#define MAX_BYTES 1024 + +extern void ppc_spe_sha256_transform(u32 *state, const u8 *src, u32 blocks); + +static void spe_begin(void) +{ + /* We just start SPE operations and will save SPE registers later. */ + preempt_disable(); + enable_kernel_spe(); +} + +static void spe_end(void) +{ + disable_kernel_spe(); + /* reenable preemption */ + preempt_enable(); +} + +void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS], + const u8 *data, size_t nblocks) +{ + do { + /* cut input data into smaller blocks */ + u32 unit = min_t(size_t, nblocks, + MAX_BYTES / SHA256_BLOCK_SIZE); + + spe_begin(); + ppc_spe_sha256_transform(state, data, unit); + spe_end(); + + data += unit * SHA256_BLOCK_SIZE; + nblocks -= unit; + } while (nblocks); +} +EXPORT_SYMBOL(sha256_blocks_arch); + +bool sha256_is_arch_optimized(void) +{ + return true; +} +EXPORT_SYMBOL(sha256_is_arch_optimized); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("SHA-256 Secure Hash Algorithm, SPE optimized"); From bf52d93865bde68896d20f2ce6c2f76d96cd678e Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 28 Apr 2025 10:00:32 -0700 Subject: [PATCH 266/382] crypto: riscv/sha256 - implement library instead of shash Instead of providing crypto_shash algorithms for the arch-optimized SHA-256 code, instead implement the SHA-256 library. This is much simpler, it makes the SHA-256 library functions be arch-optimized, and it fixes the longstanding issue where the arch-optimized SHA-256 was disabled by default. SHA-256 still remains available through crypto_shash, but individual architectures no longer need to handle it. To match sha256_blocks_arch(), change the type of the nblocks parameter of the assembly function from int to size_t. The assembly function actually already treated it as size_t. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/riscv/crypto/Kconfig | 11 -- arch/riscv/crypto/Makefile | 3 - arch/riscv/crypto/sha256-riscv64-glue.c | 125 ------------------ arch/riscv/lib/crypto/Kconfig | 7 + arch/riscv/lib/crypto/Makefile | 3 + .../sha256-riscv64-zvknha_or_zvknhb-zvkb.S | 4 +- arch/riscv/lib/crypto/sha256.c | 62 +++++++++ 7 files changed, 74 insertions(+), 141 deletions(-) delete mode 100644 arch/riscv/crypto/sha256-riscv64-glue.c rename arch/riscv/{ => lib}/crypto/sha256-riscv64-zvknha_or_zvknhb-zvkb.S (98%) create mode 100644 arch/riscv/lib/crypto/sha256.c diff --git a/arch/riscv/crypto/Kconfig b/arch/riscv/crypto/Kconfig index 4863be2a4ec2..cd9b776602f8 100644 --- a/arch/riscv/crypto/Kconfig +++ b/arch/riscv/crypto/Kconfig @@ -28,17 +28,6 @@ config CRYPTO_GHASH_RISCV64 Architecture: riscv64 using: - Zvkg vector crypto extension -config CRYPTO_SHA256_RISCV64 - tristate "Hash functions: SHA-224 and SHA-256" - depends on 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO - select CRYPTO_SHA256 - help - SHA-224 and SHA-256 secure hash algorithm (FIPS 180) - - Architecture: riscv64 using: - - Zvknha or Zvknhb vector crypto extensions - - Zvkb vector crypto extension - config CRYPTO_SHA512_RISCV64 tristate "Hash functions: SHA-384 and SHA-512" depends on 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO diff --git a/arch/riscv/crypto/Makefile b/arch/riscv/crypto/Makefile index 4ae9bf762e90..e10e8257734e 100644 --- a/arch/riscv/crypto/Makefile +++ b/arch/riscv/crypto/Makefile @@ -7,9 +7,6 @@ aes-riscv64-y := aes-riscv64-glue.o aes-riscv64-zvkned.o \ obj-$(CONFIG_CRYPTO_GHASH_RISCV64) += ghash-riscv64.o ghash-riscv64-y := ghash-riscv64-glue.o ghash-riscv64-zvkg.o -obj-$(CONFIG_CRYPTO_SHA256_RISCV64) += sha256-riscv64.o -sha256-riscv64-y := sha256-riscv64-glue.o sha256-riscv64-zvknha_or_zvknhb-zvkb.o - obj-$(CONFIG_CRYPTO_SHA512_RISCV64) += sha512-riscv64.o sha512-riscv64-y := sha512-riscv64-glue.o sha512-riscv64-zvknhb-zvkb.o diff --git a/arch/riscv/crypto/sha256-riscv64-glue.c b/arch/riscv/crypto/sha256-riscv64-glue.c deleted file mode 100644 index c998300ab843..000000000000 --- a/arch/riscv/crypto/sha256-riscv64-glue.c +++ /dev/null @@ -1,125 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * SHA-256 and SHA-224 using the RISC-V vector crypto extensions - * - * Copyright (C) 2022 VRULL GmbH - * Author: Heiko Stuebner - * - * Copyright (C) 2023 SiFive, Inc. - * Author: Jerry Shih - */ - -#include -#include -#include -#include -#include -#include -#include - -/* - * Note: the asm function only uses the 'state' field of struct sha256_state. - * It is assumed to be the first field. - */ -asmlinkage void sha256_transform_zvknha_or_zvknhb_zvkb( - struct crypto_sha256_state *state, const u8 *data, int num_blocks); - -static void sha256_block(struct crypto_sha256_state *state, const u8 *data, - int num_blocks) -{ - /* - * Ensure struct crypto_sha256_state begins directly with the SHA-256 - * 256-bit internal state, as this is what the asm function expects. - */ - BUILD_BUG_ON(offsetof(struct crypto_sha256_state, state) != 0); - - if (crypto_simd_usable()) { - kernel_vector_begin(); - sha256_transform_zvknha_or_zvknhb_zvkb(state, data, num_blocks); - kernel_vector_end(); - } else - sha256_transform_blocks(state, data, num_blocks); -} - -static int riscv64_sha256_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - return sha256_base_do_update_blocks(desc, data, len, sha256_block); -} - -static int riscv64_sha256_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - sha256_base_do_finup(desc, data, len, sha256_block); - return sha256_base_finish(desc, out); -} - -static int riscv64_sha256_digest(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - return sha256_base_init(desc) ?: - riscv64_sha256_finup(desc, data, len, out); -} - -static struct shash_alg riscv64_sha256_algs[] = { - { - .init = sha256_base_init, - .update = riscv64_sha256_update, - .finup = riscv64_sha256_finup, - .digest = riscv64_sha256_digest, - .descsize = sizeof(struct crypto_sha256_state), - .digestsize = SHA256_DIGEST_SIZE, - .base = { - .cra_blocksize = SHA256_BLOCK_SIZE, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_priority = 300, - .cra_name = "sha256", - .cra_driver_name = "sha256-riscv64-zvknha_or_zvknhb-zvkb", - .cra_module = THIS_MODULE, - }, - }, { - .init = sha224_base_init, - .update = riscv64_sha256_update, - .finup = riscv64_sha256_finup, - .descsize = sizeof(struct crypto_sha256_state), - .digestsize = SHA224_DIGEST_SIZE, - .base = { - .cra_blocksize = SHA224_BLOCK_SIZE, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_priority = 300, - .cra_name = "sha224", - .cra_driver_name = "sha224-riscv64-zvknha_or_zvknhb-zvkb", - .cra_module = THIS_MODULE, - }, - }, -}; - -static int __init riscv64_sha256_mod_init(void) -{ - /* Both zvknha and zvknhb provide the SHA-256 instructions. */ - if ((riscv_isa_extension_available(NULL, ZVKNHA) || - riscv_isa_extension_available(NULL, ZVKNHB)) && - riscv_isa_extension_available(NULL, ZVKB) && - riscv_vector_vlen() >= 128) - return crypto_register_shashes(riscv64_sha256_algs, - ARRAY_SIZE(riscv64_sha256_algs)); - - return -ENODEV; -} - -static void __exit riscv64_sha256_mod_exit(void) -{ - crypto_unregister_shashes(riscv64_sha256_algs, - ARRAY_SIZE(riscv64_sha256_algs)); -} - -module_init(riscv64_sha256_mod_init); -module_exit(riscv64_sha256_mod_exit); - -MODULE_DESCRIPTION("SHA-256 (RISC-V accelerated)"); -MODULE_AUTHOR("Heiko Stuebner "); -MODULE_LICENSE("GPL"); -MODULE_ALIAS_CRYPTO("sha256"); -MODULE_ALIAS_CRYPTO("sha224"); diff --git a/arch/riscv/lib/crypto/Kconfig b/arch/riscv/lib/crypto/Kconfig index bc7a43f33eb3..c100571feb7e 100644 --- a/arch/riscv/lib/crypto/Kconfig +++ b/arch/riscv/lib/crypto/Kconfig @@ -6,3 +6,10 @@ config CRYPTO_CHACHA_RISCV64 default CRYPTO_LIB_CHACHA select CRYPTO_ARCH_HAVE_LIB_CHACHA select CRYPTO_LIB_CHACHA_GENERIC + +config CRYPTO_SHA256_RISCV64 + tristate + depends on 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO + default CRYPTO_LIB_SHA256 + select CRYPTO_ARCH_HAVE_LIB_SHA256 + select CRYPTO_LIB_SHA256_GENERIC diff --git a/arch/riscv/lib/crypto/Makefile b/arch/riscv/lib/crypto/Makefile index e27b78f317fc..b7cb877a2c07 100644 --- a/arch/riscv/lib/crypto/Makefile +++ b/arch/riscv/lib/crypto/Makefile @@ -2,3 +2,6 @@ obj-$(CONFIG_CRYPTO_CHACHA_RISCV64) += chacha-riscv64.o chacha-riscv64-y := chacha-riscv64-glue.o chacha-riscv64-zvkb.o + +obj-$(CONFIG_CRYPTO_SHA256_RISCV64) += sha256-riscv64.o +sha256-riscv64-y := sha256.o sha256-riscv64-zvknha_or_zvknhb-zvkb.o diff --git a/arch/riscv/crypto/sha256-riscv64-zvknha_or_zvknhb-zvkb.S b/arch/riscv/lib/crypto/sha256-riscv64-zvknha_or_zvknhb-zvkb.S similarity index 98% rename from arch/riscv/crypto/sha256-riscv64-zvknha_or_zvknhb-zvkb.S rename to arch/riscv/lib/crypto/sha256-riscv64-zvknha_or_zvknhb-zvkb.S index f1f5779e4732..fad501ad0617 100644 --- a/arch/riscv/crypto/sha256-riscv64-zvknha_or_zvknhb-zvkb.S +++ b/arch/riscv/lib/crypto/sha256-riscv64-zvknha_or_zvknhb-zvkb.S @@ -106,8 +106,8 @@ sha256_4rounds \last, \k3, W3, W0, W1, W2 .endm -// void sha256_transform_zvknha_or_zvknhb_zvkb(u32 state[8], const u8 *data, -// int num_blocks); +// void sha256_transform_zvknha_or_zvknhb_zvkb(u32 state[SHA256_STATE_WORDS], +// const u8 *data, size_t nblocks); SYM_FUNC_START(sha256_transform_zvknha_or_zvknhb_zvkb) // Load the round constants into K0-K15. diff --git a/arch/riscv/lib/crypto/sha256.c b/arch/riscv/lib/crypto/sha256.c new file mode 100644 index 000000000000..18b84030f0b3 --- /dev/null +++ b/arch/riscv/lib/crypto/sha256.c @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * SHA-256 (RISC-V accelerated) + * + * Copyright (C) 2022 VRULL GmbH + * Author: Heiko Stuebner + * + * Copyright (C) 2023 SiFive, Inc. + * Author: Jerry Shih + */ + +#include +#include +#include +#include +#include +#include + +asmlinkage void sha256_transform_zvknha_or_zvknhb_zvkb( + u32 state[SHA256_STATE_WORDS], const u8 *data, size_t nblocks); + +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_extensions); + +void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS], + const u8 *data, size_t nblocks) +{ + if (static_branch_likely(&have_extensions) && crypto_simd_usable()) { + kernel_vector_begin(); + sha256_transform_zvknha_or_zvknhb_zvkb(state, data, nblocks); + kernel_vector_end(); + } else { + sha256_blocks_generic(state, data, nblocks); + } +} +EXPORT_SYMBOL(sha256_blocks_arch); + +bool sha256_is_arch_optimized(void) +{ + return static_key_enabled(&have_extensions); +} +EXPORT_SYMBOL(sha256_is_arch_optimized); + +static int __init riscv64_sha256_mod_init(void) +{ + /* Both zvknha and zvknhb provide the SHA-256 instructions. */ + if ((riscv_isa_extension_available(NULL, ZVKNHA) || + riscv_isa_extension_available(NULL, ZVKNHB)) && + riscv_isa_extension_available(NULL, ZVKB) && + riscv_vector_vlen() >= 128) + static_branch_enable(&have_extensions); + return 0; +} +arch_initcall(riscv64_sha256_mod_init); + +static void __exit riscv64_sha256_mod_exit(void) +{ +} +module_exit(riscv64_sha256_mod_exit); + +MODULE_DESCRIPTION("SHA-256 (RISC-V accelerated)"); +MODULE_AUTHOR("Heiko Stuebner "); +MODULE_LICENSE("GPL"); From b9eac03edcf83e25cd2c21be18e7733249fd75c1 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 28 Apr 2025 10:00:33 -0700 Subject: [PATCH 267/382] crypto: s390/sha256 - implement library instead of shash Instead of providing crypto_shash algorithms for the arch-optimized SHA-256 code, instead implement the SHA-256 library. This is much simpler, it makes the SHA-256 library functions be arch-optimized, and it fixes the longstanding issue where the arch-optimized SHA-256 was disabled by default. SHA-256 still remains available through crypto_shash, but individual architectures no longer need to handle it. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/s390/configs/debug_defconfig | 1 - arch/s390/configs/defconfig | 1 - arch/s390/crypto/Kconfig | 10 --- arch/s390/crypto/Makefile | 1 - arch/s390/crypto/sha256_s390.c | 144 ------------------------------ arch/s390/lib/crypto/Kconfig | 6 ++ arch/s390/lib/crypto/Makefile | 2 + arch/s390/lib/crypto/sha256.c | 47 ++++++++++ 8 files changed, 55 insertions(+), 157 deletions(-) delete mode 100644 arch/s390/crypto/sha256_s390.c create mode 100644 arch/s390/lib/crypto/sha256.c diff --git a/arch/s390/configs/debug_defconfig b/arch/s390/configs/debug_defconfig index 6f2c9ce1b154..de69faa4d94f 100644 --- a/arch/s390/configs/debug_defconfig +++ b/arch/s390/configs/debug_defconfig @@ -795,7 +795,6 @@ CONFIG_CRYPTO_USER_API_RNG=m CONFIG_CRYPTO_USER_API_AEAD=m CONFIG_CRYPTO_SHA512_S390=m CONFIG_CRYPTO_SHA1_S390=m -CONFIG_CRYPTO_SHA256_S390=m CONFIG_CRYPTO_SHA3_256_S390=m CONFIG_CRYPTO_SHA3_512_S390=m CONFIG_CRYPTO_GHASH_S390=m diff --git a/arch/s390/configs/defconfig b/arch/s390/configs/defconfig index f18a7d97ac21..f12679448e97 100644 --- a/arch/s390/configs/defconfig +++ b/arch/s390/configs/defconfig @@ -782,7 +782,6 @@ CONFIG_CRYPTO_USER_API_RNG=m CONFIG_CRYPTO_USER_API_AEAD=m CONFIG_CRYPTO_SHA512_S390=m CONFIG_CRYPTO_SHA1_S390=m -CONFIG_CRYPTO_SHA256_S390=m CONFIG_CRYPTO_SHA3_256_S390=m CONFIG_CRYPTO_SHA3_512_S390=m CONFIG_CRYPTO_GHASH_S390=m diff --git a/arch/s390/crypto/Kconfig b/arch/s390/crypto/Kconfig index a2bfd6eef0ca..e2c27588b21a 100644 --- a/arch/s390/crypto/Kconfig +++ b/arch/s390/crypto/Kconfig @@ -22,16 +22,6 @@ config CRYPTO_SHA1_S390 It is available as of z990. -config CRYPTO_SHA256_S390 - tristate "Hash functions: SHA-224 and SHA-256" - select CRYPTO_HASH - help - SHA-224 and SHA-256 secure hash algorithms (FIPS 180) - - Architecture: s390 - - It is available as of z9. - config CRYPTO_SHA3_256_S390 tristate "Hash functions: SHA3-224 and SHA3-256" select CRYPTO_HASH diff --git a/arch/s390/crypto/Makefile b/arch/s390/crypto/Makefile index e3853774e1a3..21757d86cd49 100644 --- a/arch/s390/crypto/Makefile +++ b/arch/s390/crypto/Makefile @@ -4,7 +4,6 @@ # obj-$(CONFIG_CRYPTO_SHA1_S390) += sha1_s390.o sha_common.o -obj-$(CONFIG_CRYPTO_SHA256_S390) += sha256_s390.o sha_common.o obj-$(CONFIG_CRYPTO_SHA512_S390) += sha512_s390.o sha_common.o obj-$(CONFIG_CRYPTO_SHA3_256_S390) += sha3_256_s390.o sha_common.o obj-$(CONFIG_CRYPTO_SHA3_512_S390) += sha3_512_s390.o sha_common.o diff --git a/arch/s390/crypto/sha256_s390.c b/arch/s390/crypto/sha256_s390.c deleted file mode 100644 index e6876c49414d..000000000000 --- a/arch/s390/crypto/sha256_s390.c +++ /dev/null @@ -1,144 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0+ -/* - * Cryptographic API. - * - * s390 implementation of the SHA256 and SHA224 Secure Hash Algorithm. - * - * s390 Version: - * Copyright IBM Corp. 2005, 2011 - * Author(s): Jan Glauber (jang@de.ibm.com) - */ -#include -#include -#include -#include -#include -#include -#include - -#include "sha.h" - -static int s390_sha256_init(struct shash_desc *desc) -{ - struct s390_sha_ctx *sctx = shash_desc_ctx(desc); - - sctx->state[0] = SHA256_H0; - sctx->state[1] = SHA256_H1; - sctx->state[2] = SHA256_H2; - sctx->state[3] = SHA256_H3; - sctx->state[4] = SHA256_H4; - sctx->state[5] = SHA256_H5; - sctx->state[6] = SHA256_H6; - sctx->state[7] = SHA256_H7; - sctx->count = 0; - sctx->func = CPACF_KIMD_SHA_256; - - return 0; -} - -static int sha256_export(struct shash_desc *desc, void *out) -{ - struct s390_sha_ctx *sctx = shash_desc_ctx(desc); - struct crypto_sha256_state *octx = out; - - octx->count = sctx->count; - memcpy(octx->state, sctx->state, sizeof(octx->state)); - return 0; -} - -static int sha256_import(struct shash_desc *desc, const void *in) -{ - struct s390_sha_ctx *sctx = shash_desc_ctx(desc); - const struct crypto_sha256_state *ictx = in; - - sctx->count = ictx->count; - memcpy(sctx->state, ictx->state, sizeof(ictx->state)); - sctx->func = CPACF_KIMD_SHA_256; - return 0; -} - -static struct shash_alg sha256_alg = { - .digestsize = SHA256_DIGEST_SIZE, - .init = s390_sha256_init, - .update = s390_sha_update_blocks, - .finup = s390_sha_finup, - .export = sha256_export, - .import = sha256_import, - .descsize = S390_SHA_CTX_SIZE, - .statesize = sizeof(struct crypto_sha256_state), - .base = { - .cra_name = "sha256", - .cra_driver_name= "sha256-s390", - .cra_priority = 300, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, - .cra_blocksize = SHA256_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}; - -static int s390_sha224_init(struct shash_desc *desc) -{ - struct s390_sha_ctx *sctx = shash_desc_ctx(desc); - - sctx->state[0] = SHA224_H0; - sctx->state[1] = SHA224_H1; - sctx->state[2] = SHA224_H2; - sctx->state[3] = SHA224_H3; - sctx->state[4] = SHA224_H4; - sctx->state[5] = SHA224_H5; - sctx->state[6] = SHA224_H6; - sctx->state[7] = SHA224_H7; - sctx->count = 0; - sctx->func = CPACF_KIMD_SHA_256; - - return 0; -} - -static struct shash_alg sha224_alg = { - .digestsize = SHA224_DIGEST_SIZE, - .init = s390_sha224_init, - .update = s390_sha_update_blocks, - .finup = s390_sha_finup, - .export = sha256_export, - .import = sha256_import, - .descsize = S390_SHA_CTX_SIZE, - .statesize = sizeof(struct crypto_sha256_state), - .base = { - .cra_name = "sha224", - .cra_driver_name= "sha224-s390", - .cra_priority = 300, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, - .cra_blocksize = SHA224_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}; - -static int __init sha256_s390_init(void) -{ - int ret; - - if (!cpacf_query_func(CPACF_KIMD, CPACF_KIMD_SHA_256)) - return -ENODEV; - ret = crypto_register_shash(&sha256_alg); - if (ret < 0) - goto out; - ret = crypto_register_shash(&sha224_alg); - if (ret < 0) - crypto_unregister_shash(&sha256_alg); -out: - return ret; -} - -static void __exit sha256_s390_fini(void) -{ - crypto_unregister_shash(&sha224_alg); - crypto_unregister_shash(&sha256_alg); -} - -module_cpu_feature_match(S390_CPU_FEATURE_MSA, sha256_s390_init); -module_exit(sha256_s390_fini); - -MODULE_ALIAS_CRYPTO("sha256"); -MODULE_ALIAS_CRYPTO("sha224"); -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("SHA256 and SHA224 Secure Hash Algorithm"); diff --git a/arch/s390/lib/crypto/Kconfig b/arch/s390/lib/crypto/Kconfig index 069b355fe51a..e3f855ef4393 100644 --- a/arch/s390/lib/crypto/Kconfig +++ b/arch/s390/lib/crypto/Kconfig @@ -5,3 +5,9 @@ config CRYPTO_CHACHA_S390 default CRYPTO_LIB_CHACHA select CRYPTO_LIB_CHACHA_GENERIC select CRYPTO_ARCH_HAVE_LIB_CHACHA + +config CRYPTO_SHA256_S390 + tristate + default CRYPTO_LIB_SHA256 + select CRYPTO_ARCH_HAVE_LIB_SHA256 + select CRYPTO_LIB_SHA256_GENERIC diff --git a/arch/s390/lib/crypto/Makefile b/arch/s390/lib/crypto/Makefile index 06c2cf77178e..920197967f46 100644 --- a/arch/s390/lib/crypto/Makefile +++ b/arch/s390/lib/crypto/Makefile @@ -2,3 +2,5 @@ obj-$(CONFIG_CRYPTO_CHACHA_S390) += chacha_s390.o chacha_s390-y := chacha-glue.o chacha-s390.o + +obj-$(CONFIG_CRYPTO_SHA256_S390) += sha256.o diff --git a/arch/s390/lib/crypto/sha256.c b/arch/s390/lib/crypto/sha256.c new file mode 100644 index 000000000000..50c592ce7a5d --- /dev/null +++ b/arch/s390/lib/crypto/sha256.c @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * SHA-256 optimized using the CP Assist for Cryptographic Functions (CPACF) + * + * Copyright 2025 Google LLC + */ +#include +#include +#include +#include +#include + +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_cpacf_sha256); + +void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS], + const u8 *data, size_t nblocks) +{ + if (static_branch_likely(&have_cpacf_sha256)) + cpacf_kimd(CPACF_KIMD_SHA_256, state, data, + nblocks * SHA256_BLOCK_SIZE); + else + sha256_blocks_generic(state, data, nblocks); +} +EXPORT_SYMBOL(sha256_blocks_arch); + +bool sha256_is_arch_optimized(void) +{ + return static_key_enabled(&have_cpacf_sha256); +} +EXPORT_SYMBOL(sha256_is_arch_optimized); + +static int __init sha256_s390_mod_init(void) +{ + if (cpu_have_feature(S390_CPU_FEATURE_MSA) && + cpacf_query_func(CPACF_KIMD, CPACF_KIMD_SHA_256)) + static_branch_enable(&have_cpacf_sha256); + return 0; +} +arch_initcall(sha256_s390_mod_init); + +static void __exit sha256_s390_mod_exit(void) +{ +} +module_exit(sha256_s390_mod_exit); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("SHA-256 using the CP Assist for Cryptographic Functions (CPACF)"); From 77735920ca9d6456a8135b30396b460791ace262 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 28 Apr 2025 10:00:34 -0700 Subject: [PATCH 268/382] crypto: sparc - move opcodes.h into asm directory Since arch/sparc/crypto/opcodes.h is now needed outside the arch/sparc/crypto/ directory, move it into arch/sparc/include/asm/ so that it can be included as . Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/sparc/crypto/aes_asm.S | 3 +-- arch/sparc/crypto/aes_glue.c | 3 +-- arch/sparc/crypto/camellia_asm.S | 3 +-- arch/sparc/crypto/camellia_glue.c | 3 +-- arch/sparc/crypto/des_asm.S | 3 +-- arch/sparc/crypto/des_glue.c | 3 +-- arch/sparc/crypto/md5_asm.S | 3 +-- arch/sparc/crypto/md5_glue.c | 3 +-- arch/sparc/crypto/sha1_asm.S | 3 +-- arch/sparc/crypto/sha1_glue.c | 3 +-- arch/sparc/crypto/sha256_asm.S | 3 +-- arch/sparc/crypto/sha256_glue.c | 3 +-- arch/sparc/crypto/sha512_asm.S | 3 +-- arch/sparc/crypto/sha512_glue.c | 3 +-- arch/sparc/{crypto => include/asm}/opcodes.h | 6 +++--- arch/sparc/lib/crc32c_asm.S | 3 +-- 16 files changed, 18 insertions(+), 33 deletions(-) rename arch/sparc/{crypto => include/asm}/opcodes.h (96%) diff --git a/arch/sparc/crypto/aes_asm.S b/arch/sparc/crypto/aes_asm.S index 155cefb98520..f291174a72a1 100644 --- a/arch/sparc/crypto/aes_asm.S +++ b/arch/sparc/crypto/aes_asm.S @@ -1,9 +1,8 @@ /* SPDX-License-Identifier: GPL-2.0 */ #include +#include #include -#include "opcodes.h" - #define ENCRYPT_TWO_ROUNDS(KEY_BASE, I0, I1, T0, T1) \ AES_EROUND01(KEY_BASE + 0, I0, I1, T0) \ AES_EROUND23(KEY_BASE + 2, I0, I1, T1) \ diff --git a/arch/sparc/crypto/aes_glue.c b/arch/sparc/crypto/aes_glue.c index 683150830356..359f22643b05 100644 --- a/arch/sparc/crypto/aes_glue.c +++ b/arch/sparc/crypto/aes_glue.c @@ -27,11 +27,10 @@ #include #include +#include #include #include -#include "opcodes.h" - struct aes_ops { void (*encrypt)(const u64 *key, const u32 *input, u32 *output); void (*decrypt)(const u64 *key, const u32 *input, u32 *output); diff --git a/arch/sparc/crypto/camellia_asm.S b/arch/sparc/crypto/camellia_asm.S index dcdc9193fcd7..8471b346ef54 100644 --- a/arch/sparc/crypto/camellia_asm.S +++ b/arch/sparc/crypto/camellia_asm.S @@ -1,9 +1,8 @@ /* SPDX-License-Identifier: GPL-2.0 */ #include +#include #include -#include "opcodes.h" - #define CAMELLIA_6ROUNDS(KEY_BASE, I0, I1) \ CAMELLIA_F(KEY_BASE + 0, I1, I0, I1) \ CAMELLIA_F(KEY_BASE + 2, I0, I1, I0) \ diff --git a/arch/sparc/crypto/camellia_glue.c b/arch/sparc/crypto/camellia_glue.c index aaa9714378e6..e7a1e1c42b99 100644 --- a/arch/sparc/crypto/camellia_glue.c +++ b/arch/sparc/crypto/camellia_glue.c @@ -15,11 +15,10 @@ #include #include +#include #include #include -#include "opcodes.h" - #define CAMELLIA_MIN_KEY_SIZE 16 #define CAMELLIA_MAX_KEY_SIZE 32 #define CAMELLIA_BLOCK_SIZE 16 diff --git a/arch/sparc/crypto/des_asm.S b/arch/sparc/crypto/des_asm.S index 7157468a679d..d534446cbef9 100644 --- a/arch/sparc/crypto/des_asm.S +++ b/arch/sparc/crypto/des_asm.S @@ -1,9 +1,8 @@ /* SPDX-License-Identifier: GPL-2.0 */ #include +#include #include -#include "opcodes.h" - .align 32 ENTRY(des_sparc64_key_expand) /* %o0=input_key, %o1=output_key */ diff --git a/arch/sparc/crypto/des_glue.c b/arch/sparc/crypto/des_glue.c index a499102bf706..e50ec4cd57cd 100644 --- a/arch/sparc/crypto/des_glue.c +++ b/arch/sparc/crypto/des_glue.c @@ -16,11 +16,10 @@ #include #include +#include #include #include -#include "opcodes.h" - struct des_sparc64_ctx { u64 encrypt_expkey[DES_EXPKEY_WORDS / 2]; u64 decrypt_expkey[DES_EXPKEY_WORDS / 2]; diff --git a/arch/sparc/crypto/md5_asm.S b/arch/sparc/crypto/md5_asm.S index 7a6637455f37..60b544e4d205 100644 --- a/arch/sparc/crypto/md5_asm.S +++ b/arch/sparc/crypto/md5_asm.S @@ -1,9 +1,8 @@ /* SPDX-License-Identifier: GPL-2.0 */ #include +#include #include -#include "opcodes.h" - ENTRY(md5_sparc64_transform) /* %o0 = digest, %o1 = data, %o2 = rounds */ VISEntryHalf diff --git a/arch/sparc/crypto/md5_glue.c b/arch/sparc/crypto/md5_glue.c index 5b018c6a376c..b3615f0cdf62 100644 --- a/arch/sparc/crypto/md5_glue.c +++ b/arch/sparc/crypto/md5_glue.c @@ -15,6 +15,7 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include +#include #include #include #include @@ -24,8 +25,6 @@ #include #include -#include "opcodes.h" - struct sparc_md5_state { __le32 hash[MD5_HASH_WORDS]; u64 byte_count; diff --git a/arch/sparc/crypto/sha1_asm.S b/arch/sparc/crypto/sha1_asm.S index 7d8bf354f0e7..00b46bac1b08 100644 --- a/arch/sparc/crypto/sha1_asm.S +++ b/arch/sparc/crypto/sha1_asm.S @@ -1,9 +1,8 @@ /* SPDX-License-Identifier: GPL-2.0 */ #include +#include #include -#include "opcodes.h" - ENTRY(sha1_sparc64_transform) /* %o0 = digest, %o1 = data, %o2 = rounds */ VISEntryHalf diff --git a/arch/sparc/crypto/sha1_glue.c b/arch/sparc/crypto/sha1_glue.c index ec5a06948e0d..ef19d5023b1b 100644 --- a/arch/sparc/crypto/sha1_glue.c +++ b/arch/sparc/crypto/sha1_glue.c @@ -12,6 +12,7 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include +#include #include #include #include @@ -19,8 +20,6 @@ #include #include -#include "opcodes.h" - asmlinkage void sha1_sparc64_transform(struct sha1_state *digest, const u8 *data, int rounds); diff --git a/arch/sparc/crypto/sha256_asm.S b/arch/sparc/crypto/sha256_asm.S index 0b39ec7d7ca2..8ce88611e98a 100644 --- a/arch/sparc/crypto/sha256_asm.S +++ b/arch/sparc/crypto/sha256_asm.S @@ -1,9 +1,8 @@ /* SPDX-License-Identifier: GPL-2.0 */ #include +#include #include -#include "opcodes.h" - ENTRY(sha256_sparc64_transform) /* %o0 = digest, %o1 = data, %o2 = rounds */ VISEntryHalf diff --git a/arch/sparc/crypto/sha256_glue.c b/arch/sparc/crypto/sha256_glue.c index ddb250242faf..25008603a986 100644 --- a/arch/sparc/crypto/sha256_glue.c +++ b/arch/sparc/crypto/sha256_glue.c @@ -12,6 +12,7 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include +#include #include #include #include @@ -19,8 +20,6 @@ #include #include -#include "opcodes.h" - asmlinkage void sha256_sparc64_transform(u32 *digest, const char *data, unsigned int rounds); diff --git a/arch/sparc/crypto/sha512_asm.S b/arch/sparc/crypto/sha512_asm.S index b2f6e6728802..9932b4fe1b59 100644 --- a/arch/sparc/crypto/sha512_asm.S +++ b/arch/sparc/crypto/sha512_asm.S @@ -1,9 +1,8 @@ /* SPDX-License-Identifier: GPL-2.0 */ #include +#include #include -#include "opcodes.h" - ENTRY(sha512_sparc64_transform) /* %o0 = digest, %o1 = data, %o2 = rounds */ VISEntry diff --git a/arch/sparc/crypto/sha512_glue.c b/arch/sparc/crypto/sha512_glue.c index 1d0e1f98ca46..47b9277b6877 100644 --- a/arch/sparc/crypto/sha512_glue.c +++ b/arch/sparc/crypto/sha512_glue.c @@ -11,6 +11,7 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include +#include #include #include #include @@ -18,8 +19,6 @@ #include #include -#include "opcodes.h" - asmlinkage void sha512_sparc64_transform(u64 *digest, const char *data, unsigned int rounds); diff --git a/arch/sparc/crypto/opcodes.h b/arch/sparc/include/asm/opcodes.h similarity index 96% rename from arch/sparc/crypto/opcodes.h rename to arch/sparc/include/asm/opcodes.h index 417b6a10a337..ebfda6eb49b2 100644 --- a/arch/sparc/crypto/opcodes.h +++ b/arch/sparc/include/asm/opcodes.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _OPCODES_H -#define _OPCODES_H +#ifndef _SPARC_ASM_OPCODES_H +#define _SPARC_ASM_OPCODES_H #define SPARC_CR_OPCODE_PRIORITY 300 @@ -97,4 +97,4 @@ #define MOVXTOD_G7_F62 \ .word 0xbfb02307; -#endif /* _OPCODES_H */ +#endif /* _SPARC_ASM_OPCODES_H */ diff --git a/arch/sparc/lib/crc32c_asm.S b/arch/sparc/lib/crc32c_asm.S index ee454fa6aed6..4db873850f44 100644 --- a/arch/sparc/lib/crc32c_asm.S +++ b/arch/sparc/lib/crc32c_asm.S @@ -1,10 +1,9 @@ /* SPDX-License-Identifier: GPL-2.0 */ #include +#include #include #include -#include "../crypto/opcodes.h" - ENTRY(crc32c_sparc64) /* %o0=crc32p, %o1=data_ptr, %o2=len */ VISEntryHalf From 699618d422759fe096a28ad2778d79807a5384ce Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 28 Apr 2025 10:00:35 -0700 Subject: [PATCH 269/382] crypto: sparc/sha256 - implement library instead of shash Instead of providing crypto_shash algorithms for the arch-optimized SHA-256 code, instead implement the SHA-256 library. This is much simpler, it makes the SHA-256 library functions be arch-optimized, and it fixes the longstanding issue where the arch-optimized SHA-256 was disabled by default. SHA-256 still remains available through crypto_shash, but individual architectures no longer need to handle it. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/sparc/crypto/Kconfig | 10 -- arch/sparc/crypto/Makefile | 2 - arch/sparc/crypto/sha256_glue.c | 128 ----------------------- arch/sparc/lib/Makefile | 1 + arch/sparc/lib/crypto/Kconfig | 8 ++ arch/sparc/lib/crypto/Makefile | 4 + arch/sparc/lib/crypto/sha256.c | 64 ++++++++++++ arch/sparc/{ => lib}/crypto/sha256_asm.S | 2 +- lib/crypto/Kconfig | 3 + 9 files changed, 81 insertions(+), 141 deletions(-) delete mode 100644 arch/sparc/crypto/sha256_glue.c create mode 100644 arch/sparc/lib/crypto/Kconfig create mode 100644 arch/sparc/lib/crypto/Makefile create mode 100644 arch/sparc/lib/crypto/sha256.c rename arch/sparc/{ => lib}/crypto/sha256_asm.S (96%) diff --git a/arch/sparc/crypto/Kconfig b/arch/sparc/crypto/Kconfig index e858597de89d..a6ba319c42dc 100644 --- a/arch/sparc/crypto/Kconfig +++ b/arch/sparc/crypto/Kconfig @@ -36,16 +36,6 @@ config CRYPTO_SHA1_SPARC64 Architecture: sparc64 -config CRYPTO_SHA256_SPARC64 - tristate "Hash functions: SHA-224 and SHA-256" - depends on SPARC64 - select CRYPTO_SHA256 - select CRYPTO_HASH - help - SHA-224 and SHA-256 secure hash algorithms (FIPS 180) - - Architecture: sparc64 using crypto instructions, when available - config CRYPTO_SHA512_SPARC64 tristate "Hash functions: SHA-384 and SHA-512" depends on SPARC64 diff --git a/arch/sparc/crypto/Makefile b/arch/sparc/crypto/Makefile index a2d7fca40cb4..701c39edb0d7 100644 --- a/arch/sparc/crypto/Makefile +++ b/arch/sparc/crypto/Makefile @@ -4,7 +4,6 @@ # obj-$(CONFIG_CRYPTO_SHA1_SPARC64) += sha1-sparc64.o -obj-$(CONFIG_CRYPTO_SHA256_SPARC64) += sha256-sparc64.o obj-$(CONFIG_CRYPTO_SHA512_SPARC64) += sha512-sparc64.o obj-$(CONFIG_CRYPTO_MD5_SPARC64) += md5-sparc64.o @@ -13,7 +12,6 @@ obj-$(CONFIG_CRYPTO_DES_SPARC64) += des-sparc64.o obj-$(CONFIG_CRYPTO_CAMELLIA_SPARC64) += camellia-sparc64.o sha1-sparc64-y := sha1_asm.o sha1_glue.o -sha256-sparc64-y := sha256_asm.o sha256_glue.o sha512-sparc64-y := sha512_asm.o sha512_glue.o md5-sparc64-y := md5_asm.o md5_glue.o diff --git a/arch/sparc/crypto/sha256_glue.c b/arch/sparc/crypto/sha256_glue.c deleted file mode 100644 index 25008603a986..000000000000 --- a/arch/sparc/crypto/sha256_glue.c +++ /dev/null @@ -1,128 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* Glue code for SHA256 hashing optimized for sparc64 crypto opcodes. - * - * This is based largely upon crypto/sha256_generic.c - * - * Copyright (c) Jean-Luc Cooke - * Copyright (c) Andrew McDonald - * Copyright (c) 2002 James Morris - * SHA224 Support Copyright 2007 Intel Corporation - */ - -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt - -#include -#include -#include -#include -#include -#include -#include -#include - -asmlinkage void sha256_sparc64_transform(u32 *digest, const char *data, - unsigned int rounds); - -static void sha256_block(struct crypto_sha256_state *sctx, const u8 *src, - int blocks) -{ - sha256_sparc64_transform(sctx->state, src, blocks); -} - -static int sha256_sparc64_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - return sha256_base_do_update_blocks(desc, data, len, sha256_block); -} - -static int sha256_sparc64_finup(struct shash_desc *desc, const u8 *src, - unsigned int len, u8 *out) -{ - sha256_base_do_finup(desc, src, len, sha256_block); - return sha256_base_finish(desc, out); -} - -static struct shash_alg sha256_alg = { - .digestsize = SHA256_DIGEST_SIZE, - .init = sha256_base_init, - .update = sha256_sparc64_update, - .finup = sha256_sparc64_finup, - .descsize = sizeof(struct crypto_sha256_state), - .base = { - .cra_name = "sha256", - .cra_driver_name= "sha256-sparc64", - .cra_priority = SPARC_CR_OPCODE_PRIORITY, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_blocksize = SHA256_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}; - -static struct shash_alg sha224_alg = { - .digestsize = SHA224_DIGEST_SIZE, - .init = sha224_base_init, - .update = sha256_sparc64_update, - .finup = sha256_sparc64_finup, - .descsize = sizeof(struct crypto_sha256_state), - .base = { - .cra_name = "sha224", - .cra_driver_name= "sha224-sparc64", - .cra_priority = SPARC_CR_OPCODE_PRIORITY, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_blocksize = SHA224_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}; - -static bool __init sparc64_has_sha256_opcode(void) -{ - unsigned long cfr; - - if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO)) - return false; - - __asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr)); - if (!(cfr & CFR_SHA256)) - return false; - - return true; -} - -static int __init sha256_sparc64_mod_init(void) -{ - if (sparc64_has_sha256_opcode()) { - int ret = crypto_register_shash(&sha224_alg); - if (ret < 0) - return ret; - - ret = crypto_register_shash(&sha256_alg); - if (ret < 0) { - crypto_unregister_shash(&sha224_alg); - return ret; - } - - pr_info("Using sparc64 sha256 opcode optimized SHA-256/SHA-224 implementation\n"); - return 0; - } - pr_info("sparc64 sha256 opcode not available.\n"); - return -ENODEV; -} - -static void __exit sha256_sparc64_mod_fini(void) -{ - crypto_unregister_shash(&sha224_alg); - crypto_unregister_shash(&sha256_alg); -} - -module_init(sha256_sparc64_mod_init); -module_exit(sha256_sparc64_mod_fini); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("SHA-224 and SHA-256 Secure Hash Algorithm, sparc64 sha256 opcode accelerated"); - -MODULE_ALIAS_CRYPTO("sha224"); -MODULE_ALIAS_CRYPTO("sha256"); - -#include "crop_devid.c" diff --git a/arch/sparc/lib/Makefile b/arch/sparc/lib/Makefile index 5724d0f356eb..98887dc295a1 100644 --- a/arch/sparc/lib/Makefile +++ b/arch/sparc/lib/Makefile @@ -4,6 +4,7 @@ asflags-y := -ansi -DST_DIV0=0x02 +obj-y += crypto/ lib-$(CONFIG_SPARC32) += ashrdi3.o lib-$(CONFIG_SPARC32) += memcpy.o memset.o lib-y += strlen.o diff --git a/arch/sparc/lib/crypto/Kconfig b/arch/sparc/lib/crypto/Kconfig new file mode 100644 index 000000000000..e5c3e4d3dba6 --- /dev/null +++ b/arch/sparc/lib/crypto/Kconfig @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-2.0-only + +config CRYPTO_SHA256_SPARC64 + tristate + depends on SPARC64 + default CRYPTO_LIB_SHA256 + select CRYPTO_ARCH_HAVE_LIB_SHA256 + select CRYPTO_LIB_SHA256_GENERIC diff --git a/arch/sparc/lib/crypto/Makefile b/arch/sparc/lib/crypto/Makefile new file mode 100644 index 000000000000..75ee244ad6f7 --- /dev/null +++ b/arch/sparc/lib/crypto/Makefile @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0-only + +obj-$(CONFIG_CRYPTO_SHA256_SPARC64) += sha256-sparc64.o +sha256-sparc64-y := sha256.o sha256_asm.o diff --git a/arch/sparc/lib/crypto/sha256.c b/arch/sparc/lib/crypto/sha256.c new file mode 100644 index 000000000000..6f118a23d210 --- /dev/null +++ b/arch/sparc/lib/crypto/sha256.c @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * SHA-256 accelerated using the sparc64 sha256 opcodes + * + * Copyright (c) Jean-Luc Cooke + * Copyright (c) Andrew McDonald + * Copyright (c) 2002 James Morris + * SHA224 Support Copyright 2007 Intel Corporation + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include + +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_sha256_opcodes); + +asmlinkage void sha256_sparc64_transform(u32 state[SHA256_STATE_WORDS], + const u8 *data, size_t nblocks); + +void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS], + const u8 *data, size_t nblocks) +{ + if (static_branch_likely(&have_sha256_opcodes)) + sha256_sparc64_transform(state, data, nblocks); + else + sha256_blocks_generic(state, data, nblocks); +} +EXPORT_SYMBOL(sha256_blocks_arch); + +bool sha256_is_arch_optimized(void) +{ + return static_key_enabled(&have_sha256_opcodes); +} +EXPORT_SYMBOL(sha256_is_arch_optimized); + +static int __init sha256_sparc64_mod_init(void) +{ + unsigned long cfr; + + if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO)) + return 0; + + __asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr)); + if (!(cfr & CFR_SHA256)) + return 0; + + static_branch_enable(&have_sha256_opcodes); + pr_info("Using sparc64 sha256 opcode optimized SHA-256/SHA-224 implementation\n"); + return 0; +} +arch_initcall(sha256_sparc64_mod_init); + +static void __exit sha256_sparc64_mod_exit(void) +{ +} +module_exit(sha256_sparc64_mod_exit); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("SHA-256 accelerated using the sparc64 sha256 opcodes"); diff --git a/arch/sparc/crypto/sha256_asm.S b/arch/sparc/lib/crypto/sha256_asm.S similarity index 96% rename from arch/sparc/crypto/sha256_asm.S rename to arch/sparc/lib/crypto/sha256_asm.S index 8ce88611e98a..ddcdd3daf31e 100644 --- a/arch/sparc/crypto/sha256_asm.S +++ b/arch/sparc/lib/crypto/sha256_asm.S @@ -4,7 +4,7 @@ #include ENTRY(sha256_sparc64_transform) - /* %o0 = digest, %o1 = data, %o2 = rounds */ + /* %o0 = state, %o1 = data, %o2 = nblocks */ VISEntryHalf ld [%o0 + 0x00], %f0 ld [%o0 + 0x04], %f1 diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index 7fe678047939..6319358b38c2 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -181,6 +181,9 @@ endif if S390 source "arch/s390/lib/crypto/Kconfig" endif +if SPARC +source "arch/sparc/lib/crypto/Kconfig" +endif if X86 source "arch/x86/lib/crypto/Kconfig" endif From 11d7956d526f72da82b575389a2d8f9eb15dea6c Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 28 Apr 2025 10:00:36 -0700 Subject: [PATCH 270/382] crypto: x86/sha256 - implement library instead of shash Instead of providing crypto_shash algorithms for the arch-optimized SHA-256 code, instead implement the SHA-256 library. This is much simpler, it makes the SHA-256 library functions be arch-optimized, and it fixes the longstanding issue where the arch-optimized SHA-256 was disabled by default. SHA-256 still remains available through crypto_shash, but individual architectures no longer need to handle it. To match sha256_blocks_arch(), change the type of the nblocks parameter of the assembly functions from int to size_t. The assembly functions actually already treated it as size_t. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/x86/crypto/Kconfig | 14 - arch/x86/crypto/Makefile | 3 - arch/x86/crypto/sha256_ssse3_glue.c | 432 ------------------ arch/x86/lib/crypto/Kconfig | 7 + arch/x86/lib/crypto/Makefile | 3 + arch/x86/{ => lib}/crypto/sha256-avx-asm.S | 12 +- arch/x86/{ => lib}/crypto/sha256-avx2-asm.S | 12 +- .../crypto/sha256-ni-asm.S} | 36 +- arch/x86/{ => lib}/crypto/sha256-ssse3-asm.S | 14 +- arch/x86/lib/crypto/sha256.c | 74 +++ 10 files changed, 118 insertions(+), 489 deletions(-) delete mode 100644 arch/x86/crypto/sha256_ssse3_glue.c rename arch/x86/{ => lib}/crypto/sha256-avx-asm.S (98%) rename arch/x86/{ => lib}/crypto/sha256-avx2-asm.S (98%) rename arch/x86/{crypto/sha256_ni_asm.S => lib/crypto/sha256-ni-asm.S} (85%) rename arch/x86/{ => lib}/crypto/sha256-ssse3-asm.S (98%) create mode 100644 arch/x86/lib/crypto/sha256.c diff --git a/arch/x86/crypto/Kconfig b/arch/x86/crypto/Kconfig index 9e941362e4cd..56cfdc79e2c6 100644 --- a/arch/x86/crypto/Kconfig +++ b/arch/x86/crypto/Kconfig @@ -390,20 +390,6 @@ config CRYPTO_SHA1_SSSE3 - AVX2 (Advanced Vector Extensions 2) - SHA-NI (SHA Extensions New Instructions) -config CRYPTO_SHA256_SSSE3 - tristate "Hash functions: SHA-224 and SHA-256 (SSSE3/AVX/AVX2/SHA-NI)" - depends on 64BIT - select CRYPTO_SHA256 - select CRYPTO_HASH - help - SHA-224 and SHA-256 secure hash algorithms (FIPS 180) - - Architecture: x86_64 using: - - SSSE3 (Supplemental SSE3) - - AVX (Advanced Vector Extensions) - - AVX2 (Advanced Vector Extensions 2) - - SHA-NI (SHA Extensions New Instructions) - config CRYPTO_SHA512_SSSE3 tristate "Hash functions: SHA-384 and SHA-512 (SSSE3/AVX/AVX2)" depends on 64BIT diff --git a/arch/x86/crypto/Makefile b/arch/x86/crypto/Makefile index fad59a6c6c26..aa289a9e0153 100644 --- a/arch/x86/crypto/Makefile +++ b/arch/x86/crypto/Makefile @@ -54,9 +54,6 @@ endif obj-$(CONFIG_CRYPTO_SHA1_SSSE3) += sha1-ssse3.o sha1-ssse3-y := sha1_avx2_x86_64_asm.o sha1_ssse3_asm.o sha1_ni_asm.o sha1_ssse3_glue.o -obj-$(CONFIG_CRYPTO_SHA256_SSSE3) += sha256-ssse3.o -sha256-ssse3-y := sha256-ssse3-asm.o sha256-avx-asm.o sha256-avx2-asm.o sha256_ni_asm.o sha256_ssse3_glue.o - obj-$(CONFIG_CRYPTO_SHA512_SSSE3) += sha512-ssse3.o sha512-ssse3-y := sha512-ssse3-asm.o sha512-avx-asm.o sha512-avx2-asm.o sha512_ssse3_glue.o diff --git a/arch/x86/crypto/sha256_ssse3_glue.c b/arch/x86/crypto/sha256_ssse3_glue.c deleted file mode 100644 index a5d3be00550b..000000000000 --- a/arch/x86/crypto/sha256_ssse3_glue.c +++ /dev/null @@ -1,432 +0,0 @@ -/* - * Cryptographic API. - * - * Glue code for the SHA256 Secure Hash Algorithm assembler implementations - * using SSSE3, AVX, AVX2, and SHA-NI instructions. - * - * This file is based on sha256_generic.c - * - * Copyright (C) 2013 Intel Corporation. - * - * Author: - * Tim Chen - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the Free - * Software Foundation; either version 2 of the License, or (at your option) - * any later version. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - - -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt - -#include -#include -#include -#include -#include -#include -#include - -asmlinkage void sha256_transform_ssse3(struct crypto_sha256_state *state, - const u8 *data, int blocks); - -static const struct x86_cpu_id module_cpu_ids[] = { - X86_MATCH_FEATURE(X86_FEATURE_SHA_NI, NULL), - X86_MATCH_FEATURE(X86_FEATURE_AVX2, NULL), - X86_MATCH_FEATURE(X86_FEATURE_AVX, NULL), - X86_MATCH_FEATURE(X86_FEATURE_SSSE3, NULL), - {} -}; -MODULE_DEVICE_TABLE(x86cpu, module_cpu_ids); - -static int _sha256_update(struct shash_desc *desc, const u8 *data, - unsigned int len, - sha256_block_fn *sha256_xform) -{ - int remain; - - /* - * Make sure struct crypto_sha256_state begins directly with the SHA256 - * 256-bit internal state, as this is what the asm functions expect. - */ - BUILD_BUG_ON(offsetof(struct crypto_sha256_state, state) != 0); - - kernel_fpu_begin(); - remain = sha256_base_do_update_blocks(desc, data, len, sha256_xform); - kernel_fpu_end(); - - return remain; -} - -static int sha256_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out, sha256_block_fn *sha256_xform) -{ - kernel_fpu_begin(); - sha256_base_do_finup(desc, data, len, sha256_xform); - kernel_fpu_end(); - - return sha256_base_finish(desc, out); -} - -static int sha256_ssse3_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - return _sha256_update(desc, data, len, sha256_transform_ssse3); -} - -static int sha256_ssse3_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - return sha256_finup(desc, data, len, out, sha256_transform_ssse3); -} - -static int sha256_ssse3_digest(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - return sha256_base_init(desc) ?: - sha256_ssse3_finup(desc, data, len, out); -} - -static struct shash_alg sha256_ssse3_algs[] = { { - .digestsize = SHA256_DIGEST_SIZE, - .init = sha256_base_init, - .update = sha256_ssse3_update, - .finup = sha256_ssse3_finup, - .digest = sha256_ssse3_digest, - .descsize = sizeof(struct crypto_sha256_state), - .base = { - .cra_name = "sha256", - .cra_driver_name = "sha256-ssse3", - .cra_priority = 150, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_blocksize = SHA256_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}, { - .digestsize = SHA224_DIGEST_SIZE, - .init = sha224_base_init, - .update = sha256_ssse3_update, - .finup = sha256_ssse3_finup, - .descsize = sizeof(struct crypto_sha256_state), - .base = { - .cra_name = "sha224", - .cra_driver_name = "sha224-ssse3", - .cra_priority = 150, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_blocksize = SHA224_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -} }; - -static int register_sha256_ssse3(void) -{ - if (boot_cpu_has(X86_FEATURE_SSSE3)) - return crypto_register_shashes(sha256_ssse3_algs, - ARRAY_SIZE(sha256_ssse3_algs)); - return 0; -} - -static void unregister_sha256_ssse3(void) -{ - if (boot_cpu_has(X86_FEATURE_SSSE3)) - crypto_unregister_shashes(sha256_ssse3_algs, - ARRAY_SIZE(sha256_ssse3_algs)); -} - -asmlinkage void sha256_transform_avx(struct crypto_sha256_state *state, - const u8 *data, int blocks); - -static int sha256_avx_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - return _sha256_update(desc, data, len, sha256_transform_avx); -} - -static int sha256_avx_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - return sha256_finup(desc, data, len, out, sha256_transform_avx); -} - -static int sha256_avx_digest(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - return sha256_base_init(desc) ?: - sha256_avx_finup(desc, data, len, out); -} - -static struct shash_alg sha256_avx_algs[] = { { - .digestsize = SHA256_DIGEST_SIZE, - .init = sha256_base_init, - .update = sha256_avx_update, - .finup = sha256_avx_finup, - .digest = sha256_avx_digest, - .descsize = sizeof(struct crypto_sha256_state), - .base = { - .cra_name = "sha256", - .cra_driver_name = "sha256-avx", - .cra_priority = 160, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_blocksize = SHA256_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}, { - .digestsize = SHA224_DIGEST_SIZE, - .init = sha224_base_init, - .update = sha256_avx_update, - .finup = sha256_avx_finup, - .descsize = sizeof(struct crypto_sha256_state), - .base = { - .cra_name = "sha224", - .cra_driver_name = "sha224-avx", - .cra_priority = 160, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_blocksize = SHA224_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -} }; - -static bool avx_usable(void) -{ - if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL)) { - if (boot_cpu_has(X86_FEATURE_AVX)) - pr_info("AVX detected but unusable.\n"); - return false; - } - - return true; -} - -static int register_sha256_avx(void) -{ - if (avx_usable()) - return crypto_register_shashes(sha256_avx_algs, - ARRAY_SIZE(sha256_avx_algs)); - return 0; -} - -static void unregister_sha256_avx(void) -{ - if (avx_usable()) - crypto_unregister_shashes(sha256_avx_algs, - ARRAY_SIZE(sha256_avx_algs)); -} - -asmlinkage void sha256_transform_rorx(struct crypto_sha256_state *state, - const u8 *data, int blocks); - -static int sha256_avx2_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - return _sha256_update(desc, data, len, sha256_transform_rorx); -} - -static int sha256_avx2_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - return sha256_finup(desc, data, len, out, sha256_transform_rorx); -} - -static int sha256_avx2_digest(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - return sha256_base_init(desc) ?: - sha256_avx2_finup(desc, data, len, out); -} - -static struct shash_alg sha256_avx2_algs[] = { { - .digestsize = SHA256_DIGEST_SIZE, - .init = sha256_base_init, - .update = sha256_avx2_update, - .finup = sha256_avx2_finup, - .digest = sha256_avx2_digest, - .descsize = sizeof(struct crypto_sha256_state), - .base = { - .cra_name = "sha256", - .cra_driver_name = "sha256-avx2", - .cra_priority = 170, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_blocksize = SHA256_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}, { - .digestsize = SHA224_DIGEST_SIZE, - .init = sha224_base_init, - .update = sha256_avx2_update, - .finup = sha256_avx2_finup, - .descsize = sizeof(struct crypto_sha256_state), - .base = { - .cra_name = "sha224", - .cra_driver_name = "sha224-avx2", - .cra_priority = 170, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_blocksize = SHA224_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -} }; - -static bool avx2_usable(void) -{ - if (avx_usable() && boot_cpu_has(X86_FEATURE_AVX2) && - boot_cpu_has(X86_FEATURE_BMI2)) - return true; - - return false; -} - -static int register_sha256_avx2(void) -{ - if (avx2_usable()) - return crypto_register_shashes(sha256_avx2_algs, - ARRAY_SIZE(sha256_avx2_algs)); - return 0; -} - -static void unregister_sha256_avx2(void) -{ - if (avx2_usable()) - crypto_unregister_shashes(sha256_avx2_algs, - ARRAY_SIZE(sha256_avx2_algs)); -} - -asmlinkage void sha256_ni_transform(struct crypto_sha256_state *digest, - const u8 *data, int rounds); - -static int sha256_ni_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - return _sha256_update(desc, data, len, sha256_ni_transform); -} - -static int sha256_ni_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - return sha256_finup(desc, data, len, out, sha256_ni_transform); -} - -static int sha256_ni_digest(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - return sha256_base_init(desc) ?: - sha256_ni_finup(desc, data, len, out); -} - -static struct shash_alg sha256_ni_algs[] = { { - .digestsize = SHA256_DIGEST_SIZE, - .init = sha256_base_init, - .update = sha256_ni_update, - .finup = sha256_ni_finup, - .digest = sha256_ni_digest, - .descsize = sizeof(struct crypto_sha256_state), - .base = { - .cra_name = "sha256", - .cra_driver_name = "sha256-ni", - .cra_priority = 250, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_blocksize = SHA256_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}, { - .digestsize = SHA224_DIGEST_SIZE, - .init = sha224_base_init, - .update = sha256_ni_update, - .finup = sha256_ni_finup, - .descsize = sizeof(struct crypto_sha256_state), - .base = { - .cra_name = "sha224", - .cra_driver_name = "sha224-ni", - .cra_priority = 250, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_blocksize = SHA224_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -} }; - -static int register_sha256_ni(void) -{ - if (boot_cpu_has(X86_FEATURE_SHA_NI)) - return crypto_register_shashes(sha256_ni_algs, - ARRAY_SIZE(sha256_ni_algs)); - return 0; -} - -static void unregister_sha256_ni(void) -{ - if (boot_cpu_has(X86_FEATURE_SHA_NI)) - crypto_unregister_shashes(sha256_ni_algs, - ARRAY_SIZE(sha256_ni_algs)); -} - -static int __init sha256_ssse3_mod_init(void) -{ - if (!x86_match_cpu(module_cpu_ids)) - return -ENODEV; - - if (register_sha256_ssse3()) - goto fail; - - if (register_sha256_avx()) { - unregister_sha256_ssse3(); - goto fail; - } - - if (register_sha256_avx2()) { - unregister_sha256_avx(); - unregister_sha256_ssse3(); - goto fail; - } - - if (register_sha256_ni()) { - unregister_sha256_avx2(); - unregister_sha256_avx(); - unregister_sha256_ssse3(); - goto fail; - } - - return 0; -fail: - return -ENODEV; -} - -static void __exit sha256_ssse3_mod_fini(void) -{ - unregister_sha256_ni(); - unregister_sha256_avx2(); - unregister_sha256_avx(); - unregister_sha256_ssse3(); -} - -module_init(sha256_ssse3_mod_init); -module_exit(sha256_ssse3_mod_fini); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("SHA256 Secure Hash Algorithm, Supplemental SSE3 accelerated"); - -MODULE_ALIAS_CRYPTO("sha256"); -MODULE_ALIAS_CRYPTO("sha256-ssse3"); -MODULE_ALIAS_CRYPTO("sha256-avx"); -MODULE_ALIAS_CRYPTO("sha256-avx2"); -MODULE_ALIAS_CRYPTO("sha224"); -MODULE_ALIAS_CRYPTO("sha224-ssse3"); -MODULE_ALIAS_CRYPTO("sha224-avx"); -MODULE_ALIAS_CRYPTO("sha224-avx2"); -MODULE_ALIAS_CRYPTO("sha256-ni"); -MODULE_ALIAS_CRYPTO("sha224-ni"); diff --git a/arch/x86/lib/crypto/Kconfig b/arch/x86/lib/crypto/Kconfig index 546fe2afe0b5..e344579db3d8 100644 --- a/arch/x86/lib/crypto/Kconfig +++ b/arch/x86/lib/crypto/Kconfig @@ -24,3 +24,10 @@ config CRYPTO_POLY1305_X86_64 depends on 64BIT default CRYPTO_LIB_POLY1305 select CRYPTO_ARCH_HAVE_LIB_POLY1305 + +config CRYPTO_SHA256_X86_64 + tristate + depends on 64BIT + default CRYPTO_LIB_SHA256 + select CRYPTO_ARCH_HAVE_LIB_SHA256 + select CRYPTO_LIB_SHA256_GENERIC diff --git a/arch/x86/lib/crypto/Makefile b/arch/x86/lib/crypto/Makefile index c2ff8c5f1046..abceca3d31c0 100644 --- a/arch/x86/lib/crypto/Makefile +++ b/arch/x86/lib/crypto/Makefile @@ -10,6 +10,9 @@ obj-$(CONFIG_CRYPTO_POLY1305_X86_64) += poly1305-x86_64.o poly1305-x86_64-y := poly1305-x86_64-cryptogams.o poly1305_glue.o targets += poly1305-x86_64-cryptogams.S +obj-$(CONFIG_CRYPTO_SHA256_X86_64) += sha256-x86_64.o +sha256-x86_64-y := sha256.o sha256-ssse3-asm.o sha256-avx-asm.o sha256-avx2-asm.o sha256-ni-asm.o + quiet_cmd_perlasm = PERLASM $@ cmd_perlasm = $(PERL) $< > $@ diff --git a/arch/x86/crypto/sha256-avx-asm.S b/arch/x86/lib/crypto/sha256-avx-asm.S similarity index 98% rename from arch/x86/crypto/sha256-avx-asm.S rename to arch/x86/lib/crypto/sha256-avx-asm.S index 53de72bdd851..0d7b2c3e45d9 100644 --- a/arch/x86/crypto/sha256-avx-asm.S +++ b/arch/x86/lib/crypto/sha256-avx-asm.S @@ -48,7 +48,7 @@ ######################################################################## #include -#include +#include ## assume buffers not aligned #define VMOVDQ vmovdqu @@ -341,13 +341,13 @@ a = TMP_ .endm ######################################################################## -## void sha256_transform_avx(state sha256_state *state, const u8 *data, int blocks) -## arg 1 : pointer to state -## arg 2 : pointer to input data -## arg 3 : Num blocks +## void sha256_transform_avx(u32 state[SHA256_STATE_WORDS], +## const u8 *data, size_t nblocks); ######################################################################## .text -SYM_TYPED_FUNC_START(sha256_transform_avx) +SYM_FUNC_START(sha256_transform_avx) + ANNOTATE_NOENDBR # since this is called only via static_call + pushq %rbx pushq %r12 pushq %r13 diff --git a/arch/x86/crypto/sha256-avx2-asm.S b/arch/x86/lib/crypto/sha256-avx2-asm.S similarity index 98% rename from arch/x86/crypto/sha256-avx2-asm.S rename to arch/x86/lib/crypto/sha256-avx2-asm.S index 0bbec1c75cd0..25d3380321ec 100644 --- a/arch/x86/crypto/sha256-avx2-asm.S +++ b/arch/x86/lib/crypto/sha256-avx2-asm.S @@ -49,7 +49,7 @@ ######################################################################## #include -#include +#include ## assume buffers not aligned #define VMOVDQ vmovdqu @@ -518,13 +518,13 @@ STACK_SIZE = _CTX + _CTX_SIZE .endm ######################################################################## -## void sha256_transform_rorx(struct sha256_state *state, const u8 *data, int blocks) -## arg 1 : pointer to state -## arg 2 : pointer to input data -## arg 3 : Num blocks +## void sha256_transform_rorx(u32 state[SHA256_STATE_WORDS], +## const u8 *data, size_t nblocks); ######################################################################## .text -SYM_TYPED_FUNC_START(sha256_transform_rorx) +SYM_FUNC_START(sha256_transform_rorx) + ANNOTATE_NOENDBR # since this is called only via static_call + pushq %rbx pushq %r12 pushq %r13 diff --git a/arch/x86/crypto/sha256_ni_asm.S b/arch/x86/lib/crypto/sha256-ni-asm.S similarity index 85% rename from arch/x86/crypto/sha256_ni_asm.S rename to arch/x86/lib/crypto/sha256-ni-asm.S index d515a55a3bc1..d3548206cf3d 100644 --- a/arch/x86/crypto/sha256_ni_asm.S +++ b/arch/x86/lib/crypto/sha256-ni-asm.S @@ -54,9 +54,9 @@ */ #include -#include +#include -#define DIGEST_PTR %rdi /* 1st arg */ +#define STATE_PTR %rdi /* 1st arg */ #define DATA_PTR %rsi /* 2nd arg */ #define NUM_BLKS %rdx /* 3rd arg */ @@ -98,24 +98,20 @@ .endm /* - * Intel SHA Extensions optimized implementation of a SHA-256 update function + * Intel SHA Extensions optimized implementation of a SHA-256 block function * - * The function takes a pointer to the current hash values, a pointer to the - * input data, and a number of 64 byte blocks to process. Once all blocks have - * been processed, the digest pointer is updated with the resulting hash value. - * The function only processes complete blocks, there is no functionality to - * store partial blocks. All message padding and hash value initialization must - * be done outside the update function. + * This function takes a pointer to the current SHA-256 state, a pointer to the + * input data, and the number of 64-byte blocks to process. Once all blocks + * have been processed, the state is updated with the new state. This function + * only processes complete blocks. State initialization, buffering of partial + * blocks, and digest finalization is expected to be handled elsewhere. * - * void sha256_ni_transform(uint32_t *digest, const void *data, - uint32_t numBlocks); - * digest : pointer to digest - * data: pointer to input data - * numBlocks: Number of blocks to process + * void sha256_ni_transform(u32 state[SHA256_STATE_WORDS], + * const u8 *data, size_t nblocks); */ - .text -SYM_TYPED_FUNC_START(sha256_ni_transform) +SYM_FUNC_START(sha256_ni_transform) + ANNOTATE_NOENDBR # since this is called only via static_call shl $6, NUM_BLKS /* convert to bytes */ jz .Ldone_hash @@ -126,8 +122,8 @@ SYM_TYPED_FUNC_START(sha256_ni_transform) * Need to reorder these appropriately * DCBA, HGFE -> ABEF, CDGH */ - movdqu 0*16(DIGEST_PTR), STATE0 /* DCBA */ - movdqu 1*16(DIGEST_PTR), STATE1 /* HGFE */ + movdqu 0*16(STATE_PTR), STATE0 /* DCBA */ + movdqu 1*16(STATE_PTR), STATE1 /* HGFE */ movdqa STATE0, TMP punpcklqdq STATE1, STATE0 /* FEBA */ @@ -166,8 +162,8 @@ SYM_TYPED_FUNC_START(sha256_ni_transform) pshufd $0xB1, STATE0, STATE0 /* HGFE */ pshufd $0x1B, STATE1, STATE1 /* DCBA */ - movdqu STATE1, 0*16(DIGEST_PTR) - movdqu STATE0, 1*16(DIGEST_PTR) + movdqu STATE1, 0*16(STATE_PTR) + movdqu STATE0, 1*16(STATE_PTR) .Ldone_hash: diff --git a/arch/x86/crypto/sha256-ssse3-asm.S b/arch/x86/lib/crypto/sha256-ssse3-asm.S similarity index 98% rename from arch/x86/crypto/sha256-ssse3-asm.S rename to arch/x86/lib/crypto/sha256-ssse3-asm.S index 93264ee44543..7f24a4cdcb25 100644 --- a/arch/x86/crypto/sha256-ssse3-asm.S +++ b/arch/x86/lib/crypto/sha256-ssse3-asm.S @@ -47,7 +47,7 @@ ######################################################################## #include -#include +#include ## assume buffers not aligned #define MOVDQ movdqu @@ -348,15 +348,13 @@ a = TMP_ .endm ######################################################################## -## void sha256_transform_ssse3(struct sha256_state *state, const u8 *data, -## int blocks); -## arg 1 : pointer to state -## (struct sha256_state is assumed to begin with u32 state[8]) -## arg 2 : pointer to input data -## arg 3 : Num blocks +## void sha256_transform_ssse3(u32 state[SHA256_STATE_WORDS], +## const u8 *data, size_t nblocks); ######################################################################## .text -SYM_TYPED_FUNC_START(sha256_transform_ssse3) +SYM_FUNC_START(sha256_transform_ssse3) + ANNOTATE_NOENDBR # since this is called only via static_call + pushq %rbx pushq %r12 pushq %r13 diff --git a/arch/x86/lib/crypto/sha256.c b/arch/x86/lib/crypto/sha256.c new file mode 100644 index 000000000000..47865b5cd94b --- /dev/null +++ b/arch/x86/lib/crypto/sha256.c @@ -0,0 +1,74 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * SHA-256 optimized for x86_64 + * + * Copyright 2025 Google LLC + */ +#include +#include +#include +#include +#include +#include + +asmlinkage void sha256_transform_ssse3(u32 state[SHA256_STATE_WORDS], + const u8 *data, size_t nblocks); +asmlinkage void sha256_transform_avx(u32 state[SHA256_STATE_WORDS], + const u8 *data, size_t nblocks); +asmlinkage void sha256_transform_rorx(u32 state[SHA256_STATE_WORDS], + const u8 *data, size_t nblocks); +asmlinkage void sha256_ni_transform(u32 state[SHA256_STATE_WORDS], + const u8 *data, size_t nblocks); + +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_sha256_x86); + +DEFINE_STATIC_CALL(sha256_blocks_x86, sha256_transform_ssse3); + +void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS], + const u8 *data, size_t nblocks) +{ + if (static_branch_likely(&have_sha256_x86) && crypto_simd_usable()) { + kernel_fpu_begin(); + static_call(sha256_blocks_x86)(state, data, nblocks); + kernel_fpu_end(); + } else { + sha256_blocks_generic(state, data, nblocks); + } +} +EXPORT_SYMBOL(sha256_blocks_arch); + +bool sha256_is_arch_optimized(void) +{ + return static_key_enabled(&have_sha256_x86); +} +EXPORT_SYMBOL(sha256_is_arch_optimized); + +static int __init sha256_x86_mod_init(void) +{ + if (boot_cpu_has(X86_FEATURE_SHA_NI)) { + static_call_update(sha256_blocks_x86, sha256_ni_transform); + } else if (cpu_has_xfeatures(XFEATURE_MASK_SSE | + XFEATURE_MASK_YMM, NULL) && + boot_cpu_has(X86_FEATURE_AVX)) { + if (boot_cpu_has(X86_FEATURE_AVX2) && + boot_cpu_has(X86_FEATURE_BMI2)) + static_call_update(sha256_blocks_x86, + sha256_transform_rorx); + else + static_call_update(sha256_blocks_x86, + sha256_transform_avx); + } else if (!boot_cpu_has(X86_FEATURE_SSSE3)) { + return 0; + } + static_branch_enable(&have_sha256_x86); + return 0; +} +arch_initcall(sha256_x86_mod_init); + +static void __exit sha256_x86_mod_exit(void) +{ +} +module_exit(sha256_x86_mod_exit); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("SHA-256 optimized for x86_64"); From 5aab01777fd38ff820bd7ae6cbc15d03b85de92f Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 28 Apr 2025 10:00:37 -0700 Subject: [PATCH 271/382] crypto: sha256 - remove sha256_base.h sha256_base.h is no longer used, so remove it. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- include/crypto/sha256_base.h | 151 ----------------------------------- 1 file changed, 151 deletions(-) delete mode 100644 include/crypto/sha256_base.h diff --git a/include/crypto/sha256_base.h b/include/crypto/sha256_base.h deleted file mode 100644 index 804361731a7a..000000000000 --- a/include/crypto/sha256_base.h +++ /dev/null @@ -1,151 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * sha256_base.h - core logic for SHA-256 implementations - * - * Copyright (C) 2015 Linaro Ltd - */ - -#ifndef _CRYPTO_SHA256_BASE_H -#define _CRYPTO_SHA256_BASE_H - -#include -#include -#include -#include -#include -#include -#include - -typedef void (sha256_block_fn)(struct crypto_sha256_state *sst, u8 const *src, - int blocks); - -static inline int sha224_base_init(struct shash_desc *desc) -{ - struct sha256_state *sctx = shash_desc_ctx(desc); - - sha224_init(sctx); - return 0; -} - -static inline int sha256_base_init(struct shash_desc *desc) -{ - struct sha256_state *sctx = shash_desc_ctx(desc); - - sha256_init(sctx); - return 0; -} - -static inline int lib_sha256_base_do_update(struct sha256_state *sctx, - const u8 *data, - unsigned int len, - sha256_block_fn *block_fn) -{ - unsigned int partial = sctx->count % SHA256_BLOCK_SIZE; - - sctx->count += len; - BLOCK_HASH_UPDATE_BLOCKS(block_fn, &sctx->ctx, data, len, - SHA256_BLOCK_SIZE, sctx->buf, partial); - return 0; -} - -static inline int lib_sha256_base_do_update_blocks( - struct crypto_sha256_state *sctx, const u8 *data, unsigned int len, - sha256_block_fn *block_fn) -{ - unsigned int remain = len - round_down(len, SHA256_BLOCK_SIZE); - - sctx->count += len - remain; - block_fn(sctx, data, len / SHA256_BLOCK_SIZE); - return remain; -} - -static inline int sha256_base_do_update_blocks( - struct shash_desc *desc, const u8 *data, unsigned int len, - sha256_block_fn *block_fn) -{ - return lib_sha256_base_do_update_blocks(shash_desc_ctx(desc), data, - len, block_fn); -} - -static inline int lib_sha256_base_do_finup(struct crypto_sha256_state *sctx, - const u8 *src, unsigned int len, - sha256_block_fn *block_fn) -{ - unsigned int bit_offset = SHA256_BLOCK_SIZE / 8 - 1; - union { - __be64 b64[SHA256_BLOCK_SIZE / 4]; - u8 u8[SHA256_BLOCK_SIZE * 2]; - } block = {}; - - if (len >= bit_offset * 8) - bit_offset += SHA256_BLOCK_SIZE / 8; - memcpy(&block, src, len); - block.u8[len] = 0x80; - sctx->count += len; - block.b64[bit_offset] = cpu_to_be64(sctx->count << 3); - block_fn(sctx, block.u8, (bit_offset + 1) * 8 / SHA256_BLOCK_SIZE); - memzero_explicit(&block, sizeof(block)); - - return 0; -} - -static inline int sha256_base_do_finup(struct shash_desc *desc, - const u8 *src, unsigned int len, - sha256_block_fn *block_fn) -{ - struct crypto_sha256_state *sctx = shash_desc_ctx(desc); - - if (len >= SHA256_BLOCK_SIZE) { - int remain; - - remain = lib_sha256_base_do_update_blocks(sctx, src, len, - block_fn); - src += len - remain; - len = remain; - } - return lib_sha256_base_do_finup(sctx, src, len, block_fn); -} - -static inline int lib_sha256_base_do_finalize(struct sha256_state *sctx, - sha256_block_fn *block_fn) -{ - unsigned int partial = sctx->count % SHA256_BLOCK_SIZE; - struct crypto_sha256_state *state = (void *)sctx; - - sctx->count -= partial; - return lib_sha256_base_do_finup(state, sctx->buf, partial, block_fn); -} - -static inline int __sha256_base_finish(u32 state[SHA256_DIGEST_SIZE / 4], - u8 *out, unsigned int digest_size) -{ - __be32 *digest = (__be32 *)out; - int i; - - for (i = 0; digest_size > 0; i++, digest_size -= sizeof(__be32)) - put_unaligned_be32(state[i], digest++); - return 0; -} - -static inline void lib_sha256_base_finish(struct sha256_state *sctx, u8 *out, - unsigned int digest_size) -{ - __sha256_base_finish(sctx->state, out, digest_size); - memzero_explicit(sctx, sizeof(*sctx)); -} - -static inline int sha256_base_finish(struct shash_desc *desc, u8 *out) -{ - unsigned int digest_size = crypto_shash_digestsize(desc->tfm); - struct crypto_sha256_state *sctx = shash_desc_ctx(desc); - - return __sha256_base_finish(sctx->state, out, digest_size); -} - -static inline void sha256_transform_blocks(struct crypto_sha256_state *sst, - const u8 *input, int blocks) -{ - sha256_blocks_generic(sst->state, input, blocks); -} - -#endif /* _CRYPTO_SHA256_BASE_H */ From 7350fef56b7629a539897c332d7324629580671d Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 28 Apr 2025 10:00:38 -0700 Subject: [PATCH 272/382] crypto: lib/sha256 - improve function prototypes Follow best practices by changing the length parameters to size_t and explicitly specifying the length of the output digest arrays. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- include/crypto/sha2.h | 8 ++++---- lib/crypto/sha256.c | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/include/crypto/sha2.h b/include/crypto/sha2.h index 9a56286d736d..9853cd2d1291 100644 --- a/include/crypto/sha2.h +++ b/include/crypto/sha2.h @@ -100,9 +100,9 @@ static inline void sha256_init(struct sha256_state *sctx) sctx->state[7] = SHA256_H7; sctx->count = 0; } -void sha256_update(struct sha256_state *sctx, const u8 *data, unsigned int len); -void sha256_final(struct sha256_state *sctx, u8 *out); -void sha256(const u8 *data, unsigned int len, u8 *out); +void sha256_update(struct sha256_state *sctx, const u8 *data, size_t len); +void sha256_final(struct sha256_state *sctx, u8 out[SHA256_DIGEST_SIZE]); +void sha256(const u8 *data, size_t len, u8 out[SHA256_DIGEST_SIZE]); static inline void sha224_init(struct sha256_state *sctx) { @@ -117,6 +117,6 @@ static inline void sha224_init(struct sha256_state *sctx) sctx->count = 0; } /* Simply use sha256_update as it is equivalent to sha224_update. */ -void sha224_final(struct sha256_state *sctx, u8 *out); +void sha224_final(struct sha256_state *sctx, u8 out[SHA224_DIGEST_SIZE]); #endif /* _CRYPTO_SHA2_H */ diff --git a/lib/crypto/sha256.c b/lib/crypto/sha256.c index 4b19cf977ef1..563f09c9f381 100644 --- a/lib/crypto/sha256.c +++ b/lib/crypto/sha256.c @@ -70,7 +70,7 @@ static inline void __sha256_update(struct sha256_state *sctx, const u8 *data, memcpy(&sctx->buf[partial], data, len); } -void sha256_update(struct sha256_state *sctx, const u8 *data, unsigned int len) +void sha256_update(struct sha256_state *sctx, const u8 *data, size_t len) { __sha256_update(sctx, data, len, false); } @@ -101,19 +101,19 @@ static inline void __sha256_final(struct sha256_state *sctx, u8 *out, memzero_explicit(sctx, sizeof(*sctx)); } -void sha256_final(struct sha256_state *sctx, u8 *out) +void sha256_final(struct sha256_state *sctx, u8 out[SHA256_DIGEST_SIZE]) { __sha256_final(sctx, out, SHA256_DIGEST_SIZE, false); } EXPORT_SYMBOL(sha256_final); -void sha224_final(struct sha256_state *sctx, u8 *out) +void sha224_final(struct sha256_state *sctx, u8 out[SHA224_DIGEST_SIZE]) { __sha256_final(sctx, out, SHA224_DIGEST_SIZE, false); } EXPORT_SYMBOL(sha224_final); -void sha256(const u8 *data, unsigned int len, u8 *out) +void sha256(const u8 *data, size_t len, u8 out[SHA256_DIGEST_SIZE]) { struct sha256_state sctx; From 04c535ab156bf6c08aeb04d39bbcff0f53b7a0a7 Mon Sep 17 00:00:00 2001 From: Suman Kumar Chakraborty Date: Mon, 28 Apr 2025 18:24:26 +0100 Subject: [PATCH 273/382] crypto: qat - include qat_common in top Makefile To ensure proper functionality, each specific driver needs to access functions located in the qat_common folder. Move the include path for qat_common to the top-level Makefile. This eliminates the need for redundant include directives in the Makefiles of individual drivers. Suggested-by: Andy Shevchenko Signed-off-by: Suman Kumar Chakraborty Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/intel/qat/Makefile | 1 + drivers/crypto/intel/qat/qat_420xx/Makefile | 1 - drivers/crypto/intel/qat/qat_4xxx/Makefile | 1 - drivers/crypto/intel/qat/qat_c3xxx/Makefile | 1 - drivers/crypto/intel/qat/qat_c3xxxvf/Makefile | 1 - drivers/crypto/intel/qat/qat_c62x/Makefile | 1 - drivers/crypto/intel/qat/qat_c62xvf/Makefile | 1 - drivers/crypto/intel/qat/qat_dh895xcc/Makefile | 1 - drivers/crypto/intel/qat/qat_dh895xccvf/Makefile | 1 - 9 files changed, 1 insertion(+), 8 deletions(-) diff --git a/drivers/crypto/intel/qat/Makefile b/drivers/crypto/intel/qat/Makefile index 235b69f4f3f7..1eda8dc18515 100644 --- a/drivers/crypto/intel/qat/Makefile +++ b/drivers/crypto/intel/qat/Makefile @@ -1,4 +1,5 @@ # SPDX-License-Identifier: GPL-2.0 +subdir-ccflags-y := -I$(src)/qat_common obj-$(CONFIG_CRYPTO_DEV_QAT) += qat_common/ obj-$(CONFIG_CRYPTO_DEV_QAT_DH895xCC) += qat_dh895xcc/ obj-$(CONFIG_CRYPTO_DEV_QAT_C3XXX) += qat_c3xxx/ diff --git a/drivers/crypto/intel/qat/qat_420xx/Makefile b/drivers/crypto/intel/qat/qat_420xx/Makefile index 72b24b1804cf..f6df54d2993e 100644 --- a/drivers/crypto/intel/qat/qat_420xx/Makefile +++ b/drivers/crypto/intel/qat/qat_420xx/Makefile @@ -1,4 +1,3 @@ # SPDX-License-Identifier: GPL-2.0-only -ccflags-y := -I $(src)/../qat_common obj-$(CONFIG_CRYPTO_DEV_QAT_420XX) += qat_420xx.o qat_420xx-y := adf_drv.o adf_420xx_hw_data.o diff --git a/drivers/crypto/intel/qat/qat_4xxx/Makefile b/drivers/crypto/intel/qat/qat_4xxx/Makefile index e8480bb80dee..188b611445e6 100644 --- a/drivers/crypto/intel/qat/qat_4xxx/Makefile +++ b/drivers/crypto/intel/qat/qat_4xxx/Makefile @@ -1,4 +1,3 @@ # SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) -ccflags-y := -I $(src)/../qat_common obj-$(CONFIG_CRYPTO_DEV_QAT_4XXX) += qat_4xxx.o qat_4xxx-y := adf_drv.o adf_4xxx_hw_data.o diff --git a/drivers/crypto/intel/qat/qat_c3xxx/Makefile b/drivers/crypto/intel/qat/qat_c3xxx/Makefile index d9e568572da8..43604c025f0c 100644 --- a/drivers/crypto/intel/qat/qat_c3xxx/Makefile +++ b/drivers/crypto/intel/qat/qat_c3xxx/Makefile @@ -1,4 +1,3 @@ # SPDX-License-Identifier: GPL-2.0-only -ccflags-y := -I $(src)/../qat_common obj-$(CONFIG_CRYPTO_DEV_QAT_C3XXX) += qat_c3xxx.o qat_c3xxx-y := adf_drv.o adf_c3xxx_hw_data.o diff --git a/drivers/crypto/intel/qat/qat_c3xxxvf/Makefile b/drivers/crypto/intel/qat/qat_c3xxxvf/Makefile index 31a908a211ac..03f6745b4aa2 100644 --- a/drivers/crypto/intel/qat/qat_c3xxxvf/Makefile +++ b/drivers/crypto/intel/qat/qat_c3xxxvf/Makefile @@ -1,4 +1,3 @@ # SPDX-License-Identifier: GPL-2.0-only -ccflags-y := -I $(src)/../qat_common obj-$(CONFIG_CRYPTO_DEV_QAT_C3XXXVF) += qat_c3xxxvf.o qat_c3xxxvf-y := adf_drv.o adf_c3xxxvf_hw_data.o diff --git a/drivers/crypto/intel/qat/qat_c62x/Makefile b/drivers/crypto/intel/qat/qat_c62x/Makefile index cbdaaa135e84..f3d722bef088 100644 --- a/drivers/crypto/intel/qat/qat_c62x/Makefile +++ b/drivers/crypto/intel/qat/qat_c62x/Makefile @@ -1,4 +1,3 @@ # SPDX-License-Identifier: GPL-2.0-only -ccflags-y := -I $(src)/../qat_common obj-$(CONFIG_CRYPTO_DEV_QAT_C62X) += qat_c62x.o qat_c62x-y := adf_drv.o adf_c62x_hw_data.o diff --git a/drivers/crypto/intel/qat/qat_c62xvf/Makefile b/drivers/crypto/intel/qat/qat_c62xvf/Makefile index 60e499b041ec..ed7f3f722d99 100644 --- a/drivers/crypto/intel/qat/qat_c62xvf/Makefile +++ b/drivers/crypto/intel/qat/qat_c62xvf/Makefile @@ -1,4 +1,3 @@ # SPDX-License-Identifier: GPL-2.0-only -ccflags-y := -I $(src)/../qat_common obj-$(CONFIG_CRYPTO_DEV_QAT_C62XVF) += qat_c62xvf.o qat_c62xvf-y := adf_drv.o adf_c62xvf_hw_data.o diff --git a/drivers/crypto/intel/qat/qat_dh895xcc/Makefile b/drivers/crypto/intel/qat/qat_dh895xcc/Makefile index 5bf5c890c362..1427fe76f171 100644 --- a/drivers/crypto/intel/qat/qat_dh895xcc/Makefile +++ b/drivers/crypto/intel/qat/qat_dh895xcc/Makefile @@ -1,4 +1,3 @@ # SPDX-License-Identifier: GPL-2.0-only -ccflags-y := -I $(src)/../qat_common obj-$(CONFIG_CRYPTO_DEV_QAT_DH895xCC) += qat_dh895xcc.o qat_dh895xcc-y := adf_drv.o adf_dh895xcc_hw_data.o diff --git a/drivers/crypto/intel/qat/qat_dh895xccvf/Makefile b/drivers/crypto/intel/qat/qat_dh895xccvf/Makefile index 93f9c81edf09..c2fdb6e0f68f 100644 --- a/drivers/crypto/intel/qat/qat_dh895xccvf/Makefile +++ b/drivers/crypto/intel/qat/qat_dh895xccvf/Makefile @@ -1,4 +1,3 @@ # SPDX-License-Identifier: GPL-2.0-only -ccflags-y := -I $(src)/../qat_common obj-$(CONFIG_CRYPTO_DEV_QAT_DH895xCCVF) += qat_dh895xccvf.o qat_dh895xccvf-y := adf_drv.o adf_dh895xccvf_hw_data.o From 4a7e1e815c5de24339254eafb6f17cfe0d885a60 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Tue, 29 Apr 2025 16:42:20 +0800 Subject: [PATCH 274/382] crypto: rmd160 - Use API partial block handling Use the Crypto API partial block handling. Signed-off-by: Herbert Xu --- crypto/rmd160.c | 86 ++++++++++++++++++++----------------------------- 1 file changed, 35 insertions(+), 51 deletions(-) diff --git a/crypto/rmd160.c b/crypto/rmd160.c index c5fe4034b153..890d29e46a76 100644 --- a/crypto/rmd160.c +++ b/crypto/rmd160.c @@ -9,18 +9,14 @@ * Copyright (c) 2008 Adrian-Ken Rueegsegger */ #include -#include +#include #include -#include -#include -#include - +#include #include "ripemd.h" struct rmd160_ctx { u64 byte_count; u32 state[5]; - __le32 buffer[16]; }; #define K1 RMD_K1 @@ -265,72 +261,59 @@ static int rmd160_init(struct shash_desc *desc) rctx->state[3] = RMD_H3; rctx->state[4] = RMD_H4; - memset(rctx->buffer, 0, sizeof(rctx->buffer)); - return 0; } static int rmd160_update(struct shash_desc *desc, const u8 *data, unsigned int len) { + int remain = len - round_down(len, RMD160_BLOCK_SIZE); struct rmd160_ctx *rctx = shash_desc_ctx(desc); - const u32 avail = sizeof(rctx->buffer) - (rctx->byte_count & 0x3f); + __le32 buffer[RMD160_BLOCK_SIZE / 4]; - rctx->byte_count += len; + rctx->byte_count += len - remain; - /* Enough space in buffer? If so copy and we're done */ - if (avail > len) { - memcpy((char *)rctx->buffer + (sizeof(rctx->buffer) - avail), - data, len); - goto out; - } + do { + memcpy(buffer, data, sizeof(buffer)); + rmd160_transform(rctx->state, buffer); + data += sizeof(buffer); + len -= sizeof(buffer); + } while (len >= sizeof(buffer)); - memcpy((char *)rctx->buffer + (sizeof(rctx->buffer) - avail), - data, avail); - - rmd160_transform(rctx->state, rctx->buffer); - data += avail; - len -= avail; - - while (len >= sizeof(rctx->buffer)) { - memcpy(rctx->buffer, data, sizeof(rctx->buffer)); - rmd160_transform(rctx->state, rctx->buffer); - data += sizeof(rctx->buffer); - len -= sizeof(rctx->buffer); - } - - memcpy(rctx->buffer, data, len); - -out: - return 0; + memzero_explicit(buffer, sizeof(buffer)); + return remain; } /* Add padding and return the message digest. */ -static int rmd160_final(struct shash_desc *desc, u8 *out) +static int rmd160_finup(struct shash_desc *desc, const u8 *src, + unsigned int len, u8 *out) { + unsigned int bit_offset = RMD160_BLOCK_SIZE / 8 - 1; struct rmd160_ctx *rctx = shash_desc_ctx(desc); - u32 i, index, padlen; - __le64 bits; + union { + __le64 l64[RMD160_BLOCK_SIZE / 4]; + __le32 l32[RMD160_BLOCK_SIZE / 2]; + u8 u8[RMD160_BLOCK_SIZE * 2]; + } block = {}; __le32 *dst = (__le32 *)out; - static const u8 padding[64] = { 0x80, }; + u32 i; - bits = cpu_to_le64(rctx->byte_count << 3); + rctx->byte_count += len; + if (len >= bit_offset * 8) + bit_offset += RMD160_BLOCK_SIZE / 8; + memcpy(&block, src, len); + block.u8[len] = 0x80; + block.l64[bit_offset] = cpu_to_le64(rctx->byte_count << 3); - /* Pad out to 56 mod 64 */ - index = rctx->byte_count & 0x3f; - padlen = (index < 56) ? (56 - index) : ((64+56) - index); - rmd160_update(desc, padding, padlen); - - /* Append length */ - rmd160_update(desc, (const u8 *)&bits, sizeof(bits)); + rmd160_transform(rctx->state, block.l32); + if (bit_offset > RMD160_BLOCK_SIZE / 8) + rmd160_transform(rctx->state, + block.l32 + RMD160_BLOCK_SIZE / 4); + memzero_explicit(&block, sizeof(block)); /* Store state in digest */ for (i = 0; i < 5; i++) dst[i] = cpu_to_le32p(&rctx->state[i]); - - /* Wipe context */ - memset(rctx, 0, sizeof(*rctx)); - return 0; } @@ -338,11 +321,12 @@ static struct shash_alg alg = { .digestsize = RMD160_DIGEST_SIZE, .init = rmd160_init, .update = rmd160_update, - .final = rmd160_final, + .finup = rmd160_finup, .descsize = sizeof(struct rmd160_ctx), .base = { .cra_name = "rmd160", .cra_driver_name = "rmd160-generic", + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_blocksize = RMD160_BLOCK_SIZE, .cra_module = THIS_MODULE, } From 0e2392b6513cb138e2915ad08e33ac029358e957 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Tue, 29 Apr 2025 16:43:22 +0800 Subject: [PATCH 275/382] crypto: streebog - Use API partial block handling Use the Crypto API partial block handling. Signed-off-by: Herbert Xu --- crypto/streebog_generic.c | 71 ++++++++++++++------------------------- include/crypto/streebog.h | 5 --- 2 files changed, 25 insertions(+), 51 deletions(-) diff --git a/crypto/streebog_generic.c b/crypto/streebog_generic.c index dc625ffc54ad..99c07e0c10ba 100644 --- a/crypto/streebog_generic.c +++ b/crypto/streebog_generic.c @@ -13,9 +13,10 @@ */ #include -#include -#include #include +#include +#include +#include static const struct streebog_uint512 buffer0 = { { 0, 0, 0, 0, 0, 0, 0, 0 @@ -919,17 +920,6 @@ static int streebog_init(struct shash_desc *desc) return 0; } -static void streebog_pad(struct streebog_state *ctx) -{ - if (ctx->fillsize >= STREEBOG_BLOCK_SIZE) - return; - - memset(ctx->buffer + ctx->fillsize, 0, - sizeof(ctx->buffer) - ctx->fillsize); - - ctx->buffer[ctx->fillsize] = 1; -} - static void streebog_add512(const struct streebog_uint512 *x, const struct streebog_uint512 *y, struct streebog_uint512 *r) @@ -984,16 +974,23 @@ static void streebog_stage2(struct streebog_state *ctx, const u8 *data) streebog_add512(&ctx->Sigma, &m, &ctx->Sigma); } -static void streebog_stage3(struct streebog_state *ctx) +static void streebog_stage3(struct streebog_state *ctx, const u8 *src, + unsigned int len) { struct streebog_uint512 buf = { { 0 } }; + union { + u8 buffer[STREEBOG_BLOCK_SIZE]; + struct streebog_uint512 m; + } u = {}; - buf.qword[0] = cpu_to_le64(ctx->fillsize << 3); - streebog_pad(ctx); + buf.qword[0] = cpu_to_le64(len << 3); + memcpy(u.buffer, src, len); + u.buffer[len] = 1; - streebog_g(&ctx->h, &ctx->N, &ctx->m); + streebog_g(&ctx->h, &ctx->N, &u.m); streebog_add512(&ctx->N, &buf, &ctx->N); - streebog_add512(&ctx->Sigma, &ctx->m, &ctx->Sigma); + streebog_add512(&ctx->Sigma, &u.m, &ctx->Sigma); + memzero_explicit(&u, sizeof(u)); streebog_g(&ctx->h, &buffer0, &ctx->N); streebog_g(&ctx->h, &buffer0, &ctx->Sigma); memcpy(&ctx->hash, &ctx->h, sizeof(struct streebog_uint512)); @@ -1003,42 +1000,22 @@ static int streebog_update(struct shash_desc *desc, const u8 *data, unsigned int len) { struct streebog_state *ctx = shash_desc_ctx(desc); - size_t chunksize; - if (ctx->fillsize) { - chunksize = STREEBOG_BLOCK_SIZE - ctx->fillsize; - if (chunksize > len) - chunksize = len; - memcpy(&ctx->buffer[ctx->fillsize], data, chunksize); - ctx->fillsize += chunksize; - len -= chunksize; - data += chunksize; - - if (ctx->fillsize == STREEBOG_BLOCK_SIZE) { - streebog_stage2(ctx, ctx->buffer); - ctx->fillsize = 0; - } - } - - while (len >= STREEBOG_BLOCK_SIZE) { + do { streebog_stage2(ctx, data); data += STREEBOG_BLOCK_SIZE; len -= STREEBOG_BLOCK_SIZE; - } + } while (len >= STREEBOG_BLOCK_SIZE); - if (len) { - memcpy(&ctx->buffer, data, len); - ctx->fillsize = len; - } - return 0; + return len; } -static int streebog_final(struct shash_desc *desc, u8 *digest) +static int streebog_finup(struct shash_desc *desc, const u8 *src, + unsigned int len, u8 *digest) { struct streebog_state *ctx = shash_desc_ctx(desc); - streebog_stage3(ctx); - ctx->fillsize = 0; + streebog_stage3(ctx, src, len); if (crypto_shash_digestsize(desc->tfm) == STREEBOG256_DIGEST_SIZE) memcpy(digest, &ctx->hash.qword[4], STREEBOG256_DIGEST_SIZE); else @@ -1050,11 +1027,12 @@ static struct shash_alg algs[2] = { { .digestsize = STREEBOG256_DIGEST_SIZE, .init = streebog_init, .update = streebog_update, - .final = streebog_final, + .finup = streebog_finup, .descsize = sizeof(struct streebog_state), .base = { .cra_name = "streebog256", .cra_driver_name = "streebog256-generic", + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_blocksize = STREEBOG_BLOCK_SIZE, .cra_module = THIS_MODULE, }, @@ -1062,11 +1040,12 @@ static struct shash_alg algs[2] = { { .digestsize = STREEBOG512_DIGEST_SIZE, .init = streebog_init, .update = streebog_update, - .final = streebog_final, + .finup = streebog_finup, .descsize = sizeof(struct streebog_state), .base = { .cra_name = "streebog512", .cra_driver_name = "streebog512-generic", + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_blocksize = STREEBOG_BLOCK_SIZE, .cra_module = THIS_MODULE, } diff --git a/include/crypto/streebog.h b/include/crypto/streebog.h index cae1b4a01971..570f720a113b 100644 --- a/include/crypto/streebog.h +++ b/include/crypto/streebog.h @@ -23,15 +23,10 @@ struct streebog_uint512 { }; struct streebog_state { - union { - u8 buffer[STREEBOG_BLOCK_SIZE]; - struct streebog_uint512 m; - }; struct streebog_uint512 hash; struct streebog_uint512 h; struct streebog_uint512 N; struct streebog_uint512 Sigma; - size_t fillsize; }; #endif /* !_CRYPTO_STREEBOG_H_ */ From 89490e6b80c53bf7783fe183a2fda8d0944f52d2 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Tue, 29 Apr 2025 16:49:32 +0800 Subject: [PATCH 276/382] crypto: s390/hmac - Extend hash length counters to 128 bits As sha512 requires 128-bit counters, extend the hash length counters to that length. Previously they were just 32 bits which means that a >4G sha256 hash would be incorrect. Signed-off-by: Herbert Xu --- arch/s390/crypto/hmac_s390.c | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/arch/s390/crypto/hmac_s390.c b/arch/s390/crypto/hmac_s390.c index bba9a818dfdc..e6edf1013228 100644 --- a/arch/s390/crypto/hmac_s390.c +++ b/arch/s390/crypto/hmac_s390.c @@ -72,23 +72,23 @@ struct s390_kmac_sha2_ctx { u8 param[MAX_DIGEST_SIZE + MAX_IMBL_SIZE + MAX_BLOCK_SIZE]; union s390_kmac_gr0 gr0; u8 buf[MAX_BLOCK_SIZE]; - unsigned int buflen; + u64 buflen[2]; }; /* * kmac_sha2_set_imbl - sets the input message bit-length based on the blocksize */ -static inline void kmac_sha2_set_imbl(u8 *param, unsigned int buflen, - unsigned int blocksize) +static inline void kmac_sha2_set_imbl(u8 *param, u64 buflen_lo, + u64 buflen_hi, unsigned int blocksize) { u8 *imbl = param + SHA2_IMBL_OFFSET(blocksize); switch (blocksize) { case SHA256_BLOCK_SIZE: - *(u64 *)imbl = (u64)buflen * BITS_PER_BYTE; + *(u64 *)imbl = buflen_lo * BITS_PER_BYTE; break; case SHA512_BLOCK_SIZE: - *(u128 *)imbl = (u128)buflen * BITS_PER_BYTE; + *(u128 *)imbl = (((u128)buflen_hi << 64) + buflen_lo) << 3; break; default: break; @@ -176,7 +176,8 @@ static int s390_hmac_sha2_init(struct shash_desc *desc) memcpy(ctx->param + SHA2_KEY_OFFSET(bs), tfm_ctx->key, bs); - ctx->buflen = 0; + ctx->buflen[0] = 0; + ctx->buflen[1] = 0; ctx->gr0.reg = 0; switch (crypto_shash_digestsize(desc->tfm)) { case SHA224_DIGEST_SIZE: @@ -206,8 +207,10 @@ static int s390_hmac_sha2_update(struct shash_desc *desc, unsigned int offset, n; /* check current buffer */ - offset = ctx->buflen % bs; - ctx->buflen += len; + offset = ctx->buflen[0] % bs; + ctx->buflen[0] += len; + if (ctx->buflen[0] < len) + ctx->buflen[1]++; if (offset + len < bs) goto store; @@ -243,8 +246,8 @@ static int s390_hmac_sha2_final(struct shash_desc *desc, u8 *out) unsigned int bs = crypto_shash_blocksize(desc->tfm); ctx->gr0.iimp = 0; - kmac_sha2_set_imbl(ctx->param, ctx->buflen, bs); - _cpacf_kmac(&ctx->gr0.reg, ctx->param, ctx->buf, ctx->buflen % bs); + kmac_sha2_set_imbl(ctx->param, ctx->buflen[0], ctx->buflen[1], bs); + _cpacf_kmac(&ctx->gr0.reg, ctx->param, ctx->buf, ctx->buflen[0] % bs); memcpy(out, ctx->param, crypto_shash_digestsize(desc->tfm)); return 0; @@ -262,7 +265,7 @@ static int s390_hmac_sha2_digest(struct shash_desc *desc, return rc; ctx->gr0.iimp = 0; - kmac_sha2_set_imbl(ctx->param, len, + kmac_sha2_set_imbl(ctx->param, len, 0, crypto_shash_blocksize(desc->tfm)); _cpacf_kmac(&ctx->gr0.reg, ctx->param, data, len); memcpy(out, ctx->param, ds); From ef93f1562803cd7bb8159e3abedaf7f47dce4e35 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Wed, 30 Apr 2025 16:17:02 +0800 Subject: [PATCH 277/382] Revert "crypto: run initcalls for generic implementations earlier" This reverts commit c4741b23059794bd99beef0f700103b0d983b3fd. Crypto API self-tests no longer run at registration time and now occur either at late_initcall or upon the first use. Therefore the premise of the above commit no longer exists. Revert it and subsequent additions of subsys_initcall and arch_initcall. Note that lib/crypto calls will stay at subsys_initcall (or rather downgraded from arch_initcall) because they may need to occur before Crypto API registration. Signed-off-by: Herbert Xu --- arch/arm/crypto/aes-neonbs-glue.c | 2 +- arch/arm/lib/crypto/chacha-glue.c | 2 +- arch/arm/lib/crypto/poly1305-glue.c | 2 +- arch/arm/lib/crypto/sha256.c | 2 +- arch/arm64/lib/crypto/chacha-neon-glue.c | 2 +- arch/arm64/lib/crypto/poly1305-glue.c | 2 +- arch/arm64/lib/crypto/sha256.c | 2 +- arch/powerpc/lib/crypto/chacha-p10-glue.c | 2 +- arch/powerpc/lib/crypto/poly1305-p10-glue.c | 2 +- arch/riscv/lib/crypto/chacha-riscv64-glue.c | 2 +- arch/riscv/lib/crypto/sha256.c | 2 +- arch/s390/lib/crypto/sha256.c | 2 +- arch/sparc/lib/crypto/sha256.c | 2 +- arch/x86/lib/crypto/chacha_glue.c | 2 +- arch/x86/lib/crypto/poly1305_glue.c | 2 +- arch/x86/lib/crypto/sha256.c | 2 +- crypto/842.c | 2 +- crypto/adiantum.c | 2 +- crypto/aegis128-core.c | 2 +- crypto/aes_generic.c | 2 +- crypto/algboss.c | 8 +------- crypto/ansi_cprng.c | 2 +- crypto/anubis.c | 2 +- crypto/arc4.c | 2 +- crypto/aria_generic.c | 2 +- crypto/authenc.c | 2 +- crypto/authencesn.c | 2 +- crypto/blake2b_generic.c | 2 +- crypto/blowfish_generic.c | 2 +- crypto/camellia_generic.c | 2 +- crypto/cast5_generic.c | 2 +- crypto/cast6_generic.c | 2 +- crypto/cbc.c | 2 +- crypto/ccm.c | 2 +- crypto/chacha.c | 2 +- crypto/chacha20poly1305.c | 2 +- crypto/cmac.c | 2 +- crypto/crc32_generic.c | 2 +- crypto/crc32c_generic.c | 2 +- crypto/cryptd.c | 2 +- crypto/crypto_null.c | 2 +- crypto/ctr.c | 2 +- crypto/cts.c | 2 +- crypto/curve25519-generic.c | 2 +- crypto/deflate.c | 2 +- crypto/des_generic.c | 2 +- crypto/dh.c | 2 +- crypto/drbg.c | 2 +- crypto/ecb.c | 2 +- crypto/ecdh.c | 2 +- crypto/ecdsa.c | 2 +- crypto/echainiv.c | 2 +- crypto/essiv.c | 2 +- crypto/fcrypt.c | 2 +- crypto/fips.c | 2 +- crypto/gcm.c | 2 +- crypto/ghash-generic.c | 2 +- crypto/hctr2.c | 2 +- crypto/hmac.c | 2 +- crypto/khazad.c | 2 +- crypto/krb5enc.c | 2 +- crypto/lrw.c | 2 +- crypto/lz4.c | 2 +- crypto/lz4hc.c | 2 +- crypto/lzo-rle.c | 2 +- crypto/lzo.c | 2 +- crypto/md4.c | 2 +- crypto/md5.c | 2 +- crypto/michael_mic.c | 2 +- crypto/nhpoly1305.c | 2 +- crypto/pcbc.c | 2 +- crypto/pcrypt.c | 2 +- crypto/polyval-generic.c | 2 +- crypto/rmd160.c | 2 +- crypto/rsa.c | 2 +- crypto/seed.c | 2 +- crypto/seqiv.c | 2 +- crypto/serpent_generic.c | 2 +- crypto/sha1_generic.c | 2 +- crypto/sha256.c | 2 +- crypto/sha512_generic.c | 2 +- crypto/sm3_generic.c | 2 +- crypto/sm4_generic.c | 2 +- crypto/streebog_generic.c | 2 +- crypto/tea.c | 2 +- crypto/twofish_generic.c | 2 +- crypto/wp512.c | 2 +- crypto/xcbc.c | 2 +- crypto/xctr.c | 2 +- crypto/xts.c | 2 +- crypto/xxhash_generic.c | 2 +- crypto/zstd.c | 2 +- 92 files changed, 92 insertions(+), 98 deletions(-) diff --git a/arch/arm/crypto/aes-neonbs-glue.c b/arch/arm/crypto/aes-neonbs-glue.c index 95418df97fb4..c60104dc1585 100644 --- a/arch/arm/crypto/aes-neonbs-glue.c +++ b/arch/arm/crypto/aes-neonbs-glue.c @@ -407,5 +407,5 @@ static int __init aes_init(void) return crypto_register_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); } -late_initcall(aes_init); +module_init(aes_init); module_exit(aes_exit); diff --git a/arch/arm/lib/crypto/chacha-glue.c b/arch/arm/lib/crypto/chacha-glue.c index 12afb40cf1ff..1e28736834a0 100644 --- a/arch/arm/lib/crypto/chacha-glue.c +++ b/arch/arm/lib/crypto/chacha-glue.c @@ -122,7 +122,7 @@ static int __init chacha_arm_mod_init(void) } return 0; } -arch_initcall(chacha_arm_mod_init); +subsys_initcall(chacha_arm_mod_init); static void __exit chacha_arm_mod_exit(void) { diff --git a/arch/arm/lib/crypto/poly1305-glue.c b/arch/arm/lib/crypto/poly1305-glue.c index 91da42b26d9c..2603b0771f2c 100644 --- a/arch/arm/lib/crypto/poly1305-glue.c +++ b/arch/arm/lib/crypto/poly1305-glue.c @@ -69,7 +69,7 @@ static int __init arm_poly1305_mod_init(void) static_branch_enable(&have_neon); return 0; } -arch_initcall(arm_poly1305_mod_init); +subsys_initcall(arm_poly1305_mod_init); static void __exit arm_poly1305_mod_exit(void) { diff --git a/arch/arm/lib/crypto/sha256.c b/arch/arm/lib/crypto/sha256.c index 3a8dfc304807..81612b7b9043 100644 --- a/arch/arm/lib/crypto/sha256.c +++ b/arch/arm/lib/crypto/sha256.c @@ -53,7 +53,7 @@ static int __init sha256_arm_mod_init(void) } return 0; } -arch_initcall(sha256_arm_mod_init); +subsys_initcall(sha256_arm_mod_init); static void __exit sha256_arm_mod_exit(void) { diff --git a/arch/arm64/lib/crypto/chacha-neon-glue.c b/arch/arm64/lib/crypto/chacha-neon-glue.c index 14a2836eff61..2b0de97a6daf 100644 --- a/arch/arm64/lib/crypto/chacha-neon-glue.c +++ b/arch/arm64/lib/crypto/chacha-neon-glue.c @@ -104,7 +104,7 @@ static int __init chacha_simd_mod_init(void) static_branch_enable(&have_neon); return 0; } -arch_initcall(chacha_simd_mod_init); +subsys_initcall(chacha_simd_mod_init); static void __exit chacha_simd_mod_exit(void) { diff --git a/arch/arm64/lib/crypto/poly1305-glue.c b/arch/arm64/lib/crypto/poly1305-glue.c index 681c26557336..6a661cf04821 100644 --- a/arch/arm64/lib/crypto/poly1305-glue.c +++ b/arch/arm64/lib/crypto/poly1305-glue.c @@ -62,7 +62,7 @@ static int __init neon_poly1305_mod_init(void) static_branch_enable(&have_neon); return 0; } -arch_initcall(neon_poly1305_mod_init); +subsys_initcall(neon_poly1305_mod_init); static void __exit neon_poly1305_mod_exit(void) { diff --git a/arch/arm64/lib/crypto/sha256.c b/arch/arm64/lib/crypto/sha256.c index 2bd413c586d2..354fc402c6d3 100644 --- a/arch/arm64/lib/crypto/sha256.c +++ b/arch/arm64/lib/crypto/sha256.c @@ -64,7 +64,7 @@ static int __init sha256_arm64_mod_init(void) } return 0; } -arch_initcall(sha256_arm64_mod_init); +subsys_initcall(sha256_arm64_mod_init); static void __exit sha256_arm64_mod_exit(void) { diff --git a/arch/powerpc/lib/crypto/chacha-p10-glue.c b/arch/powerpc/lib/crypto/chacha-p10-glue.c index 351ed409f9b2..51daeaf5d26e 100644 --- a/arch/powerpc/lib/crypto/chacha-p10-glue.c +++ b/arch/powerpc/lib/crypto/chacha-p10-glue.c @@ -87,7 +87,7 @@ static int __init chacha_p10_init(void) static_branch_enable(&have_p10); return 0; } -arch_initcall(chacha_p10_init); +subsys_initcall(chacha_p10_init); static void __exit chacha_p10_exit(void) { diff --git a/arch/powerpc/lib/crypto/poly1305-p10-glue.c b/arch/powerpc/lib/crypto/poly1305-p10-glue.c index 50ac802220e0..16c2a8316696 100644 --- a/arch/powerpc/lib/crypto/poly1305-p10-glue.c +++ b/arch/powerpc/lib/crypto/poly1305-p10-glue.c @@ -76,7 +76,7 @@ static int __init poly1305_p10_init(void) static_branch_enable(&have_p10); return 0; } -arch_initcall(poly1305_p10_init); +subsys_initcall(poly1305_p10_init); static void __exit poly1305_p10_exit(void) { diff --git a/arch/riscv/lib/crypto/chacha-riscv64-glue.c b/arch/riscv/lib/crypto/chacha-riscv64-glue.c index afc4e3be3cac..1740e1ca3a94 100644 --- a/arch/riscv/lib/crypto/chacha-riscv64-glue.c +++ b/arch/riscv/lib/crypto/chacha-riscv64-glue.c @@ -62,7 +62,7 @@ static int __init riscv64_chacha_mod_init(void) static_branch_enable(&use_zvkb); return 0; } -arch_initcall(riscv64_chacha_mod_init); +subsys_initcall(riscv64_chacha_mod_init); static void __exit riscv64_chacha_mod_exit(void) { diff --git a/arch/riscv/lib/crypto/sha256.c b/arch/riscv/lib/crypto/sha256.c index 18b84030f0b3..c8065c70337b 100644 --- a/arch/riscv/lib/crypto/sha256.c +++ b/arch/riscv/lib/crypto/sha256.c @@ -50,7 +50,7 @@ static int __init riscv64_sha256_mod_init(void) static_branch_enable(&have_extensions); return 0; } -arch_initcall(riscv64_sha256_mod_init); +subsys_initcall(riscv64_sha256_mod_init); static void __exit riscv64_sha256_mod_exit(void) { diff --git a/arch/s390/lib/crypto/sha256.c b/arch/s390/lib/crypto/sha256.c index 50c592ce7a5d..f1d73ea1eef6 100644 --- a/arch/s390/lib/crypto/sha256.c +++ b/arch/s390/lib/crypto/sha256.c @@ -36,7 +36,7 @@ static int __init sha256_s390_mod_init(void) static_branch_enable(&have_cpacf_sha256); return 0; } -arch_initcall(sha256_s390_mod_init); +subsys_initcall(sha256_s390_mod_init); static void __exit sha256_s390_mod_exit(void) { diff --git a/arch/sparc/lib/crypto/sha256.c b/arch/sparc/lib/crypto/sha256.c index 6f118a23d210..0394c2b72f9e 100644 --- a/arch/sparc/lib/crypto/sha256.c +++ b/arch/sparc/lib/crypto/sha256.c @@ -53,7 +53,7 @@ static int __init sha256_sparc64_mod_init(void) pr_info("Using sparc64 sha256 opcode optimized SHA-256/SHA-224 implementation\n"); return 0; } -arch_initcall(sha256_sparc64_mod_init); +subsys_initcall(sha256_sparc64_mod_init); static void __exit sha256_sparc64_mod_exit(void) { diff --git a/arch/x86/lib/crypto/chacha_glue.c b/arch/x86/lib/crypto/chacha_glue.c index 59bf63c00072..94fcefbc8827 100644 --- a/arch/x86/lib/crypto/chacha_glue.c +++ b/arch/x86/lib/crypto/chacha_glue.c @@ -174,7 +174,7 @@ static int __init chacha_simd_mod_init(void) } return 0; } -arch_initcall(chacha_simd_mod_init); +subsys_initcall(chacha_simd_mod_init); static void __exit chacha_simd_mod_exit(void) { diff --git a/arch/x86/lib/crypto/poly1305_glue.c b/arch/x86/lib/crypto/poly1305_glue.c index f799828c5809..b7e78a583e07 100644 --- a/arch/x86/lib/crypto/poly1305_glue.c +++ b/arch/x86/lib/crypto/poly1305_glue.c @@ -117,7 +117,7 @@ static int __init poly1305_simd_mod_init(void) static_branch_enable(&poly1305_use_avx512); return 0; } -arch_initcall(poly1305_simd_mod_init); +subsys_initcall(poly1305_simd_mod_init); static void __exit poly1305_simd_mod_exit(void) { diff --git a/arch/x86/lib/crypto/sha256.c b/arch/x86/lib/crypto/sha256.c index 47865b5cd94b..ea17514cb0f3 100644 --- a/arch/x86/lib/crypto/sha256.c +++ b/arch/x86/lib/crypto/sha256.c @@ -63,7 +63,7 @@ static int __init sha256_x86_mod_init(void) static_branch_enable(&have_sha256_x86); return 0; } -arch_initcall(sha256_x86_mod_init); +subsys_initcall(sha256_x86_mod_init); static void __exit sha256_x86_mod_exit(void) { diff --git a/crypto/842.c b/crypto/842.c index 881945d44328..8c257c40e2b9 100644 --- a/crypto/842.c +++ b/crypto/842.c @@ -70,7 +70,7 @@ static int __init crypto842_mod_init(void) { return crypto_register_scomp(&scomp); } -subsys_initcall(crypto842_mod_init); +module_init(crypto842_mod_init); static void __exit crypto842_mod_exit(void) { diff --git a/crypto/adiantum.c b/crypto/adiantum.c index c3ef583598b4..a6bca877c3c7 100644 --- a/crypto/adiantum.c +++ b/crypto/adiantum.c @@ -639,7 +639,7 @@ static void __exit adiantum_module_exit(void) crypto_unregister_template(&adiantum_tmpl); } -subsys_initcall(adiantum_module_init); +module_init(adiantum_module_init); module_exit(adiantum_module_exit); MODULE_DESCRIPTION("Adiantum length-preserving encryption mode"); diff --git a/crypto/aegis128-core.c b/crypto/aegis128-core.c index 72f6ee1345ef..ca80d861345d 100644 --- a/crypto/aegis128-core.c +++ b/crypto/aegis128-core.c @@ -566,7 +566,7 @@ static void __exit crypto_aegis128_module_exit(void) crypto_unregister_aead(&crypto_aegis128_alg_generic); } -subsys_initcall(crypto_aegis128_module_init); +module_init(crypto_aegis128_module_init); module_exit(crypto_aegis128_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/aes_generic.c b/crypto/aes_generic.c index 3c66d425c97b..85d2e78c8ef2 100644 --- a/crypto/aes_generic.c +++ b/crypto/aes_generic.c @@ -1311,7 +1311,7 @@ static void __exit aes_fini(void) crypto_unregister_alg(&aes_alg); } -subsys_initcall(aes_init); +module_init(aes_init); module_exit(aes_fini); MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm"); diff --git a/crypto/algboss.c b/crypto/algboss.c index a20926bfd34e..ef5c73780fc7 100644 --- a/crypto/algboss.c +++ b/crypto/algboss.c @@ -247,13 +247,7 @@ static void __exit cryptomgr_exit(void) BUG_ON(err); } -/* - * This is arch_initcall() so that the crypto self-tests are run on algorithms - * registered early by subsys_initcall(). subsys_initcall() is needed for - * generic implementations so that they're available for comparison tests when - * other implementations are registered later by module_init(). - */ -arch_initcall(cryptomgr_init); +module_init(cryptomgr_init); module_exit(cryptomgr_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/ansi_cprng.c b/crypto/ansi_cprng.c index 64f57c4c4b06..153523ce6076 100644 --- a/crypto/ansi_cprng.c +++ b/crypto/ansi_cprng.c @@ -467,7 +467,7 @@ MODULE_DESCRIPTION("Software Pseudo Random Number Generator"); MODULE_AUTHOR("Neil Horman "); module_param(dbg, int, 0); MODULE_PARM_DESC(dbg, "Boolean to enable debugging (0/1 == off/on)"); -subsys_initcall(prng_mod_init); +module_init(prng_mod_init); module_exit(prng_mod_fini); MODULE_ALIAS_CRYPTO("stdrng"); MODULE_ALIAS_CRYPTO("ansi_cprng"); diff --git a/crypto/anubis.c b/crypto/anubis.c index 886e7c913688..4268c3833baa 100644 --- a/crypto/anubis.c +++ b/crypto/anubis.c @@ -694,7 +694,7 @@ static void __exit anubis_mod_fini(void) crypto_unregister_alg(&anubis_alg); } -subsys_initcall(anubis_mod_init); +module_init(anubis_mod_init); module_exit(anubis_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/arc4.c b/crypto/arc4.c index 1a4825c97c5a..1608018111d0 100644 --- a/crypto/arc4.c +++ b/crypto/arc4.c @@ -73,7 +73,7 @@ static void __exit arc4_exit(void) crypto_unregister_lskcipher(&arc4_alg); } -subsys_initcall(arc4_init); +module_init(arc4_init); module_exit(arc4_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/aria_generic.c b/crypto/aria_generic.c index bd359d3313c2..faa7900383f6 100644 --- a/crypto/aria_generic.c +++ b/crypto/aria_generic.c @@ -304,7 +304,7 @@ static void __exit aria_fini(void) crypto_unregister_alg(&aria_alg); } -subsys_initcall(aria_init); +module_init(aria_init); module_exit(aria_fini); MODULE_DESCRIPTION("ARIA Cipher Algorithm"); diff --git a/crypto/authenc.c b/crypto/authenc.c index 3aaf3ab4e360..9521ae2f112e 100644 --- a/crypto/authenc.c +++ b/crypto/authenc.c @@ -451,7 +451,7 @@ static void __exit crypto_authenc_module_exit(void) crypto_unregister_template(&crypto_authenc_tmpl); } -subsys_initcall(crypto_authenc_module_init); +module_init(crypto_authenc_module_init); module_exit(crypto_authenc_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/authencesn.c b/crypto/authencesn.c index 2cc933e2f790..b1c78313cbc1 100644 --- a/crypto/authencesn.c +++ b/crypto/authencesn.c @@ -465,7 +465,7 @@ static void __exit crypto_authenc_esn_module_exit(void) crypto_unregister_template(&crypto_authenc_esn_tmpl); } -subsys_initcall(crypto_authenc_esn_module_init); +module_init(crypto_authenc_esn_module_init); module_exit(crypto_authenc_esn_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/blake2b_generic.c b/crypto/blake2b_generic.c index 6fa38965a493..60f056217510 100644 --- a/crypto/blake2b_generic.c +++ b/crypto/blake2b_generic.c @@ -176,7 +176,7 @@ static void __exit blake2b_mod_fini(void) crypto_unregister_shashes(blake2b_algs, ARRAY_SIZE(blake2b_algs)); } -subsys_initcall(blake2b_mod_init); +module_init(blake2b_mod_init); module_exit(blake2b_mod_fini); MODULE_AUTHOR("David Sterba "); diff --git a/crypto/blowfish_generic.c b/crypto/blowfish_generic.c index 0146bc762c09..f3c5f9b09850 100644 --- a/crypto/blowfish_generic.c +++ b/crypto/blowfish_generic.c @@ -124,7 +124,7 @@ static void __exit blowfish_mod_fini(void) crypto_unregister_alg(&alg); } -subsys_initcall(blowfish_mod_init); +module_init(blowfish_mod_init); module_exit(blowfish_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/camellia_generic.c b/crypto/camellia_generic.c index 197fcf3abc89..ee4336a04b93 100644 --- a/crypto/camellia_generic.c +++ b/crypto/camellia_generic.c @@ -1064,7 +1064,7 @@ static void __exit camellia_fini(void) crypto_unregister_alg(&camellia_alg); } -subsys_initcall(camellia_init); +module_init(camellia_init); module_exit(camellia_fini); MODULE_DESCRIPTION("Camellia Cipher Algorithm"); diff --git a/crypto/cast5_generic.c b/crypto/cast5_generic.c index f3e57775fa02..f68330793e0c 100644 --- a/crypto/cast5_generic.c +++ b/crypto/cast5_generic.c @@ -531,7 +531,7 @@ static void __exit cast5_mod_fini(void) crypto_unregister_alg(&alg); } -subsys_initcall(cast5_mod_init); +module_init(cast5_mod_init); module_exit(cast5_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/cast6_generic.c b/crypto/cast6_generic.c index 11b725b12f27..4c08c42646f0 100644 --- a/crypto/cast6_generic.c +++ b/crypto/cast6_generic.c @@ -271,7 +271,7 @@ static void __exit cast6_mod_fini(void) crypto_unregister_alg(&alg); } -subsys_initcall(cast6_mod_init); +module_init(cast6_mod_init); module_exit(cast6_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/cbc.c b/crypto/cbc.c index e81918ca68b7..ed3df6246765 100644 --- a/crypto/cbc.c +++ b/crypto/cbc.c @@ -179,7 +179,7 @@ static void __exit crypto_cbc_module_exit(void) crypto_unregister_template(&crypto_cbc_tmpl); } -subsys_initcall(crypto_cbc_module_init); +module_init(crypto_cbc_module_init); module_exit(crypto_cbc_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/ccm.c b/crypto/ccm.c index f3f455e4908b..2ae929ffdef8 100644 --- a/crypto/ccm.c +++ b/crypto/ccm.c @@ -929,7 +929,7 @@ static void __exit crypto_ccm_module_exit(void) ARRAY_SIZE(crypto_ccm_tmpls)); } -subsys_initcall(crypto_ccm_module_init); +module_init(crypto_ccm_module_init); module_exit(crypto_ccm_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/chacha.c b/crypto/chacha.c index 5103bc0b2881..28a8ad6197ab 100644 --- a/crypto/chacha.c +++ b/crypto/chacha.c @@ -243,7 +243,7 @@ static void __exit crypto_chacha_mod_fini(void) crypto_unregister_skciphers(algs, num_algs); } -subsys_initcall(crypto_chacha_mod_init); +module_init(crypto_chacha_mod_init); module_exit(crypto_chacha_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/chacha20poly1305.c b/crypto/chacha20poly1305.c index b29f66ba1e2f..b4b5a7198d84 100644 --- a/crypto/chacha20poly1305.c +++ b/crypto/chacha20poly1305.c @@ -476,7 +476,7 @@ static void __exit chacha20poly1305_module_exit(void) ARRAY_SIZE(rfc7539_tmpls)); } -subsys_initcall(chacha20poly1305_module_init); +module_init(chacha20poly1305_module_init); module_exit(chacha20poly1305_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/cmac.c b/crypto/cmac.c index f297042a324b..1b03964abe00 100644 --- a/crypto/cmac.c +++ b/crypto/cmac.c @@ -251,7 +251,7 @@ static void __exit crypto_cmac_module_exit(void) crypto_unregister_template(&crypto_cmac_tmpl); } -subsys_initcall(crypto_cmac_module_init); +module_init(crypto_cmac_module_init); module_exit(crypto_cmac_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/crc32_generic.c b/crypto/crc32_generic.c index 783a30b27398..cc371d42601f 100644 --- a/crypto/crc32_generic.c +++ b/crypto/crc32_generic.c @@ -172,7 +172,7 @@ static void __exit crc32_mod_fini(void) crypto_unregister_shashes(algs, num_algs); } -subsys_initcall(crc32_mod_init); +module_init(crc32_mod_init); module_exit(crc32_mod_fini); MODULE_AUTHOR("Alexander Boyko "); diff --git a/crypto/crc32c_generic.c b/crypto/crc32c_generic.c index b1a36d32dc50..e5377898414a 100644 --- a/crypto/crc32c_generic.c +++ b/crypto/crc32c_generic.c @@ -212,7 +212,7 @@ static void __exit crc32c_mod_fini(void) crypto_unregister_shashes(algs, num_algs); } -subsys_initcall(crc32c_mod_init); +module_init(crc32c_mod_init); module_exit(crc32c_mod_fini); MODULE_AUTHOR("Clay Haapala "); diff --git a/crypto/cryptd.c b/crypto/cryptd.c index 31d022d47f7a..5bb6f8d88cc2 100644 --- a/crypto/cryptd.c +++ b/crypto/cryptd.c @@ -1138,7 +1138,7 @@ static void __exit cryptd_exit(void) crypto_unregister_template(&cryptd_tmpl); } -subsys_initcall(cryptd_init); +module_init(cryptd_init); module_exit(cryptd_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/crypto_null.c b/crypto/crypto_null.c index ced90f88ee07..5822753b0995 100644 --- a/crypto/crypto_null.c +++ b/crypto/crypto_null.c @@ -210,7 +210,7 @@ static void __exit crypto_null_mod_fini(void) crypto_unregister_skcipher(&skcipher_null); } -subsys_initcall(crypto_null_mod_init); +module_init(crypto_null_mod_init); module_exit(crypto_null_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/ctr.c b/crypto/ctr.c index 97a947b0a876..a388f0ceb3a0 100644 --- a/crypto/ctr.c +++ b/crypto/ctr.c @@ -350,7 +350,7 @@ static void __exit crypto_ctr_module_exit(void) ARRAY_SIZE(crypto_ctr_tmpls)); } -subsys_initcall(crypto_ctr_module_init); +module_init(crypto_ctr_module_init); module_exit(crypto_ctr_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/cts.c b/crypto/cts.c index f5b42156b6c7..48898d5e24ff 100644 --- a/crypto/cts.c +++ b/crypto/cts.c @@ -402,7 +402,7 @@ static void __exit crypto_cts_module_exit(void) crypto_unregister_template(&crypto_cts_tmpl); } -subsys_initcall(crypto_cts_module_init); +module_init(crypto_cts_module_init); module_exit(crypto_cts_module_exit); MODULE_LICENSE("Dual BSD/GPL"); diff --git a/crypto/curve25519-generic.c b/crypto/curve25519-generic.c index 68a673262e04..f3e56e73c66c 100644 --- a/crypto/curve25519-generic.c +++ b/crypto/curve25519-generic.c @@ -82,7 +82,7 @@ static void __exit curve25519_exit(void) crypto_unregister_kpp(&curve25519_alg); } -subsys_initcall(curve25519_init); +module_init(curve25519_init); module_exit(curve25519_exit); MODULE_ALIAS_CRYPTO("curve25519"); diff --git a/crypto/deflate.c b/crypto/deflate.c index 0d2b64d96d6e..7eb1a5c44ee0 100644 --- a/crypto/deflate.c +++ b/crypto/deflate.c @@ -243,7 +243,7 @@ static void __exit deflate_mod_fini(void) crypto_acomp_free_streams(&deflate_streams); } -subsys_initcall(deflate_mod_init); +module_init(deflate_mod_init); module_exit(deflate_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/des_generic.c b/crypto/des_generic.c index 1274e18d3eb9..fce341400914 100644 --- a/crypto/des_generic.c +++ b/crypto/des_generic.c @@ -122,7 +122,7 @@ static void __exit des_generic_mod_fini(void) crypto_unregister_algs(des_algs, ARRAY_SIZE(des_algs)); } -subsys_initcall(des_generic_mod_init); +module_init(des_generic_mod_init); module_exit(des_generic_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/dh.c b/crypto/dh.c index afc0fd847761..8250eeeebd0f 100644 --- a/crypto/dh.c +++ b/crypto/dh.c @@ -920,7 +920,7 @@ static void __exit dh_exit(void) crypto_unregister_kpp(&dh); } -subsys_initcall(dh_init); +module_init(dh_init); module_exit(dh_exit); MODULE_ALIAS_CRYPTO("dh"); MODULE_LICENSE("GPL"); diff --git a/crypto/drbg.c b/crypto/drbg.c index f28dfc2511a2..dbe4c8bb5ceb 100644 --- a/crypto/drbg.c +++ b/crypto/drbg.c @@ -2132,7 +2132,7 @@ static void __exit drbg_exit(void) crypto_unregister_rngs(drbg_algs, (ARRAY_SIZE(drbg_cores) * 2)); } -subsys_initcall(drbg_init); +module_init(drbg_init); module_exit(drbg_exit); #ifndef CRYPTO_DRBG_HASH_STRING #define CRYPTO_DRBG_HASH_STRING "" diff --git a/crypto/ecb.c b/crypto/ecb.c index 95d7e972865a..cd1b20456dad 100644 --- a/crypto/ecb.c +++ b/crypto/ecb.c @@ -219,7 +219,7 @@ static void __exit crypto_ecb_module_exit(void) crypto_unregister_template(&crypto_ecb_tmpl); } -subsys_initcall(crypto_ecb_module_init); +module_init(crypto_ecb_module_init); module_exit(crypto_ecb_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/ecdh.c b/crypto/ecdh.c index 72cfd1590156..9f0b93b3166d 100644 --- a/crypto/ecdh.c +++ b/crypto/ecdh.c @@ -240,7 +240,7 @@ static void __exit ecdh_exit(void) crypto_unregister_kpp(&ecdh_nist_p384); } -subsys_initcall(ecdh_init); +module_init(ecdh_init); module_exit(ecdh_exit); MODULE_ALIAS_CRYPTO("ecdh"); MODULE_LICENSE("GPL"); diff --git a/crypto/ecdsa.c b/crypto/ecdsa.c index a70b60a90a3c..ce8e4364842f 100644 --- a/crypto/ecdsa.c +++ b/crypto/ecdsa.c @@ -334,7 +334,7 @@ static void __exit ecdsa_exit(void) crypto_unregister_sig(&ecdsa_nist_p521); } -subsys_initcall(ecdsa_init); +module_init(ecdsa_init); module_exit(ecdsa_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/echainiv.c b/crypto/echainiv.c index 69686668625e..1913be8dfbba 100644 --- a/crypto/echainiv.c +++ b/crypto/echainiv.c @@ -157,7 +157,7 @@ static void __exit echainiv_module_exit(void) crypto_unregister_template(&echainiv_tmpl); } -subsys_initcall(echainiv_module_init); +module_init(echainiv_module_init); module_exit(echainiv_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/essiv.c b/crypto/essiv.c index cfce8ef7ab1f..d003b78fcd85 100644 --- a/crypto/essiv.c +++ b/crypto/essiv.c @@ -641,7 +641,7 @@ static void __exit essiv_module_exit(void) crypto_unregister_template(&essiv_tmpl); } -subsys_initcall(essiv_module_init); +module_init(essiv_module_init); module_exit(essiv_module_exit); MODULE_DESCRIPTION("ESSIV skcipher/aead wrapper for block encryption"); diff --git a/crypto/fcrypt.c b/crypto/fcrypt.c index 95a16e88899b..80036835cec5 100644 --- a/crypto/fcrypt.c +++ b/crypto/fcrypt.c @@ -411,7 +411,7 @@ static void __exit fcrypt_mod_fini(void) crypto_unregister_alg(&fcrypt_alg); } -subsys_initcall(fcrypt_mod_init); +module_init(fcrypt_mod_init); module_exit(fcrypt_mod_fini); MODULE_LICENSE("Dual BSD/GPL"); diff --git a/crypto/fips.c b/crypto/fips.c index 2fa3a9ee61a1..e88a604cb42b 100644 --- a/crypto/fips.c +++ b/crypto/fips.c @@ -95,5 +95,5 @@ static void __exit fips_exit(void) crypto_proc_fips_exit(); } -subsys_initcall(fips_init); +module_init(fips_init); module_exit(fips_exit); diff --git a/crypto/gcm.c b/crypto/gcm.c index 84f7c23d14e4..54ca9faf0e0c 100644 --- a/crypto/gcm.c +++ b/crypto/gcm.c @@ -1152,7 +1152,7 @@ static void __exit crypto_gcm_module_exit(void) ARRAY_SIZE(crypto_gcm_tmpls)); } -subsys_initcall(crypto_gcm_module_init); +module_init(crypto_gcm_module_init); module_exit(crypto_gcm_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/ghash-generic.c b/crypto/ghash-generic.c index b5fc20a0dafc..e5803c249c12 100644 --- a/crypto/ghash-generic.c +++ b/crypto/ghash-generic.c @@ -153,7 +153,7 @@ static void __exit ghash_mod_exit(void) crypto_unregister_shash(&ghash_alg); } -subsys_initcall(ghash_mod_init); +module_init(ghash_mod_init); module_exit(ghash_mod_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/hctr2.c b/crypto/hctr2.c index cbcd673be481..c8932777bba8 100644 --- a/crypto/hctr2.c +++ b/crypto/hctr2.c @@ -570,7 +570,7 @@ static void __exit hctr2_module_exit(void) ARRAY_SIZE(hctr2_tmpls)); } -subsys_initcall(hctr2_module_init); +module_init(hctr2_module_init); module_exit(hctr2_module_exit); MODULE_DESCRIPTION("HCTR2 length-preserving encryption mode"); diff --git a/crypto/hmac.c b/crypto/hmac.c index dfb153511865..ba36ddf50037 100644 --- a/crypto/hmac.c +++ b/crypto/hmac.c @@ -257,7 +257,7 @@ static void __exit hmac_module_exit(void) crypto_unregister_template(&hmac_tmpl); } -subsys_initcall(hmac_module_init); +module_init(hmac_module_init); module_exit(hmac_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/khazad.c b/crypto/khazad.c index 7ad338ca2c18..024264ee9cd1 100644 --- a/crypto/khazad.c +++ b/crypto/khazad.c @@ -871,7 +871,7 @@ static void __exit khazad_mod_fini(void) } -subsys_initcall(khazad_mod_init); +module_init(khazad_mod_init); module_exit(khazad_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/krb5enc.c b/crypto/krb5enc.c index d07769bf149e..a1de55994d92 100644 --- a/crypto/krb5enc.c +++ b/crypto/krb5enc.c @@ -496,7 +496,7 @@ static void __exit crypto_krb5enc_module_exit(void) crypto_unregister_template(&crypto_krb5enc_tmpl); } -subsys_initcall(crypto_krb5enc_module_init); +module_init(crypto_krb5enc_module_init); module_exit(crypto_krb5enc_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/lrw.c b/crypto/lrw.c index 391ae0f7641f..e7f0368f8c97 100644 --- a/crypto/lrw.c +++ b/crypto/lrw.c @@ -420,7 +420,7 @@ static void __exit lrw_module_exit(void) crypto_unregister_template(&lrw_tmpl); } -subsys_initcall(lrw_module_init); +module_init(lrw_module_init); module_exit(lrw_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/lz4.c b/crypto/lz4.c index 9661ed01692f..7a984ae5ae52 100644 --- a/crypto/lz4.c +++ b/crypto/lz4.c @@ -89,7 +89,7 @@ static void __exit lz4_mod_fini(void) crypto_unregister_scomp(&scomp); } -subsys_initcall(lz4_mod_init); +module_init(lz4_mod_init); module_exit(lz4_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/lz4hc.c b/crypto/lz4hc.c index a637fddc1ccd..9c61d05b6214 100644 --- a/crypto/lz4hc.c +++ b/crypto/lz4hc.c @@ -87,7 +87,7 @@ static void __exit lz4hc_mod_fini(void) crypto_unregister_scomp(&scomp); } -subsys_initcall(lz4hc_mod_init); +module_init(lz4hc_mod_init); module_exit(lz4hc_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/lzo-rle.c b/crypto/lzo-rle.c index e7efcf107179..ba013f2d5090 100644 --- a/crypto/lzo-rle.c +++ b/crypto/lzo-rle.c @@ -91,7 +91,7 @@ static void __exit lzorle_mod_fini(void) crypto_unregister_scomp(&scomp); } -subsys_initcall(lzorle_mod_init); +module_init(lzorle_mod_init); module_exit(lzorle_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/lzo.c b/crypto/lzo.c index f1b36a1ca6f6..7867e2c67c4e 100644 --- a/crypto/lzo.c +++ b/crypto/lzo.c @@ -91,7 +91,7 @@ static void __exit lzo_mod_fini(void) crypto_unregister_scomp(&scomp); } -subsys_initcall(lzo_mod_init); +module_init(lzo_mod_init); module_exit(lzo_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/md4.c b/crypto/md4.c index 2e7f2f319f95..55bf47e23c13 100644 --- a/crypto/md4.c +++ b/crypto/md4.c @@ -233,7 +233,7 @@ static void __exit md4_mod_fini(void) crypto_unregister_shash(&alg); } -subsys_initcall(md4_mod_init); +module_init(md4_mod_init); module_exit(md4_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/md5.c b/crypto/md5.c index 994005cd977d..32c0819f5118 100644 --- a/crypto/md5.c +++ b/crypto/md5.c @@ -216,7 +216,7 @@ static void __exit md5_mod_fini(void) crypto_unregister_shash(&alg); } -subsys_initcall(md5_mod_init); +module_init(md5_mod_init); module_exit(md5_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/michael_mic.c b/crypto/michael_mic.c index 0d14e980d4d6..69ad35f524d7 100644 --- a/crypto/michael_mic.c +++ b/crypto/michael_mic.c @@ -167,7 +167,7 @@ static void __exit michael_mic_exit(void) } -subsys_initcall(michael_mic_init); +module_init(michael_mic_init); module_exit(michael_mic_exit); MODULE_LICENSE("GPL v2"); diff --git a/crypto/nhpoly1305.c b/crypto/nhpoly1305.c index a661d4f667cd..2b648615b5ec 100644 --- a/crypto/nhpoly1305.c +++ b/crypto/nhpoly1305.c @@ -245,7 +245,7 @@ static void __exit nhpoly1305_mod_exit(void) crypto_unregister_shash(&nhpoly1305_alg); } -subsys_initcall(nhpoly1305_mod_init); +module_init(nhpoly1305_mod_init); module_exit(nhpoly1305_mod_exit); MODULE_DESCRIPTION("NHPoly1305 ε-almost-∆-universal hash function"); diff --git a/crypto/pcbc.c b/crypto/pcbc.c index 9d2e56d6744a..d092717ea4fc 100644 --- a/crypto/pcbc.c +++ b/crypto/pcbc.c @@ -186,7 +186,7 @@ static void __exit crypto_pcbc_module_exit(void) crypto_unregister_template(&crypto_pcbc_tmpl); } -subsys_initcall(crypto_pcbc_module_init); +module_init(crypto_pcbc_module_init); module_exit(crypto_pcbc_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/pcrypt.c b/crypto/pcrypt.c index 7fc79e7dce44..c33d29a523e0 100644 --- a/crypto/pcrypt.c +++ b/crypto/pcrypt.c @@ -381,7 +381,7 @@ static void __exit pcrypt_exit(void) kset_unregister(pcrypt_kset); } -subsys_initcall(pcrypt_init); +module_init(pcrypt_init); module_exit(pcrypt_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/polyval-generic.c b/crypto/polyval-generic.c index ffd174e75420..db8adb56e4ca 100644 --- a/crypto/polyval-generic.c +++ b/crypto/polyval-generic.c @@ -196,7 +196,7 @@ static void __exit polyval_mod_exit(void) crypto_unregister_shash(&polyval_alg); } -subsys_initcall(polyval_mod_init); +module_init(polyval_mod_init); module_exit(polyval_mod_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/rmd160.c b/crypto/rmd160.c index 890d29e46a76..9860b60c9be4 100644 --- a/crypto/rmd160.c +++ b/crypto/rmd160.c @@ -342,7 +342,7 @@ static void __exit rmd160_mod_fini(void) crypto_unregister_shash(&alg); } -subsys_initcall(rmd160_mod_init); +module_init(rmd160_mod_init); module_exit(rmd160_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/rsa.c b/crypto/rsa.c index b7d21529c552..6c7734083c98 100644 --- a/crypto/rsa.c +++ b/crypto/rsa.c @@ -430,7 +430,7 @@ static void __exit rsa_exit(void) crypto_unregister_akcipher(&rsa); } -subsys_initcall(rsa_init); +module_init(rsa_init); module_exit(rsa_exit); MODULE_ALIAS_CRYPTO("rsa"); MODULE_LICENSE("GPL"); diff --git a/crypto/seed.c b/crypto/seed.c index d05d8ed909fa..815391f213de 100644 --- a/crypto/seed.c +++ b/crypto/seed.c @@ -460,7 +460,7 @@ static void __exit seed_fini(void) crypto_unregister_alg(&seed_alg); } -subsys_initcall(seed_init); +module_init(seed_init); module_exit(seed_fini); MODULE_DESCRIPTION("SEED Cipher Algorithm"); diff --git a/crypto/seqiv.c b/crypto/seqiv.c index 17e11d51ddc3..a17ef5184398 100644 --- a/crypto/seqiv.c +++ b/crypto/seqiv.c @@ -179,7 +179,7 @@ static void __exit seqiv_module_exit(void) crypto_unregister_template(&seqiv_tmpl); } -subsys_initcall(seqiv_module_init); +module_init(seqiv_module_init); module_exit(seqiv_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/serpent_generic.c b/crypto/serpent_generic.c index f6ef187be6fe..b21e7606c652 100644 --- a/crypto/serpent_generic.c +++ b/crypto/serpent_generic.c @@ -599,7 +599,7 @@ static void __exit serpent_mod_fini(void) crypto_unregister_alg(&srp_alg); } -subsys_initcall(serpent_mod_init); +module_init(serpent_mod_init); module_exit(serpent_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/sha1_generic.c b/crypto/sha1_generic.c index 7a3c837923b5..024e8043bab0 100644 --- a/crypto/sha1_generic.c +++ b/crypto/sha1_generic.c @@ -77,7 +77,7 @@ static void __exit sha1_generic_mod_fini(void) crypto_unregister_shash(&alg); } -subsys_initcall(sha1_generic_mod_init); +module_init(sha1_generic_mod_init); module_exit(sha1_generic_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/sha256.c b/crypto/sha256.c index c2588d08ee3e..47ad7e4cc55f 100644 --- a/crypto/sha256.c +++ b/crypto/sha256.c @@ -224,7 +224,7 @@ static int __init crypto_sha256_mod_init(void) num_algs /= 2; return crypto_register_shashes(algs, ARRAY_SIZE(algs)); } -subsys_initcall(crypto_sha256_mod_init); +module_init(crypto_sha256_mod_init); static void __exit crypto_sha256_mod_exit(void) { diff --git a/crypto/sha512_generic.c b/crypto/sha512_generic.c index bfea65f4181c..7368173f545e 100644 --- a/crypto/sha512_generic.c +++ b/crypto/sha512_generic.c @@ -205,7 +205,7 @@ static void __exit sha512_generic_mod_fini(void) crypto_unregister_shashes(sha512_algs, ARRAY_SIZE(sha512_algs)); } -subsys_initcall(sha512_generic_mod_init); +module_init(sha512_generic_mod_init); module_exit(sha512_generic_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/sm3_generic.c b/crypto/sm3_generic.c index 4fb6957c2f0c..7529139fcc96 100644 --- a/crypto/sm3_generic.c +++ b/crypto/sm3_generic.c @@ -62,7 +62,7 @@ static void __exit sm3_generic_mod_fini(void) crypto_unregister_shash(&sm3_alg); } -subsys_initcall(sm3_generic_mod_init); +module_init(sm3_generic_mod_init); module_exit(sm3_generic_mod_fini); MODULE_LICENSE("GPL v2"); diff --git a/crypto/sm4_generic.c b/crypto/sm4_generic.c index 7df86369ac00..d57444e8428c 100644 --- a/crypto/sm4_generic.c +++ b/crypto/sm4_generic.c @@ -83,7 +83,7 @@ static void __exit sm4_fini(void) crypto_unregister_alg(&sm4_alg); } -subsys_initcall(sm4_init); +module_init(sm4_init); module_exit(sm4_fini); MODULE_DESCRIPTION("SM4 Cipher Algorithm"); diff --git a/crypto/streebog_generic.c b/crypto/streebog_generic.c index 99c07e0c10ba..57bbf70f4c22 100644 --- a/crypto/streebog_generic.c +++ b/crypto/streebog_generic.c @@ -1061,7 +1061,7 @@ static void __exit streebog_mod_fini(void) crypto_unregister_shashes(algs, ARRAY_SIZE(algs)); } -subsys_initcall(streebog_mod_init); +module_init(streebog_mod_init); module_exit(streebog_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/tea.c b/crypto/tea.c index b315da8c89eb..cb05140e3470 100644 --- a/crypto/tea.c +++ b/crypto/tea.c @@ -255,7 +255,7 @@ MODULE_ALIAS_CRYPTO("tea"); MODULE_ALIAS_CRYPTO("xtea"); MODULE_ALIAS_CRYPTO("xeta"); -subsys_initcall(tea_mod_init); +module_init(tea_mod_init); module_exit(tea_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/twofish_generic.c b/crypto/twofish_generic.c index 19f2b365e140..368018cfa9bf 100644 --- a/crypto/twofish_generic.c +++ b/crypto/twofish_generic.c @@ -187,7 +187,7 @@ static void __exit twofish_mod_fini(void) crypto_unregister_alg(&alg); } -subsys_initcall(twofish_mod_init); +module_init(twofish_mod_init); module_exit(twofish_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/wp512.c b/crypto/wp512.c index 07994e5ebf4e..41f13d490333 100644 --- a/crypto/wp512.c +++ b/crypto/wp512.c @@ -1169,7 +1169,7 @@ MODULE_ALIAS_CRYPTO("wp512"); MODULE_ALIAS_CRYPTO("wp384"); MODULE_ALIAS_CRYPTO("wp256"); -subsys_initcall(wp512_mod_init); +module_init(wp512_mod_init); module_exit(wp512_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/xcbc.c b/crypto/xcbc.c index 970ff581dc58..6c5f6766fdd6 100644 --- a/crypto/xcbc.c +++ b/crypto/xcbc.c @@ -199,7 +199,7 @@ static void __exit crypto_xcbc_module_exit(void) crypto_unregister_template(&crypto_xcbc_tmpl); } -subsys_initcall(crypto_xcbc_module_init); +module_init(crypto_xcbc_module_init); module_exit(crypto_xcbc_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/xctr.c b/crypto/xctr.c index 9c536ab6d2e5..607ab82cb19b 100644 --- a/crypto/xctr.c +++ b/crypto/xctr.c @@ -182,7 +182,7 @@ static void __exit crypto_xctr_module_exit(void) crypto_unregister_template(&crypto_xctr_tmpl); } -subsys_initcall(crypto_xctr_module_init); +module_init(crypto_xctr_module_init); module_exit(crypto_xctr_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/xts.c b/crypto/xts.c index 31529c9ef08f..1a9edd55a3a2 100644 --- a/crypto/xts.c +++ b/crypto/xts.c @@ -466,7 +466,7 @@ static void __exit xts_module_exit(void) crypto_unregister_template(&xts_tmpl); } -subsys_initcall(xts_module_init); +module_init(xts_module_init); module_exit(xts_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/xxhash_generic.c b/crypto/xxhash_generic.c index ac206ad4184d..175bb7ae0fcd 100644 --- a/crypto/xxhash_generic.c +++ b/crypto/xxhash_generic.c @@ -96,7 +96,7 @@ static void __exit xxhash_mod_fini(void) crypto_unregister_shash(&alg); } -subsys_initcall(xxhash_mod_init); +module_init(xxhash_mod_init); module_exit(xxhash_mod_fini); MODULE_AUTHOR("Nikolay Borisov "); diff --git a/crypto/zstd.c b/crypto/zstd.c index 90bb4f36f846..7570e11b4ee6 100644 --- a/crypto/zstd.c +++ b/crypto/zstd.c @@ -196,7 +196,7 @@ static void __exit zstd_mod_fini(void) crypto_unregister_scomp(&scomp); } -subsys_initcall(zstd_mod_init); +module_init(zstd_mod_init); module_exit(zstd_mod_fini); MODULE_LICENSE("GPL"); From ce026b35b7b62094b640d3781d09bbd5b132b326 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Wed, 30 Apr 2025 16:25:55 +0800 Subject: [PATCH 278/382] crypto: x86/blake2s - Include linux/init.h Explicitly include linux/init.h rather than pulling it through potluck. Signed-off-by: Herbert Xu --- arch/x86/lib/crypto/blake2s-glue.c | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/arch/x86/lib/crypto/blake2s-glue.c b/arch/x86/lib/crypto/blake2s-glue.c index 00f84f29cc8c..adc296cd17c9 100644 --- a/arch/x86/lib/crypto/blake2s-glue.c +++ b/arch/x86/lib/crypto/blake2s-glue.c @@ -3,17 +3,15 @@ * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. */ -#include - -#include -#include -#include -#include - #include #include #include #include +#include +#include +#include +#include +#include asmlinkage void blake2s_compress_ssse3(struct blake2s_state *state, const u8 *block, const size_t nblocks, From b8ca43806292d7094103c2af44fbb9824874b19e Mon Sep 17 00:00:00 2001 From: George Abraham P Date: Wed, 30 Apr 2025 12:34:43 +0100 Subject: [PATCH 279/382] crypto: qat - rename and relocate timer logic Rename adf_gen4_timer.c to adf_timer.c and adf_gen4_timer.h to adf_timer.h to make the files generation-agnostic. This includes renaming the start() and stop() timer APIs and macro definitions to be generic, allowing for reuse across different device generations. This does not introduce any functional changes. Signed-off-by: George Abraham P Reviewed-by: Giovanni Cabiddu Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- .../intel/qat/qat_420xx/adf_420xx_hw_data.c | 6 +++--- .../intel/qat/qat_4xxx/adf_4xxx_hw_data.c | 6 +++--- drivers/crypto/intel/qat/qat_common/Makefile | 2 +- .../{adf_gen4_timer.c => adf_timer.c} | 18 +++++++++--------- .../{adf_gen4_timer.h => adf_timer.h} | 10 +++++----- 5 files changed, 21 insertions(+), 21 deletions(-) rename drivers/crypto/intel/qat/qat_common/{adf_gen4_timer.c => adf_timer.c} (78%) rename drivers/crypto/intel/qat/qat_common/{adf_gen4_timer.h => adf_timer.h} (58%) diff --git a/drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c b/drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c index 795f4598400b..5817b3164185 100644 --- a/drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c +++ b/drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c @@ -15,9 +15,9 @@ #include #include #include -#include #include #include +#include #include "adf_420xx_hw_data.h" #include "icp_qat_hw.h" @@ -468,8 +468,8 @@ void adf_init_hw_data_420xx(struct adf_hw_device_data *hw_data, u32 dev_id) hw_data->enable_pm = adf_gen4_enable_pm; hw_data->handle_pm_interrupt = adf_gen4_handle_pm_interrupt; hw_data->dev_config = adf_gen4_dev_config; - hw_data->start_timer = adf_gen4_timer_start; - hw_data->stop_timer = adf_gen4_timer_stop; + hw_data->start_timer = adf_timer_start; + hw_data->stop_timer = adf_timer_stop; hw_data->get_hb_clock = adf_gen4_get_heartbeat_clock; hw_data->num_hb_ctrs = ADF_NUM_HB_CNT_PER_AE; hw_data->clock_frequency = ADF_420XX_AE_FREQ; diff --git a/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c b/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c index 7d4c366aa8b2..2d89d4a3a7b9 100644 --- a/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c +++ b/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c @@ -15,9 +15,9 @@ #include #include #include "adf_gen4_ras.h" -#include #include #include +#include #include "adf_4xxx_hw_data.h" #include "icp_qat_hw.h" @@ -454,8 +454,8 @@ void adf_init_hw_data_4xxx(struct adf_hw_device_data *hw_data, u32 dev_id) hw_data->enable_pm = adf_gen4_enable_pm; hw_data->handle_pm_interrupt = adf_gen4_handle_pm_interrupt; hw_data->dev_config = adf_gen4_dev_config; - hw_data->start_timer = adf_gen4_timer_start; - hw_data->stop_timer = adf_gen4_timer_stop; + hw_data->start_timer = adf_timer_start; + hw_data->stop_timer = adf_timer_stop; hw_data->get_hb_clock = adf_gen4_get_heartbeat_clock; hw_data->num_hb_ctrs = ADF_NUM_HB_CNT_PER_AE; hw_data->clock_frequency = ADF_4XXX_AE_FREQ; diff --git a/drivers/crypto/intel/qat/qat_common/Makefile b/drivers/crypto/intel/qat/qat_common/Makefile index af5df29fd2e3..0370eaad42b1 100644 --- a/drivers/crypto/intel/qat/qat_common/Makefile +++ b/drivers/crypto/intel/qat/qat_common/Makefile @@ -19,7 +19,6 @@ intel_qat-y := adf_accel_engine.o \ adf_gen4_hw_data.o \ adf_gen4_pm.o \ adf_gen4_ras.o \ - adf_gen4_timer.o \ adf_gen4_vf_mig.o \ adf_hw_arbiter.o \ adf_init.o \ @@ -30,6 +29,7 @@ intel_qat-y := adf_accel_engine.o \ adf_sysfs.o \ adf_sysfs_ras_counters.o \ adf_sysfs_rl.o \ + adf_timer.o \ adf_transport.o \ qat_algs.o \ qat_algs_send.o \ diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen4_timer.c b/drivers/crypto/intel/qat/qat_common/adf_timer.c similarity index 78% rename from drivers/crypto/intel/qat/qat_common/adf_gen4_timer.c rename to drivers/crypto/intel/qat/qat_common/adf_timer.c index 35ccb91d6ec1..8962a49f145a 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_gen4_timer.c +++ b/drivers/crypto/intel/qat/qat_common/adf_timer.c @@ -12,9 +12,9 @@ #include "adf_admin.h" #include "adf_accel_devices.h" #include "adf_common_drv.h" -#include "adf_gen4_timer.h" +#include "adf_timer.h" -#define ADF_GEN4_TIMER_PERIOD_MS 200 +#define ADF_DEFAULT_TIMER_PERIOD_MS 200 /* This periodic update is used to trigger HB, RL & TL fw events */ static void work_handler(struct work_struct *work) @@ -27,16 +27,16 @@ static void work_handler(struct work_struct *work) accel_dev = timer_ctx->accel_dev; adf_misc_wq_queue_delayed_work(&timer_ctx->work_ctx, - msecs_to_jiffies(ADF_GEN4_TIMER_PERIOD_MS)); + msecs_to_jiffies(ADF_DEFAULT_TIMER_PERIOD_MS)); time_periods = div_u64(ktime_ms_delta(ktime_get_real(), timer_ctx->initial_ktime), - ADF_GEN4_TIMER_PERIOD_MS); + ADF_DEFAULT_TIMER_PERIOD_MS); if (adf_send_admin_tim_sync(accel_dev, time_periods)) dev_err(&GET_DEV(accel_dev), "Failed to synchronize qat timer\n"); } -int adf_gen4_timer_start(struct adf_accel_dev *accel_dev) +int adf_timer_start(struct adf_accel_dev *accel_dev) { struct adf_timer *timer_ctx; @@ -50,13 +50,13 @@ int adf_gen4_timer_start(struct adf_accel_dev *accel_dev) INIT_DELAYED_WORK(&timer_ctx->work_ctx, work_handler); adf_misc_wq_queue_delayed_work(&timer_ctx->work_ctx, - msecs_to_jiffies(ADF_GEN4_TIMER_PERIOD_MS)); + msecs_to_jiffies(ADF_DEFAULT_TIMER_PERIOD_MS)); return 0; } -EXPORT_SYMBOL_GPL(adf_gen4_timer_start); +EXPORT_SYMBOL_GPL(adf_timer_start); -void adf_gen4_timer_stop(struct adf_accel_dev *accel_dev) +void adf_timer_stop(struct adf_accel_dev *accel_dev) { struct adf_timer *timer_ctx = accel_dev->timer; @@ -68,4 +68,4 @@ void adf_gen4_timer_stop(struct adf_accel_dev *accel_dev) kfree(timer_ctx); accel_dev->timer = NULL; } -EXPORT_SYMBOL_GPL(adf_gen4_timer_stop); +EXPORT_SYMBOL_GPL(adf_timer_stop); diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen4_timer.h b/drivers/crypto/intel/qat/qat_common/adf_timer.h similarity index 58% rename from drivers/crypto/intel/qat/qat_common/adf_gen4_timer.h rename to drivers/crypto/intel/qat/qat_common/adf_timer.h index 66a709e7b358..68e5136d6ba1 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_gen4_timer.h +++ b/drivers/crypto/intel/qat/qat_common/adf_timer.h @@ -1,8 +1,8 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* Copyright(c) 2023 Intel Corporation */ -#ifndef ADF_GEN4_TIMER_H_ -#define ADF_GEN4_TIMER_H_ +#ifndef ADF_TIMER_H_ +#define ADF_TIMER_H_ #include #include @@ -15,7 +15,7 @@ struct adf_timer { ktime_t initial_ktime; }; -int adf_gen4_timer_start(struct adf_accel_dev *accel_dev); -void adf_gen4_timer_stop(struct adf_accel_dev *accel_dev); +int adf_timer_start(struct adf_accel_dev *accel_dev); +void adf_timer_stop(struct adf_accel_dev *accel_dev); -#endif /* ADF_GEN4_TIMER_H_ */ +#endif /* ADF_TIMER_H_ */ From cd0e7160f80f5fd3ebb6481ddf4f562838cad4bf Mon Sep 17 00:00:00 2001 From: Suman Kumar Chakraborty Date: Wed, 30 Apr 2025 12:34:44 +0100 Subject: [PATCH 280/382] crypto: qat - refactor compression template logic The logic that generates the compression templates, which are used by to submit compression requests to the QAT device, is very similar between QAT devices and diverges mainly on the HW generation-specific configuration word. This makes the logic that generates the compression and decompression templates common between GEN2 and GEN4 devices and abstracts the generation-specific logic to the generation-specific implementations. The adf_gen2_dc.c and adf_gen4_dc.c have been replaced by adf_dc.c, and the generation-specific logic has been reduced and moved to adf_gen2_hw_data.c and adf_gen4_hw_data.c. This does not introduce any functional change. Co-developed-by: Vijay Sundar Selvamani Signed-off-by: Vijay Sundar Selvamani Signed-off-by: Suman Kumar Chakraborty Reviewed-by: Giovanni Cabiddu Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- .../intel/qat/qat_420xx/adf_420xx_hw_data.c | 1 - .../intel/qat/qat_4xxx/adf_4xxx_hw_data.c | 1 - .../intel/qat/qat_c3xxx/adf_c3xxx_hw_data.c | 1 - .../qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c | 1 - .../intel/qat/qat_c62x/adf_c62x_hw_data.c | 1 - .../intel/qat/qat_c62xvf/adf_c62xvf_hw_data.c | 1 - drivers/crypto/intel/qat/qat_common/Makefile | 3 +- .../intel/qat/qat_common/adf_accel_devices.h | 4 +- .../qat_common/{adf_gen2_dc.c => adf_dc.c} | 47 +++++------ drivers/crypto/intel/qat/qat_common/adf_dc.h | 17 ++++ .../crypto/intel/qat/qat_common/adf_gen2_dc.h | 10 --- .../intel/qat/qat_common/adf_gen2_hw_data.c | 57 +++++++++++++ .../intel/qat/qat_common/adf_gen2_hw_data.h | 1 + .../crypto/intel/qat/qat_common/adf_gen4_dc.c | 83 ------------------- .../crypto/intel/qat/qat_common/adf_gen4_dc.h | 10 --- .../intel/qat/qat_common/adf_gen4_hw_data.c | 70 ++++++++++++++++ .../intel/qat/qat_common/adf_gen4_hw_data.h | 2 + .../intel/qat/qat_common/qat_comp_algs.c | 5 +- .../intel/qat/qat_common/qat_compression.c | 1 - .../intel/qat/qat_common/qat_compression.h | 1 - .../qat/qat_dh895xcc/adf_dh895xcc_hw_data.c | 1 - .../qat_dh895xccvf/adf_dh895xccvf_hw_data.c | 1 - 22 files changed, 173 insertions(+), 146 deletions(-) rename drivers/crypto/intel/qat/qat_common/{adf_gen2_dc.c => adf_dc.c} (61%) create mode 100644 drivers/crypto/intel/qat/qat_common/adf_dc.h delete mode 100644 drivers/crypto/intel/qat/qat_common/adf_gen2_dc.h delete mode 100644 drivers/crypto/intel/qat/qat_common/adf_gen4_dc.c delete mode 100644 drivers/crypto/intel/qat/qat_common/adf_gen4_dc.h diff --git a/drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c b/drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c index 5817b3164185..7c3c0f561c95 100644 --- a/drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c +++ b/drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c @@ -9,7 +9,6 @@ #include #include #include -#include #include #include #include diff --git a/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c b/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c index 2d89d4a3a7b9..bd0b1b1015c0 100644 --- a/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c +++ b/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c @@ -9,7 +9,6 @@ #include #include #include -#include #include #include #include diff --git a/drivers/crypto/intel/qat/qat_c3xxx/adf_c3xxx_hw_data.c b/drivers/crypto/intel/qat/qat_c3xxx/adf_c3xxx_hw_data.c index 9425af26d34c..07f2c42a68f5 100644 --- a/drivers/crypto/intel/qat/qat_c3xxx/adf_c3xxx_hw_data.c +++ b/drivers/crypto/intel/qat/qat_c3xxx/adf_c3xxx_hw_data.c @@ -5,7 +5,6 @@ #include #include #include -#include #include #include #include diff --git a/drivers/crypto/intel/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c b/drivers/crypto/intel/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c index f73d9a4a9ab7..db3c33fa1881 100644 --- a/drivers/crypto/intel/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c +++ b/drivers/crypto/intel/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c @@ -3,7 +3,6 @@ #include #include #include -#include #include #include #include diff --git a/drivers/crypto/intel/qat/qat_c62x/adf_c62x_hw_data.c b/drivers/crypto/intel/qat/qat_c62x/adf_c62x_hw_data.c index 1a2f36b603fb..0b410b41474d 100644 --- a/drivers/crypto/intel/qat/qat_c62x/adf_c62x_hw_data.c +++ b/drivers/crypto/intel/qat/qat_c62x/adf_c62x_hw_data.c @@ -5,7 +5,6 @@ #include #include #include -#include #include #include #include diff --git a/drivers/crypto/intel/qat/qat_c62xvf/adf_c62xvf_hw_data.c b/drivers/crypto/intel/qat/qat_c62xvf/adf_c62xvf_hw_data.c index 29e53b41a895..7f00035d3661 100644 --- a/drivers/crypto/intel/qat/qat_c62xvf/adf_c62xvf_hw_data.c +++ b/drivers/crypto/intel/qat/qat_c62xvf/adf_c62xvf_hw_data.c @@ -3,7 +3,6 @@ #include #include #include -#include #include #include #include diff --git a/drivers/crypto/intel/qat/qat_common/Makefile b/drivers/crypto/intel/qat/qat_common/Makefile index 0370eaad42b1..0a9da7398a78 100644 --- a/drivers/crypto/intel/qat/qat_common/Makefile +++ b/drivers/crypto/intel/qat/qat_common/Makefile @@ -8,13 +8,12 @@ intel_qat-y := adf_accel_engine.o \ adf_cfg_services.o \ adf_clock.o \ adf_ctl_drv.o \ + adf_dc.o \ adf_dev_mgr.o \ adf_gen2_config.o \ - adf_gen2_dc.o \ adf_gen2_hw_csr_data.o \ adf_gen2_hw_data.o \ adf_gen4_config.o \ - adf_gen4_dc.o \ adf_gen4_hw_csr_data.o \ adf_gen4_hw_data.o \ adf_gen4_pm.o \ diff --git a/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h b/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h index 1e301a20c244..a39f506322f6 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h @@ -12,6 +12,7 @@ #include #include #include "adf_cfg_common.h" +#include "adf_dc.h" #include "adf_rl.h" #include "adf_telemetry.h" #include "adf_pfvf_msg.h" @@ -267,7 +268,8 @@ struct adf_pfvf_ops { }; struct adf_dc_ops { - void (*build_deflate_ctx)(void *ctx); + int (*build_comp_block)(void *ctx, enum adf_dc_algo algo); + int (*build_decomp_block)(void *ctx, enum adf_dc_algo algo); }; struct qat_migdev_ops { diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen2_dc.c b/drivers/crypto/intel/qat/qat_common/adf_dc.c similarity index 61% rename from drivers/crypto/intel/qat/qat_common/adf_gen2_dc.c rename to drivers/crypto/intel/qat/qat_common/adf_dc.c index 47261b1c1da6..4beb4b7dbf0e 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_gen2_dc.c +++ b/drivers/crypto/intel/qat/qat_common/adf_dc.c @@ -1,22 +1,21 @@ // SPDX-License-Identifier: GPL-2.0-only /* Copyright(c) 2022 Intel Corporation */ #include "adf_accel_devices.h" -#include "adf_gen2_dc.h" +#include "adf_dc.h" #include "icp_qat_fw_comp.h" -static void qat_comp_build_deflate_ctx(void *ctx) +int qat_comp_build_ctx(struct adf_accel_dev *accel_dev, void *ctx, enum adf_dc_algo algo) { - struct icp_qat_fw_comp_req *req_tmpl = (struct icp_qat_fw_comp_req *)ctx; - struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr; - struct icp_qat_fw_comp_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars; - struct icp_qat_fw_comp_req_params *req_pars = &req_tmpl->comp_pars; + struct icp_qat_fw_comp_req *req_tmpl = ctx; struct icp_qat_fw_comp_cd_hdr *comp_cd_ctrl = &req_tmpl->comp_cd_ctrl; + struct icp_qat_fw_comp_req_params *req_pars = &req_tmpl->comp_pars; + struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr; + int ret; memset(req_tmpl, 0, sizeof(*req_tmpl)); header->hdr_flags = ICP_QAT_FW_COMN_HDR_FLAGS_BUILD(ICP_QAT_FW_COMN_REQ_FLAG_SET); header->service_type = ICP_QAT_FW_COMN_REQ_CPM_FW_COMP; - header->service_cmd_id = ICP_QAT_FW_COMP_CMD_STATIC; header->comn_req_flags = ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_CD_FLD_TYPE_16BYTE_DATA, QAT_COMN_PTR_TYPE_SGL); @@ -26,12 +25,14 @@ static void qat_comp_build_deflate_ctx(void *ctx) ICP_QAT_FW_COMP_NOT_ENH_AUTO_SELECT_BEST, ICP_QAT_FW_COMP_NOT_DISABLE_TYPE0_ENH_AUTO_SELECT_BEST, ICP_QAT_FW_COMP_ENABLE_SECURE_RAM_USED_AS_INTMD_BUF); - cd_pars->u.sl.comp_slice_cfg_word[0] = - ICP_QAT_HW_COMPRESSION_CONFIG_BUILD(ICP_QAT_HW_COMPRESSION_DIR_COMPRESS, - ICP_QAT_HW_COMPRESSION_DELAYED_MATCH_DISABLED, - ICP_QAT_HW_COMPRESSION_ALGO_DEFLATE, - ICP_QAT_HW_COMPRESSION_DEPTH_1, - ICP_QAT_HW_COMPRESSION_FILE_TYPE_0); + + /* Build HW config block for compression */ + ret = GET_DC_OPS(accel_dev)->build_comp_block(ctx, algo); + if (ret) { + dev_err(&GET_DEV(accel_dev), "Failed to build compression block\n"); + return ret; + } + req_pars->crc.legacy.initial_adler = COMP_CPR_INITIAL_ADLER; req_pars->crc.legacy.initial_crc32 = COMP_CPR_INITIAL_CRC; req_pars->req_par_flags = @@ -52,19 +53,11 @@ static void qat_comp_build_deflate_ctx(void *ctx) /* Fill second half of the template for decompression */ memcpy(req_tmpl + 1, req_tmpl, sizeof(*req_tmpl)); req_tmpl++; - header = &req_tmpl->comn_hdr; - header->service_cmd_id = ICP_QAT_FW_COMP_CMD_DECOMPRESS; - cd_pars = &req_tmpl->cd_pars; - cd_pars->u.sl.comp_slice_cfg_word[0] = - ICP_QAT_HW_COMPRESSION_CONFIG_BUILD(ICP_QAT_HW_COMPRESSION_DIR_DECOMPRESS, - ICP_QAT_HW_COMPRESSION_DELAYED_MATCH_DISABLED, - ICP_QAT_HW_COMPRESSION_ALGO_DEFLATE, - ICP_QAT_HW_COMPRESSION_DEPTH_1, - ICP_QAT_HW_COMPRESSION_FILE_TYPE_0); -} -void adf_gen2_init_dc_ops(struct adf_dc_ops *dc_ops) -{ - dc_ops->build_deflate_ctx = qat_comp_build_deflate_ctx; + /* Build HW config block for decompression */ + ret = GET_DC_OPS(accel_dev)->build_decomp_block(req_tmpl, algo); + if (ret) + dev_err(&GET_DEV(accel_dev), "Failed to build decompression block\n"); + + return ret; } -EXPORT_SYMBOL_GPL(adf_gen2_init_dc_ops); diff --git a/drivers/crypto/intel/qat/qat_common/adf_dc.h b/drivers/crypto/intel/qat/qat_common/adf_dc.h new file mode 100644 index 000000000000..6cb5e09054a6 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_dc.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2025 Intel Corporation */ +#ifndef ADF_DC_H +#define ADF_DC_H + +struct adf_accel_dev; + +enum adf_dc_algo { + QAT_DEFLATE, + QAT_LZ4, + QAT_LZ4S, + QAT_ZSTD, +}; + +int qat_comp_build_ctx(struct adf_accel_dev *accel_dev, void *ctx, enum adf_dc_algo algo); + +#endif /* ADF_DC_H */ diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen2_dc.h b/drivers/crypto/intel/qat/qat_common/adf_gen2_dc.h deleted file mode 100644 index 6eae023354d7..000000000000 --- a/drivers/crypto/intel/qat/qat_common/adf_gen2_dc.h +++ /dev/null @@ -1,10 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* Copyright(c) 2022 Intel Corporation */ -#ifndef ADF_GEN2_DC_H -#define ADF_GEN2_DC_H - -#include "adf_accel_devices.h" - -void adf_gen2_init_dc_ops(struct adf_dc_ops *dc_ops); - -#endif /* ADF_GEN2_DC_H */ diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen2_hw_data.c b/drivers/crypto/intel/qat/qat_common/adf_gen2_hw_data.c index 2b263442c856..6a505e9a5cf9 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_gen2_hw_data.c +++ b/drivers/crypto/intel/qat/qat_common/adf_gen2_hw_data.c @@ -1,7 +1,9 @@ // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) /* Copyright(c) 2020 Intel Corporation */ #include "adf_common_drv.h" +#include "adf_dc.h" #include "adf_gen2_hw_data.h" +#include "icp_qat_fw_comp.h" #include "icp_qat_hw.h" #include @@ -169,3 +171,58 @@ void adf_gen2_set_ssm_wdtimer(struct adf_accel_dev *accel_dev) } } EXPORT_SYMBOL_GPL(adf_gen2_set_ssm_wdtimer); + +static int adf_gen2_build_comp_block(void *ctx, enum adf_dc_algo algo) +{ + struct icp_qat_fw_comp_req *req_tmpl = ctx; + struct icp_qat_fw_comp_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars; + struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr; + + switch (algo) { + case QAT_DEFLATE: + header->service_cmd_id = ICP_QAT_FW_COMP_CMD_STATIC; + break; + default: + return -EINVAL; + } + + cd_pars->u.sl.comp_slice_cfg_word[0] = + ICP_QAT_HW_COMPRESSION_CONFIG_BUILD(ICP_QAT_HW_COMPRESSION_DIR_COMPRESS, + ICP_QAT_HW_COMPRESSION_DELAYED_MATCH_DISABLED, + ICP_QAT_HW_COMPRESSION_ALGO_DEFLATE, + ICP_QAT_HW_COMPRESSION_DEPTH_1, + ICP_QAT_HW_COMPRESSION_FILE_TYPE_0); + + return 0; +} + +static int adf_gen2_build_decomp_block(void *ctx, enum adf_dc_algo algo) +{ + struct icp_qat_fw_comp_req *req_tmpl = ctx; + struct icp_qat_fw_comp_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars; + struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr; + + switch (algo) { + case QAT_DEFLATE: + header->service_cmd_id = ICP_QAT_FW_COMP_CMD_DECOMPRESS; + break; + default: + return -EINVAL; + } + + cd_pars->u.sl.comp_slice_cfg_word[0] = + ICP_QAT_HW_COMPRESSION_CONFIG_BUILD(ICP_QAT_HW_COMPRESSION_DIR_DECOMPRESS, + ICP_QAT_HW_COMPRESSION_DELAYED_MATCH_DISABLED, + ICP_QAT_HW_COMPRESSION_ALGO_DEFLATE, + ICP_QAT_HW_COMPRESSION_DEPTH_1, + ICP_QAT_HW_COMPRESSION_FILE_TYPE_0); + + return 0; +} + +void adf_gen2_init_dc_ops(struct adf_dc_ops *dc_ops) +{ + dc_ops->build_comp_block = adf_gen2_build_comp_block; + dc_ops->build_decomp_block = adf_gen2_build_decomp_block; +} +EXPORT_SYMBOL_GPL(adf_gen2_init_dc_ops); diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen2_hw_data.h b/drivers/crypto/intel/qat/qat_common/adf_gen2_hw_data.h index 708e9186127b..59bad368a921 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_gen2_hw_data.h +++ b/drivers/crypto/intel/qat/qat_common/adf_gen2_hw_data.h @@ -88,5 +88,6 @@ void adf_gen2_get_arb_info(struct arb_info *arb_info); void adf_gen2_enable_ints(struct adf_accel_dev *accel_dev); u32 adf_gen2_get_accel_cap(struct adf_accel_dev *accel_dev); void adf_gen2_set_ssm_wdtimer(struct adf_accel_dev *accel_dev); +void adf_gen2_init_dc_ops(struct adf_dc_ops *dc_ops); #endif diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen4_dc.c b/drivers/crypto/intel/qat/qat_common/adf_gen4_dc.c deleted file mode 100644 index 5859238e37de..000000000000 --- a/drivers/crypto/intel/qat/qat_common/adf_gen4_dc.c +++ /dev/null @@ -1,83 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* Copyright(c) 2022 Intel Corporation */ -#include "adf_accel_devices.h" -#include "icp_qat_fw_comp.h" -#include "icp_qat_hw_20_comp.h" -#include "adf_gen4_dc.h" - -static void qat_comp_build_deflate(void *ctx) -{ - struct icp_qat_fw_comp_req *req_tmpl = - (struct icp_qat_fw_comp_req *)ctx; - struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr; - struct icp_qat_fw_comp_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars; - struct icp_qat_fw_comp_req_params *req_pars = &req_tmpl->comp_pars; - struct icp_qat_hw_comp_20_config_csr_upper hw_comp_upper_csr = {0}; - struct icp_qat_hw_comp_20_config_csr_lower hw_comp_lower_csr = {0}; - struct icp_qat_hw_decomp_20_config_csr_lower hw_decomp_lower_csr = {0}; - u32 upper_val; - u32 lower_val; - - memset(req_tmpl, 0, sizeof(*req_tmpl)); - header->hdr_flags = - ICP_QAT_FW_COMN_HDR_FLAGS_BUILD(ICP_QAT_FW_COMN_REQ_FLAG_SET); - header->service_type = ICP_QAT_FW_COMN_REQ_CPM_FW_COMP; - header->service_cmd_id = ICP_QAT_FW_COMP_CMD_STATIC; - header->comn_req_flags = - ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_CD_FLD_TYPE_16BYTE_DATA, - QAT_COMN_PTR_TYPE_SGL); - header->serv_specif_flags = - ICP_QAT_FW_COMP_FLAGS_BUILD(ICP_QAT_FW_COMP_STATELESS_SESSION, - ICP_QAT_FW_COMP_AUTO_SELECT_BEST, - ICP_QAT_FW_COMP_NOT_ENH_AUTO_SELECT_BEST, - ICP_QAT_FW_COMP_NOT_DISABLE_TYPE0_ENH_AUTO_SELECT_BEST, - ICP_QAT_FW_COMP_ENABLE_SECURE_RAM_USED_AS_INTMD_BUF); - hw_comp_lower_csr.skip_ctrl = ICP_QAT_HW_COMP_20_BYTE_SKIP_3BYTE_LITERAL; - hw_comp_lower_csr.algo = ICP_QAT_HW_COMP_20_HW_COMP_FORMAT_ILZ77; - hw_comp_lower_csr.lllbd = ICP_QAT_HW_COMP_20_LLLBD_CTRL_LLLBD_ENABLED; - hw_comp_lower_csr.sd = ICP_QAT_HW_COMP_20_SEARCH_DEPTH_LEVEL_1; - hw_comp_lower_csr.hash_update = ICP_QAT_HW_COMP_20_SKIP_HASH_UPDATE_DONT_ALLOW; - hw_comp_lower_csr.edmm = ICP_QAT_HW_COMP_20_EXTENDED_DELAY_MATCH_MODE_EDMM_ENABLED; - hw_comp_upper_csr.nice = ICP_QAT_HW_COMP_20_CONFIG_CSR_NICE_PARAM_DEFAULT_VAL; - hw_comp_upper_csr.lazy = ICP_QAT_HW_COMP_20_CONFIG_CSR_LAZY_PARAM_DEFAULT_VAL; - - upper_val = ICP_QAT_FW_COMP_20_BUILD_CONFIG_UPPER(hw_comp_upper_csr); - lower_val = ICP_QAT_FW_COMP_20_BUILD_CONFIG_LOWER(hw_comp_lower_csr); - - cd_pars->u.sl.comp_slice_cfg_word[0] = lower_val; - cd_pars->u.sl.comp_slice_cfg_word[1] = upper_val; - - req_pars->crc.legacy.initial_adler = COMP_CPR_INITIAL_ADLER; - req_pars->crc.legacy.initial_crc32 = COMP_CPR_INITIAL_CRC; - req_pars->req_par_flags = - ICP_QAT_FW_COMP_REQ_PARAM_FLAGS_BUILD(ICP_QAT_FW_COMP_SOP, - ICP_QAT_FW_COMP_EOP, - ICP_QAT_FW_COMP_BFINAL, - ICP_QAT_FW_COMP_CNV, - ICP_QAT_FW_COMP_CNV_RECOVERY, - ICP_QAT_FW_COMP_NO_CNV_DFX, - ICP_QAT_FW_COMP_CRC_MODE_LEGACY, - ICP_QAT_FW_COMP_NO_XXHASH_ACC, - ICP_QAT_FW_COMP_CNV_ERROR_NONE, - ICP_QAT_FW_COMP_NO_APPEND_CRC, - ICP_QAT_FW_COMP_NO_DROP_DATA); - - /* Fill second half of the template for decompression */ - memcpy(req_tmpl + 1, req_tmpl, sizeof(*req_tmpl)); - req_tmpl++; - header = &req_tmpl->comn_hdr; - header->service_cmd_id = ICP_QAT_FW_COMP_CMD_DECOMPRESS; - cd_pars = &req_tmpl->cd_pars; - - hw_decomp_lower_csr.algo = ICP_QAT_HW_DECOMP_20_HW_DECOMP_FORMAT_DEFLATE; - lower_val = ICP_QAT_FW_DECOMP_20_BUILD_CONFIG_LOWER(hw_decomp_lower_csr); - - cd_pars->u.sl.comp_slice_cfg_word[0] = lower_val; - cd_pars->u.sl.comp_slice_cfg_word[1] = 0; -} - -void adf_gen4_init_dc_ops(struct adf_dc_ops *dc_ops) -{ - dc_ops->build_deflate_ctx = qat_comp_build_deflate; -} -EXPORT_SYMBOL_GPL(adf_gen4_init_dc_ops); diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen4_dc.h b/drivers/crypto/intel/qat/qat_common/adf_gen4_dc.h deleted file mode 100644 index 0b1a6774412e..000000000000 --- a/drivers/crypto/intel/qat/qat_common/adf_gen4_dc.h +++ /dev/null @@ -1,10 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* Copyright(c) 2022 Intel Corporation */ -#ifndef ADF_GEN4_DC_H -#define ADF_GEN4_DC_H - -#include "adf_accel_devices.h" - -void adf_gen4_init_dc_ops(struct adf_dc_ops *dc_ops); - -#endif /* ADF_GEN4_DC_H */ diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_data.c b/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_data.c index 099949a2421c..0406cb09c5bb 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_data.c +++ b/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_data.c @@ -9,6 +9,8 @@ #include "adf_fw_config.h" #include "adf_gen4_hw_data.h" #include "adf_gen4_pm.h" +#include "icp_qat_fw_comp.h" +#include "icp_qat_hw_20_comp.h" u32 adf_gen4_get_accel_mask(struct adf_hw_device_data *self) { @@ -663,3 +665,71 @@ int adf_gen4_bank_state_restore(struct adf_accel_dev *accel_dev, u32 bank_number return ret; } EXPORT_SYMBOL_GPL(adf_gen4_bank_state_restore); + +static int adf_gen4_build_comp_block(void *ctx, enum adf_dc_algo algo) +{ + struct icp_qat_fw_comp_req *req_tmpl = ctx; + struct icp_qat_fw_comp_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars; + struct icp_qat_hw_comp_20_config_csr_upper hw_comp_upper_csr = { }; + struct icp_qat_hw_comp_20_config_csr_lower hw_comp_lower_csr = { }; + struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr; + u32 upper_val; + u32 lower_val; + + switch (algo) { + case QAT_DEFLATE: + header->service_cmd_id = ICP_QAT_FW_COMP_CMD_DYNAMIC; + break; + default: + return -EINVAL; + } + + hw_comp_lower_csr.skip_ctrl = ICP_QAT_HW_COMP_20_BYTE_SKIP_3BYTE_LITERAL; + hw_comp_lower_csr.algo = ICP_QAT_HW_COMP_20_HW_COMP_FORMAT_ILZ77; + hw_comp_lower_csr.lllbd = ICP_QAT_HW_COMP_20_LLLBD_CTRL_LLLBD_ENABLED; + hw_comp_lower_csr.sd = ICP_QAT_HW_COMP_20_SEARCH_DEPTH_LEVEL_1; + hw_comp_lower_csr.hash_update = ICP_QAT_HW_COMP_20_SKIP_HASH_UPDATE_DONT_ALLOW; + hw_comp_lower_csr.edmm = ICP_QAT_HW_COMP_20_EXTENDED_DELAY_MATCH_MODE_EDMM_ENABLED; + hw_comp_upper_csr.nice = ICP_QAT_HW_COMP_20_CONFIG_CSR_NICE_PARAM_DEFAULT_VAL; + hw_comp_upper_csr.lazy = ICP_QAT_HW_COMP_20_CONFIG_CSR_LAZY_PARAM_DEFAULT_VAL; + + upper_val = ICP_QAT_FW_COMP_20_BUILD_CONFIG_UPPER(hw_comp_upper_csr); + lower_val = ICP_QAT_FW_COMP_20_BUILD_CONFIG_LOWER(hw_comp_lower_csr); + + cd_pars->u.sl.comp_slice_cfg_word[0] = lower_val; + cd_pars->u.sl.comp_slice_cfg_word[1] = upper_val; + + return 0; +} + +static int adf_gen4_build_decomp_block(void *ctx, enum adf_dc_algo algo) +{ + struct icp_qat_fw_comp_req *req_tmpl = ctx; + struct icp_qat_hw_decomp_20_config_csr_lower hw_decomp_lower_csr = { }; + struct icp_qat_fw_comp_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars; + struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr; + u32 lower_val; + + switch (algo) { + case QAT_DEFLATE: + header->service_cmd_id = ICP_QAT_FW_COMP_CMD_DECOMPRESS; + break; + default: + return -EINVAL; + } + + hw_decomp_lower_csr.algo = ICP_QAT_HW_DECOMP_20_HW_DECOMP_FORMAT_DEFLATE; + lower_val = ICP_QAT_FW_DECOMP_20_BUILD_CONFIG_LOWER(hw_decomp_lower_csr); + + cd_pars->u.sl.comp_slice_cfg_word[0] = lower_val; + cd_pars->u.sl.comp_slice_cfg_word[1] = 0; + + return 0; +} + +void adf_gen4_init_dc_ops(struct adf_dc_ops *dc_ops) +{ + dc_ops->build_comp_block = adf_gen4_build_comp_block; + dc_ops->build_decomp_block = adf_gen4_build_decomp_block; +} +EXPORT_SYMBOL_GPL(adf_gen4_init_dc_ops); diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_data.h b/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_data.h index 51fc2eaa263e..e4f4d5fa616d 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_data.h +++ b/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_data.h @@ -7,6 +7,7 @@ #include "adf_accel_devices.h" #include "adf_cfg_common.h" +#include "adf_dc.h" /* PCIe configuration space */ #define ADF_GEN4_BAR_MASK (BIT(0) | BIT(2) | BIT(4)) @@ -180,5 +181,6 @@ int adf_gen4_bank_state_save(struct adf_accel_dev *accel_dev, u32 bank_number, int adf_gen4_bank_state_restore(struct adf_accel_dev *accel_dev, u32 bank_number, struct bank_state *state); bool adf_gen4_services_supported(unsigned long service_mask); +void adf_gen4_init_dc_ops(struct adf_dc_ops *dc_ops); #endif diff --git a/drivers/crypto/intel/qat/qat_common/qat_comp_algs.c b/drivers/crypto/intel/qat/qat_common/qat_comp_algs.c index a0a29b97a749..8b123472b71c 100644 --- a/drivers/crypto/intel/qat/qat_common/qat_comp_algs.c +++ b/drivers/crypto/intel/qat/qat_common/qat_comp_algs.c @@ -8,6 +8,7 @@ #include #include "adf_accel_devices.h" #include "adf_common_drv.h" +#include "adf_dc.h" #include "qat_bl.h" #include "qat_comp_req.h" #include "qat_compression.h" @@ -145,9 +146,7 @@ static int qat_comp_alg_init_tfm(struct crypto_acomp *acomp_tfm) return -EINVAL; ctx->inst = inst; - ctx->inst->build_deflate_ctx(ctx->comp_ctx); - - return 0; + return qat_comp_build_ctx(inst->accel_dev, ctx->comp_ctx, QAT_DEFLATE); } static void qat_comp_alg_exit_tfm(struct crypto_acomp *acomp_tfm) diff --git a/drivers/crypto/intel/qat/qat_common/qat_compression.c b/drivers/crypto/intel/qat/qat_common/qat_compression.c index 7842a9f22178..c285b45b8679 100644 --- a/drivers/crypto/intel/qat/qat_common/qat_compression.c +++ b/drivers/crypto/intel/qat/qat_common/qat_compression.c @@ -144,7 +144,6 @@ static int qat_compression_create_instances(struct adf_accel_dev *accel_dev) inst->id = i; atomic_set(&inst->refctr, 0); inst->accel_dev = accel_dev; - inst->build_deflate_ctx = GET_DC_OPS(accel_dev)->build_deflate_ctx; snprintf(key, sizeof(key), ADF_DC "%d" ADF_RING_DC_BANK_NUM, i); ret = adf_cfg_get_param_value(accel_dev, SEC, key, val); diff --git a/drivers/crypto/intel/qat/qat_common/qat_compression.h b/drivers/crypto/intel/qat/qat_common/qat_compression.h index aebac2302dcf..5ced3ed0e5ea 100644 --- a/drivers/crypto/intel/qat/qat_common/qat_compression.h +++ b/drivers/crypto/intel/qat/qat_common/qat_compression.h @@ -20,7 +20,6 @@ struct qat_compression_instance { atomic_t refctr; struct qat_instance_backlog backlog; struct adf_dc_data *dc_data; - void (*build_deflate_ctx)(void *ctx); }; static inline bool adf_hw_dev_has_compression(struct adf_accel_dev *accel_dev) diff --git a/drivers/crypto/intel/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c b/drivers/crypto/intel/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c index bf9e8f34f451..5b4bd0ba1ccb 100644 --- a/drivers/crypto/intel/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c +++ b/drivers/crypto/intel/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c @@ -4,7 +4,6 @@ #include #include #include -#include #include #include #include diff --git a/drivers/crypto/intel/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c b/drivers/crypto/intel/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c index bc59c1473eef..828456c43b76 100644 --- a/drivers/crypto/intel/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c +++ b/drivers/crypto/intel/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c @@ -3,7 +3,6 @@ #include #include #include -#include #include #include #include From 7e673b80f420eb00660e01bf82befa8eaf652b28 Mon Sep 17 00:00:00 2001 From: Suman Kumar Chakraborty Date: Wed, 30 Apr 2025 12:34:45 +0100 Subject: [PATCH 281/382] crypto: qat - use pr_fmt() in qat uclo.c Add pr_fmt() to qat uclo.c logging and update the debug and error messages to utilize it accordingly. This does not introduce any functional changes. Signed-off-by: Suman Kumar Chakraborty Reviewed-by: Giovanni Cabiddu Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- .../crypto/intel/qat/qat_common/qat_uclo.c | 135 +++++++++--------- 1 file changed, 65 insertions(+), 70 deletions(-) diff --git a/drivers/crypto/intel/qat/qat_common/qat_uclo.c b/drivers/crypto/intel/qat/qat_common/qat_uclo.c index 620300e70238..9dc26f5d2d01 100644 --- a/drivers/crypto/intel/qat/qat_common/qat_uclo.c +++ b/drivers/crypto/intel/qat/qat_common/qat_uclo.c @@ -1,5 +1,8 @@ // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) /* Copyright(c) 2014 - 2020 Intel Corporation */ + +#define pr_fmt(fmt) "QAT: " fmt + #include #include #include @@ -60,7 +63,7 @@ static int qat_uclo_free_ae_data(struct icp_qat_uclo_aedata *ae_data) unsigned int i; if (!ae_data) { - pr_err("QAT: bad argument, ae_data is NULL\n"); + pr_err("bad argument, ae_data is NULL\n"); return -EINVAL; } @@ -87,12 +90,11 @@ static int qat_uclo_check_uof_format(struct icp_qat_uof_filehdr *hdr) int min = hdr->min_ver & 0xff; if (hdr->file_id != ICP_QAT_UOF_FID) { - pr_err("QAT: Invalid header 0x%x\n", hdr->file_id); + pr_err("Invalid header 0x%x\n", hdr->file_id); return -EINVAL; } if (min != ICP_QAT_UOF_MINVER || maj != ICP_QAT_UOF_MAJVER) { - pr_err("QAT: bad UOF version, major 0x%x, minor 0x%x\n", - maj, min); + pr_err("bad UOF version, major 0x%x, minor 0x%x\n", maj, min); return -EINVAL; } return 0; @@ -104,20 +106,19 @@ static int qat_uclo_check_suof_format(struct icp_qat_suof_filehdr *suof_hdr) int min = suof_hdr->min_ver & 0xff; if (suof_hdr->file_id != ICP_QAT_SUOF_FID) { - pr_err("QAT: invalid header 0x%x\n", suof_hdr->file_id); + pr_err("invalid header 0x%x\n", suof_hdr->file_id); return -EINVAL; } if (suof_hdr->fw_type != 0) { - pr_err("QAT: unsupported firmware type\n"); + pr_err("unsupported firmware type\n"); return -EINVAL; } if (suof_hdr->num_chunks <= 0x1) { - pr_err("QAT: SUOF chunk amount is incorrect\n"); + pr_err("SUOF chunk amount is incorrect\n"); return -EINVAL; } if (maj != ICP_QAT_SUOF_MAJVER || min != ICP_QAT_SUOF_MINVER) { - pr_err("QAT: bad SUOF version, major 0x%x, minor 0x%x\n", - maj, min); + pr_err("bad SUOF version, major 0x%x, minor 0x%x\n", maj, min); return -EINVAL; } return 0; @@ -224,24 +225,24 @@ static int qat_uclo_fetch_initmem_ae(struct icp_qat_fw_loader_handle *handle, char *str; if ((init_mem->addr + init_mem->num_in_bytes) > (size_range << 0x2)) { - pr_err("QAT: initmem is out of range"); + pr_err("initmem is out of range"); return -EINVAL; } if (init_mem->scope != ICP_QAT_UOF_LOCAL_SCOPE) { - pr_err("QAT: Memory scope for init_mem error\n"); + pr_err("Memory scope for init_mem error\n"); return -EINVAL; } str = qat_uclo_get_string(&obj_handle->str_table, init_mem->sym_name); if (!str) { - pr_err("QAT: AE name assigned in UOF init table is NULL\n"); + pr_err("AE name assigned in UOF init table is NULL\n"); return -EINVAL; } if (qat_uclo_parse_num(str, ae)) { - pr_err("QAT: Parse num for AE number failed\n"); + pr_err("Parse num for AE number failed\n"); return -EINVAL; } if (*ae >= ICP_QAT_UCLO_MAX_AE) { - pr_err("QAT: ae %d out of range\n", *ae); + pr_err("ae %d out of range\n", *ae); return -EINVAL; } return 0; @@ -357,8 +358,7 @@ static int qat_uclo_init_ae_memory(struct icp_qat_fw_loader_handle *handle, return -EINVAL; break; default: - pr_err("QAT: initmem region error. region type=0x%x\n", - init_mem->region); + pr_err("initmem region error. region type=0x%x\n", init_mem->region); return -EINVAL; } return 0; @@ -432,7 +432,7 @@ static int qat_uclo_init_memory(struct icp_qat_fw_loader_handle *handle) for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { if (qat_hal_batch_wr_lm(handle, ae, obj_handle->lm_init_tab[ae])) { - pr_err("QAT: fail to batch init lmem for AE %d\n", ae); + pr_err("fail to batch init lmem for AE %d\n", ae); return -EINVAL; } qat_uclo_cleanup_batch_init_list(handle, @@ -540,26 +540,26 @@ qat_uclo_check_image_compat(struct icp_qat_uof_encap_obj *encap_uof_obj, code_page->imp_expr_tab_offset); if (uc_var_tab->entry_num || imp_var_tab->entry_num || imp_expr_tab->entry_num) { - pr_err("QAT: UOF can't contain imported variable to be parsed\n"); + pr_err("UOF can't contain imported variable to be parsed\n"); return -EINVAL; } neigh_reg_tab = (struct icp_qat_uof_objtable *) (encap_uof_obj->beg_uof + code_page->neigh_reg_tab_offset); if (neigh_reg_tab->entry_num) { - pr_err("QAT: UOF can't contain neighbor register table\n"); + pr_err("UOF can't contain neighbor register table\n"); return -EINVAL; } if (image->numpages > 1) { - pr_err("QAT: UOF can't contain multiple pages\n"); + pr_err("UOF can't contain multiple pages\n"); return -EINVAL; } if (ICP_QAT_SHARED_USTORE_MODE(image->ae_mode)) { - pr_err("QAT: UOF can't use shared control store feature\n"); + pr_err("UOF can't use shared control store feature\n"); return -EFAULT; } if (RELOADABLE_CTX_SHARED_MODE(image->ae_mode)) { - pr_err("QAT: UOF can't use reloadable feature\n"); + pr_err("UOF can't use reloadable feature\n"); return -EFAULT; } return 0; @@ -678,7 +678,7 @@ static int qat_uclo_map_ae(struct icp_qat_fw_loader_handle *handle, int max_ae) } } if (!mflag) { - pr_err("QAT: uimage uses AE not set\n"); + pr_err("uimage uses AE not set\n"); return -EINVAL; } return 0; @@ -738,8 +738,7 @@ qat_uclo_get_dev_type(struct icp_qat_fw_loader_handle *handle) case PCI_DEVICE_ID_INTEL_QAT_420XX: return ICP_QAT_AC_4XXX_A_DEV_TYPE; default: - pr_err("QAT: unsupported device 0x%x\n", - handle->pci_dev->device); + pr_err("unsupported device 0x%x\n", handle->pci_dev->device); return 0; } } @@ -749,7 +748,7 @@ static int qat_uclo_check_uof_compat(struct icp_qat_uclo_objhandle *obj_handle) unsigned int maj_ver, prod_type = obj_handle->prod_type; if (!(prod_type & obj_handle->encap_uof_obj.obj_hdr->ac_dev_type)) { - pr_err("QAT: UOF type 0x%x doesn't match with platform 0x%x\n", + pr_err("UOF type 0x%x doesn't match with platform 0x%x\n", obj_handle->encap_uof_obj.obj_hdr->ac_dev_type, prod_type); return -EINVAL; @@ -757,7 +756,7 @@ static int qat_uclo_check_uof_compat(struct icp_qat_uclo_objhandle *obj_handle) maj_ver = obj_handle->prod_rev & 0xff; if (obj_handle->encap_uof_obj.obj_hdr->max_cpu_ver < maj_ver || obj_handle->encap_uof_obj.obj_hdr->min_cpu_ver > maj_ver) { - pr_err("QAT: UOF majVer 0x%x out of range\n", maj_ver); + pr_err("UOF majVer 0x%x out of range\n", maj_ver); return -EINVAL; } return 0; @@ -800,7 +799,7 @@ static int qat_uclo_init_reg(struct icp_qat_fw_loader_handle *handle, case ICP_NEIGH_REL: return qat_hal_init_nn(handle, ae, ctx_mask, reg_addr, value); default: - pr_err("QAT: UOF uses not supported reg type 0x%x\n", reg_type); + pr_err("UOF uses not supported reg type 0x%x\n", reg_type); return -EFAULT; } return 0; @@ -836,8 +835,7 @@ static int qat_uclo_init_reg_sym(struct icp_qat_fw_loader_handle *handle, case ICP_QAT_UOF_INIT_REG_CTX: /* check if ctx is appropriate for the ctxMode */ if (!((1 << init_regsym->ctx) & ctx_mask)) { - pr_err("QAT: invalid ctx num = 0x%x\n", - init_regsym->ctx); + pr_err("invalid ctx num = 0x%x\n", init_regsym->ctx); return -EINVAL; } qat_uclo_init_reg(handle, ae, @@ -849,10 +847,10 @@ static int qat_uclo_init_reg_sym(struct icp_qat_fw_loader_handle *handle, exp_res); break; case ICP_QAT_UOF_INIT_EXPR: - pr_err("QAT: INIT_EXPR feature not supported\n"); + pr_err("INIT_EXPR feature not supported\n"); return -EINVAL; case ICP_QAT_UOF_INIT_EXPR_ENDIAN_SWAP: - pr_err("QAT: INIT_EXPR_ENDIAN_SWAP feature not supported\n"); + pr_err("INIT_EXPR_ENDIAN_SWAP feature not supported\n"); return -EINVAL; default: break; @@ -872,7 +870,7 @@ static int qat_uclo_init_globals(struct icp_qat_fw_loader_handle *handle) return 0; if (obj_handle->init_mem_tab.entry_num) { if (qat_uclo_init_memory(handle)) { - pr_err("QAT: initialize memory failed\n"); + pr_err("initialize memory failed\n"); return -EINVAL; } } @@ -901,40 +899,40 @@ static int qat_hal_set_modes(struct icp_qat_fw_loader_handle *handle, mode = ICP_QAT_CTX_MODE(uof_image->ae_mode); ret = qat_hal_set_ae_ctx_mode(handle, ae, mode); if (ret) { - pr_err("QAT: qat_hal_set_ae_ctx_mode error\n"); + pr_err("qat_hal_set_ae_ctx_mode error\n"); return ret; } if (handle->chip_info->nn) { mode = ICP_QAT_NN_MODE(uof_image->ae_mode); ret = qat_hal_set_ae_nn_mode(handle, ae, mode); if (ret) { - pr_err("QAT: qat_hal_set_ae_nn_mode error\n"); + pr_err("qat_hal_set_ae_nn_mode error\n"); return ret; } } mode = ICP_QAT_LOC_MEM0_MODE(uof_image->ae_mode); ret = qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM0, mode); if (ret) { - pr_err("QAT: qat_hal_set_ae_lm_mode LMEM0 error\n"); + pr_err("qat_hal_set_ae_lm_mode LMEM0 error\n"); return ret; } mode = ICP_QAT_LOC_MEM1_MODE(uof_image->ae_mode); ret = qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM1, mode); if (ret) { - pr_err("QAT: qat_hal_set_ae_lm_mode LMEM1 error\n"); + pr_err("qat_hal_set_ae_lm_mode LMEM1 error\n"); return ret; } if (handle->chip_info->lm2lm3) { mode = ICP_QAT_LOC_MEM2_MODE(uof_image->ae_mode); ret = qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM2, mode); if (ret) { - pr_err("QAT: qat_hal_set_ae_lm_mode LMEM2 error\n"); + pr_err("qat_hal_set_ae_lm_mode LMEM2 error\n"); return ret; } mode = ICP_QAT_LOC_MEM3_MODE(uof_image->ae_mode); ret = qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM3, mode); if (ret) { - pr_err("QAT: qat_hal_set_ae_lm_mode LMEM3 error\n"); + pr_err("qat_hal_set_ae_lm_mode LMEM3 error\n"); return ret; } mode = ICP_QAT_LOC_TINDEX_MODE(uof_image->ae_mode); @@ -998,7 +996,7 @@ static int qat_uclo_parse_uof_obj(struct icp_qat_fw_loader_handle *handle) obj_handle->prod_rev = PID_MAJOR_REV | (PID_MINOR_REV & handle->hal_handle->revision_id); if (qat_uclo_check_uof_compat(obj_handle)) { - pr_err("QAT: UOF incompatible\n"); + pr_err("UOF incompatible\n"); return -EINVAL; } obj_handle->uword_buf = kcalloc(UWORD_CPYBUF_SIZE, sizeof(u64), @@ -1009,7 +1007,7 @@ static int qat_uclo_parse_uof_obj(struct icp_qat_fw_loader_handle *handle) if (!obj_handle->obj_hdr->file_buff || !qat_uclo_map_str_table(obj_handle->obj_hdr, ICP_QAT_UOF_STRT, &obj_handle->str_table)) { - pr_err("QAT: UOF doesn't have effective images\n"); + pr_err("UOF doesn't have effective images\n"); goto out_err; } obj_handle->uimage_num = @@ -1018,7 +1016,7 @@ static int qat_uclo_parse_uof_obj(struct icp_qat_fw_loader_handle *handle) if (!obj_handle->uimage_num) goto out_err; if (qat_uclo_map_ae(handle, handle->hal_handle->ae_max_num)) { - pr_err("QAT: Bad object\n"); + pr_err("Bad object\n"); goto out_check_uof_aemask_err; } qat_uclo_init_uword_num(handle); @@ -1051,7 +1049,7 @@ static int qat_uclo_map_suof_file_hdr(struct icp_qat_fw_loader_handle *handle, check_sum = qat_uclo_calc_str_checksum((char *)&suof_ptr->min_ver, min_ver_offset); if (check_sum != suof_ptr->check_sum) { - pr_err("QAT: incorrect SUOF checksum\n"); + pr_err("incorrect SUOF checksum\n"); return -EINVAL; } suof_handle->check_sum = suof_ptr->check_sum; @@ -1113,14 +1111,13 @@ static int qat_uclo_check_simg_compat(struct icp_qat_fw_loader_handle *handle, prod_rev = PID_MAJOR_REV | (PID_MINOR_REV & handle->hal_handle->revision_id); if (img_ae_mode->dev_type != prod_type) { - pr_err("QAT: incompatible product type %x\n", - img_ae_mode->dev_type); + pr_err("incompatible product type %x\n", img_ae_mode->dev_type); return -EINVAL; } maj_ver = prod_rev & 0xff; if (maj_ver > img_ae_mode->devmax_ver || maj_ver < img_ae_mode->devmin_ver) { - pr_err("QAT: incompatible device majver 0x%x\n", maj_ver); + pr_err("incompatible device majver 0x%x\n", maj_ver); return -EINVAL; } return 0; @@ -1163,7 +1160,7 @@ static int qat_uclo_map_suof(struct icp_qat_fw_loader_handle *handle, struct icp_qat_suof_img_hdr img_header; if (!suof_ptr || suof_size == 0) { - pr_err("QAT: input parameter SUOF pointer/size is NULL\n"); + pr_err("input parameter SUOF pointer/size is NULL\n"); return -EINVAL; } if (qat_uclo_check_suof_format(suof_ptr)) @@ -1237,7 +1234,7 @@ static int qat_uclo_auth_fw(struct icp_qat_fw_loader_handle *handle, return 0; } while (retry++ < FW_AUTH_MAX_RETRY); auth_fail: - pr_err("QAT: authentication error (FCU_STATUS = 0x%x),retry = %d\n", + pr_err("authentication error (FCU_STATUS = 0x%x),retry = %d\n", fcu_sts & FCU_AUTH_STS_MASK, retry); return -EINVAL; } @@ -1273,14 +1270,13 @@ static int qat_uclo_broadcast_load_fw(struct icp_qat_fw_loader_handle *handle, fcu_sts_csr = handle->chip_info->fcu_sts_csr; fcu_loaded_csr = handle->chip_info->fcu_loaded_ae_csr; } else { - pr_err("Chip 0x%x doesn't support broadcast load\n", - handle->pci_dev->device); + pr_err("Chip 0x%x doesn't support broadcast load\n", handle->pci_dev->device); return -EINVAL; } for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { if (qat_hal_check_ae_active(handle, (unsigned char)ae)) { - pr_err("QAT: Broadcast load failed. AE is not enabled or active.\n"); + pr_err("Broadcast load failed. AE is not enabled or active.\n"); return -EINVAL; } @@ -1312,7 +1308,7 @@ static int qat_uclo_broadcast_load_fw(struct icp_qat_fw_loader_handle *handle, } while (retry++ < FW_AUTH_MAX_RETRY); if (retry > FW_AUTH_MAX_RETRY) { - pr_err("QAT: broadcast load failed timeout %d\n", retry); + pr_err("broadcast load failed timeout %d\n", retry); return -EINVAL; } } @@ -1397,13 +1393,13 @@ static int qat_uclo_check_image(struct icp_qat_fw_loader_handle *handle, if (size > ICP_QAT_CSS_RSA3K_MAX_IMAGE_LEN) goto err; } else { - pr_err("QAT: Unsupported firmware type\n"); + pr_err("Unsupported firmware type\n"); return -EINVAL; } return 0; err: - pr_err("QAT: Invalid %s firmware image\n", fw_type_name); + pr_err("Invalid %s firmware image\n", fw_type_name); return -EINVAL; } @@ -1422,12 +1418,12 @@ static int qat_uclo_map_auth_fw(struct icp_qat_fw_loader_handle *handle, ret = qat_uclo_simg_alloc(handle, &img_desc, ICP_QAT_CSS_RSA4K_MAX_IMAGE_LEN); if (ret) { - pr_err("QAT: error, allocate continuous dram fail\n"); + pr_err("error, allocate continuous dram fail\n"); return ret; } if (!IS_ALIGNED(img_desc.dram_size, 8) || !img_desc.dram_bus_addr) { - pr_debug("QAT: invalid address\n"); + pr_debug("invalid address\n"); qat_uclo_simg_free(handle, &img_desc); return -EINVAL; } @@ -1489,7 +1485,7 @@ static int qat_uclo_map_auth_fw(struct icp_qat_fw_loader_handle *handle, auth_desc->img_len = size - ICP_QAT_AE_IMG_OFFSET(handle); if (bus_addr + auth_desc->img_len > img_desc.dram_bus_addr + ICP_QAT_CSS_RSA4K_MAX_IMAGE_LEN) { - pr_err("QAT: insufficient memory size for authentication data\n"); + pr_err("insufficient memory size for authentication data\n"); qat_uclo_simg_free(handle, &img_desc); return -ENOMEM; } @@ -1546,7 +1542,7 @@ static int qat_uclo_load_fw(struct icp_qat_fw_loader_handle *handle, if (!((desc->ae_mask >> i) & 0x1)) continue; if (qat_hal_check_ae_active(handle, i)) { - pr_err("QAT: AE %d is active\n", i); + pr_err("AE %d is active\n", i); return -EINVAL; } SET_CAP_CSR(handle, fcu_ctl_csr, @@ -1566,7 +1562,7 @@ static int qat_uclo_load_fw(struct icp_qat_fw_loader_handle *handle, } } while (retry++ < FW_AUTH_MAX_RETRY); if (retry > FW_AUTH_MAX_RETRY) { - pr_err("QAT: firmware load failed timeout %x\n", retry); + pr_err("firmware load failed timeout %x\n", retry); return -EINVAL; } } @@ -1584,7 +1580,7 @@ static int qat_uclo_map_suof_obj(struct icp_qat_fw_loader_handle *handle, handle->sobj_handle = suof_handle; if (qat_uclo_map_suof(handle, addr_ptr, mem_size)) { qat_uclo_del_suof(handle); - pr_err("QAT: map SUOF failed\n"); + pr_err("map SUOF failed\n"); return -EINVAL; } return 0; @@ -1608,7 +1604,7 @@ int qat_uclo_wr_mimage(struct icp_qat_fw_loader_handle *handle, qat_uclo_ummap_auth_fw(handle, &desc); } else { if (handle->chip_info->mmp_sram_size < mem_size) { - pr_err("QAT: MMP size is too large: 0x%x\n", mem_size); + pr_err("MMP size is too large: 0x%x\n", mem_size); return -EFBIG; } qat_uclo_wr_sram_by_words(handle, 0, addr_ptr, mem_size); @@ -1634,7 +1630,7 @@ static int qat_uclo_map_uof_obj(struct icp_qat_fw_loader_handle *handle, objhdl->obj_hdr = qat_uclo_map_chunk((char *)objhdl->obj_buf, filehdr, ICP_QAT_UOF_OBJS); if (!objhdl->obj_hdr) { - pr_err("QAT: object file chunk is null\n"); + pr_err("object file chunk is null\n"); goto out_objhdr_err; } handle->obj_handle = objhdl; @@ -1669,7 +1665,7 @@ static int qat_uclo_map_mof_file_hdr(struct icp_qat_fw_loader_handle *handle, checksum = qat_uclo_calc_str_checksum(&mof_ptr->min_ver, min_ver_offset); if (checksum != mof_ptr->checksum) { - pr_err("QAT: incorrect MOF checksum\n"); + pr_err("incorrect MOF checksum\n"); return -EINVAL; } @@ -1705,7 +1701,7 @@ static int qat_uclo_seek_obj_inside_mof(struct icp_qat_mof_handle *mobj_handle, } } - pr_err("QAT: object %s is not found inside MOF\n", obj_name); + pr_err("object %s is not found inside MOF\n", obj_name); return -EINVAL; } @@ -1722,7 +1718,7 @@ static int qat_uclo_map_obj_from_mof(struct icp_qat_mof_handle *mobj_handle, ICP_QAT_MOF_OBJ_CHUNKID_LEN)) { obj = mobj_handle->sobjs_hdr + obj_chunkhdr->offset; } else { - pr_err("QAT: unsupported chunk id\n"); + pr_err("unsupported chunk id\n"); return -EINVAL; } mobj_hdr->obj_buf = obj; @@ -1783,7 +1779,7 @@ static int qat_uclo_map_objs_from_mof(struct icp_qat_mof_handle *mobj_handle) } if ((uobj_chunk_num + sobj_chunk_num) != *valid_chunk) { - pr_err("QAT: inconsistent UOF/SUOF chunk amount\n"); + pr_err("inconsistent UOF/SUOF chunk amount\n"); return -EINVAL; } return 0; @@ -1824,17 +1820,16 @@ static int qat_uclo_check_mof_format(struct icp_qat_mof_file_hdr *mof_hdr) int min = mof_hdr->min_ver & 0xff; if (mof_hdr->file_id != ICP_QAT_MOF_FID) { - pr_err("QAT: invalid header 0x%x\n", mof_hdr->file_id); + pr_err("invalid header 0x%x\n", mof_hdr->file_id); return -EINVAL; } if (mof_hdr->num_chunks <= 0x1) { - pr_err("QAT: MOF chunk amount is incorrect\n"); + pr_err("MOF chunk amount is incorrect\n"); return -EINVAL; } if (maj != ICP_QAT_MOF_MAJVER || min != ICP_QAT_MOF_MINVER) { - pr_err("QAT: bad MOF version, major 0x%x, minor 0x%x\n", - maj, min); + pr_err("bad MOF version, major 0x%x, minor 0x%x\n", maj, min); return -EINVAL; } return 0; From 98943958a4895299c4f54719f4ef34b950a2ca4c Mon Sep 17 00:00:00 2001 From: Jack Xu Date: Wed, 30 Apr 2025 12:34:46 +0100 Subject: [PATCH 282/382] crypto: qat - refactor FW signing algorithm The current implementation is designed to support single FW signing authentication only. Refactor the implementation to support other FW signing methods. This does not include any functional change. Co-developed-by: Suman Kumar Chakraborty Signed-off-by: Suman Kumar Chakraborty Signed-off-by: Jack Xu Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- .../crypto/intel/qat/qat_common/qat_uclo.c | 154 ++++++++++-------- 1 file changed, 84 insertions(+), 70 deletions(-) diff --git a/drivers/crypto/intel/qat/qat_common/qat_uclo.c b/drivers/crypto/intel/qat/qat_common/qat_uclo.c index 9dc26f5d2d01..d7f2ceb81f1f 100644 --- a/drivers/crypto/intel/qat/qat_common/qat_uclo.c +++ b/drivers/crypto/intel/qat/qat_common/qat_uclo.c @@ -1033,6 +1033,23 @@ out_err: return -EFAULT; } +static unsigned int qat_uclo_simg_hdr2sign_len(struct icp_qat_fw_loader_handle *handle) +{ + return ICP_QAT_AE_IMG_OFFSET(handle); +} + +static unsigned int qat_uclo_simg_hdr2cont_len(struct icp_qat_fw_loader_handle *handle) +{ + return ICP_QAT_AE_IMG_OFFSET(handle); +} + +static unsigned int qat_uclo_simg_fw_type(struct icp_qat_fw_loader_handle *handle, void *img_ptr) +{ + struct icp_qat_css_hdr *hdr = img_ptr; + + return hdr->fw_type; +} + static int qat_uclo_map_suof_file_hdr(struct icp_qat_fw_loader_handle *handle, struct icp_qat_suof_filehdr *suof_ptr, int suof_size) @@ -1064,9 +1081,9 @@ static void qat_uclo_map_simg(struct icp_qat_fw_loader_handle *handle, struct icp_qat_suof_chunk_hdr *suof_chunk_hdr) { struct icp_qat_suof_handle *suof_handle = handle->sobj_handle; - unsigned int offset = ICP_QAT_AE_IMG_OFFSET(handle); - struct icp_qat_simg_ae_mode *ae_mode; + unsigned int offset = qat_uclo_simg_hdr2cont_len(handle); struct icp_qat_suof_objhdr *suof_objhdr; + struct icp_qat_simg_ae_mode *ae_mode; suof_img_hdr->simg_buf = (suof_handle->suof_buf + suof_chunk_hdr->offset + @@ -1362,21 +1379,24 @@ static void qat_uclo_ummap_auth_fw(struct icp_qat_fw_loader_handle *handle, } static int qat_uclo_check_image(struct icp_qat_fw_loader_handle *handle, - char *image, unsigned int size, + void *image, unsigned int size, unsigned int fw_type) { char *fw_type_name = fw_type ? "MMP" : "AE"; unsigned int css_dword_size = sizeof(u32); + unsigned int header_len, simg_type; + struct icp_qat_css_hdr *css_hdr; if (handle->chip_info->fw_auth) { - struct icp_qat_css_hdr *css_hdr = (struct icp_qat_css_hdr *)image; - unsigned int header_len = ICP_QAT_AE_IMG_OFFSET(handle); + header_len = qat_uclo_simg_hdr2sign_len(handle); + simg_type = qat_uclo_simg_fw_type(handle, image); + css_hdr = image; if ((css_hdr->header_len * css_dword_size) != header_len) goto err; if ((css_hdr->size * css_dword_size) != size) goto err; - if (fw_type != css_hdr->fw_type) + if (fw_type != simg_type) goto err; if (size <= header_len) goto err; @@ -1403,113 +1423,85 @@ err: return -EINVAL; } -static int qat_uclo_map_auth_fw(struct icp_qat_fw_loader_handle *handle, - char *image, unsigned int size, - struct icp_qat_fw_auth_desc **desc) +static int qat_uclo_build_auth_desc_RSA(struct icp_qat_fw_loader_handle *handle, + char *image, unsigned int size, + struct icp_firml_dram_desc *dram_desc, + unsigned int fw_type, struct icp_qat_fw_auth_desc **desc) { struct icp_qat_css_hdr *css_hdr = (struct icp_qat_css_hdr *)image; - struct icp_qat_fw_auth_desc *auth_desc; - struct icp_qat_auth_chunk *auth_chunk; - u64 virt_addr, bus_addr, virt_base; - unsigned int simg_offset = sizeof(*auth_chunk); struct icp_qat_simg_ae_mode *simg_ae_mode; - struct icp_firml_dram_desc img_desc; - int ret; + struct icp_qat_fw_auth_desc *auth_desc; + char *virt_addr, *virt_base; + u64 bus_addr; - ret = qat_uclo_simg_alloc(handle, &img_desc, ICP_QAT_CSS_RSA4K_MAX_IMAGE_LEN); - if (ret) { - pr_err("error, allocate continuous dram fail\n"); - return ret; - } - - if (!IS_ALIGNED(img_desc.dram_size, 8) || !img_desc.dram_bus_addr) { - pr_debug("invalid address\n"); - qat_uclo_simg_free(handle, &img_desc); - return -EINVAL; - } - - auth_chunk = img_desc.dram_base_addr_v; - auth_chunk->chunk_size = img_desc.dram_size; - auth_chunk->chunk_bus_addr = img_desc.dram_bus_addr; - virt_base = (uintptr_t)img_desc.dram_base_addr_v + simg_offset; - bus_addr = img_desc.dram_bus_addr + simg_offset; - auth_desc = img_desc.dram_base_addr_v; - auth_desc->css_hdr_high = (unsigned int)(bus_addr >> BITS_PER_TYPE(u32)); - auth_desc->css_hdr_low = (unsigned int)bus_addr; + virt_base = dram_desc->dram_base_addr_v; + virt_base += sizeof(struct icp_qat_auth_chunk); + bus_addr = dram_desc->dram_bus_addr + sizeof(struct icp_qat_auth_chunk); + auth_desc = dram_desc->dram_base_addr_v; + auth_desc->css_hdr_high = upper_32_bits(bus_addr); + auth_desc->css_hdr_low = lower_32_bits(bus_addr); virt_addr = virt_base; - memcpy((void *)(uintptr_t)virt_addr, image, sizeof(*css_hdr)); + memcpy(virt_addr, image, sizeof(*css_hdr)); /* pub key */ bus_addr = ADD_ADDR(auth_desc->css_hdr_high, auth_desc->css_hdr_low) + sizeof(*css_hdr); virt_addr = virt_addr + sizeof(*css_hdr); - auth_desc->fwsk_pub_high = (unsigned int)(bus_addr >> BITS_PER_TYPE(u32)); - auth_desc->fwsk_pub_low = (unsigned int)bus_addr; + auth_desc->fwsk_pub_high = upper_32_bits(bus_addr); + auth_desc->fwsk_pub_low = lower_32_bits(bus_addr); - memcpy((void *)(uintptr_t)virt_addr, - (void *)(image + sizeof(*css_hdr)), - ICP_QAT_CSS_FWSK_MODULUS_LEN(handle)); + memcpy(virt_addr, image + sizeof(*css_hdr), ICP_QAT_CSS_FWSK_MODULUS_LEN(handle)); /* padding */ memset((void *)(uintptr_t)(virt_addr + ICP_QAT_CSS_FWSK_MODULUS_LEN(handle)), 0, ICP_QAT_CSS_FWSK_PAD_LEN(handle)); /* exponent */ - memcpy((void *)(uintptr_t)(virt_addr + ICP_QAT_CSS_FWSK_MODULUS_LEN(handle) + - ICP_QAT_CSS_FWSK_PAD_LEN(handle)), - (void *)(image + sizeof(*css_hdr) + - ICP_QAT_CSS_FWSK_MODULUS_LEN(handle)), - sizeof(unsigned int)); + memcpy(virt_addr + ICP_QAT_CSS_FWSK_MODULUS_LEN(handle) + + ICP_QAT_CSS_FWSK_PAD_LEN(handle), image + sizeof(*css_hdr) + + ICP_QAT_CSS_FWSK_MODULUS_LEN(handle), sizeof(unsigned int)); /* signature */ bus_addr = ADD_ADDR(auth_desc->fwsk_pub_high, auth_desc->fwsk_pub_low) + ICP_QAT_CSS_FWSK_PUB_LEN(handle); virt_addr = virt_addr + ICP_QAT_CSS_FWSK_PUB_LEN(handle); - auth_desc->signature_high = (unsigned int)(bus_addr >> BITS_PER_TYPE(u32)); - auth_desc->signature_low = (unsigned int)bus_addr; + auth_desc->signature_high = upper_32_bits(bus_addr); + auth_desc->signature_low = lower_32_bits(bus_addr); - memcpy((void *)(uintptr_t)virt_addr, - (void *)(image + sizeof(*css_hdr) + - ICP_QAT_CSS_FWSK_MODULUS_LEN(handle) + - ICP_QAT_CSS_FWSK_EXPONENT_LEN(handle)), - ICP_QAT_CSS_SIGNATURE_LEN(handle)); + memcpy(virt_addr, image + sizeof(*css_hdr) + ICP_QAT_CSS_FWSK_MODULUS_LEN(handle) + + ICP_QAT_CSS_FWSK_EXPONENT_LEN(handle), ICP_QAT_CSS_SIGNATURE_LEN(handle)); bus_addr = ADD_ADDR(auth_desc->signature_high, auth_desc->signature_low) + ICP_QAT_CSS_SIGNATURE_LEN(handle); virt_addr += ICP_QAT_CSS_SIGNATURE_LEN(handle); - auth_desc->img_high = (unsigned int)(bus_addr >> BITS_PER_TYPE(u32)); - auth_desc->img_low = (unsigned int)bus_addr; - auth_desc->img_len = size - ICP_QAT_AE_IMG_OFFSET(handle); - if (bus_addr + auth_desc->img_len > img_desc.dram_bus_addr + - ICP_QAT_CSS_RSA4K_MAX_IMAGE_LEN) { + auth_desc->img_high = upper_32_bits(bus_addr); + auth_desc->img_low = lower_32_bits(bus_addr); + auth_desc->img_len = size - qat_uclo_simg_hdr2sign_len(handle); + if (bus_addr + auth_desc->img_len > + dram_desc->dram_bus_addr + ICP_QAT_CSS_RSA4K_MAX_IMAGE_LEN) { pr_err("insufficient memory size for authentication data\n"); - qat_uclo_simg_free(handle, &img_desc); + qat_uclo_simg_free(handle, dram_desc); return -ENOMEM; } - memcpy((void *)(uintptr_t)virt_addr, - (void *)(image + ICP_QAT_AE_IMG_OFFSET(handle)), - auth_desc->img_len); + memcpy(virt_addr, image + qat_uclo_simg_hdr2sign_len(handle), auth_desc->img_len); virt_addr = virt_base; /* AE firmware */ - if (((struct icp_qat_css_hdr *)(uintptr_t)virt_addr)->fw_type == - CSS_AE_FIRMWARE) { + if (fw_type == CSS_AE_FIRMWARE) { auth_desc->img_ae_mode_data_high = auth_desc->img_high; auth_desc->img_ae_mode_data_low = auth_desc->img_low; bus_addr = ADD_ADDR(auth_desc->img_ae_mode_data_high, auth_desc->img_ae_mode_data_low) + sizeof(struct icp_qat_simg_ae_mode); - auth_desc->img_ae_init_data_high = - (unsigned int)(bus_addr >> BITS_PER_TYPE(u32)); - auth_desc->img_ae_init_data_low = (unsigned int)bus_addr; + auth_desc->img_ae_init_data_high = upper_32_bits(bus_addr); + auth_desc->img_ae_init_data_low = lower_32_bits(bus_addr); bus_addr += ICP_QAT_SIMG_AE_INIT_SEQ_LEN; - auth_desc->img_ae_insts_high = - (unsigned int)(bus_addr >> BITS_PER_TYPE(u32)); - auth_desc->img_ae_insts_low = (unsigned int)bus_addr; + auth_desc->img_ae_insts_high = upper_32_bits(bus_addr); + auth_desc->img_ae_insts_low = lower_32_bits(bus_addr); virt_addr += sizeof(struct icp_qat_css_hdr); virt_addr += ICP_QAT_CSS_FWSK_PUB_LEN(handle); virt_addr += ICP_QAT_CSS_SIGNATURE_LEN(handle); @@ -1523,6 +1515,28 @@ static int qat_uclo_map_auth_fw(struct icp_qat_fw_loader_handle *handle, return 0; } +static int qat_uclo_map_auth_fw(struct icp_qat_fw_loader_handle *handle, + char *image, unsigned int size, + struct icp_qat_fw_auth_desc **desc) +{ + struct icp_qat_auth_chunk *auth_chunk; + struct icp_firml_dram_desc img_desc; + unsigned int simg_fw_type; + int ret; + + ret = qat_uclo_simg_alloc(handle, &img_desc, ICP_QAT_CSS_RSA4K_MAX_IMAGE_LEN); + if (ret) + return ret; + + simg_fw_type = qat_uclo_simg_fw_type(handle, image); + auth_chunk = img_desc.dram_base_addr_v; + auth_chunk->chunk_size = img_desc.dram_size; + auth_chunk->chunk_bus_addr = img_desc.dram_bus_addr; + + return qat_uclo_build_auth_desc_RSA(handle, image, size, &img_desc, + simg_fw_type, desc); +} + static int qat_uclo_load_fw(struct icp_qat_fw_loader_handle *handle, struct icp_qat_fw_auth_desc *desc) { From e7b73261096749d5fa3320528f86daecd08a0556 Mon Sep 17 00:00:00 2001 From: Jack Xu Date: Wed, 30 Apr 2025 12:34:47 +0100 Subject: [PATCH 283/382] crypto: qat - add GEN6 firmware loader Add support for the QAT GEN6 devices in the firmware loader. This includes handling firmware images signed with the RSA 3K and the XMSS algorithms. Co-developed-by: Suman Kumar Chakraborty Signed-off-by: Suman Kumar Chakraborty Signed-off-by: Jack Xu Signed-off-by: Herbert Xu --- .../intel/qat/qat_common/adf_accel_devices.h | 2 + .../qat/qat_common/icp_qat_fw_loader_handle.h | 1 + .../intel/qat/qat_common/icp_qat_uclo.h | 23 +++ drivers/crypto/intel/qat/qat_common/qat_hal.c | 3 + .../crypto/intel/qat/qat_common/qat_uclo.c | 154 +++++++++++++++++- 5 files changed, 176 insertions(+), 7 deletions(-) diff --git a/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h b/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h index a39f506322f6..ed8b85360573 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h @@ -34,6 +34,8 @@ #define PCI_DEVICE_ID_INTEL_QAT_402XXIOV 0x4945 #define PCI_DEVICE_ID_INTEL_QAT_420XX 0x4946 #define PCI_DEVICE_ID_INTEL_QAT_420XXIOV 0x4947 +#define PCI_DEVICE_ID_INTEL_QAT_6XXX 0x4948 + #define ADF_DEVICE_FUSECTL_OFFSET 0x40 #define ADF_DEVICE_LEGFUSE_OFFSET 0x4C #define ADF_DEVICE_FUSECTL_MASK 0x80000000 diff --git a/drivers/crypto/intel/qat/qat_common/icp_qat_fw_loader_handle.h b/drivers/crypto/intel/qat/qat_common/icp_qat_fw_loader_handle.h index 7eb5daef4f88..6887930c7995 100644 --- a/drivers/crypto/intel/qat/qat_common/icp_qat_fw_loader_handle.h +++ b/drivers/crypto/intel/qat/qat_common/icp_qat_fw_loader_handle.h @@ -35,6 +35,7 @@ struct icp_qat_fw_loader_chip_info { u32 wakeup_event_val; bool fw_auth; bool css_3k; + bool dual_sign; bool tgroup_share_ustore; u32 fcu_ctl_csr; u32 fcu_sts_csr; diff --git a/drivers/crypto/intel/qat/qat_common/icp_qat_uclo.h b/drivers/crypto/intel/qat/qat_common/icp_qat_uclo.h index 1c7bcd8e4055..6313c35eff0c 100644 --- a/drivers/crypto/intel/qat/qat_common/icp_qat_uclo.h +++ b/drivers/crypto/intel/qat/qat_common/icp_qat_uclo.h @@ -7,6 +7,7 @@ #define ICP_QAT_AC_C62X_DEV_TYPE 0x01000000 #define ICP_QAT_AC_C3XXX_DEV_TYPE 0x02000000 #define ICP_QAT_AC_4XXX_A_DEV_TYPE 0x08000000 +#define ICP_QAT_AC_6XXX_DEV_TYPE 0x80000000 #define ICP_QAT_UCLO_MAX_AE 17 #define ICP_QAT_UCLO_MAX_CTX 8 #define ICP_QAT_UCLO_MAX_UIMAGE (ICP_QAT_UCLO_MAX_AE * ICP_QAT_UCLO_MAX_CTX) @@ -81,6 +82,21 @@ #define ICP_QAT_CSS_RSA4K_MAX_IMAGE_LEN 0x40000 #define ICP_QAT_CSS_RSA3K_MAX_IMAGE_LEN 0x30000 +/* All lengths below are in bytes */ +#define ICP_QAT_DUALSIGN_OPAQUE_HDR_LEN 12 +#define ICP_QAT_DUALSIGN_OPAQUE_HDR_ALIGN_LEN 16 +#define ICP_QAT_DUALSIGN_OPAQUE_DATA_LEN 3540 +#define ICP_QAT_DUALSIGN_XMSS_PUBKEY_LEN 64 +#define ICP_QAT_DUALSIGN_XMSS_SIG_LEN 2692 +#define ICP_QAT_DUALSIGN_XMSS_SIG_ALIGN_LEN 2696 +#define ICP_QAT_DUALSIGN_MISC_INFO_LEN 16 +#define ICP_QAT_DUALSIGN_FW_TYPE_LEN 7 +#define ICP_QAT_DUALSIGN_MODULE_TYPE 0x14 +#define ICP_QAT_DUALSIGN_HDR_LEN 0x375 +#define ICP_QAT_DUALSIGN_HDR_VER 0x40001 +#define ICP_QAT_DUALSIGN_HDR_LEN_OFFSET 4 +#define ICP_QAT_DUALSIGN_HDR_VER_OFFSET 8 + #define ICP_QAT_CTX_MODE(ae_mode) ((ae_mode) & 0xf) #define ICP_QAT_NN_MODE(ae_mode) (((ae_mode) >> 0x4) & 0xf) #define ICP_QAT_SHARED_USTORE_MODE(ae_mode) (((ae_mode) >> 0xb) & 0x1) @@ -440,6 +456,13 @@ struct icp_qat_fw_auth_desc { unsigned int img_ae_init_data_low; unsigned int img_ae_insts_high; unsigned int img_ae_insts_low; + unsigned int cpp_mask; + unsigned int reserved; + unsigned int xmss_pubkey_high; + unsigned int xmss_pubkey_low; + unsigned int xmss_sig_high; + unsigned int xmss_sig_low; + unsigned int reserved2[2]; }; struct icp_qat_auth_chunk { diff --git a/drivers/crypto/intel/qat/qat_common/qat_hal.c b/drivers/crypto/intel/qat/qat_common/qat_hal.c index 841c1d7d3ffe..da4eca6e1633 100644 --- a/drivers/crypto/intel/qat/qat_common/qat_hal.c +++ b/drivers/crypto/intel/qat/qat_common/qat_hal.c @@ -698,6 +698,7 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, case PCI_DEVICE_ID_INTEL_QAT_401XX: case PCI_DEVICE_ID_INTEL_QAT_402XX: case PCI_DEVICE_ID_INTEL_QAT_420XX: + case PCI_DEVICE_ID_INTEL_QAT_6XXX: handle->chip_info->mmp_sram_size = 0; handle->chip_info->nn = false; handle->chip_info->lm2lm3 = true; @@ -712,6 +713,8 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, handle->chip_info->wakeup_event_val = 0x80000000; handle->chip_info->fw_auth = true; handle->chip_info->css_3k = true; + if (handle->pci_dev->device == PCI_DEVICE_ID_INTEL_QAT_6XXX) + handle->chip_info->dual_sign = true; handle->chip_info->tgroup_share_ustore = true; handle->chip_info->fcu_ctl_csr = FCU_CONTROL_4XXX; handle->chip_info->fcu_sts_csr = FCU_STATUS_4XXX; diff --git a/drivers/crypto/intel/qat/qat_common/qat_uclo.c b/drivers/crypto/intel/qat/qat_common/qat_uclo.c index d7f2ceb81f1f..21d652a1c8ef 100644 --- a/drivers/crypto/intel/qat/qat_common/qat_uclo.c +++ b/drivers/crypto/intel/qat/qat_common/qat_uclo.c @@ -10,6 +10,7 @@ #include #include #include +#include #include "adf_accel_devices.h" #include "adf_common_drv.h" #include "icp_qat_uclo.h" @@ -737,6 +738,8 @@ qat_uclo_get_dev_type(struct icp_qat_fw_loader_handle *handle) case PCI_DEVICE_ID_INTEL_QAT_402XX: case PCI_DEVICE_ID_INTEL_QAT_420XX: return ICP_QAT_AC_4XXX_A_DEV_TYPE; + case PCI_DEVICE_ID_INTEL_QAT_6XXX: + return ICP_QAT_AC_6XXX_DEV_TYPE; default: pr_err("unsupported device 0x%x\n", handle->pci_dev->device); return 0; @@ -1035,17 +1038,30 @@ out_err: static unsigned int qat_uclo_simg_hdr2sign_len(struct icp_qat_fw_loader_handle *handle) { + if (handle->chip_info->dual_sign) + return ICP_QAT_DUALSIGN_OPAQUE_DATA_LEN; + return ICP_QAT_AE_IMG_OFFSET(handle); } static unsigned int qat_uclo_simg_hdr2cont_len(struct icp_qat_fw_loader_handle *handle) { + if (handle->chip_info->dual_sign) + return ICP_QAT_DUALSIGN_OPAQUE_DATA_LEN + ICP_QAT_DUALSIGN_MISC_INFO_LEN; + return ICP_QAT_AE_IMG_OFFSET(handle); } static unsigned int qat_uclo_simg_fw_type(struct icp_qat_fw_loader_handle *handle, void *img_ptr) { struct icp_qat_css_hdr *hdr = img_ptr; + char *fw_hdr = img_ptr; + unsigned int offset; + + if (handle->chip_info->dual_sign) { + offset = qat_uclo_simg_hdr2sign_len(handle) + ICP_QAT_DUALSIGN_FW_TYPE_LEN; + return *(fw_hdr + offset); + } return hdr->fw_type; } @@ -1390,16 +1406,27 @@ static int qat_uclo_check_image(struct icp_qat_fw_loader_handle *handle, if (handle->chip_info->fw_auth) { header_len = qat_uclo_simg_hdr2sign_len(handle); simg_type = qat_uclo_simg_fw_type(handle, image); - css_hdr = image; - if ((css_hdr->header_len * css_dword_size) != header_len) - goto err; - if ((css_hdr->size * css_dword_size) != size) - goto err; + + if (handle->chip_info->dual_sign) { + if (css_hdr->module_type != ICP_QAT_DUALSIGN_MODULE_TYPE) + goto err; + if (css_hdr->header_len != ICP_QAT_DUALSIGN_HDR_LEN) + goto err; + if (css_hdr->header_ver != ICP_QAT_DUALSIGN_HDR_VER) + goto err; + } else { + if (css_hdr->header_len * css_dword_size != header_len) + goto err; + if (css_hdr->size * css_dword_size != size) + goto err; + if (size <= header_len) + goto err; + } + if (fw_type != simg_type) goto err; - if (size <= header_len) - goto err; + size -= header_len; } @@ -1515,6 +1542,115 @@ static int qat_uclo_build_auth_desc_RSA(struct icp_qat_fw_loader_handle *handle, return 0; } +static int qat_uclo_build_auth_desc_dualsign(struct icp_qat_fw_loader_handle *handle, + char *image, unsigned int size, + struct icp_firml_dram_desc *dram_desc, + unsigned int fw_type, + struct icp_qat_fw_auth_desc **desc) +{ + struct icp_qat_simg_ae_mode *simg_ae_mode; + struct icp_qat_fw_auth_desc *auth_desc; + unsigned int chunk_offset, img_offset; + u64 bus_addr, addr; + char *virt_addr; + + virt_addr = dram_desc->dram_base_addr_v; + virt_addr += sizeof(struct icp_qat_auth_chunk); + bus_addr = dram_desc->dram_bus_addr + sizeof(struct icp_qat_auth_chunk); + + auth_desc = dram_desc->dram_base_addr_v; + auth_desc->img_len = size - qat_uclo_simg_hdr2sign_len(handle); + auth_desc->css_hdr_high = upper_32_bits(bus_addr); + auth_desc->css_hdr_low = lower_32_bits(bus_addr); + memcpy(virt_addr, image, ICP_QAT_DUALSIGN_OPAQUE_HDR_LEN); + + img_offset = ICP_QAT_DUALSIGN_OPAQUE_HDR_LEN; + chunk_offset = ICP_QAT_DUALSIGN_OPAQUE_HDR_ALIGN_LEN; + + /* RSA pub key */ + addr = bus_addr + chunk_offset; + auth_desc->fwsk_pub_high = upper_32_bits(addr); + auth_desc->fwsk_pub_low = lower_32_bits(addr); + memcpy(virt_addr + chunk_offset, image + img_offset, ICP_QAT_CSS_FWSK_MODULUS_LEN(handle)); + + img_offset += ICP_QAT_CSS_FWSK_MODULUS_LEN(handle); + chunk_offset += ICP_QAT_CSS_FWSK_MODULUS_LEN(handle); + /* RSA padding */ + memset(virt_addr + chunk_offset, 0, ICP_QAT_CSS_FWSK_PAD_LEN(handle)); + + chunk_offset += ICP_QAT_CSS_FWSK_PAD_LEN(handle); + /* RSA exponent */ + memcpy(virt_addr + chunk_offset, image + img_offset, ICP_QAT_CSS_FWSK_EXPONENT_LEN(handle)); + + img_offset += ICP_QAT_CSS_FWSK_EXPONENT_LEN(handle); + chunk_offset += ICP_QAT_CSS_FWSK_EXPONENT_LEN(handle); + /* RSA signature */ + addr = bus_addr + chunk_offset; + auth_desc->signature_high = upper_32_bits(addr); + auth_desc->signature_low = lower_32_bits(addr); + memcpy(virt_addr + chunk_offset, image + img_offset, ICP_QAT_CSS_SIGNATURE_LEN(handle)); + + img_offset += ICP_QAT_CSS_SIGNATURE_LEN(handle); + chunk_offset += ICP_QAT_CSS_SIGNATURE_LEN(handle); + /* XMSS pubkey */ + addr = bus_addr + chunk_offset; + auth_desc->xmss_pubkey_high = upper_32_bits(addr); + auth_desc->xmss_pubkey_low = lower_32_bits(addr); + memcpy(virt_addr + chunk_offset, image + img_offset, ICP_QAT_DUALSIGN_XMSS_PUBKEY_LEN); + + img_offset += ICP_QAT_DUALSIGN_XMSS_PUBKEY_LEN; + chunk_offset += ICP_QAT_DUALSIGN_XMSS_PUBKEY_LEN; + /* XMSS signature */ + addr = bus_addr + chunk_offset; + auth_desc->xmss_sig_high = upper_32_bits(addr); + auth_desc->xmss_sig_low = lower_32_bits(addr); + memcpy(virt_addr + chunk_offset, image + img_offset, ICP_QAT_DUALSIGN_XMSS_SIG_LEN); + + img_offset += ICP_QAT_DUALSIGN_XMSS_SIG_LEN; + chunk_offset += ICP_QAT_DUALSIGN_XMSS_SIG_ALIGN_LEN; + + if (dram_desc->dram_size < (chunk_offset + auth_desc->img_len)) { + pr_err("auth chunk memory size is not enough to store data\n"); + return -ENOMEM; + } + + /* Signed data */ + addr = bus_addr + chunk_offset; + auth_desc->img_high = upper_32_bits(addr); + auth_desc->img_low = lower_32_bits(addr); + memcpy(virt_addr + chunk_offset, image + img_offset, auth_desc->img_len); + + chunk_offset += ICP_QAT_DUALSIGN_MISC_INFO_LEN; + /* AE firmware */ + if (fw_type == CSS_AE_FIRMWARE) { + /* AE mode data */ + addr = bus_addr + chunk_offset; + auth_desc->img_ae_mode_data_high = upper_32_bits(addr); + auth_desc->img_ae_mode_data_low = lower_32_bits(addr); + simg_ae_mode = + (struct icp_qat_simg_ae_mode *)(virt_addr + chunk_offset); + auth_desc->ae_mask = simg_ae_mode->ae_mask & handle->cfg_ae_mask; + + chunk_offset += sizeof(struct icp_qat_simg_ae_mode); + /* AE init seq */ + addr = bus_addr + chunk_offset; + auth_desc->img_ae_init_data_high = upper_32_bits(addr); + auth_desc->img_ae_init_data_low = lower_32_bits(addr); + + chunk_offset += ICP_QAT_SIMG_AE_INIT_SEQ_LEN; + /* AE instructions */ + addr = bus_addr + chunk_offset; + auth_desc->img_ae_insts_high = upper_32_bits(addr); + auth_desc->img_ae_insts_low = lower_32_bits(addr); + } else { + addr = bus_addr + chunk_offset; + auth_desc->img_ae_insts_high = upper_32_bits(addr); + auth_desc->img_ae_insts_low = lower_32_bits(addr); + } + *desc = auth_desc; + return 0; +} + static int qat_uclo_map_auth_fw(struct icp_qat_fw_loader_handle *handle, char *image, unsigned int size, struct icp_qat_fw_auth_desc **desc) @@ -1533,6 +1669,10 @@ static int qat_uclo_map_auth_fw(struct icp_qat_fw_loader_handle *handle, auth_chunk->chunk_size = img_desc.dram_size; auth_chunk->chunk_bus_addr = img_desc.dram_bus_addr; + if (handle->chip_info->dual_sign) + return qat_uclo_build_auth_desc_dualsign(handle, image, size, &img_desc, + simg_fw_type, desc); + return qat_uclo_build_auth_desc_RSA(handle, image, size, &img_desc, simg_fw_type, desc); } From 61e152873adb00c32f6d87c04ab3b8ac6e14495f Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Wed, 30 Apr 2025 12:34:48 +0100 Subject: [PATCH 284/382] crypto: qat - export adf_get_service_mask() Export the function adf_get_service_mask() as it will be used by the qat_6xxx driver to configure the device. Signed-off-by: Giovanni Cabiddu Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- drivers/crypto/intel/qat/qat_common/adf_cfg_services.c | 3 ++- drivers/crypto/intel/qat/qat_common/adf_cfg_services.h | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/crypto/intel/qat/qat_common/adf_cfg_services.c b/drivers/crypto/intel/qat/qat_common/adf_cfg_services.c index 30abcd9e1283..c39871291da7 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_cfg_services.c +++ b/drivers/crypto/intel/qat/qat_common/adf_cfg_services.c @@ -116,7 +116,7 @@ int adf_parse_service_string(struct adf_accel_dev *accel_dev, const char *in, return adf_service_mask_to_string(mask, out, out_len); } -static int adf_get_service_mask(struct adf_accel_dev *accel_dev, unsigned long *mask) +int adf_get_service_mask(struct adf_accel_dev *accel_dev, unsigned long *mask) { char services[ADF_CFG_MAX_VAL_LEN_IN_BYTES] = { }; size_t len; @@ -138,6 +138,7 @@ static int adf_get_service_mask(struct adf_accel_dev *accel_dev, unsigned long * return ret; } +EXPORT_SYMBOL_GPL(adf_get_service_mask); int adf_get_service_enabled(struct adf_accel_dev *accel_dev) { diff --git a/drivers/crypto/intel/qat/qat_common/adf_cfg_services.h b/drivers/crypto/intel/qat/qat_common/adf_cfg_services.h index f6bafc15cbc6..3742c450878f 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_cfg_services.h +++ b/drivers/crypto/intel/qat/qat_common/adf_cfg_services.h @@ -32,5 +32,6 @@ enum { int adf_parse_service_string(struct adf_accel_dev *accel_dev, const char *in, size_t in_len, char *out, size_t out_len); int adf_get_service_enabled(struct adf_accel_dev *accel_dev); +int adf_get_service_mask(struct adf_accel_dev *accel_dev, unsigned long *mask); #endif From 7db55726450af0373b128e944f263b848eaa7dc2 Mon Sep 17 00:00:00 2001 From: Suman Kumar Chakraborty Date: Wed, 30 Apr 2025 12:34:49 +0100 Subject: [PATCH 285/382] crypto: qat - expose configuration functions The functions related to compression and crypto configurations were previously declared static, restricting the visibility to the defining source file. Remove the static qualifier, allowing it to be used in other files as needed. This is necessary for sharing this configuration functions with other QAT generations. Signed-off-by: Suman Kumar Chakraborty Reviewed-by: Giovanni Cabiddu Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- drivers/crypto/intel/qat/qat_common/adf_gen4_config.c | 6 +++--- drivers/crypto/intel/qat/qat_common/adf_gen4_config.h | 3 +++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen4_config.c b/drivers/crypto/intel/qat/qat_common/adf_gen4_config.c index f97e7a880f3a..afcdfdd0a37a 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_gen4_config.c +++ b/drivers/crypto/intel/qat/qat_common/adf_gen4_config.c @@ -11,7 +11,7 @@ #include "qat_compression.h" #include "qat_crypto.h" -static int adf_crypto_dev_config(struct adf_accel_dev *accel_dev) +int adf_crypto_dev_config(struct adf_accel_dev *accel_dev) { char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES]; int banks = GET_MAX_BANKS(accel_dev); @@ -117,7 +117,7 @@ err: return ret; } -static int adf_comp_dev_config(struct adf_accel_dev *accel_dev) +int adf_comp_dev_config(struct adf_accel_dev *accel_dev) { char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES]; int banks = GET_MAX_BANKS(accel_dev); @@ -187,7 +187,7 @@ err: return ret; } -static int adf_no_dev_config(struct adf_accel_dev *accel_dev) +int adf_no_dev_config(struct adf_accel_dev *accel_dev) { unsigned long val; int ret; diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen4_config.h b/drivers/crypto/intel/qat/qat_common/adf_gen4_config.h index bb87655f69a8..38a674c27e40 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_gen4_config.h +++ b/drivers/crypto/intel/qat/qat_common/adf_gen4_config.h @@ -7,5 +7,8 @@ int adf_gen4_dev_config(struct adf_accel_dev *accel_dev); int adf_gen4_cfg_dev_init(struct adf_accel_dev *accel_dev); +int adf_crypto_dev_config(struct adf_accel_dev *accel_dev); +int adf_comp_dev_config(struct adf_accel_dev *accel_dev); +int adf_no_dev_config(struct adf_accel_dev *accel_dev); #endif From 0fdc836a371deedc9b604016486e692d49373362 Mon Sep 17 00:00:00 2001 From: Suman Kumar Chakraborty Date: Wed, 30 Apr 2025 12:34:50 +0100 Subject: [PATCH 286/382] crypto: qat - export adf_init_admin_pm() Export the function adf_init_admin_pm() as it will be used by the qat_6xxx driver to send the power management initialization messages to the firmware. Signed-off-by: Suman Kumar Chakraborty Reviewed-by: Giovanni Cabiddu Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- drivers/crypto/intel/qat/qat_common/adf_admin.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/crypto/intel/qat/qat_common/adf_admin.c b/drivers/crypto/intel/qat/qat_common/adf_admin.c index acad526eb741..573388c37100 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_admin.c +++ b/drivers/crypto/intel/qat/qat_common/adf_admin.c @@ -449,6 +449,7 @@ int adf_init_admin_pm(struct adf_accel_dev *accel_dev, u32 idle_delay) return adf_send_admin(accel_dev, &req, &resp, ae_mask); } +EXPORT_SYMBOL_GPL(adf_init_admin_pm); int adf_get_pm_info(struct adf_accel_dev *accel_dev, dma_addr_t p_state_addr, size_t buff_size) From f14a2de55ad9cea49391bb42fc2d113d6cac16a4 Mon Sep 17 00:00:00 2001 From: Suman Kumar Chakraborty Date: Wed, 30 Apr 2025 12:34:51 +0100 Subject: [PATCH 287/382] crypto: qat - update firmware api Update the firmware API to have partial decomp as an argument. Modify the firmware descriptor to support auto-select best and partial decompress. Define the maximal auto-select best value. Define the mask and bit position for the partial decompress field in the firmware descriptor. Signed-off-by: Suman Kumar Chakraborty Reviewed-by: Giovanni Cabiddu Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- drivers/crypto/intel/qat/qat_common/adf_dc.c | 3 ++- .../intel/qat/qat_common/icp_qat_fw_comp.h | 23 ++++++++++++++++--- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/drivers/crypto/intel/qat/qat_common/adf_dc.c b/drivers/crypto/intel/qat/qat_common/adf_dc.c index 4beb4b7dbf0e..3e8fb4e3ed97 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_dc.c +++ b/drivers/crypto/intel/qat/qat_common/adf_dc.c @@ -46,7 +46,8 @@ int qat_comp_build_ctx(struct adf_accel_dev *accel_dev, void *ctx, enum adf_dc_a ICP_QAT_FW_COMP_NO_XXHASH_ACC, ICP_QAT_FW_COMP_CNV_ERROR_NONE, ICP_QAT_FW_COMP_NO_APPEND_CRC, - ICP_QAT_FW_COMP_NO_DROP_DATA); + ICP_QAT_FW_COMP_NO_DROP_DATA, + ICP_QAT_FW_COMP_NO_PARTIAL_DECOMPRESS); ICP_QAT_FW_COMN_NEXT_ID_SET(comp_cd_ctrl, ICP_QAT_FW_SLICE_DRAM_WR); ICP_QAT_FW_COMN_CURR_ID_SET(comp_cd_ctrl, ICP_QAT_FW_SLICE_COMP); diff --git a/drivers/crypto/intel/qat/qat_common/icp_qat_fw_comp.h b/drivers/crypto/intel/qat/qat_common/icp_qat_fw_comp.h index 04f645957e28..81969c515a17 100644 --- a/drivers/crypto/intel/qat/qat_common/icp_qat_fw_comp.h +++ b/drivers/crypto/intel/qat/qat_common/icp_qat_fw_comp.h @@ -44,6 +44,7 @@ enum icp_qat_fw_comp_20_cmd_id { #define ICP_QAT_FW_COMP_RET_DISABLE_TYPE0_HEADER_DATA_MASK 0x1 #define ICP_QAT_FW_COMP_DISABLE_SECURE_RAM_AS_INTMD_BUF_BITPOS 7 #define ICP_QAT_FW_COMP_DISABLE_SECURE_RAM_AS_INTMD_BUF_MASK 0x1 +#define ICP_QAT_FW_COMP_AUTO_SELECT_BEST_MAX_VALUE 0xFFFFFFFF #define ICP_QAT_FW_COMP_FLAGS_BUILD(sesstype, autoselect, enhanced_asb, \ ret_uncomp, secure_ram) \ @@ -117,7 +118,7 @@ struct icp_qat_fw_comp_req_params { #define ICP_QAT_FW_COMP_REQ_PARAM_FLAGS_BUILD(sop, eop, bfinal, cnv, cnvnr, \ cnvdfx, crc, xxhash_acc, \ cnv_error_type, append_crc, \ - drop_data) \ + drop_data, partial_decomp) \ ((((sop) & ICP_QAT_FW_COMP_SOP_MASK) << \ ICP_QAT_FW_COMP_SOP_BITPOS) | \ (((eop) & ICP_QAT_FW_COMP_EOP_MASK) << \ @@ -139,7 +140,9 @@ struct icp_qat_fw_comp_req_params { (((append_crc) & ICP_QAT_FW_COMP_APPEND_CRC_MASK) \ << ICP_QAT_FW_COMP_APPEND_CRC_BITPOS) | \ (((drop_data) & ICP_QAT_FW_COMP_DROP_DATA_MASK) \ - << ICP_QAT_FW_COMP_DROP_DATA_BITPOS)) + << ICP_QAT_FW_COMP_DROP_DATA_BITPOS) | \ + (((partial_decomp) & ICP_QAT_FW_COMP_PARTIAL_DECOMP_MASK) \ + << ICP_QAT_FW_COMP_PARTIAL_DECOMP_BITPOS)) #define ICP_QAT_FW_COMP_NOT_SOP 0 #define ICP_QAT_FW_COMP_SOP 1 @@ -161,6 +164,8 @@ struct icp_qat_fw_comp_req_params { #define ICP_QAT_FW_COMP_NO_APPEND_CRC 0 #define ICP_QAT_FW_COMP_DROP_DATA 1 #define ICP_QAT_FW_COMP_NO_DROP_DATA 0 +#define ICP_QAT_FW_COMP_PARTIAL_DECOMPRESS 1 +#define ICP_QAT_FW_COMP_NO_PARTIAL_DECOMPRESS 0 #define ICP_QAT_FW_COMP_SOP_BITPOS 0 #define ICP_QAT_FW_COMP_SOP_MASK 0x1 #define ICP_QAT_FW_COMP_EOP_BITPOS 1 @@ -189,6 +194,8 @@ struct icp_qat_fw_comp_req_params { #define ICP_QAT_FW_COMP_APPEND_CRC_MASK 0x1 #define ICP_QAT_FW_COMP_DROP_DATA_BITPOS 25 #define ICP_QAT_FW_COMP_DROP_DATA_MASK 0x1 +#define ICP_QAT_FW_COMP_PARTIAL_DECOMP_BITPOS 27 +#define ICP_QAT_FW_COMP_PARTIAL_DECOMP_MASK 0x1 #define ICP_QAT_FW_COMP_SOP_GET(flags) \ QAT_FIELD_GET(flags, ICP_QAT_FW_COMP_SOP_BITPOS, \ @@ -281,8 +288,18 @@ struct icp_qat_fw_comp_req { union { struct icp_qat_fw_xlt_req_params xlt_pars; __u32 resrvd1[ICP_QAT_FW_NUM_LONGWORDS_2]; + struct { + __u32 partial_decompress_length; + __u32 partial_decompress_offset; + } partial_decompress; } u1; - __u32 resrvd2[ICP_QAT_FW_NUM_LONGWORDS_2]; + union { + __u32 resrvd2[ICP_QAT_FW_NUM_LONGWORDS_2]; + struct { + __u32 asb_value; + __u32 reserved; + } asb_threshold; + } u3; struct icp_qat_fw_comp_cd_hdr comp_cd_ctrl; union { struct icp_qat_fw_xlt_cd_hdr xlt_cd_ctrl; From 942028bc2fd9f35b1d54be21c06d7de5c40cf6b5 Mon Sep 17 00:00:00 2001 From: Suman Kumar Chakraborty Date: Wed, 30 Apr 2025 12:34:52 +0100 Subject: [PATCH 288/382] crypto: qat - add firmware headers for GEN6 devices Add firmware headers related to compression that define macros for building the hardware configuration word, along with bitfields related to algorithm settings. Signed-off-by: Suman Kumar Chakraborty Reviewed-by: Giovanni Cabiddu Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- .../intel/qat/qat_common/icp_qat_hw_51_comp.h | 99 ++++++ .../qat/qat_common/icp_qat_hw_51_comp_defs.h | 318 ++++++++++++++++++ 2 files changed, 417 insertions(+) create mode 100644 drivers/crypto/intel/qat/qat_common/icp_qat_hw_51_comp.h create mode 100644 drivers/crypto/intel/qat/qat_common/icp_qat_hw_51_comp_defs.h diff --git a/drivers/crypto/intel/qat/qat_common/icp_qat_hw_51_comp.h b/drivers/crypto/intel/qat/qat_common/icp_qat_hw_51_comp.h new file mode 100644 index 000000000000..dce639152345 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/icp_qat_hw_51_comp.h @@ -0,0 +1,99 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2025 Intel Corporation */ +#ifndef ICP_QAT_HW_51_COMP_H_ +#define ICP_QAT_HW_51_COMP_H_ + +#include + +#include "icp_qat_fw.h" +#include "icp_qat_hw_51_comp_defs.h" + +struct icp_qat_hw_comp_51_config_csr_lower { + enum icp_qat_hw_comp_51_abd abd; + enum icp_qat_hw_comp_51_lllbd_ctrl lllbd; + enum icp_qat_hw_comp_51_search_depth sd; + enum icp_qat_hw_comp_51_min_match_control mmctrl; + enum icp_qat_hw_comp_51_lz4_block_checksum lbc; +}; + +static inline u32 +ICP_QAT_FW_COMP_51_BUILD_CONFIG_LOWER(struct icp_qat_hw_comp_51_config_csr_lower csr) +{ + u32 val32 = 0; + + QAT_FIELD_SET(val32, csr.abd, + ICP_QAT_HW_COMP_51_CONFIG_CSR_ABD_BITPOS, + ICP_QAT_HW_COMP_51_CONFIG_CSR_ABD_MASK); + QAT_FIELD_SET(val32, csr.lllbd, + ICP_QAT_HW_COMP_51_CONFIG_CSR_LLLBD_CTRL_BITPOS, + ICP_QAT_HW_COMP_51_CONFIG_CSR_LLLBD_CTRL_MASK); + QAT_FIELD_SET(val32, csr.sd, + ICP_QAT_HW_COMP_51_CONFIG_CSR_SEARCH_DEPTH_BITPOS, + ICP_QAT_HW_COMP_51_CONFIG_CSR_SEARCH_DEPTH_MASK); + QAT_FIELD_SET(val32, csr.mmctrl, + ICP_QAT_HW_COMP_51_CONFIG_CSR_MIN_MATCH_CONTROL_BITPOS, + ICP_QAT_HW_COMP_51_CONFIG_CSR_MIN_MATCH_CONTROL_MASK); + QAT_FIELD_SET(val32, csr.lbc, + ICP_QAT_HW_COMP_51_CONFIG_CSR_LZ4_BLOCK_CHECKSUM_BITPOS, + ICP_QAT_HW_COMP_51_CONFIG_CSR_LZ4_BLOCK_CHECKSUM_MASK); + + return val32; +} + +struct icp_qat_hw_comp_51_config_csr_upper { + enum icp_qat_hw_comp_51_dmm_algorithm edmm; + enum icp_qat_hw_comp_51_bms bms; + enum icp_qat_hw_comp_51_scb_mode_reset_mask scb_mode_reset; +}; + +static inline u32 +ICP_QAT_FW_COMP_51_BUILD_CONFIG_UPPER(struct icp_qat_hw_comp_51_config_csr_upper csr) +{ + u32 val32 = 0; + + QAT_FIELD_SET(val32, csr.edmm, + ICP_QAT_HW_COMP_51_CONFIG_CSR_DMM_ALGORITHM_BITPOS, + ICP_QAT_HW_COMP_51_CONFIG_CSR_DMM_ALGORITHM_MASK); + QAT_FIELD_SET(val32, csr.bms, + ICP_QAT_HW_COMP_51_CONFIG_CSR_BMS_BITPOS, + ICP_QAT_HW_COMP_51_CONFIG_CSR_BMS_MASK); + QAT_FIELD_SET(val32, csr.scb_mode_reset, + ICP_QAT_HW_COMP_51_CONFIG_CSR_SCB_MODE_RESET_MASK_BITPOS, + ICP_QAT_HW_COMP_51_CONFIG_CSR_SCB_MODE_RESET_MASK_MASK); + + return val32; +} + +struct icp_qat_hw_decomp_51_config_csr_lower { + enum icp_qat_hw_decomp_51_lz4_block_checksum lbc; +}; + +static inline u32 +ICP_QAT_FW_DECOMP_51_BUILD_CONFIG_LOWER(struct icp_qat_hw_decomp_51_config_csr_lower csr) +{ + u32 val32 = 0; + + QAT_FIELD_SET(val32, csr.lbc, + ICP_QAT_HW_DECOMP_51_CONFIG_CSR_LZ4_BLOCK_CHECKSUM_BITPOS, + ICP_QAT_HW_DECOMP_51_CONFIG_CSR_LZ4_BLOCK_CHECKSUM_MASK); + + return val32; +} + +struct icp_qat_hw_decomp_51_config_csr_upper { + enum icp_qat_hw_decomp_51_bms bms; +}; + +static inline u32 +ICP_QAT_FW_DECOMP_51_BUILD_CONFIG_UPPER(struct icp_qat_hw_decomp_51_config_csr_upper csr) +{ + u32 val32 = 0; + + QAT_FIELD_SET(val32, csr.bms, + ICP_QAT_HW_DECOMP_51_CONFIG_CSR_BMS_BITPOS, + ICP_QAT_HW_DECOMP_51_CONFIG_CSR_BMS_MASK); + + return val32; +} + +#endif /* ICP_QAT_HW_51_COMP_H_ */ diff --git a/drivers/crypto/intel/qat/qat_common/icp_qat_hw_51_comp_defs.h b/drivers/crypto/intel/qat/qat_common/icp_qat_hw_51_comp_defs.h new file mode 100644 index 000000000000..e745688c5da4 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/icp_qat_hw_51_comp_defs.h @@ -0,0 +1,318 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2025 Intel Corporation */ +#ifndef ICP_QAT_HW_51_COMP_DEFS_H_ +#define ICP_QAT_HW_51_COMP_DEFS_H_ + +#include + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_SOM_CONTROL_BITPOS 28 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_SOM_CONTROL_MASK GENMASK(1, 0) +enum icp_qat_hw_comp_51_som_control { + ICP_QAT_HW_COMP_51_SOM_CONTROL_NORMAL_MODE = 0x0, + ICP_QAT_HW_COMP_51_SOM_CONTROL_DICTIONARY_MODE = 0x1, + ICP_QAT_HW_COMP_51_SOM_CONTROL_INPUT_CRC = 0x2, + ICP_QAT_HW_COMP_51_SOM_CONTROL_RESERVED_MODE = 0x3, +}; + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_SOM_CONTROL_DEFAULT_VAL \ + ICP_QAT_HW_COMP_51_SOM_CONTROL_NORMAL_MODE +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_SKIP_HASH_RD_CONTROL_BITPOS 27 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_SKIP_HASH_RD_CONTROL_MASK GENMASK(0, 0) +enum icp_qat_hw_comp_51_skip_hash_rd_control { + ICP_QAT_HW_COMP_51_SKIP_HASH_RD_CONTROL_NO_SKIP = 0x0, + ICP_QAT_HW_COMP_51_SKIP_HASH_RD_CONTROL_SKIP_HASH_READS = 0x1, +}; + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_SKIP_HASH_RD_CONTROL_DEFAULT_VAL \ + ICP_QAT_HW_COMP_51_SKIP_HASH_RD_CONTROL_NO_SKIP +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_BYPASS_COMPRESSION_BITPOS 25 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_BYPASS_COMPRESSION_MASK GENMASK(0, 0) +enum icp_qat_hw_comp_51_bypass_compression { + ICP_QAT_HW_COMP_51_BYPASS_COMPRESSION_DISABLED = 0x0, + ICP_QAT_HW_COMP_51_BYPASS_COMPRESSION_ENABLED = 0x1, +}; + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_BYPASS_COMPRESSION_DEFAULT_VAL \ + ICP_QAT_HW_COMP_51_BYPASS_COMPRESSION_DISABLED +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_DMM_ALGORITHM_BITPOS 22 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_DMM_ALGORITHM_MASK GENMASK(0, 0) +enum icp_qat_hw_comp_51_dmm_algorithm { + ICP_QAT_HW_COMP_51_DMM_ALGORITHM_EDMM_ENABLED = 0x0, + ICP_QAT_HW_COMP_51_DMM_ALGORITHM_ZSTD_DMM_LITE = 0x1, +}; + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_DMM_ALGORITHM_DEFAULT_VAL \ + ICP_QAT_HW_COMP_51_DMM_ALGORITHM_EDMM_ENABLED +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_TOKEN_FUSION_INTERNAL_ONLY_BITPOS 21 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_TOKEN_FUSION_INTERNAL_ONLY_MASK GENMASK(0, 0) +enum icp_qat_hw_comp_51_token_fusion_internal_only { + ICP_QAT_HW_COMP_51_TOKEN_FUSION_INTERNAL_ONLY_ENABLED = 0x0, + ICP_QAT_HW_COMP_51_TOKEN_FUSION_INTERNAL_ONLY_DISABLED = 0x1, +}; + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_TOKEN_FUSION_INTERNAL_ONLY_DEFAULT_VAL \ + ICP_QAT_HW_COMP_51_TOKEN_FUSION_INTERNAL_ONLY_ENABLED +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_BMS_BITPOS 19 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_BMS_MASK GENMASK(1, 0) +enum icp_qat_hw_comp_51_bms { + ICP_QAT_HW_COMP_51_BMS_BMS_64KB = 0x0, + ICP_QAT_HW_COMP_51_BMS_BMS_256KB = 0x1, + ICP_QAT_HW_COMP_51_BMS_BMS_1MB = 0x2, + ICP_QAT_HW_COMP_51_BMS_BMS_4MB = 0x3, +}; + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_BMS_DEFAULT_VAL \ + ICP_QAT_HW_COMP_51_BMS_BMS_64KB +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_SCB_MODE_RESET_MASK_BITPOS 18 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_SCB_MODE_RESET_MASK_MASK GENMASK(0, 0) +enum icp_qat_hw_comp_51_scb_mode_reset_mask { + ICP_QAT_HW_COMP_51_SCB_MODE_RESET_MASK_DO_NOT_RESET_HB_HT = 0x0, + ICP_QAT_HW_COMP_51_SCB_MODE_RESET_MASK_RESET_HB_HT = 0x1, +}; + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_SCB_MODE_RESET_MASK_DEFAULT_VAL \ + ICP_QAT_HW_COMP_51_SCB_MODE_RESET_MASK_DO_NOT_RESET_HB_HT +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_ZSTD_FRAME_GEN_DEC_EN_BITPOS 2 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_ZSTD_FRAME_GEN_DEC_EN_MASK GENMASK(0, 0) +enum icp_qat_hw_comp_51_zstd_frame_gen_dec_en { + ICP_QAT_HW_COMP_51_ZSTD_FRAME_GEN_DEC_EN_ZSTD_FRAME_HDR_DISABLE = 0x0, + ICP_QAT_HW_COMP_51_ZSTD_FRAME_GEN_DEC_EN_ZSTD_FRAME_HDR_ENABLE = 0x1, +}; + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_ZSTD_FRAME_GEN_DEC_EN_DEFAULT_VAL \ + ICP_QAT_HW_COMP_51_ZSTD_FRAME_GEN_DEC_EN_ZSTD_FRAME_HDR_ENABLE +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_CNV_DISABLE_BITPOS 1 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_CNV_DISABLE_MASK GENMASK(0, 0) +enum icp_qat_hw_comp_51_cnv_disable { + ICP_QAT_HW_COMP_51_CNV_DISABLE_CNV_ENABLED = 0x0, + ICP_QAT_HW_COMP_51_CNV_DISABLE_CNV_DISABLED = 0x1, +}; + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_CNV_DISABLE_DEFAULT_VAL \ + ICP_QAT_HW_COMP_51_CNV_DISABLE_CNV_ENABLED +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_ASB_DISABLE_BITPOS 0 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_ASB_DISABLE_MASK GENMASK(0, 0) +enum icp_qat_hw_comp_51_asb_disable { + ICP_QAT_HW_COMP_51_ASB_DISABLE_ASB_ENABLED = 0x0, + ICP_QAT_HW_COMP_51_ASB_DISABLE_ASB_DISABLED = 0x1, +}; + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_ASB_DISABLE_DEFAULT_VAL \ + ICP_QAT_HW_COMP_51_ASB_DISABLE_ASB_ENABLED +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_SPEC_DECODER_INTERNAL_ONLY_BITPOS 21 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_SPEC_DECODER_INTERNAL_ONLY_MASK GENMASK(0, 0) +enum icp_qat_hw_comp_51_spec_decoder_internal_only { + ICP_QAT_HW_COMP_51_SPEC_DECODER_INTERNAL_ONLY_NORMAL = 0x0, + ICP_QAT_HW_COMP_51_SPEC_DECODER_INTERNAL_ONLY_DISABLED = 0x1, +}; + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_SPEC_DECODER_INTERNAL_ONLY_DEFAULT_VAL \ + ICP_QAT_HW_COMP_51_SPEC_DECODER_INTERNAL_ONLY_NORMAL +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_MINI_XCAM_INTERNAL_ONLY_BITPOS 20 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_MINI_XCAM_INTERNAL_ONLY_MASK GENMASK(0, 0) +enum icp_qat_hw_comp_51_mini_xcam_internal_only { + ICP_QAT_HW_COMP_51_MINI_XCAM_INTERNAL_ONLY_NORMAL = 0x0, + ICP_QAT_HW_COMP_51_MINI_XCAM_INTERNAL_ONLY_DISABLED = 0x1, +}; + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_MINI_XCAM_INTERNAL_ONLY_DEFAULT_VAL \ + ICP_QAT_HW_COMP_51_MINI_XCAM_INTERNAL_ONLY_NORMAL +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_REP_OFF_ENC_INTERNAL_ONLY_BITPOS 19 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_REP_OFF_ENC_INTERNAL_ONLY_MASK GENMASK(0, 0) +enum icp_qat_hw_comp_51_rep_off_enc_internal_only { + ICP_QAT_HW_COMP_51_REP_OFF_ENC_INTERNAL_ONLY_ENABLED = 0x0, + ICP_QAT_HW_COMP_51_REP_OFF_ENC_INTERNAL_ONLY_DISABLED = 0x1, +}; + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_REP_OFF_ENC_INTERNAL_ONLY_DEFAULT_VAL \ + ICP_QAT_HW_COMP_51_REP_OFF_ENC_INTERNAL_ONLY_ENABLED +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_PROG_BLOCK_DROP_INTERNAL_ONLY_BITPOS 18 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_PROG_BLOCK_DROP_INTERNAL_ONLY_MASK GENMASK(0, 0) +enum icp_qat_hw_comp_51_prog_block_drop_internal_only { + ICP_QAT_HW_COMP_51_PROG_BLOCK_DROP_INTERNAL_ONLY_DISABLE = 0x0, + ICP_QAT_HW_COMP_51_PROG_BLOCK_DROP_INTERNAL_ONLY_ENABLE = 0x1, +}; + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_PROG_BLOCK_DROP_INTERNAL_ONLY_DEFAULT_VAL \ + ICP_QAT_HW_COMP_51_PROG_BLOCK_DROP_INTERNAL_ONLY_DISABLE +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_SKIP_HASH_OVERRIDE_INTERNAL_ONLY_BITPOS 17 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_SKIP_HASH_OVERRIDE_INTERNAL_ONLY_MASK GENMASK(0, 0) +enum icp_qat_hw_comp_51_skip_hash_override_internal_only { + ICP_QAT_HW_COMP_51_SKIP_HASH_OVERRIDE_INTERNAL_ONLY_DETERMINE_HASH_PARAMS = 0x0, + ICP_QAT_HW_COMP_51_SKIP_HASH_OVERRIDE_INTERNAL_ONLY_OVERRIDE_HASH_PARAMS = 0x1, +}; + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_SKIP_HASH_OVERRIDE_INTERNAL_ONLY_DEFAULT_VAL \ + ICP_QAT_HW_COMP_51_SKIP_HASH_OVERRIDE_INTERNAL_ONLY_DETERMINE_HASH_PARAMS +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_HBS_BITPOS 14 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_HBS_MASK GENMASK(2, 0) +enum icp_qat_hw_comp_51_hbs { + ICP_QAT_HW_COMP_51_HBS_32KB = 0x0, + ICP_QAT_HW_COMP_51_HBS_64KB = 0x1, +}; + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_HBS_DEFAULT_VAL \ + ICP_QAT_HW_COMP_51_HBS_32KB +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_ABD_BITPOS 13 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_ABD_MASK GENMASK(0, 0) +enum icp_qat_hw_comp_51_abd { + ICP_QAT_HW_COMP_51_ABD_ABD_ENABLED = 0x0, + ICP_QAT_HW_COMP_51_ABD_ABD_DISABLED = 0x1, +}; + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_ABD_DEFAULT_VAL \ + ICP_QAT_HW_COMP_51_ABD_ABD_ENABLED +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_LLLBD_CTRL_BITPOS 12 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_LLLBD_CTRL_MASK GENMASK(0, 0) +enum icp_qat_hw_comp_51_lllbd_ctrl { + ICP_QAT_HW_COMP_51_LLLBD_CTRL_LLLBD_ENABLED = 0x0, + ICP_QAT_HW_COMP_51_LLLBD_CTRL_LLLBD_DISABLED = 0x1, +}; + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_LLLBD_CTRL_DEFAULT_VAL \ + ICP_QAT_HW_COMP_51_LLLBD_CTRL_LLLBD_ENABLED +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_SEARCH_DEPTH_BITPOS 8 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_SEARCH_DEPTH_MASK GENMASK(3, 0) +enum icp_qat_hw_comp_51_search_depth { + ICP_QAT_HW_COMP_51_SEARCH_DEPTH_LEVEL_1 = 0x1, + ICP_QAT_HW_COMP_51_SEARCH_DEPTH_LEVEL_6 = 0x3, + ICP_QAT_HW_COMP_51_SEARCH_DEPTH_LEVEL_9 = 0x4, + ICP_QAT_HW_COMP_51_SEARCH_DEPTH_LEVEL_10 = 0x4, +}; + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_SEARCH_DEPTH_DEFAULT_VAL \ + ICP_QAT_HW_COMP_51_SEARCH_DEPTH_LEVEL_1 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_FORMAT_BITPOS 5 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_FORMAT_MASK GENMASK(2, 0) +enum icp_qat_hw_comp_51_format { + ICP_QAT_HW_COMP_51_FORMAT_ILZ77 = 0x1, + ICP_QAT_HW_COMP_51_FORMAT_LZ4 = 0x2, + ICP_QAT_HW_COMP_51_FORMAT_LZ4s = 0x3, + ICP_QAT_HW_COMP_51_FORMAT_ZSTD = 0x4, +}; + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_FORMAT_DEFAULT_VAL \ + ICP_QAT_HW_COMP_51_FORMAT_ILZ77 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_MIN_MATCH_CONTROL_BITPOS 4 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_MIN_MATCH_CONTROL_MASK GENMASK(0, 0) +enum icp_qat_hw_comp_51_min_match_control { + ICP_QAT_HW_COMP_51_MIN_MATCH_CONTROL_MATCH_3B = 0x0, + ICP_QAT_HW_COMP_51_MIN_MATCH_CONTROL_MATCH_4B = 0x1, +}; + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_MIN_MATCH_CONTROL_DEFAULT_VAL \ + ICP_QAT_HW_COMP_51_MIN_MATCH_CONTROL_MATCH_3B +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_SKIP_HASH_COLLISION_BITPOS 3 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_SKIP_HASH_COLLISION_MASK GENMASK(0, 0) +enum icp_qat_hw_comp_51_skip_hash_collision { + ICP_QAT_HW_COMP_51_SKIP_HASH_COLLISION_ALLOW = 0x0, + ICP_QAT_HW_COMP_51_SKIP_HASH_COLLISION_DONT_ALLOW = 0x1, +}; + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_SKIP_HASH_COLLISION_DEFAULT_VAL \ + ICP_QAT_HW_COMP_51_SKIP_HASH_COLLISION_ALLOW +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_SKIP_HASH_UPDATE_BITPOS 2 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_SKIP_HASH_UPDATE_MASK GENMASK(0, 0) +enum icp_qat_hw_comp_51_skip_hash_update { + ICP_QAT_HW_COMP_51_SKIP_HASH_UPDATE_ALLOW = 0x0, + ICP_QAT_HW_COMP_51_SKIP_HASH_UPDATE_DONT_ALLOW = 0x1, +}; + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_SKIP_HASH_UPDATE_DEFAULT_VAL \ + ICP_QAT_HW_COMP_51_SKIP_HASH_UPDATE_ALLOW +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_BYTE_SKIP_BITPOS 1 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_BYTE_SKIP_MASK GENMASK(0, 0) +enum icp_qat_hw_comp_51_byte_skip { + ICP_QAT_HW_COMP_51_BYTE_SKIP_3BYTE_TOKEN = 0x0, + ICP_QAT_HW_COMP_51_BYTE_SKIP_3BYTE_LITERAL = 0x1, +}; + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_BYTE_SKIP_DEFAULT_VAL \ + ICP_QAT_HW_COMP_51_BYTE_SKIP_3BYTE_TOKEN +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_LZ4_BLOCK_CHECKSUM_BITPOS 0 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_LZ4_BLOCK_CHECKSUM_MASK GENMASK(0, 0) +enum icp_qat_hw_comp_51_lz4_block_checksum { + ICP_QAT_HW_COMP_51_LZ4_BLOCK_CHECKSUM_ABSENT = 0x0, + ICP_QAT_HW_COMP_51_LZ4_BLOCK_CHECKSUM_PRESENT = 0x1, +}; + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_LZ4_BLOCK_CHECKSUM_DEFAULT_VAL \ + ICP_QAT_HW_COMP_51_LZ4_BLOCK_CHECKSUM_ABSENT +#define ICP_QAT_HW_DECOMP_51_CONFIG_CSR_DISCARD_DATA_BITPOS 26 +#define ICP_QAT_HW_DECOMP_51_CONFIG_CSR_DISCARD_DATA_MASK GENMASK(0, 0) +enum icp_qat_hw_decomp_51_discard_data { + ICP_QAT_HW_DECOMP_51_DISCARD_DATA_DISABLED = 0x0, + ICP_QAT_HW_DECOMP_51_DISCARD_DATA_ENABLED = 0x1, +}; + +#define ICP_QAT_HW_DECOMP_51_CONFIG_CSR_DISCARD_DATA_DEFAULT_VAL \ + ICP_QAT_HW_DECOMP_51_DISCARD_DATA_DISABLED +#define ICP_QAT_HW_DECOMP_51_CONFIG_CSR_BMS_BITPOS 19 +#define ICP_QAT_HW_DECOMP_51_CONFIG_CSR_BMS_MASK GENMASK(1, 0) +enum icp_qat_hw_decomp_51_bms { + ICP_QAT_HW_DECOMP_51_BMS_BMS_64KB = 0x0, + ICP_QAT_HW_DECOMP_51_BMS_BMS_256KB = 0x1, + ICP_QAT_HW_DECOMP_51_BMS_BMS_1MB = 0x2, + ICP_QAT_HW_DECOMP_51_BMS_BMS_4MB = 0x3, +}; + +#define ICP_QAT_HW_DECOMP_51_CONFIG_CSR_BMS_DEFAULT_VAL \ + ICP_QAT_HW_DECOMP_51_BMS_BMS_64KB +#define ICP_QAT_HW_DECOMP_51_CONFIG_CSR_ZSTD_FRAME_GEN_DEC_EN_BITPOS 2 +#define ICP_QAT_HW_DECOMP_51_CONFIG_CSR_ZSTD_FRAME_GEN_DEC_EN_MASK GENMASK(0, 0) +enum icp_qat_hw_decomp_51_zstd_frame_gen_dec_en { + ICP_QAT_HW_DECOMP_51_ZSTD_FRAME_GEN_DEC_EN_ZSTD_FRAME_HDR_DISABLE = 0x0, + ICP_QAT_HW_DECOMP_51_ZSTD_FRAME_GEN_DEC_EN_ZSTD_FRAME_HDR_ENABLE = 0x1, +}; + +#define ICP_QAT_HW_DECOMP_51_CONFIG_CSR_ZSTD_FRAME_GEN_DEC_EN_DEFAULT_VAL \ + ICP_QAT_HW_DECOMP_51_ZSTD_FRAME_GEN_DEC_EN_ZSTD_FRAME_HDR_ENABLE +#define ICP_QAT_HW_DECOMP_51_CONFIG_CSR_SPEC_DECODER_INTERNAL_ONLY_BITPOS 21 +#define ICP_QAT_HW_DECOMP_51_CONFIG_CSR_SPEC_DECODER_INTERNAL_ONLY_MASK GENMASK(0, 0) +enum icp_qat_hw_decomp_51_spec_decoder_internal_only { + ICP_QAT_HW_DECOMP_51_SPEC_DECODER_INTERNAL_ONLY_NORMAL = 0x0, + ICP_QAT_HW_DECOMP_51_SPEC_DECODER_INTERNAL_ONLY_DISABLED = 0x1, +}; + +#define ICP_QAT_HW_DECOMP_51_CONFIG_CSR_SPEC_DECODER_INTERNAL_ONLY_DEFAULT_VAL \ + ICP_QAT_HW_DECOMP_51_SPEC_DECODER_INTERNAL_ONLY_NORMAL +#define ICP_QAT_HW_DECOMP_51_CONFIG_CSR_MINI_XCAM_INTERNAL_ONLY_BITPOS 20 +#define ICP_QAT_HW_DECOMP_51_CONFIG_CSR_MINI_XCAM_INTERNAL_ONLY_MASK GENMASK(0, 0) +enum icp_qat_hw_decomp_51_mini_xcam_internal_only { + ICP_QAT_HW_DECOMP_51_MINI_XCAM_INTERNAL_ONLY_NORMAL = 0x0, + ICP_QAT_HW_DECOMP_51_MINI_XCAM_INTERNAL_ONLY_DISABLED = 0x1, +}; + +#define ICP_QAT_HW_DECOMP_51_CONFIG_CSR_MINI_XCAM_INTERNAL_ONLY_DEFAULT_VAL \ + ICP_QAT_HW_DECOMP_51_MINI_XCAM_INTERNAL_ONLY_NORMAL +#define ICP_QAT_HW_DECOMP_51_CONFIG_CSR_HBS_BITPOS 14 +#define ICP_QAT_HW_DECOMP_51_CONFIG_CSR_HBS_MASK GENMASK(2, 0) +enum icp_qat_hw_decomp_51_hbs { + ICP_QAT_HW_DECOMP_51_HBS_32KB = 0x0, + ICP_QAT_HW_DECOMP_51_HBS_64KB = 0x1, +}; + +#define ICP_QAT_HW_DECOMP_51_CONFIG_CSR_HBS_DEFAULT_VAL \ + ICP_QAT_HW_DECOMP_51_HBS_32KB +#define ICP_QAT_HW_DECOMP_51_CONFIG_CSR_FORMAT_BITPOS 5 +#define ICP_QAT_HW_DECOMP_51_CONFIG_CSR_FORMAT_MASK GENMASK(2, 0) +enum icp_qat_hw_decomp_51_format { + ICP_QAT_HW_DECOMP_51_FORMAT_ILZ77 = 0x1, + ICP_QAT_HW_DECOMP_51_FORMAT_LZ4 = 0x2, + ICP_QAT_HW_DECOMP_51_FORMAT_RESERVED = 0x3, + ICP_QAT_HW_DECOMP_51_FORMAT_ZSTD = 0x4, +}; + +#define ICP_QAT_HW_DECOMP_51_CONFIG_CSR_FORMAT_DEFAULT_VAL \ + ICP_QAT_HW_DECOMP_51_FORMAT_ILZ77 +#define ICP_QAT_HW_DECOMP_51_CONFIG_CSR_LZ4_BLOCK_CHECKSUM_BITPOS 0 +#define ICP_QAT_HW_DECOMP_51_CONFIG_CSR_LZ4_BLOCK_CHECKSUM_MASK GENMASK(0, 0) +enum icp_qat_hw_decomp_51_lz4_block_checksum { + ICP_QAT_HW_DECOMP_51_LZ4_BLOCK_CHECKSUM_ABSENT = 0x0, + ICP_QAT_HW_DECOMP_51_LZ4_BLOCK_CHECKSUM_PRESENT = 0x1, +}; + +#define ICP_QAT_HW_DECOMP_51_CONFIG_CSR_LZ4_BLOCK_CHECKSUM_DEFAULT_VAL \ + ICP_QAT_HW_DECOMP_51_LZ4_BLOCK_CHECKSUM_ABSENT + +#endif /* ICP_QAT_HW_51_COMP_DEFS_H_ */ From 17fd7514ae68c541bf952876ce2005cb4a53f283 Mon Sep 17 00:00:00 2001 From: Laurent M Coquerel Date: Wed, 30 Apr 2025 12:34:53 +0100 Subject: [PATCH 289/382] crypto: qat - add qat_6xxx driver Add a new driver, qat_6xxx, to support QAT GEN6 devices. QAT GEN6 devices are a follow-on generation of GEN4 devices and differently from the previous generation, they can support all three services (symmetric, asymmetric, and data compression) concurrently. In order to have the qat_6xxx driver to reuse some of the GEN4 logic, a new abstraction layer has been introduced to bridge the two implementations. This allows to avoid code duplication and to keep the qat_6xxx driver isolated from the GEN4 logic. This approach has been used for the PF to VF logic and the HW CSR access logic. Signed-off-by: Laurent M Coquerel Co-developed-by: George Abraham P Signed-off-by: George Abraham P Co-developed-by: Karthikeyan Gopal Signed-off-by: Karthikeyan Gopal Co-developed-by: Suman Kumar Chakraborty Signed-off-by: Suman Kumar Chakraborty Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/intel/qat/Kconfig | 12 + drivers/crypto/intel/qat/Makefile | 1 + drivers/crypto/intel/qat/qat_6xxx/Makefile | 3 + .../intel/qat/qat_6xxx/adf_6xxx_hw_data.c | 843 ++++++++++++++++++ .../intel/qat/qat_6xxx/adf_6xxx_hw_data.h | 148 +++ drivers/crypto/intel/qat/qat_6xxx/adf_drv.c | 224 +++++ drivers/crypto/intel/qat/qat_common/Makefile | 1 + .../intel/qat/qat_common/adf_accel_devices.h | 2 + .../intel/qat/qat_common/adf_cfg_common.h | 1 + .../intel/qat/qat_common/adf_fw_config.h | 1 + .../crypto/intel/qat/qat_common/adf_gen6_pm.h | 28 + .../intel/qat/qat_common/adf_gen6_shared.c | 49 + .../intel/qat/qat_common/adf_gen6_shared.h | 15 + 13 files changed, 1328 insertions(+) create mode 100644 drivers/crypto/intel/qat/qat_6xxx/Makefile create mode 100644 drivers/crypto/intel/qat/qat_6xxx/adf_6xxx_hw_data.c create mode 100644 drivers/crypto/intel/qat/qat_6xxx/adf_6xxx_hw_data.h create mode 100644 drivers/crypto/intel/qat/qat_6xxx/adf_drv.c create mode 100644 drivers/crypto/intel/qat/qat_common/adf_gen6_pm.h create mode 100644 drivers/crypto/intel/qat/qat_common/adf_gen6_shared.c create mode 100644 drivers/crypto/intel/qat/qat_common/adf_gen6_shared.h diff --git a/drivers/crypto/intel/qat/Kconfig b/drivers/crypto/intel/qat/Kconfig index 02fb8abe4e6e..359c61f0c8a1 100644 --- a/drivers/crypto/intel/qat/Kconfig +++ b/drivers/crypto/intel/qat/Kconfig @@ -70,6 +70,18 @@ config CRYPTO_DEV_QAT_420XX To compile this as a module, choose M here: the module will be called qat_420xx. +config CRYPTO_DEV_QAT_6XXX + tristate "Support for Intel(R) QuickAssist Technology QAT_6XXX" + depends on (X86 || COMPILE_TEST) + depends on PCI + select CRYPTO_DEV_QAT + help + Support for Intel(R) QuickAssist Technology QAT_6xxx + for accelerating crypto and compression workloads. + + To compile this as a module, choose M here: the module + will be called qat_6xxx. + config CRYPTO_DEV_QAT_DH895xCCVF tristate "Support for Intel(R) DH895xCC Virtual Function" depends on PCI && (!CPU_BIG_ENDIAN || COMPILE_TEST) diff --git a/drivers/crypto/intel/qat/Makefile b/drivers/crypto/intel/qat/Makefile index 1eda8dc18515..abef14207afa 100644 --- a/drivers/crypto/intel/qat/Makefile +++ b/drivers/crypto/intel/qat/Makefile @@ -6,6 +6,7 @@ obj-$(CONFIG_CRYPTO_DEV_QAT_C3XXX) += qat_c3xxx/ obj-$(CONFIG_CRYPTO_DEV_QAT_C62X) += qat_c62x/ obj-$(CONFIG_CRYPTO_DEV_QAT_4XXX) += qat_4xxx/ obj-$(CONFIG_CRYPTO_DEV_QAT_420XX) += qat_420xx/ +obj-$(CONFIG_CRYPTO_DEV_QAT_6XXX) += qat_6xxx/ obj-$(CONFIG_CRYPTO_DEV_QAT_DH895xCCVF) += qat_dh895xccvf/ obj-$(CONFIG_CRYPTO_DEV_QAT_C3XXXVF) += qat_c3xxxvf/ obj-$(CONFIG_CRYPTO_DEV_QAT_C62XVF) += qat_c62xvf/ diff --git a/drivers/crypto/intel/qat/qat_6xxx/Makefile b/drivers/crypto/intel/qat/qat_6xxx/Makefile new file mode 100644 index 000000000000..4b4de67cb0c2 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_6xxx/Makefile @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_CRYPTO_DEV_QAT_6XXX) += qat_6xxx.o +qat_6xxx-y := adf_drv.o adf_6xxx_hw_data.o diff --git a/drivers/crypto/intel/qat/qat_6xxx/adf_6xxx_hw_data.c b/drivers/crypto/intel/qat/qat_6xxx/adf_6xxx_hw_data.c new file mode 100644 index 000000000000..73d479383b1f --- /dev/null +++ b/drivers/crypto/intel/qat/qat_6xxx/adf_6xxx_hw_data.c @@ -0,0 +1,843 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2025 Intel Corporation */ +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "adf_6xxx_hw_data.h" +#include "icp_qat_fw_comp.h" +#include "icp_qat_hw_51_comp.h" + +#define RP_GROUP_0_MASK (BIT(0) | BIT(2)) +#define RP_GROUP_1_MASK (BIT(1) | BIT(3)) +#define RP_GROUP_ALL_MASK (RP_GROUP_0_MASK | RP_GROUP_1_MASK) + +#define ADF_AE_GROUP_0 GENMASK(3, 0) +#define ADF_AE_GROUP_1 GENMASK(7, 4) +#define ADF_AE_GROUP_2 BIT(8) + +struct adf_ring_config { + u32 ring_mask; + enum adf_cfg_service_type ring_type; + const unsigned long *thrd_mask; +}; + +static u32 rmask_two_services[] = { + RP_GROUP_0_MASK, + RP_GROUP_1_MASK, +}; + +enum adf_gen6_rps { + RP0 = 0, + RP1 = 1, + RP2 = 2, + RP3 = 3, + RP_MAX = RP3 +}; + +/* + * thrd_mask_[sym|asym|cpr|dcc]: these static arrays define the thread + * configuration for handling requests of specific services across the + * accelerator engines. Each element in an array corresponds to an + * accelerator engine, with the value being a bitmask that specifies which + * threads within that engine are capable of processing the particular service. + * + * For example, a value of 0x0C means that threads 2 and 3 are enabled for the + * service in the respective accelerator engine. + */ +static const unsigned long thrd_mask_sym[ADF_6XXX_MAX_ACCELENGINES] = { + 0x0C, 0x0C, 0x0C, 0x0C, 0x1C, 0x1C, 0x1C, 0x1C, 0x00 +}; + +static const unsigned long thrd_mask_asym[ADF_6XXX_MAX_ACCELENGINES] = { + 0x70, 0x70, 0x70, 0x70, 0x60, 0x60, 0x60, 0x60, 0x00 +}; + +static const unsigned long thrd_mask_cpr[ADF_6XXX_MAX_ACCELENGINES] = { + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x00 +}; + +static const unsigned long thrd_mask_dcc[ADF_6XXX_MAX_ACCELENGINES] = { + 0x00, 0x00, 0x00, 0x00, 0x07, 0x07, 0x03, 0x03, 0x00 +}; + +static const char *const adf_6xxx_fw_objs[] = { + [ADF_FW_CY_OBJ] = ADF_6XXX_CY_OBJ, + [ADF_FW_DC_OBJ] = ADF_6XXX_DC_OBJ, + [ADF_FW_ADMIN_OBJ] = ADF_6XXX_ADMIN_OBJ, +}; + +static const struct adf_fw_config adf_default_fw_config[] = { + { ADF_AE_GROUP_1, ADF_FW_DC_OBJ }, + { ADF_AE_GROUP_0, ADF_FW_CY_OBJ }, + { ADF_AE_GROUP_2, ADF_FW_ADMIN_OBJ }, +}; + +static struct adf_hw_device_class adf_6xxx_class = { + .name = ADF_6XXX_DEVICE_NAME, + .type = DEV_6XXX, +}; + +static bool services_supported(unsigned long mask) +{ + int num_svc; + + if (mask >= BIT(SVC_BASE_COUNT)) + return false; + + num_svc = hweight_long(mask); + switch (num_svc) { + case ADF_ONE_SERVICE: + return true; + case ADF_TWO_SERVICES: + case ADF_THREE_SERVICES: + return !test_bit(SVC_DCC, &mask); + default: + return false; + } +} + +static int get_service(unsigned long *mask) +{ + if (test_and_clear_bit(SVC_ASYM, mask)) + return SVC_ASYM; + + if (test_and_clear_bit(SVC_SYM, mask)) + return SVC_SYM; + + if (test_and_clear_bit(SVC_DC, mask)) + return SVC_DC; + + if (test_and_clear_bit(SVC_DCC, mask)) + return SVC_DCC; + + return -EINVAL; +} + +static enum adf_cfg_service_type get_ring_type(enum adf_services service) +{ + switch (service) { + case SVC_SYM: + return SYM; + case SVC_ASYM: + return ASYM; + case SVC_DC: + case SVC_DCC: + return COMP; + default: + return UNUSED; + } +} + +static const unsigned long *get_thrd_mask(enum adf_services service) +{ + switch (service) { + case SVC_SYM: + return thrd_mask_sym; + case SVC_ASYM: + return thrd_mask_asym; + case SVC_DC: + return thrd_mask_cpr; + case SVC_DCC: + return thrd_mask_dcc; + default: + return NULL; + } +} + +static int get_rp_config(struct adf_accel_dev *accel_dev, struct adf_ring_config *rp_config, + unsigned int *num_services) +{ + unsigned int i, nservices; + unsigned long mask; + int ret, service; + + ret = adf_get_service_mask(accel_dev, &mask); + if (ret) + return ret; + + nservices = hweight_long(mask); + if (nservices > MAX_NUM_CONCURR_SVC) + return -EINVAL; + + for (i = 0; i < nservices; i++) { + service = get_service(&mask); + if (service < 0) + return service; + + rp_config[i].ring_type = get_ring_type(service); + rp_config[i].thrd_mask = get_thrd_mask(service); + + /* + * If there is only one service enabled, use all ring pairs for + * that service. + * If there are two services enabled, use ring pairs 0 and 2 for + * one service and ring pairs 1 and 3 for the other service. + */ + switch (nservices) { + case ADF_ONE_SERVICE: + rp_config[i].ring_mask = RP_GROUP_ALL_MASK; + break; + case ADF_TWO_SERVICES: + rp_config[i].ring_mask = rmask_two_services[i]; + break; + case ADF_THREE_SERVICES: + rp_config[i].ring_mask = BIT(i); + + /* If ASYM is enabled, use additional ring pair */ + if (service == SVC_ASYM) + rp_config[i].ring_mask |= BIT(RP3); + + break; + default: + return -EINVAL; + } + } + + *num_services = nservices; + + return 0; +} + +static u32 adf_gen6_get_arb_mask(struct adf_accel_dev *accel_dev, unsigned int ae) +{ + struct adf_ring_config rp_config[MAX_NUM_CONCURR_SVC]; + unsigned int num_services, i, thrd; + u32 ring_mask, thd2arb_mask = 0; + const unsigned long *p_mask; + + if (get_rp_config(accel_dev, rp_config, &num_services)) + return 0; + + /* + * The thd2arb_mask maps ring pairs to threads within an accelerator engine. + * It ensures that jobs submitted to ring pairs are scheduled on threads capable + * of handling the specified service type. + * + * Each group of 4 bits in the mask corresponds to a thread, with each bit + * indicating whether a job from a ring pair can be scheduled on that thread. + * The use of 4 bits is due to the organization of ring pairs into groups of + * four, where each group shares the same configuration. + */ + for (i = 0; i < num_services; i++) { + p_mask = &rp_config[i].thrd_mask[ae]; + ring_mask = rp_config[i].ring_mask; + + for_each_set_bit(thrd, p_mask, ADF_NUM_THREADS_PER_AE) + thd2arb_mask |= ring_mask << (thrd * 4); + } + + return thd2arb_mask; +} + +static u16 get_ring_to_svc_map(struct adf_accel_dev *accel_dev) +{ + enum adf_cfg_service_type rps[ADF_GEN6_NUM_BANKS_PER_VF] = { }; + struct adf_ring_config rp_config[MAX_NUM_CONCURR_SVC]; + unsigned int num_services, rp_num, i; + unsigned long cfg_mask; + u16 ring_to_svc_map; + + if (get_rp_config(accel_dev, rp_config, &num_services)) + return 0; + + /* + * Loop through the configured services and populate the `rps` array that + * contains what service that particular ring pair can handle (i.e. symmetric + * crypto, asymmetric crypto, data compression or compression chaining). + */ + for (i = 0; i < num_services; i++) { + cfg_mask = rp_config[i].ring_mask; + for_each_set_bit(rp_num, &cfg_mask, ADF_GEN6_NUM_BANKS_PER_VF) + rps[rp_num] = rp_config[i].ring_type; + } + + /* + * The ring_mask is structured into segments of 3 bits, with each + * segment representing the service configuration for a specific ring pair. + * Since ring pairs are organized into groups of 4, the ring_mask contains 4 + * such 3-bit segments, each corresponding to one ring pair. + * + * The device has 64 ring pairs, which are organized in groups of 4, namely + * 16 groups. Each group has the same configuration, represented here by + * `ring_to_svc_map`. + */ + ring_to_svc_map = rps[RP0] << ADF_CFG_SERV_RING_PAIR_0_SHIFT | + rps[RP1] << ADF_CFG_SERV_RING_PAIR_1_SHIFT | + rps[RP2] << ADF_CFG_SERV_RING_PAIR_2_SHIFT | + rps[RP3] << ADF_CFG_SERV_RING_PAIR_3_SHIFT; + + return ring_to_svc_map; +} + +static u32 get_accel_mask(struct adf_hw_device_data *self) +{ + return ADF_GEN6_ACCELERATORS_MASK; +} + +static u32 get_num_accels(struct adf_hw_device_data *self) +{ + return ADF_GEN6_MAX_ACCELERATORS; +} + +static u32 get_num_aes(struct adf_hw_device_data *self) +{ + return self ? hweight32(self->ae_mask) : 0; +} + +static u32 get_misc_bar_id(struct adf_hw_device_data *self) +{ + return ADF_GEN6_PMISC_BAR; +} + +static u32 get_etr_bar_id(struct adf_hw_device_data *self) +{ + return ADF_GEN6_ETR_BAR; +} + +static u32 get_sram_bar_id(struct adf_hw_device_data *self) +{ + return ADF_GEN6_SRAM_BAR; +} + +static enum dev_sku_info get_sku(struct adf_hw_device_data *self) +{ + return DEV_SKU_1; +} + +static void get_arb_info(struct arb_info *arb_info) +{ + arb_info->arb_cfg = ADF_GEN6_ARB_CONFIG; + arb_info->arb_offset = ADF_GEN6_ARB_OFFSET; + arb_info->wt2sam_offset = ADF_GEN6_ARB_WRK_2_SER_MAP_OFFSET; +} + +static void get_admin_info(struct admin_info *admin_csrs_info) +{ + admin_csrs_info->mailbox_offset = ADF_GEN6_MAILBOX_BASE_OFFSET; + admin_csrs_info->admin_msg_ur = ADF_GEN6_ADMINMSGUR_OFFSET; + admin_csrs_info->admin_msg_lr = ADF_GEN6_ADMINMSGLR_OFFSET; +} + +static u32 get_heartbeat_clock(struct adf_hw_device_data *self) +{ + return ADF_GEN6_COUNTER_FREQ; +} + +static void enable_error_correction(struct adf_accel_dev *accel_dev) +{ + void __iomem *csr = adf_get_pmisc_base(accel_dev); + + /* + * Enable all error notification bits in errsou3 except VFLR + * notification on host. + */ + ADF_CSR_WR(csr, ADF_GEN6_ERRMSK3, ADF_GEN6_VFLNOTIFY); +} + +static void enable_ints(struct adf_accel_dev *accel_dev) +{ + void __iomem *addr = adf_get_pmisc_base(accel_dev); + + /* Enable bundle interrupts */ + ADF_CSR_WR(addr, ADF_GEN6_SMIAPF_RP_X0_MASK_OFFSET, 0); + ADF_CSR_WR(addr, ADF_GEN6_SMIAPF_RP_X1_MASK_OFFSET, 0); + + /* Enable misc interrupts */ + ADF_CSR_WR(addr, ADF_GEN6_SMIAPF_MASK_OFFSET, 0); +} + +static void set_ssm_wdtimer(struct adf_accel_dev *accel_dev) +{ + void __iomem *addr = adf_get_pmisc_base(accel_dev); + u64 val_pke = ADF_SSM_WDT_PKE_DEFAULT_VALUE; + u64 val = ADF_SSM_WDT_DEFAULT_VALUE; + + /* Enable watchdog timer for sym and dc */ + ADF_CSR_WR64_LO_HI(addr, ADF_SSMWDTATHL_OFFSET, ADF_SSMWDTATHH_OFFSET, val); + ADF_CSR_WR64_LO_HI(addr, ADF_SSMWDTCNVL_OFFSET, ADF_SSMWDTCNVH_OFFSET, val); + ADF_CSR_WR64_LO_HI(addr, ADF_SSMWDTUCSL_OFFSET, ADF_SSMWDTUCSH_OFFSET, val); + ADF_CSR_WR64_LO_HI(addr, ADF_SSMWDTDCPRL_OFFSET, ADF_SSMWDTDCPRH_OFFSET, val); + + /* Enable watchdog timer for pke */ + ADF_CSR_WR64_LO_HI(addr, ADF_SSMWDTPKEL_OFFSET, ADF_SSMWDTPKEH_OFFSET, val_pke); +} + +/* + * The vector routing table is used to select the MSI-X entry to use for each + * interrupt source. + * The first ADF_GEN6_ETR_MAX_BANKS entries correspond to ring interrupts. + * The final entry corresponds to VF2PF or error interrupts. + * This vector table could be used to configure one MSI-X entry to be shared + * between multiple interrupt sources. + * + * The default routing is set to have a one to one correspondence between the + * interrupt source and the MSI-X entry used. + */ +static void set_msix_default_rttable(struct adf_accel_dev *accel_dev) +{ + void __iomem *csr = adf_get_pmisc_base(accel_dev); + unsigned int i; + + for (i = 0; i <= ADF_GEN6_ETR_MAX_BANKS; i++) + ADF_CSR_WR(csr, ADF_GEN6_MSIX_RTTABLE_OFFSET(i), i); +} + +static int reset_ring_pair(void __iomem *csr, u32 bank_number) +{ + u32 status; + int ret; + + /* + * Write rpresetctl register BIT(0) as 1. + * Since rpresetctl registers have no RW fields, no need to preserve + * values for other bits. Just write directly. + */ + ADF_CSR_WR(csr, ADF_WQM_CSR_RPRESETCTL(bank_number), + ADF_WQM_CSR_RPRESETCTL_RESET); + + /* Read rpresetsts register and wait for rp reset to complete */ + ret = read_poll_timeout(ADF_CSR_RD, status, + status & ADF_WQM_CSR_RPRESETSTS_STATUS, + ADF_RPRESET_POLL_DELAY_US, + ADF_RPRESET_POLL_TIMEOUT_US, true, + csr, ADF_WQM_CSR_RPRESETSTS(bank_number)); + if (ret) + return ret; + + /* When ring pair reset is done, clear rpresetsts */ + ADF_CSR_WR(csr, ADF_WQM_CSR_RPRESETSTS(bank_number), ADF_WQM_CSR_RPRESETSTS_STATUS); + + return 0; +} + +static int ring_pair_reset(struct adf_accel_dev *accel_dev, u32 bank_number) +{ + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + void __iomem *csr = adf_get_etr_base(accel_dev); + int ret; + + if (bank_number >= hw_data->num_banks) + return -EINVAL; + + dev_dbg(&GET_DEV(accel_dev), "ring pair reset for bank:%d\n", bank_number); + + ret = reset_ring_pair(csr, bank_number); + if (ret) + dev_err(&GET_DEV(accel_dev), "ring pair reset failed (timeout)\n"); + else + dev_dbg(&GET_DEV(accel_dev), "ring pair reset successful\n"); + + return ret; +} + +static int build_comp_block(void *ctx, enum adf_dc_algo algo) +{ + struct icp_qat_fw_comp_req *req_tmpl = ctx; + struct icp_qat_fw_comp_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars; + struct icp_qat_hw_comp_51_config_csr_lower hw_comp_lower_csr = { }; + struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr; + u32 lower_val; + + switch (algo) { + case QAT_DEFLATE: + header->service_cmd_id = ICP_QAT_FW_COMP_CMD_DYNAMIC; + break; + default: + return -EINVAL; + } + + hw_comp_lower_csr.lllbd = ICP_QAT_HW_COMP_51_LLLBD_CTRL_LLLBD_DISABLED; + hw_comp_lower_csr.sd = ICP_QAT_HW_COMP_51_SEARCH_DEPTH_LEVEL_1; + lower_val = ICP_QAT_FW_COMP_51_BUILD_CONFIG_LOWER(hw_comp_lower_csr); + cd_pars->u.sl.comp_slice_cfg_word[0] = lower_val; + cd_pars->u.sl.comp_slice_cfg_word[1] = 0; + + return 0; +} + +static int build_decomp_block(void *ctx, enum adf_dc_algo algo) +{ + struct icp_qat_fw_comp_req *req_tmpl = ctx; + struct icp_qat_fw_comp_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars; + struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr; + + switch (algo) { + case QAT_DEFLATE: + header->service_cmd_id = ICP_QAT_FW_COMP_CMD_DECOMPRESS; + break; + default: + return -EINVAL; + } + + cd_pars->u.sl.comp_slice_cfg_word[0] = 0; + cd_pars->u.sl.comp_slice_cfg_word[1] = 0; + + return 0; +} + +static void adf_gen6_init_dc_ops(struct adf_dc_ops *dc_ops) +{ + dc_ops->build_comp_block = build_comp_block; + dc_ops->build_decomp_block = build_decomp_block; +} + +static int adf_gen6_init_thd2arb_map(struct adf_accel_dev *accel_dev) +{ + struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev); + u32 *thd2arb_map = hw_data->thd_to_arb_map; + unsigned int i; + + for (i = 0; i < hw_data->num_engines; i++) { + thd2arb_map[i] = adf_gen6_get_arb_mask(accel_dev, i); + dev_dbg(&GET_DEV(accel_dev), "ME:%d arb_mask:%#x\n", i, thd2arb_map[i]); + } + + return 0; +} + +static void set_vc_csr_for_bank(void __iomem *csr, u32 bank_number) +{ + u32 value; + + /* + * After each PF FLR, for each of the 64 ring pairs in the PF, the + * driver must program the ringmodectl CSRs. + */ + value = ADF_CSR_RD(csr, ADF_GEN6_CSR_RINGMODECTL(bank_number)); + value |= FIELD_PREP(ADF_GEN6_RINGMODECTL_TC_MASK, ADF_GEN6_RINGMODECTL_TC_DEFAULT); + value |= FIELD_PREP(ADF_GEN6_RINGMODECTL_TC_EN_MASK, ADF_GEN6_RINGMODECTL_TC_EN_OP1); + ADF_CSR_WR(csr, ADF_GEN6_CSR_RINGMODECTL(bank_number), value); +} + +static int set_vc_config(struct adf_accel_dev *accel_dev) +{ + struct pci_dev *pdev = accel_to_pci_dev(accel_dev); + u32 value; + int err; + + /* + * After each PF FLR, the driver must program the Port Virtual Channel (VC) + * Control Registers. + * Read PVC0CTL then write the masked values. + */ + pci_read_config_dword(pdev, ADF_GEN6_PVC0CTL_OFFSET, &value); + value |= FIELD_PREP(ADF_GEN6_PVC0CTL_TCVCMAP_MASK, ADF_GEN6_PVC0CTL_TCVCMAP_DEFAULT); + err = pci_write_config_dword(pdev, ADF_GEN6_PVC0CTL_OFFSET, value); + if (err) { + dev_err(&GET_DEV(accel_dev), "pci write to PVC0CTL failed\n"); + return pcibios_err_to_errno(err); + } + + /* Read PVC1CTL then write masked values */ + pci_read_config_dword(pdev, ADF_GEN6_PVC1CTL_OFFSET, &value); + value |= FIELD_PREP(ADF_GEN6_PVC1CTL_TCVCMAP_MASK, ADF_GEN6_PVC1CTL_TCVCMAP_DEFAULT); + value |= FIELD_PREP(ADF_GEN6_PVC1CTL_VCEN_MASK, ADF_GEN6_PVC1CTL_VCEN_ON); + err = pci_write_config_dword(pdev, ADF_GEN6_PVC1CTL_OFFSET, value); + if (err) + dev_err(&GET_DEV(accel_dev), "pci write to PVC1CTL failed\n"); + + return pcibios_err_to_errno(err); +} + +static int adf_gen6_set_vc(struct adf_accel_dev *accel_dev) +{ + struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev); + void __iomem *csr = adf_get_etr_base(accel_dev); + u32 i; + + for (i = 0; i < hw_data->num_banks; i++) { + dev_dbg(&GET_DEV(accel_dev), "set virtual channels for bank:%d\n", i); + set_vc_csr_for_bank(csr, i); + } + + return set_vc_config(accel_dev); +} + +static u32 get_ae_mask(struct adf_hw_device_data *self) +{ + unsigned long fuses = self->fuses[ADF_FUSECTL4]; + u32 mask = ADF_6XXX_ACCELENGINES_MASK; + + /* + * If bit 0 is set in the fuses, the first 4 engines are disabled. + * If bit 4 is set, the second group of 4 engines are disabled. + * If bit 8 is set, the admin engine (bit 8) is disabled. + */ + if (test_bit(0, &fuses)) + mask &= ~ADF_AE_GROUP_0; + + if (test_bit(4, &fuses)) + mask &= ~ADF_AE_GROUP_1; + + if (test_bit(8, &fuses)) + mask &= ~ADF_AE_GROUP_2; + + return mask; +} + +static u32 get_accel_cap(struct adf_accel_dev *accel_dev) +{ + u32 capabilities_sym, capabilities_asym; + u32 capabilities_dc; + unsigned long mask; + u32 caps = 0; + u32 fusectl1; + + fusectl1 = GET_HW_DATA(accel_dev)->fuses[ADF_FUSECTL1]; + + /* Read accelerator capabilities mask */ + capabilities_sym = ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC | + ICP_ACCEL_CAPABILITIES_CIPHER | + ICP_ACCEL_CAPABILITIES_AUTHENTICATION | + ICP_ACCEL_CAPABILITIES_SHA3 | + ICP_ACCEL_CAPABILITIES_SHA3_EXT | + ICP_ACCEL_CAPABILITIES_CHACHA_POLY | + ICP_ACCEL_CAPABILITIES_AESGCM_SPC | + ICP_ACCEL_CAPABILITIES_AES_V2; + + /* A set bit in fusectl1 means the corresponding feature is OFF in this SKU */ + if (fusectl1 & ICP_ACCEL_GEN6_MASK_UCS_SLICE) { + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC; + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_CIPHER; + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_CHACHA_POLY; + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_AESGCM_SPC; + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_AES_V2; + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_CIPHER; + } + if (fusectl1 & ICP_ACCEL_GEN6_MASK_AUTH_SLICE) { + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_AUTHENTICATION; + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_SHA3; + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_SHA3_EXT; + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_CIPHER; + } + + capabilities_asym = 0; + + capabilities_dc = ICP_ACCEL_CAPABILITIES_COMPRESSION | + ICP_ACCEL_CAPABILITIES_LZ4_COMPRESSION | + ICP_ACCEL_CAPABILITIES_LZ4S_COMPRESSION | + ICP_ACCEL_CAPABILITIES_CNV_INTEGRITY64; + + if (fusectl1 & ICP_ACCEL_GEN6_MASK_CPR_SLICE) { + capabilities_dc &= ~ICP_ACCEL_CAPABILITIES_COMPRESSION; + capabilities_dc &= ~ICP_ACCEL_CAPABILITIES_LZ4_COMPRESSION; + capabilities_dc &= ~ICP_ACCEL_CAPABILITIES_LZ4S_COMPRESSION; + capabilities_dc &= ~ICP_ACCEL_CAPABILITIES_CNV_INTEGRITY64; + } + + if (adf_get_service_mask(accel_dev, &mask)) + return 0; + + if (test_bit(SVC_ASYM, &mask)) + caps |= capabilities_asym; + if (test_bit(SVC_SYM, &mask)) + caps |= capabilities_sym; + if (test_bit(SVC_DC, &mask)) + caps |= capabilities_dc; + if (test_bit(SVC_DCC, &mask)) { + /* + * Sym capabilities are available for chaining operations, + * but sym crypto instances cannot be supported + */ + caps = capabilities_dc | capabilities_sym; + caps &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC; + } + + return caps; +} + +static u32 uof_get_num_objs(struct adf_accel_dev *accel_dev) +{ + return ARRAY_SIZE(adf_default_fw_config); +} + +static const char *uof_get_name(struct adf_accel_dev *accel_dev, u32 obj_num) +{ + int num_fw_objs = ARRAY_SIZE(adf_6xxx_fw_objs); + int id; + + id = adf_default_fw_config[obj_num].obj; + if (id >= num_fw_objs) + return NULL; + + return adf_6xxx_fw_objs[id]; +} + +static const char *uof_get_name_6xxx(struct adf_accel_dev *accel_dev, u32 obj_num) +{ + return uof_get_name(accel_dev, obj_num); +} + +static int uof_get_obj_type(struct adf_accel_dev *accel_dev, u32 obj_num) +{ + if (obj_num >= uof_get_num_objs(accel_dev)) + return -EINVAL; + + return adf_default_fw_config[obj_num].obj; +} + +static u32 uof_get_ae_mask(struct adf_accel_dev *accel_dev, u32 obj_num) +{ + return adf_default_fw_config[obj_num].ae_mask; +} + +static const u32 *adf_get_arbiter_mapping(struct adf_accel_dev *accel_dev) +{ + if (adf_gen6_init_thd2arb_map(accel_dev)) + dev_warn(&GET_DEV(accel_dev), + "Failed to generate thread to arbiter mapping"); + + return GET_HW_DATA(accel_dev)->thd_to_arb_map; +} + +static int adf_init_device(struct adf_accel_dev *accel_dev) +{ + void __iomem *addr = adf_get_pmisc_base(accel_dev); + u32 status; + u32 csr; + int ret; + + /* Temporarily mask PM interrupt */ + csr = ADF_CSR_RD(addr, ADF_GEN6_ERRMSK2); + csr |= ADF_GEN6_PM_SOU; + ADF_CSR_WR(addr, ADF_GEN6_ERRMSK2, csr); + + /* Set DRV_ACTIVE bit to power up the device */ + ADF_CSR_WR(addr, ADF_GEN6_PM_INTERRUPT, ADF_GEN6_PM_DRV_ACTIVE); + + /* Poll status register to make sure the device is powered up */ + ret = read_poll_timeout(ADF_CSR_RD, status, + status & ADF_GEN6_PM_INIT_STATE, + ADF_GEN6_PM_POLL_DELAY_US, + ADF_GEN6_PM_POLL_TIMEOUT_US, true, addr, + ADF_GEN6_PM_STATUS); + if (ret) { + dev_err(&GET_DEV(accel_dev), "Failed to power up the device\n"); + return ret; + } + + dev_dbg(&GET_DEV(accel_dev), "Setting virtual channels for device qat_dev%d\n", + accel_dev->accel_id); + + ret = adf_gen6_set_vc(accel_dev); + if (ret) + dev_err(&GET_DEV(accel_dev), "Failed to set virtual channels\n"); + + return ret; +} + +static int enable_pm(struct adf_accel_dev *accel_dev) +{ + return adf_init_admin_pm(accel_dev, ADF_GEN6_PM_DEFAULT_IDLE_FILTER); +} + +static int dev_config(struct adf_accel_dev *accel_dev) +{ + int ret; + + ret = adf_cfg_section_add(accel_dev, ADF_KERNEL_SEC); + if (ret) + return ret; + + ret = adf_cfg_section_add(accel_dev, "Accelerator0"); + if (ret) + return ret; + + switch (adf_get_service_enabled(accel_dev)) { + case SVC_DC: + case SVC_DCC: + ret = adf_gen6_comp_dev_config(accel_dev); + break; + default: + ret = adf_gen6_no_dev_config(accel_dev); + break; + } + if (ret) + return ret; + + __set_bit(ADF_STATUS_CONFIGURED, &accel_dev->status); + + return ret; +} + +void adf_init_hw_data_6xxx(struct adf_hw_device_data *hw_data) +{ + hw_data->dev_class = &adf_6xxx_class; + hw_data->instance_id = adf_6xxx_class.instances++; + hw_data->num_banks = ADF_GEN6_ETR_MAX_BANKS; + hw_data->num_banks_per_vf = ADF_GEN6_NUM_BANKS_PER_VF; + hw_data->num_rings_per_bank = ADF_GEN6_NUM_RINGS_PER_BANK; + hw_data->num_accel = ADF_GEN6_MAX_ACCELERATORS; + hw_data->num_engines = ADF_6XXX_MAX_ACCELENGINES; + hw_data->num_logical_accel = 1; + hw_data->tx_rx_gap = ADF_GEN6_RX_RINGS_OFFSET; + hw_data->tx_rings_mask = ADF_GEN6_TX_RINGS_MASK; + hw_data->ring_to_svc_map = 0; + hw_data->alloc_irq = adf_isr_resource_alloc; + hw_data->free_irq = adf_isr_resource_free; + hw_data->enable_error_correction = enable_error_correction; + hw_data->get_accel_mask = get_accel_mask; + hw_data->get_ae_mask = get_ae_mask; + hw_data->get_num_accels = get_num_accels; + hw_data->get_num_aes = get_num_aes; + hw_data->get_sram_bar_id = get_sram_bar_id; + hw_data->get_etr_bar_id = get_etr_bar_id; + hw_data->get_misc_bar_id = get_misc_bar_id; + hw_data->get_arb_info = get_arb_info; + hw_data->get_admin_info = get_admin_info; + hw_data->get_accel_cap = get_accel_cap; + hw_data->get_sku = get_sku; + hw_data->init_admin_comms = adf_init_admin_comms; + hw_data->exit_admin_comms = adf_exit_admin_comms; + hw_data->send_admin_init = adf_send_admin_init; + hw_data->init_arb = adf_init_arb; + hw_data->exit_arb = adf_exit_arb; + hw_data->get_arb_mapping = adf_get_arbiter_mapping; + hw_data->enable_ints = enable_ints; + hw_data->reset_device = adf_reset_flr; + hw_data->admin_ae_mask = ADF_6XXX_ADMIN_AE_MASK; + hw_data->fw_name = ADF_6XXX_FW; + hw_data->fw_mmp_name = ADF_6XXX_MMP; + hw_data->uof_get_name = uof_get_name_6xxx; + hw_data->uof_get_num_objs = uof_get_num_objs; + hw_data->uof_get_obj_type = uof_get_obj_type; + hw_data->uof_get_ae_mask = uof_get_ae_mask; + hw_data->set_msix_rttable = set_msix_default_rttable; + hw_data->set_ssm_wdtimer = set_ssm_wdtimer; + hw_data->get_ring_to_svc_map = get_ring_to_svc_map; + hw_data->disable_iov = adf_disable_sriov; + hw_data->ring_pair_reset = ring_pair_reset; + hw_data->dev_config = dev_config; + hw_data->get_hb_clock = get_heartbeat_clock; + hw_data->num_hb_ctrs = ADF_NUM_HB_CNT_PER_AE; + hw_data->start_timer = adf_timer_start; + hw_data->stop_timer = adf_timer_stop; + hw_data->init_device = adf_init_device; + hw_data->enable_pm = enable_pm; + hw_data->services_supported = services_supported; + + adf_gen6_init_hw_csr_ops(&hw_data->csr_ops); + adf_gen6_init_pf_pfvf_ops(&hw_data->pfvf_ops); + adf_gen6_init_dc_ops(&hw_data->dc_ops); +} + +void adf_clean_hw_data_6xxx(struct adf_hw_device_data *hw_data) +{ + if (hw_data->dev_class->instances) + hw_data->dev_class->instances--; +} diff --git a/drivers/crypto/intel/qat/qat_6xxx/adf_6xxx_hw_data.h b/drivers/crypto/intel/qat/qat_6xxx/adf_6xxx_hw_data.h new file mode 100644 index 000000000000..78e2e2c5816e --- /dev/null +++ b/drivers/crypto/intel/qat/qat_6xxx/adf_6xxx_hw_data.h @@ -0,0 +1,148 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2025 Intel Corporation */ +#ifndef ADF_6XXX_HW_DATA_H_ +#define ADF_6XXX_HW_DATA_H_ + +#include +#include +#include + +#include "adf_accel_devices.h" +#include "adf_cfg_common.h" +#include "adf_dc.h" + +/* PCIe configuration space */ +#define ADF_GEN6_BAR_MASK (BIT(0) | BIT(2) | BIT(4)) +#define ADF_GEN6_SRAM_BAR 0 +#define ADF_GEN6_PMISC_BAR 1 +#define ADF_GEN6_ETR_BAR 2 +#define ADF_6XXX_MAX_ACCELENGINES 9 + +/* Clocks frequency */ +#define ADF_GEN6_COUNTER_FREQ (100 * HZ_PER_MHZ) + +/* Physical function fuses */ +#define ADF_GEN6_FUSECTL0_OFFSET 0x2C8 +#define ADF_GEN6_FUSECTL1_OFFSET 0x2CC +#define ADF_GEN6_FUSECTL4_OFFSET 0x2D8 + +/* Accelerators */ +#define ADF_GEN6_ACCELERATORS_MASK 0x1 +#define ADF_GEN6_MAX_ACCELERATORS 1 + +/* MSI-X interrupt */ +#define ADF_GEN6_SMIAPF_RP_X0_MASK_OFFSET 0x41A040 +#define ADF_GEN6_SMIAPF_RP_X1_MASK_OFFSET 0x41A044 +#define ADF_GEN6_SMIAPF_MASK_OFFSET 0x41A084 +#define ADF_GEN6_MSIX_RTTABLE_OFFSET(i) (0x409000 + ((i) * 4)) + +/* Bank and ring configuration */ +#define ADF_GEN6_NUM_RINGS_PER_BANK 2 +#define ADF_GEN6_NUM_BANKS_PER_VF 4 +#define ADF_GEN6_ETR_MAX_BANKS 64 +#define ADF_GEN6_RX_RINGS_OFFSET 1 +#define ADF_GEN6_TX_RINGS_MASK 0x1 + +/* Arbiter configuration */ +#define ADF_GEN6_ARB_CONFIG (BIT(31) | BIT(6) | BIT(0)) +#define ADF_GEN6_ARB_OFFSET 0x000 +#define ADF_GEN6_ARB_WRK_2_SER_MAP_OFFSET 0x400 + +/* Admin interface configuration */ +#define ADF_GEN6_ADMINMSGUR_OFFSET 0x500574 +#define ADF_GEN6_ADMINMSGLR_OFFSET 0x500578 +#define ADF_GEN6_MAILBOX_BASE_OFFSET 0x600970 + +/* + * Watchdog timers + * Timeout is in cycles. Clock speed may vary across products but this + * value should be a few milli-seconds. + */ +#define ADF_SSM_WDT_DEFAULT_VALUE 0x7000000ULL +#define ADF_SSM_WDT_PKE_DEFAULT_VALUE 0x8000000ULL +#define ADF_SSMWDTATHL_OFFSET 0x5208 +#define ADF_SSMWDTATHH_OFFSET 0x520C +#define ADF_SSMWDTCNVL_OFFSET 0x5408 +#define ADF_SSMWDTCNVH_OFFSET 0x540C +#define ADF_SSMWDTUCSL_OFFSET 0x5808 +#define ADF_SSMWDTUCSH_OFFSET 0x580C +#define ADF_SSMWDTDCPRL_OFFSET 0x5A08 +#define ADF_SSMWDTDCPRH_OFFSET 0x5A0C +#define ADF_SSMWDTPKEL_OFFSET 0x5E08 +#define ADF_SSMWDTPKEH_OFFSET 0x5E0C + +/* Ring reset */ +#define ADF_RPRESET_POLL_TIMEOUT_US (5 * USEC_PER_SEC) +#define ADF_RPRESET_POLL_DELAY_US 20 +#define ADF_WQM_CSR_RPRESETCTL_RESET BIT(0) +#define ADF_WQM_CSR_RPRESETCTL(bank) (0x6000 + (bank) * 8) +#define ADF_WQM_CSR_RPRESETSTS_STATUS BIT(0) +#define ADF_WQM_CSR_RPRESETSTS(bank) (ADF_WQM_CSR_RPRESETCTL(bank) + 4) + +/* Controls and sets up the corresponding ring mode of operation */ +#define ADF_GEN6_CSR_RINGMODECTL(bank) (0x9000 + (bank) * 4) + +/* Specifies the traffic class to use for the transactions to/from the ring */ +#define ADF_GEN6_RINGMODECTL_TC_MASK GENMASK(18, 16) +#define ADF_GEN6_RINGMODECTL_TC_DEFAULT 0x7 + +/* Specifies usage of tc for the transactions to/from this ring */ +#define ADF_GEN6_RINGMODECTL_TC_EN_MASK GENMASK(20, 19) + +/* + * Use the value programmed in the tc field for request descriptor + * and metadata read transactions + */ +#define ADF_GEN6_RINGMODECTL_TC_EN_OP1 0x1 + +/* VC0 Resource Control Register */ +#define ADF_GEN6_PVC0CTL_OFFSET 0x204 +#define ADF_GEN6_PVC0CTL_TCVCMAP_OFFSET 1 +#define ADF_GEN6_PVC0CTL_TCVCMAP_MASK GENMASK(7, 1) +#define ADF_GEN6_PVC0CTL_TCVCMAP_DEFAULT 0x7F + +/* VC1 Resource Control Register */ +#define ADF_GEN6_PVC1CTL_OFFSET 0x210 +#define ADF_GEN6_PVC1CTL_TCVCMAP_OFFSET 1 +#define ADF_GEN6_PVC1CTL_TCVCMAP_MASK GENMASK(7, 1) +#define ADF_GEN6_PVC1CTL_TCVCMAP_DEFAULT 0x40 +#define ADF_GEN6_PVC1CTL_VCEN_OFFSET 31 +#define ADF_GEN6_PVC1CTL_VCEN_MASK BIT(31) +/* RW bit: 0x1 - enables a Virtual Channel, 0x0 - disables */ +#define ADF_GEN6_PVC1CTL_VCEN_ON 0x1 + +/* Error source mask registers */ +#define ADF_GEN6_ERRMSK0 0x41A210 +#define ADF_GEN6_ERRMSK1 0x41A214 +#define ADF_GEN6_ERRMSK2 0x41A218 +#define ADF_GEN6_ERRMSK3 0x41A21C + +#define ADF_GEN6_VFLNOTIFY BIT(7) + +/* Number of heartbeat counter pairs */ +#define ADF_NUM_HB_CNT_PER_AE ADF_NUM_THREADS_PER_AE + +/* Physical function fuses */ +#define ADF_6XXX_ACCELENGINES_MASK GENMASK(8, 0) +#define ADF_6XXX_ADMIN_AE_MASK GENMASK(8, 8) + +/* Firmware binaries */ +#define ADF_6XXX_FW "qat_6xxx.bin" +#define ADF_6XXX_MMP "qat_6xxx_mmp.bin" +#define ADF_6XXX_CY_OBJ "qat_6xxx_cy.bin" +#define ADF_6XXX_DC_OBJ "qat_6xxx_dc.bin" +#define ADF_6XXX_ADMIN_OBJ "qat_6xxx_admin.bin" + +enum icp_qat_gen6_slice_mask { + ICP_ACCEL_GEN6_MASK_UCS_SLICE = BIT(0), + ICP_ACCEL_GEN6_MASK_AUTH_SLICE = BIT(1), + ICP_ACCEL_GEN6_MASK_PKE_SLICE = BIT(2), + ICP_ACCEL_GEN6_MASK_CPR_SLICE = BIT(3), + ICP_ACCEL_GEN6_MASK_DCPRZ_SLICE = BIT(4), + ICP_ACCEL_GEN6_MASK_WCP_WAT_SLICE = BIT(6), +}; + +void adf_init_hw_data_6xxx(struct adf_hw_device_data *hw_data); +void adf_clean_hw_data_6xxx(struct adf_hw_device_data *hw_data); + +#endif /* ADF_6XXX_HW_DATA_H_ */ diff --git a/drivers/crypto/intel/qat/qat_6xxx/adf_drv.c b/drivers/crypto/intel/qat/qat_6xxx/adf_drv.c new file mode 100644 index 000000000000..2531c337e0dd --- /dev/null +++ b/drivers/crypto/intel/qat/qat_6xxx/adf_drv.c @@ -0,0 +1,224 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2025 Intel Corporation */ +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "adf_gen6_shared.h" +#include "adf_6xxx_hw_data.h" + +static int bar_map[] = { + 0, /* SRAM */ + 2, /* PMISC */ + 4, /* ETR */ +}; + +static void adf_device_down(void *accel_dev) +{ + adf_dev_down(accel_dev); +} + +static void adf_dbgfs_cleanup(void *accel_dev) +{ + adf_dbgfs_exit(accel_dev); +} + +static void adf_cfg_device_remove(void *accel_dev) +{ + adf_cfg_dev_remove(accel_dev); +} + +static void adf_cleanup_hw_data(void *accel_dev) +{ + struct adf_accel_dev *accel_device = accel_dev; + + if (accel_device->hw_device) { + adf_clean_hw_data_6xxx(accel_device->hw_device); + accel_device->hw_device = NULL; + } +} + +static void adf_devmgr_remove(void *accel_dev) +{ + adf_devmgr_rm_dev(accel_dev, NULL); +} + +static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) +{ + struct adf_accel_pci *accel_pci_dev; + struct adf_hw_device_data *hw_data; + struct device *dev = &pdev->dev; + struct adf_accel_dev *accel_dev; + struct adf_bar *bar; + unsigned int i; + int ret; + + if (num_possible_nodes() > 1 && dev_to_node(dev) < 0) { + /* + * If the accelerator is connected to a node with no memory + * there is no point in using the accelerator since the remote + * memory transaction will be very slow. + */ + return dev_err_probe(dev, -EINVAL, "Invalid NUMA configuration.\n"); + } + + accel_dev = devm_kzalloc(dev, sizeof(*accel_dev), GFP_KERNEL); + if (!accel_dev) + return -ENOMEM; + + INIT_LIST_HEAD(&accel_dev->crypto_list); + INIT_LIST_HEAD(&accel_dev->list); + accel_pci_dev = &accel_dev->accel_pci_dev; + accel_pci_dev->pci_dev = pdev; + accel_dev->owner = THIS_MODULE; + + hw_data = devm_kzalloc(dev, sizeof(*hw_data), GFP_KERNEL); + if (!hw_data) + return -ENOMEM; + + pci_read_config_byte(pdev, PCI_REVISION_ID, &accel_pci_dev->revid); + pci_read_config_dword(pdev, ADF_GEN6_FUSECTL4_OFFSET, &hw_data->fuses[ADF_FUSECTL4]); + pci_read_config_dword(pdev, ADF_GEN6_FUSECTL0_OFFSET, &hw_data->fuses[ADF_FUSECTL0]); + pci_read_config_dword(pdev, ADF_GEN6_FUSECTL1_OFFSET, &hw_data->fuses[ADF_FUSECTL1]); + + if (!(hw_data->fuses[ADF_FUSECTL1] & ICP_ACCEL_GEN6_MASK_WCP_WAT_SLICE)) + return dev_err_probe(dev, -EFAULT, "Wireless mode is not supported.\n"); + + /* Enable PCI device */ + ret = pcim_enable_device(pdev); + if (ret) + return dev_err_probe(dev, ret, "Cannot enable PCI device.\n"); + + ret = adf_devmgr_add_dev(accel_dev, NULL); + if (ret) + return dev_err_probe(dev, ret, "Failed to add new accelerator device.\n"); + + ret = devm_add_action_or_reset(dev, adf_devmgr_remove, accel_dev); + if (ret) + return ret; + + accel_dev->hw_device = hw_data; + adf_init_hw_data_6xxx(accel_dev->hw_device); + + ret = devm_add_action_or_reset(dev, adf_cleanup_hw_data, accel_dev); + if (ret) + return ret; + + /* Get Accelerators and Accelerator Engine masks */ + hw_data->accel_mask = hw_data->get_accel_mask(hw_data); + hw_data->ae_mask = hw_data->get_ae_mask(hw_data); + accel_pci_dev->sku = hw_data->get_sku(hw_data); + + /* If the device has no acceleration engines then ignore it */ + if (!hw_data->accel_mask || !hw_data->ae_mask || + (~hw_data->ae_mask & ADF_GEN6_ACCELERATORS_MASK)) { + ret = -EFAULT; + return dev_err_probe(dev, ret, "No acceleration units were found.\n"); + } + + /* Create device configuration table */ + ret = adf_cfg_dev_add(accel_dev); + if (ret) + return ret; + + ret = devm_add_action_or_reset(dev, adf_cfg_device_remove, accel_dev); + if (ret) + return ret; + + /* Set DMA identifier */ + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64)); + if (ret) + return dev_err_probe(dev, ret, "No usable DMA configuration.\n"); + + ret = adf_gen6_cfg_dev_init(accel_dev); + if (ret) + return dev_err_probe(dev, ret, "Failed to initialize configuration.\n"); + + /* Get accelerator capability mask */ + hw_data->accel_capabilities_mask = hw_data->get_accel_cap(accel_dev); + if (!hw_data->accel_capabilities_mask) { + ret = -EINVAL; + return dev_err_probe(dev, ret, "Failed to get capabilities mask.\n"); + } + + for (i = 0; i < ARRAY_SIZE(bar_map); i++) { + bar = &accel_pci_dev->pci_bars[i]; + + /* Map 64-bit PCIe BAR */ + bar->virt_addr = pcim_iomap_region(pdev, bar_map[i], pci_name(pdev)); + if (!bar->virt_addr) { + ret = -ENOMEM; + return dev_err_probe(dev, ret, "Failed to ioremap PCI region.\n"); + } + } + + pci_set_master(pdev); + + /* + * The PCI config space is saved at this point and will be restored + * after a Function Level Reset (FLR) as the FLR does not completely + * restore it. + */ + ret = pci_save_state(pdev); + if (ret) + return dev_err_probe(dev, ret, "Failed to save pci state.\n"); + + adf_dbgfs_init(accel_dev); + + ret = devm_add_action_or_reset(dev, adf_dbgfs_cleanup, accel_dev); + if (ret) + return ret; + + ret = adf_dev_up(accel_dev, true); + if (ret) + return ret; + + ret = devm_add_action_or_reset(dev, adf_device_down, accel_dev); + if (ret) + return ret; + + ret = adf_sysfs_init(accel_dev); + + return ret; +} + +static void adf_shutdown(struct pci_dev *pdev) +{ + struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); + + adf_dev_down(accel_dev); +} + +static const struct pci_device_id adf_pci_tbl[] = { + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QAT_6XXX) }, + { } +}; +MODULE_DEVICE_TABLE(pci, adf_pci_tbl); + +static struct pci_driver adf_driver = { + .id_table = adf_pci_tbl, + .name = ADF_6XXX_DEVICE_NAME, + .probe = adf_probe, + .shutdown = adf_shutdown, + .sriov_configure = adf_sriov_configure, + .err_handler = &adf_err_handler, +}; +module_pci_driver(adf_driver); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Intel"); +MODULE_FIRMWARE(ADF_6XXX_FW); +MODULE_FIRMWARE(ADF_6XXX_MMP); +MODULE_DESCRIPTION("Intel(R) QuickAssist Technology for GEN6 Devices"); +MODULE_SOFTDEP("pre: crypto-intel_qat"); +MODULE_IMPORT_NS("CRYPTO_QAT"); diff --git a/drivers/crypto/intel/qat/qat_common/Makefile b/drivers/crypto/intel/qat/qat_common/Makefile index 0a9da7398a78..958f9a8ac6b3 100644 --- a/drivers/crypto/intel/qat/qat_common/Makefile +++ b/drivers/crypto/intel/qat/qat_common/Makefile @@ -19,6 +19,7 @@ intel_qat-y := adf_accel_engine.o \ adf_gen4_pm.o \ adf_gen4_ras.o \ adf_gen4_vf_mig.o \ + adf_gen6_shared.o \ adf_hw_arbiter.o \ adf_init.o \ adf_isr.o \ diff --git a/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h b/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h index ed8b85360573..2ee526063213 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h @@ -26,6 +26,7 @@ #define ADF_C3XXXVF_DEVICE_NAME "c3xxxvf" #define ADF_4XXX_DEVICE_NAME "4xxx" #define ADF_420XX_DEVICE_NAME "420xx" +#define ADF_6XXX_DEVICE_NAME "6xxx" #define PCI_DEVICE_ID_INTEL_QAT_4XXX 0x4940 #define PCI_DEVICE_ID_INTEL_QAT_4XXXIOV 0x4941 #define PCI_DEVICE_ID_INTEL_QAT_401XX 0x4942 @@ -35,6 +36,7 @@ #define PCI_DEVICE_ID_INTEL_QAT_420XX 0x4946 #define PCI_DEVICE_ID_INTEL_QAT_420XXIOV 0x4947 #define PCI_DEVICE_ID_INTEL_QAT_6XXX 0x4948 +#define PCI_DEVICE_ID_INTEL_QAT_6XXX_IOV 0x4949 #define ADF_DEVICE_FUSECTL_OFFSET 0x40 #define ADF_DEVICE_LEGFUSE_OFFSET 0x4C diff --git a/drivers/crypto/intel/qat/qat_common/adf_cfg_common.h b/drivers/crypto/intel/qat/qat_common/adf_cfg_common.h index 89df3888d7ea..15fdf9854b81 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_cfg_common.h +++ b/drivers/crypto/intel/qat/qat_common/adf_cfg_common.h @@ -48,6 +48,7 @@ enum adf_device_type { DEV_C3XXXVF, DEV_4XXX, DEV_420XX, + DEV_6XXX, }; struct adf_dev_status_info { diff --git a/drivers/crypto/intel/qat/qat_common/adf_fw_config.h b/drivers/crypto/intel/qat/qat_common/adf_fw_config.h index 4f86696800c9..78957fa900b7 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_fw_config.h +++ b/drivers/crypto/intel/qat/qat_common/adf_fw_config.h @@ -8,6 +8,7 @@ enum adf_fw_objs { ADF_FW_ASYM_OBJ, ADF_FW_DC_OBJ, ADF_FW_ADMIN_OBJ, + ADF_FW_CY_OBJ, }; struct adf_fw_config { diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen6_pm.h b/drivers/crypto/intel/qat/qat_common/adf_gen6_pm.h new file mode 100644 index 000000000000..9a5b995f7ada --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_gen6_pm.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2025 Intel Corporation */ +#ifndef ADF_GEN6_PM_H +#define ADF_GEN6_PM_H + +#include +#include + +struct adf_accel_dev; + +/* Power management */ +#define ADF_GEN6_PM_POLL_DELAY_US 20 +#define ADF_GEN6_PM_POLL_TIMEOUT_US USEC_PER_SEC +#define ADF_GEN6_PM_STATUS 0x50A00C +#define ADF_GEN6_PM_INTERRUPT 0x50A028 + +/* Power management source in ERRSOU2 and ERRMSK2 */ +#define ADF_GEN6_PM_SOU BIT(18) + +/* cpm_pm_interrupt bitfields */ +#define ADF_GEN6_PM_DRV_ACTIVE BIT(20) + +#define ADF_GEN6_PM_DEFAULT_IDLE_FILTER 0x6 + +/* cpm_pm_status bitfields */ +#define ADF_GEN6_PM_INIT_STATE BIT(21) + +#endif /* ADF_GEN6_PM_H */ diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen6_shared.c b/drivers/crypto/intel/qat/qat_common/adf_gen6_shared.c new file mode 100644 index 000000000000..58a072e2f936 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_gen6_shared.c @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2025 Intel Corporation */ +#include + +#include "adf_gen4_config.h" +#include "adf_gen4_hw_csr_data.h" +#include "adf_gen4_pfvf.h" +#include "adf_gen6_shared.h" + +struct adf_accel_dev; +struct adf_pfvf_ops; +struct adf_hw_csr_ops; + +/* + * QAT GEN4 and GEN6 devices often differ in terms of supported features, + * options and internal logic. However, some of the mechanisms and register + * layout are shared between those two GENs. This file serves as an abstraction + * layer that allows to use existing GEN4 implementation that is also + * applicable to GEN6 without additional overhead and complexity. + */ +void adf_gen6_init_pf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops) +{ + adf_gen4_init_pf_pfvf_ops(pfvf_ops); +} +EXPORT_SYMBOL_GPL(adf_gen6_init_pf_pfvf_ops); + +void adf_gen6_init_hw_csr_ops(struct adf_hw_csr_ops *csr_ops) +{ + return adf_gen4_init_hw_csr_ops(csr_ops); +} +EXPORT_SYMBOL_GPL(adf_gen6_init_hw_csr_ops); + +int adf_gen6_cfg_dev_init(struct adf_accel_dev *accel_dev) +{ + return adf_gen4_cfg_dev_init(accel_dev); +} +EXPORT_SYMBOL_GPL(adf_gen6_cfg_dev_init); + +int adf_gen6_comp_dev_config(struct adf_accel_dev *accel_dev) +{ + return adf_comp_dev_config(accel_dev); +} +EXPORT_SYMBOL_GPL(adf_gen6_comp_dev_config); + +int adf_gen6_no_dev_config(struct adf_accel_dev *accel_dev) +{ + return adf_no_dev_config(accel_dev); +} +EXPORT_SYMBOL_GPL(adf_gen6_no_dev_config); diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen6_shared.h b/drivers/crypto/intel/qat/qat_common/adf_gen6_shared.h new file mode 100644 index 000000000000..bc8e71e984fc --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_gen6_shared.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2025 Intel Corporation */ +#ifndef ADF_GEN6_SHARED_H_ +#define ADF_GEN6_SHARED_H_ + +struct adf_hw_csr_ops; +struct adf_accel_dev; +struct adf_pfvf_ops; + +void adf_gen6_init_pf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops); +void adf_gen6_init_hw_csr_ops(struct adf_hw_csr_ops *csr_ops); +int adf_gen6_cfg_dev_init(struct adf_accel_dev *accel_dev); +int adf_gen6_comp_dev_config(struct adf_accel_dev *accel_dev); +int adf_gen6_no_dev_config(struct adf_accel_dev *accel_dev); +#endif/* ADF_GEN6_SHARED_H_ */ From 63ff962c058a0b32a6c512a9fcd5aae11ebcb7e1 Mon Sep 17 00:00:00 2001 From: Ovidiu Panait Date: Wed, 30 Apr 2025 14:59:53 +0300 Subject: [PATCH 290/382] crypto: rng - fix documentation for crypto_rng_alg() Current documentation states that crypto_rng_alg() returns the cra_name of the rng algorithm, but it actually returns a 'struct rng_alg' pointer from a RNG handle. Update documentation to reflect this. Signed-off-by: Ovidiu Panait Signed-off-by: Herbert Xu --- include/crypto/rng.h | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/include/crypto/rng.h b/include/crypto/rng.h index 5ac4388f50e1..f8224cc390f8 100644 --- a/include/crypto/rng.h +++ b/include/crypto/rng.h @@ -102,12 +102,10 @@ static inline struct rng_alg *__crypto_rng_alg(struct crypto_alg *alg) } /** - * crypto_rng_alg - obtain name of RNG - * @tfm: cipher handle + * crypto_rng_alg() - obtain 'struct rng_alg' pointer from RNG handle + * @tfm: RNG handle * - * Return the generic name (cra_name) of the initialized random number generator - * - * Return: generic name string + * Return: Pointer to 'struct rng_alg', derived from @tfm RNG handle */ static inline struct rng_alg *crypto_rng_alg(struct crypto_rng *tfm) { From 165ef524bbeb71ccd470e70a4e63f813fa71e7cd Mon Sep 17 00:00:00 2001 From: Nicolas Frattaroli Date: Wed, 30 Apr 2025 18:16:34 +0200 Subject: [PATCH 291/382] dt-bindings: rng: rockchip,rk3588-rng: add rk3576-rng compatible The RK3576 SoC contains another standalone TRNG implementation. While the register map and hardware is different, it has the same clocks/interrupts/resets as the RK3588's TRNG, so can go in the same binding. Add the compatible and generalise the title/description of the binding some more. Signed-off-by: Nicolas Frattaroli Acked-by: Krzysztof Kozlowski Signed-off-by: Herbert Xu --- .../devicetree/bindings/rng/rockchip,rk3588-rng.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Documentation/devicetree/bindings/rng/rockchip,rk3588-rng.yaml b/Documentation/devicetree/bindings/rng/rockchip,rk3588-rng.yaml index ca71b400bcae..fcc5be80142d 100644 --- a/Documentation/devicetree/bindings/rng/rockchip,rk3588-rng.yaml +++ b/Documentation/devicetree/bindings/rng/rockchip,rk3588-rng.yaml @@ -4,9 +4,9 @@ $id: http://devicetree.org/schemas/rng/rockchip,rk3588-rng.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Rockchip RK3588 TRNG +title: Rockchip RK3576/RK3588 TRNG -description: True Random Number Generator on Rockchip RK3588 SoC +description: True Random Number Generator on Rockchip RK3576/RK3588 SoCs maintainers: - Nicolas Frattaroli @@ -14,6 +14,7 @@ maintainers: properties: compatible: enum: + - rockchip,rk3576-rng - rockchip,rk3588-rng reg: From 8f66ccbd8f67ab41b29f54f383f8a8516be7696c Mon Sep 17 00:00:00 2001 From: Nicolas Frattaroli Date: Wed, 30 Apr 2025 18:16:35 +0200 Subject: [PATCH 292/382] hwrng: rockchip - add support for RK3576's RNG The Rockchip RK3576 SoC uses a new hardware random number generator IP. It's also used on the Rockchip RK3562 and the Rockchip RK3528. It has several modes of operation and self-checking features that are not implemented here. For starters, it has a DRNG output, which is an AES-CTR pseudo-random number generator that can be reseeded from the true entropy regularly. However, it also allows for access of the true entropy generator directly. This entropy is generated from an oscillator. There are several configuration registers which we don't touch here. The oscillator can be switched between a "CRO" and "STR" oscillator, and the length of the oscillator can be configured. The hardware also supports some automatic continuous entropy quality checking, which is also not implemented in this driver for the time being. The output as-is has been deemed sufficient to be useful: rngtest: starting FIPS tests... rngtest: bits received from input: 20000032 rngtest: FIPS 140-2 successes: 997 rngtest: FIPS 140-2 failures: 3 rngtest: FIPS 140-2(2001-10-10) Monobit: 0 rngtest: FIPS 140-2(2001-10-10) Poker: 1 rngtest: FIPS 140-2(2001-10-10) Runs: 1 rngtest: FIPS 140-2(2001-10-10) Long run: 1 rngtest: FIPS 140-2(2001-10-10) Continuous run: 0 rngtest: input channel speed: (min=17.050; avg=1897.272; max=19531250.000)Kibits/s rngtest: FIPS tests speed: (min=44.773; avg=71.179; max=96.820)Mibits/s rngtest: Program run time: 11760715 microseconds rngtest: bits received from input: 40000032 rngtest: FIPS 140-2 successes: 1997 rngtest: FIPS 140-2 failures: 3 rngtest: FIPS 140-2(2001-10-10) Monobit: 0 rngtest: FIPS 140-2(2001-10-10) Poker: 1 rngtest: FIPS 140-2(2001-10-10) Runs: 1 rngtest: FIPS 140-2(2001-10-10) Long run: 1 rngtest: FIPS 140-2(2001-10-10) Continuous run: 0 rngtest: input channel speed: (min=17.050; avg=1798.618; max=19531250.000)Kibits/s rngtest: FIPS tests speed: (min=44.773; avg=64.561; max=96.820)Mibits/s rngtest: Program run time: 23507723 microseconds Stretching the entropy can then be left up to Linux's actual entropy pool. Signed-off-by: Nicolas Frattaroli Signed-off-by: Herbert Xu --- drivers/char/hw_random/rockchip-rng.c | 73 +++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/drivers/char/hw_random/rockchip-rng.c b/drivers/char/hw_random/rockchip-rng.c index 161050591663..fb4a30b95507 100644 --- a/drivers/char/hw_random/rockchip-rng.c +++ b/drivers/char/hw_random/rockchip-rng.c @@ -93,6 +93,30 @@ #define TRNG_v1_VERSION_CODE 0x46bc /* end of TRNG_V1 register definitions */ +/* + * RKRNG register definitions + * The RKRNG IP is a stand-alone TRNG implementation (not part of a crypto IP) + * and can be found in the Rockchip RK3576, Rockchip RK3562 and Rockchip RK3528 + * SoCs. It can either output true randomness (TRNG) or "deterministic" + * randomness derived from hashing the true entropy (DRNG). This driver + * implementation uses just the true entropy, and leaves stretching the entropy + * up to Linux. + */ +#define RKRNG_CFG 0x0000 +#define RKRNG_CTRL 0x0010 +#define RKRNG_CTRL_REQ_TRNG BIT(4) +#define RKRNG_STATE 0x0014 +#define RKRNG_STATE_TRNG_RDY BIT(4) +#define RKRNG_TRNG_DATA0 0x0050 +#define RKRNG_TRNG_DATA1 0x0054 +#define RKRNG_TRNG_DATA2 0x0058 +#define RKRNG_TRNG_DATA3 0x005C +#define RKRNG_TRNG_DATA4 0x0060 +#define RKRNG_TRNG_DATA5 0x0064 +#define RKRNG_TRNG_DATA6 0x0068 +#define RKRNG_TRNG_DATA7 0x006C +#define RKRNG_READ_LEN 32 + /* Before removing this assert, give rk3588_rng_read an upper bound of 32 */ static_assert(RK_RNG_MAX_BYTE <= (TRNG_V1_RAND7 + 4 - TRNG_V1_RAND0), "You raised RK_RNG_MAX_BYTE and broke rk3588-rng, congrats."); @@ -205,6 +229,46 @@ out: return (ret < 0) ? ret : to_read; } +static int rk3576_rng_init(struct hwrng *rng) +{ + struct rk_rng *rk_rng = container_of(rng, struct rk_rng, rng); + + return rk_rng_enable_clks(rk_rng); +} + +static int rk3576_rng_read(struct hwrng *rng, void *buf, size_t max, bool wait) +{ + struct rk_rng *rk_rng = container_of(rng, struct rk_rng, rng); + size_t to_read = min_t(size_t, max, RKRNG_READ_LEN); + int ret = 0; + u32 val; + + ret = pm_runtime_resume_and_get(rk_rng->dev); + if (ret < 0) + return ret; + + rk_rng_writel(rk_rng, RKRNG_CTRL_REQ_TRNG | (RKRNG_CTRL_REQ_TRNG << 16), + RKRNG_CTRL); + + if (readl_poll_timeout(rk_rng->base + RKRNG_STATE, val, + (val & RKRNG_STATE_TRNG_RDY), RK_RNG_POLL_PERIOD_US, + RK_RNG_POLL_TIMEOUT_US)) { + dev_err(rk_rng->dev, "timed out waiting for data\n"); + ret = -ETIMEDOUT; + goto out; + } + + rk_rng_writel(rk_rng, RKRNG_STATE_TRNG_RDY, RKRNG_STATE); + + memcpy_fromio(buf, rk_rng->base + RKRNG_TRNG_DATA0, to_read); + +out: + pm_runtime_mark_last_busy(rk_rng->dev); + pm_runtime_put_sync_autosuspend(rk_rng->dev); + + return (ret < 0) ? ret : to_read; +} + static int rk3588_rng_init(struct hwrng *rng) { struct rk_rng *rk_rng = container_of(rng, struct rk_rng, rng); @@ -305,6 +369,14 @@ static const struct rk_rng_soc_data rk3568_soc_data = { .reset_optional = false, }; +static const struct rk_rng_soc_data rk3576_soc_data = { + .rk_rng_init = rk3576_rng_init, + .rk_rng_read = rk3576_rng_read, + .rk_rng_cleanup = rk3588_rng_cleanup, + .quality = 999, /* as determined by actual testing */ + .reset_optional = true, +}; + static const struct rk_rng_soc_data rk3588_soc_data = { .rk_rng_init = rk3588_rng_init, .rk_rng_read = rk3588_rng_read, @@ -397,6 +469,7 @@ static const struct dev_pm_ops rk_rng_pm_ops = { static const struct of_device_id rk_rng_dt_match[] = { { .compatible = "rockchip,rk3568-rng", .data = (void *)&rk3568_soc_data }, + { .compatible = "rockchip,rk3576-rng", .data = (void *)&rk3576_soc_data }, { .compatible = "rockchip,rk3588-rng", .data = (void *)&rk3588_soc_data }, { /* sentinel */ }, }; From 0efae411280988b43d5e5042aa047eeb276d1dac Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Thu, 1 May 2025 20:37:30 +0800 Subject: [PATCH 293/382] crypto: acomp - Clone folios properly The folios contain references to the request itself so they must be setup again in the cloned request. Fixes: 5f3437e9c89e ("crypto: acomp - Simplify folio handling") Signed-off-by: Herbert Xu --- crypto/acompress.c | 18 ++++++++++++++++++ include/crypto/acompress.h | 8 ++------ 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/crypto/acompress.c b/crypto/acompress.c index 9dea76ed4513..6ecbfc49bfa8 100644 --- a/crypto/acompress.c +++ b/crypto/acompress.c @@ -566,5 +566,23 @@ int acomp_walk_virt(struct acomp_walk *__restrict walk, } EXPORT_SYMBOL_GPL(acomp_walk_virt); +struct acomp_req *acomp_request_clone(struct acomp_req *req, + size_t total, gfp_t gfp) +{ + struct acomp_req *nreq; + + nreq = container_of(crypto_request_clone(&req->base, total, gfp), + struct acomp_req, base); + if (nreq == req) + return req; + + if (req->src == &req->chain.ssg) + nreq->src = &nreq->chain.ssg; + if (req->dst == &req->chain.dsg) + nreq->dst = &nreq->chain.dsg; + return nreq; +} +EXPORT_SYMBOL_GPL(acomp_request_clone); + MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Asynchronous compression type"); diff --git a/include/crypto/acompress.h b/include/crypto/acompress.h index f1812e92d3e3..9eacb9fa375d 100644 --- a/include/crypto/acompress.h +++ b/include/crypto/acompress.h @@ -551,11 +551,7 @@ static inline struct acomp_req *acomp_request_on_stack_init( return req; } -static inline struct acomp_req *acomp_request_clone(struct acomp_req *req, - size_t total, gfp_t gfp) -{ - return container_of(crypto_request_clone(&req->base, total, gfp), - struct acomp_req, base); -} +struct acomp_req *acomp_request_clone(struct acomp_req *req, + size_t total, gfp_t gfp); #endif From 8fd17374be8f220c26bec2b482cabf51ebbaed80 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Thu, 1 May 2025 20:37:32 +0800 Subject: [PATCH 294/382] crypto: api - Rename CRYPTO_ALG_REQ_CHAIN to CRYPTO_ALG_REQ_VIRT As chaining has been removed, all that remains of REQ_CHAIN is just virtual address support. Rename it before the reintroduction of batching creates confusion. Signed-off-by: Herbert Xu --- crypto/acompress.c | 4 ++-- crypto/ahash.c | 6 +++--- crypto/deflate.c | 2 +- crypto/scompress.c | 2 +- include/crypto/algapi.h | 4 ++-- include/crypto/internal/acompress.h | 4 ++-- include/crypto/internal/hash.h | 4 ++-- include/linux/crypto.h | 4 ++-- 8 files changed, 15 insertions(+), 15 deletions(-) diff --git a/crypto/acompress.c b/crypto/acompress.c index 6ecbfc49bfa8..be28cbfd22e3 100644 --- a/crypto/acompress.c +++ b/crypto/acompress.c @@ -287,7 +287,7 @@ int crypto_acomp_compress(struct acomp_req *req) if (acomp_req_on_stack(req) && acomp_is_async(tfm)) return -EAGAIN; - if (crypto_acomp_req_chain(tfm) || acomp_request_issg(req)) + if (crypto_acomp_req_virt(tfm) || acomp_request_issg(req)) return crypto_acomp_reqtfm(req)->compress(req); return acomp_do_req_chain(req, true); } @@ -299,7 +299,7 @@ int crypto_acomp_decompress(struct acomp_req *req) if (acomp_req_on_stack(req) && acomp_is_async(tfm)) return -EAGAIN; - if (crypto_acomp_req_chain(tfm) || acomp_request_issg(req)) + if (crypto_acomp_req_virt(tfm) || acomp_request_issg(req)) return crypto_acomp_reqtfm(req)->decompress(req); return acomp_do_req_chain(req, false); } diff --git a/crypto/ahash.c b/crypto/ahash.c index cc9885d5cfd2..57c131a13067 100644 --- a/crypto/ahash.c +++ b/crypto/ahash.c @@ -407,7 +407,7 @@ static int ahash_do_req_chain(struct ahash_request *req, u8 *page = NULL; int err; - if (crypto_ahash_req_chain(tfm) || + if (crypto_ahash_req_virt(tfm) || !update || !ahash_request_isvirt(req)) return op(req); @@ -550,7 +550,7 @@ int crypto_ahash_finup(struct ahash_request *req) if (ahash_req_on_stack(req) && ahash_is_async(tfm)) return -EAGAIN; if (!crypto_ahash_alg(tfm)->finup || - (!crypto_ahash_req_chain(tfm) && ahash_request_isvirt(req))) + (!crypto_ahash_req_virt(tfm) && ahash_request_isvirt(req))) return ahash_def_finup(req); return ahash_do_req_chain(req, crypto_ahash_alg(tfm)->finup); } @@ -622,7 +622,7 @@ int crypto_ahash_digest(struct ahash_request *req) return shash_ahash_digest(req, prepare_shash_desc(req, tfm)); if (ahash_req_on_stack(req) && ahash_is_async(tfm)) return -EAGAIN; - if (!crypto_ahash_req_chain(tfm) && ahash_request_isvirt(req)) + if (!crypto_ahash_req_virt(tfm) && ahash_request_isvirt(req)) return ahash_def_digest(req); if (crypto_ahash_get_flags(tfm) & CRYPTO_TFM_NEED_KEY) return -ENOKEY; diff --git a/crypto/deflate.c b/crypto/deflate.c index 7eb1a5c44ee0..fe8e4ad0fee1 100644 --- a/crypto/deflate.c +++ b/crypto/deflate.c @@ -228,7 +228,7 @@ static struct acomp_alg acomp = { .init = deflate_init, .base.cra_name = "deflate", .base.cra_driver_name = "deflate-generic", - .base.cra_flags = CRYPTO_ALG_REQ_CHAIN, + .base.cra_flags = CRYPTO_ALG_REQ_VIRT, .base.cra_module = THIS_MODULE, }; diff --git a/crypto/scompress.c b/crypto/scompress.c index 15148c58d648..c651e7f2197a 100644 --- a/crypto/scompress.c +++ b/crypto/scompress.c @@ -355,7 +355,7 @@ static void scomp_prepare_alg(struct scomp_alg *alg) comp_prepare_alg(&alg->calg); - base->cra_flags |= CRYPTO_ALG_REQ_CHAIN; + base->cra_flags |= CRYPTO_ALG_REQ_VIRT; } int crypto_register_scomp(struct scomp_alg *alg) diff --git a/include/crypto/algapi.h b/include/crypto/algapi.h index f5f730969d72..423e57eca351 100644 --- a/include/crypto/algapi.h +++ b/include/crypto/algapi.h @@ -255,9 +255,9 @@ static inline u32 crypto_tfm_alg_type(struct crypto_tfm *tfm) return tfm->__crt_alg->cra_flags & CRYPTO_ALG_TYPE_MASK; } -static inline bool crypto_tfm_req_chain(struct crypto_tfm *tfm) +static inline bool crypto_tfm_req_virt(struct crypto_tfm *tfm) { - return tfm->__crt_alg->cra_flags & CRYPTO_ALG_REQ_CHAIN; + return tfm->__crt_alg->cra_flags & CRYPTO_ALG_REQ_VIRT; } static inline u32 crypto_request_flags(struct crypto_async_request *req) diff --git a/include/crypto/internal/acompress.h b/include/crypto/internal/acompress.h index b72bb7a6a2b2..ffffd88bbbad 100644 --- a/include/crypto/internal/acompress.h +++ b/include/crypto/internal/acompress.h @@ -186,9 +186,9 @@ static inline bool acomp_request_isnondma(struct acomp_req *req) CRYPTO_ACOMP_REQ_DST_NONDMA); } -static inline bool crypto_acomp_req_chain(struct crypto_acomp *tfm) +static inline bool crypto_acomp_req_virt(struct crypto_acomp *tfm) { - return crypto_tfm_req_chain(&tfm->base); + return crypto_tfm_req_virt(&tfm->base); } void crypto_acomp_free_streams(struct crypto_acomp_streams *s); diff --git a/include/crypto/internal/hash.h b/include/crypto/internal/hash.h index 33d45275f5bd..e911f32f46dc 100644 --- a/include/crypto/internal/hash.h +++ b/include/crypto/internal/hash.h @@ -267,9 +267,9 @@ static inline bool ahash_request_isvirt(struct ahash_request *req) return req->base.flags & CRYPTO_AHASH_REQ_VIRT; } -static inline bool crypto_ahash_req_chain(struct crypto_ahash *tfm) +static inline bool crypto_ahash_req_virt(struct crypto_ahash *tfm) { - return crypto_tfm_req_chain(&tfm->base); + return crypto_tfm_req_virt(&tfm->base); } static inline struct crypto_ahash *crypto_ahash_fb(struct crypto_ahash *tfm) diff --git a/include/linux/crypto.h b/include/linux/crypto.h index b8d875b11193..b50f1954d1bb 100644 --- a/include/linux/crypto.h +++ b/include/linux/crypto.h @@ -133,8 +133,8 @@ */ #define CRYPTO_ALG_FIPS_INTERNAL 0x00020000 -/* Set if the algorithm supports request chains and virtual addresses. */ -#define CRYPTO_ALG_REQ_CHAIN 0x00040000 +/* Set if the algorithm supports virtual addresses. */ +#define CRYPTO_ALG_REQ_VIRT 0x00040000 /* The high bits 0xff000000 are reserved for type-specific flags. */ From 9334f427576e6d361a409959b52246b0aa10476f Mon Sep 17 00:00:00 2001 From: Ovidiu Panait Date: Thu, 1 May 2025 22:06:50 +0300 Subject: [PATCH 295/382] crypto: sun8i-ce - undo runtime PM changes during driver removal The pm_runtime_use_autosuspend() call must be undone with pm_runtime_dont_use_autosuspend() at driver exit, but this is not currently handled in the driver. To fix this issue and at the same time simplify error handling, switch to devm_pm_runtime_enable(). It will call both pm_runtime_disable() and pm_runtime_dont_use_autosuspend() during driver removal. Fixes: 06f751b61329 ("crypto: allwinner - Add sun8i-ce Crypto Engine") Signed-off-by: Ovidiu Panait Signed-off-by: Herbert Xu --- .../crypto/allwinner/sun8i-ce/sun8i-ce-core.c | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c index ec1ffda9ea32..658f520cee0c 100644 --- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c @@ -832,13 +832,12 @@ static int sun8i_ce_pm_init(struct sun8i_ce_dev *ce) err = pm_runtime_set_suspended(ce->dev); if (err) return err; - pm_runtime_enable(ce->dev); - return err; -} -static void sun8i_ce_pm_exit(struct sun8i_ce_dev *ce) -{ - pm_runtime_disable(ce->dev); + err = devm_pm_runtime_enable(ce->dev); + if (err) + return err; + + return 0; } static int sun8i_ce_get_clks(struct sun8i_ce_dev *ce) @@ -1041,7 +1040,7 @@ static int sun8i_ce_probe(struct platform_device *pdev) "sun8i-ce-ns", ce); if (err) { dev_err(ce->dev, "Cannot request CryptoEngine Non-secure IRQ (err=%d)\n", err); - goto error_irq; + goto error_pm; } err = sun8i_ce_register_algs(ce); @@ -1082,8 +1081,6 @@ static int sun8i_ce_probe(struct platform_device *pdev) return 0; error_alg: sun8i_ce_unregister_algs(ce); -error_irq: - sun8i_ce_pm_exit(ce); error_pm: sun8i_ce_free_chanlist(ce, MAXFLOW - 1); return err; @@ -1104,8 +1101,6 @@ static void sun8i_ce_remove(struct platform_device *pdev) #endif sun8i_ce_free_chanlist(ce, MAXFLOW - 1); - - sun8i_ce_pm_exit(ce); } static const struct of_device_id sun8i_ce_crypto_of_match_table[] = { From 7d2461c7616743d62be0df8f9a5f4a6de29f119a Mon Sep 17 00:00:00 2001 From: Ovidiu Panait Date: Thu, 1 May 2025 22:06:51 +0300 Subject: [PATCH 296/382] crypto: sun8i-ce-hash - use pm_runtime_resume_and_get() Replace pm_runtime_get_sync() usage with pm_runtime_resume_and_get() to simplify error handling. This is recommended in the documentation of pm_runtime_get_sync(). Signed-off-by: Ovidiu Panait Signed-off-by: Herbert Xu --- drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c index ba13fb75c05d..bef44f350167 100644 --- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c @@ -65,12 +65,11 @@ int sun8i_ce_hash_init_tfm(struct crypto_ahash *tfm) crypto_ahash_driver_name(op->fallback_tfm), CRYPTO_MAX_ALG_NAME); - err = pm_runtime_get_sync(op->ce->dev); + err = pm_runtime_resume_and_get(op->ce->dev); if (err < 0) goto error_pm; return 0; error_pm: - pm_runtime_put_noidle(op->ce->dev); crypto_free_ahash(op->fallback_tfm); return err; } From 5b90a779bc547939421bfeb333e470658ba94fb6 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 2 May 2025 13:30:53 +0800 Subject: [PATCH 297/382] crypto: lib/sha256 - Add helpers for block-based shash Add an internal sha256_finup helper and move the finalisation code from __sha256_final into it. Also add sha256_choose_blocks and CRYPTO_ARCH_HAVE_LIB_SHA256_SIMD so that the Crypto API can use the SIMD block function unconditionally. The Crypto API must not be used in hard IRQs and there is no reason to have a fallback path for hardirqs. Signed-off-by: Herbert Xu --- include/crypto/internal/sha2.h | 45 ++++++++++++++++++++++++++++++++++ lib/crypto/Kconfig | 8 ++++++ lib/crypto/sha256.c | 32 +++++++----------------- 3 files changed, 62 insertions(+), 23 deletions(-) diff --git a/include/crypto/internal/sha2.h b/include/crypto/internal/sha2.h index d641c67abcbc..fff156f66edc 100644 --- a/include/crypto/internal/sha2.h +++ b/include/crypto/internal/sha2.h @@ -3,7 +3,12 @@ #ifndef _CRYPTO_INTERNAL_SHA2_H #define _CRYPTO_INTERNAL_SHA2_H +#include #include +#include +#include +#include +#include void sha256_update_generic(struct sha256_state *sctx, const u8 *data, size_t len); @@ -24,5 +29,45 @@ void sha256_blocks_generic(u32 state[SHA256_STATE_WORDS], const u8 *data, size_t nblocks); void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS], const u8 *data, size_t nblocks); +void sha256_blocks_simd(u32 state[SHA256_STATE_WORDS], + const u8 *data, size_t nblocks); + +static inline void sha256_choose_blocks( + u32 state[SHA256_STATE_WORDS], const u8 *data, size_t nblocks, + bool force_generic, bool force_simd) +{ + if (!IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_SHA256) || force_generic) + sha256_blocks_generic(state, data, nblocks); + else if (IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_SHA256_SIMD) && + (force_simd || crypto_simd_usable())) + sha256_blocks_simd(state, data, nblocks); + else + sha256_blocks_arch(state, data, nblocks); +} + +static __always_inline void sha256_finup( + struct crypto_sha256_state *sctx, u8 buf[SHA256_BLOCK_SIZE], + size_t len, u8 out[SHA256_DIGEST_SIZE], size_t digest_size, + bool force_generic, bool force_simd) +{ + const size_t bit_offset = SHA256_BLOCK_SIZE - 8; + __be64 *bits = (__be64 *)&buf[bit_offset]; + int i; + + buf[len++] = 0x80; + if (len > bit_offset) { + memset(&buf[len], 0, SHA256_BLOCK_SIZE - len); + sha256_choose_blocks(sctx->state, buf, 1, force_generic, + force_simd); + len = 0; + } + + memset(&buf[len], 0, bit_offset - len); + *bits = cpu_to_be64(sctx->count << 3); + sha256_choose_blocks(sctx->state, buf, 1, force_generic, force_simd); + + for (i = 0; i < digest_size; i += 4) + put_unaligned_be32(sctx->state[i / 4], out + i); +} #endif /* _CRYPTO_INTERNAL_SHA2_H */ diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index 6319358b38c2..1ec1466108cc 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -150,6 +150,14 @@ config CRYPTO_ARCH_HAVE_LIB_SHA256 Declares whether the architecture provides an arch-specific accelerated implementation of the SHA-256 library interface. +config CRYPTO_ARCH_HAVE_LIB_SHA256_SIMD + bool + help + Declares whether the architecture provides an arch-specific + accelerated implementation of the SHA-256 library interface + that is SIMD-based and therefore not usable in hardirq + context. + config CRYPTO_LIB_SHA256_GENERIC tristate default CRYPTO_LIB_SHA256 if !CRYPTO_ARCH_HAVE_LIB_SHA256 diff --git a/lib/crypto/sha256.c b/lib/crypto/sha256.c index 563f09c9f381..2ced29efa181 100644 --- a/lib/crypto/sha256.c +++ b/lib/crypto/sha256.c @@ -15,7 +15,6 @@ #include #include #include -#include /* * If __DISABLE_EXPORTS is defined, then this file is being compiled for a @@ -26,14 +25,16 @@ #include "sha256-generic.c" #endif +static inline bool sha256_purgatory(void) +{ + return __is_defined(__DISABLE_EXPORTS); +} + static inline void sha256_blocks(u32 state[SHA256_STATE_WORDS], const u8 *data, size_t nblocks, bool force_generic) { -#if IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_SHA256) && !defined(__DISABLE_EXPORTS) - if (!force_generic) - return sha256_blocks_arch(state, data, nblocks); -#endif - sha256_blocks_generic(state, data, nblocks); + sha256_choose_blocks(state, data, nblocks, + force_generic || sha256_purgatory(), false); } static inline void __sha256_update(struct sha256_state *sctx, const u8 *data, @@ -79,25 +80,10 @@ EXPORT_SYMBOL(sha256_update); static inline void __sha256_final(struct sha256_state *sctx, u8 *out, size_t digest_size, bool force_generic) { - const size_t bit_offset = SHA256_BLOCK_SIZE - sizeof(__be64); - __be64 *bits = (__be64 *)&sctx->buf[bit_offset]; size_t partial = sctx->count % SHA256_BLOCK_SIZE; - size_t i; - - sctx->buf[partial++] = 0x80; - if (partial > bit_offset) { - memset(&sctx->buf[partial], 0, SHA256_BLOCK_SIZE - partial); - sha256_blocks(sctx->state, sctx->buf, 1, force_generic); - partial = 0; - } - - memset(&sctx->buf[partial], 0, bit_offset - partial); - *bits = cpu_to_be64(sctx->count << 3); - sha256_blocks(sctx->state, sctx->buf, 1, force_generic); - - for (i = 0; i < digest_size; i += 4) - put_unaligned_be32(sctx->state[i / 4], out + i); + sha256_finup(&sctx->ctx, sctx->buf, partial, out, digest_size, + force_generic || sha256_purgatory(), false); memzero_explicit(sctx, sizeof(*sctx)); } From ff8f037d394f0900597ba527388a6eb95cd02695 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 2 May 2025 13:30:56 +0800 Subject: [PATCH 298/382] crypto: sha256 - Use the partial block API for generic The shash interface already handles partial blocks, use it for sha224-generic and sha256-generic instead of going through the lib/sha256 interface. Signed-off-by: Herbert Xu --- crypto/sha256.c | 83 +++++++++++++++++++++++-------------------- include/crypto/sha2.h | 14 ++++++-- 2 files changed, 56 insertions(+), 41 deletions(-) diff --git a/crypto/sha256.c b/crypto/sha256.c index 47ad7e4cc55f..cf190114574e 100644 --- a/crypto/sha256.c +++ b/crypto/sha256.c @@ -30,15 +30,26 @@ EXPORT_SYMBOL_GPL(sha256_zero_message_hash); static int crypto_sha256_init(struct shash_desc *desc) { - sha256_init(shash_desc_ctx(desc)); + sha256_block_init(shash_desc_ctx(desc)); return 0; } +static inline int crypto_sha256_update(struct shash_desc *desc, const u8 *data, + unsigned int len, bool force_generic) +{ + struct crypto_sha256_state *sctx = shash_desc_ctx(desc); + int remain = len % SHA256_BLOCK_SIZE; + + sctx->count += len - remain; + sha256_choose_blocks(sctx->state, data, len / SHA256_BLOCK_SIZE, + force_generic, !force_generic); + return remain; +} + static int crypto_sha256_update_generic(struct shash_desc *desc, const u8 *data, unsigned int len) { - sha256_update_generic(shash_desc_ctx(desc), data, len); - return 0; + return crypto_sha256_update(desc, data, len, true); } static int crypto_sha256_update_arch(struct shash_desc *desc, const u8 *data, @@ -48,26 +59,35 @@ static int crypto_sha256_update_arch(struct shash_desc *desc, const u8 *data, return 0; } -static int crypto_sha256_final_generic(struct shash_desc *desc, u8 *out) -{ - sha256_final_generic(shash_desc_ctx(desc), out); - return 0; -} - static int crypto_sha256_final_arch(struct shash_desc *desc, u8 *out) { sha256_final(shash_desc_ctx(desc), out); return 0; } +static __always_inline int crypto_sha256_finup(struct shash_desc *desc, + const u8 *data, + unsigned int len, u8 *out, + bool force_generic) +{ + struct crypto_sha256_state *sctx = shash_desc_ctx(desc); + unsigned int remain = len; + u8 *buf; + + if (len >= SHA256_BLOCK_SIZE) + remain = crypto_sha256_update(desc, data, len, force_generic); + sctx->count += remain; + buf = memcpy(sctx + 1, data + len - remain, remain); + sha256_finup(sctx, buf, remain, out, + crypto_shash_digestsize(desc->tfm), force_generic, + !force_generic); + return 0; +} + static int crypto_sha256_finup_generic(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) { - struct sha256_state *sctx = shash_desc_ctx(desc); - - sha256_update_generic(sctx, data, len); - sha256_final_generic(sctx, out); - return 0; + return crypto_sha256_finup(desc, data, len, out, true); } static int crypto_sha256_finup_arch(struct shash_desc *desc, const u8 *data, @@ -83,12 +103,8 @@ static int crypto_sha256_finup_arch(struct shash_desc *desc, const u8 *data, static int crypto_sha256_digest_generic(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) { - struct sha256_state *sctx = shash_desc_ctx(desc); - - sha256_init(sctx); - sha256_update_generic(sctx, data, len); - sha256_final_generic(sctx, out); - return 0; + crypto_sha256_init(desc); + return crypto_sha256_finup_generic(desc, data, len, out); } static int crypto_sha256_digest_arch(struct shash_desc *desc, const u8 *data, @@ -100,13 +116,7 @@ static int crypto_sha256_digest_arch(struct shash_desc *desc, const u8 *data, static int crypto_sha224_init(struct shash_desc *desc) { - sha224_init(shash_desc_ctx(desc)); - return 0; -} - -static int crypto_sha224_final_generic(struct shash_desc *desc, u8 *out) -{ - sha224_final_generic(shash_desc_ctx(desc), out); + sha224_block_init(shash_desc_ctx(desc)); return 0; } @@ -147,35 +157,30 @@ static struct shash_alg algs[] = { .base.cra_name = "sha256", .base.cra_driver_name = "sha256-generic", .base.cra_priority = 100, + .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .base.cra_blocksize = SHA256_BLOCK_SIZE, .base.cra_module = THIS_MODULE, .digestsize = SHA256_DIGEST_SIZE, .init = crypto_sha256_init, .update = crypto_sha256_update_generic, - .final = crypto_sha256_final_generic, .finup = crypto_sha256_finup_generic, .digest = crypto_sha256_digest_generic, - .descsize = sizeof(struct sha256_state), - .statesize = sizeof(struct crypto_sha256_state) + - SHA256_BLOCK_SIZE + 1, - .import = crypto_sha256_import_lib, - .export = crypto_sha256_export_lib, + .descsize = sizeof(struct crypto_sha256_state), }, { .base.cra_name = "sha224", .base.cra_driver_name = "sha224-generic", .base.cra_priority = 100, + .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .base.cra_blocksize = SHA224_BLOCK_SIZE, .base.cra_module = THIS_MODULE, .digestsize = SHA224_DIGEST_SIZE, .init = crypto_sha224_init, .update = crypto_sha256_update_generic, - .final = crypto_sha224_final_generic, - .descsize = sizeof(struct sha256_state), - .statesize = sizeof(struct crypto_sha256_state) + - SHA256_BLOCK_SIZE + 1, - .import = crypto_sha256_import_lib, - .export = crypto_sha256_export_lib, + .finup = crypto_sha256_finup_generic, + .descsize = sizeof(struct crypto_sha256_state), }, { .base.cra_name = "sha256", diff --git a/include/crypto/sha2.h b/include/crypto/sha2.h index 9853cd2d1291..4912572578dc 100644 --- a/include/crypto/sha2.h +++ b/include/crypto/sha2.h @@ -88,7 +88,7 @@ struct sha512_state { u8 buf[SHA512_BLOCK_SIZE]; }; -static inline void sha256_init(struct sha256_state *sctx) +static inline void sha256_block_init(struct crypto_sha256_state *sctx) { sctx->state[0] = SHA256_H0; sctx->state[1] = SHA256_H1; @@ -100,11 +100,16 @@ static inline void sha256_init(struct sha256_state *sctx) sctx->state[7] = SHA256_H7; sctx->count = 0; } + +static inline void sha256_init(struct sha256_state *sctx) +{ + sha256_block_init(&sctx->ctx); +} void sha256_update(struct sha256_state *sctx, const u8 *data, size_t len); void sha256_final(struct sha256_state *sctx, u8 out[SHA256_DIGEST_SIZE]); void sha256(const u8 *data, size_t len, u8 out[SHA256_DIGEST_SIZE]); -static inline void sha224_init(struct sha256_state *sctx) +static inline void sha224_block_init(struct crypto_sha256_state *sctx) { sctx->state[0] = SHA224_H0; sctx->state[1] = SHA224_H1; @@ -116,6 +121,11 @@ static inline void sha224_init(struct sha256_state *sctx) sctx->state[7] = SHA224_H7; sctx->count = 0; } + +static inline void sha224_init(struct sha256_state *sctx) +{ + sha224_block_init(&sctx->ctx); +} /* Simply use sha256_update as it is equivalent to sha224_update. */ void sha224_final(struct sha256_state *sctx, u8 out[SHA224_DIGEST_SIZE]); From 67488527afa9a51d8967dff91af0d1ead19218a0 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 2 May 2025 13:30:58 +0800 Subject: [PATCH 299/382] crypto: arch/sha256 - Export block functions as GPL only Export the block functions as GPL only, there is no reason to let arbitrary modules use these internal functions. Signed-off-by: Herbert Xu --- arch/arm/lib/crypto/sha256.c | 4 ++-- arch/arm64/lib/crypto/sha256.c | 4 ++-- arch/mips/cavium-octeon/crypto/octeon-sha256.c | 4 ++-- arch/powerpc/lib/crypto/sha256.c | 4 ++-- arch/riscv/lib/crypto/sha256.c | 4 ++-- arch/s390/lib/crypto/sha256.c | 4 ++-- arch/sparc/lib/crypto/sha256.c | 4 ++-- arch/x86/lib/crypto/sha256.c | 4 ++-- 8 files changed, 16 insertions(+), 16 deletions(-) diff --git a/arch/arm/lib/crypto/sha256.c b/arch/arm/lib/crypto/sha256.c index 81612b7b9043..2c9cfdaaa069 100644 --- a/arch/arm/lib/crypto/sha256.c +++ b/arch/arm/lib/crypto/sha256.c @@ -35,14 +35,14 @@ void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS], sha256_block_data_order(state, data, nblocks); } } -EXPORT_SYMBOL(sha256_blocks_arch); +EXPORT_SYMBOL_GPL(sha256_blocks_arch); bool sha256_is_arch_optimized(void) { /* We always can use at least the ARM scalar implementation. */ return true; } -EXPORT_SYMBOL(sha256_is_arch_optimized); +EXPORT_SYMBOL_GPL(sha256_is_arch_optimized); static int __init sha256_arm_mod_init(void) { diff --git a/arch/arm64/lib/crypto/sha256.c b/arch/arm64/lib/crypto/sha256.c index 354fc402c6d3..fb9bff40357b 100644 --- a/arch/arm64/lib/crypto/sha256.c +++ b/arch/arm64/lib/crypto/sha256.c @@ -45,14 +45,14 @@ void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS], sha256_block_data_order(state, data, nblocks); } } -EXPORT_SYMBOL(sha256_blocks_arch); +EXPORT_SYMBOL_GPL(sha256_blocks_arch); bool sha256_is_arch_optimized(void) { /* We always can use at least the ARM64 scalar implementation. */ return true; } -EXPORT_SYMBOL(sha256_is_arch_optimized); +EXPORT_SYMBOL_GPL(sha256_is_arch_optimized); static int __init sha256_arm64_mod_init(void) { diff --git a/arch/mips/cavium-octeon/crypto/octeon-sha256.c b/arch/mips/cavium-octeon/crypto/octeon-sha256.c index f169054852bc..f93faaf1f4af 100644 --- a/arch/mips/cavium-octeon/crypto/octeon-sha256.c +++ b/arch/mips/cavium-octeon/crypto/octeon-sha256.c @@ -60,13 +60,13 @@ void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS], state64[3] = read_octeon_64bit_hash_dword(3); octeon_crypto_disable(&cop2_state, flags); } -EXPORT_SYMBOL(sha256_blocks_arch); +EXPORT_SYMBOL_GPL(sha256_blocks_arch); bool sha256_is_arch_optimized(void) { return octeon_has_crypto(); } -EXPORT_SYMBOL(sha256_is_arch_optimized); +EXPORT_SYMBOL_GPL(sha256_is_arch_optimized); MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("SHA-256 Secure Hash Algorithm (OCTEON)"); diff --git a/arch/powerpc/lib/crypto/sha256.c b/arch/powerpc/lib/crypto/sha256.c index c05023c5acdd..6b0f079587eb 100644 --- a/arch/powerpc/lib/crypto/sha256.c +++ b/arch/powerpc/lib/crypto/sha256.c @@ -58,13 +58,13 @@ void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS], nblocks -= unit; } while (nblocks); } -EXPORT_SYMBOL(sha256_blocks_arch); +EXPORT_SYMBOL_GPL(sha256_blocks_arch); bool sha256_is_arch_optimized(void) { return true; } -EXPORT_SYMBOL(sha256_is_arch_optimized); +EXPORT_SYMBOL_GPL(sha256_is_arch_optimized); MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("SHA-256 Secure Hash Algorithm, SPE optimized"); diff --git a/arch/riscv/lib/crypto/sha256.c b/arch/riscv/lib/crypto/sha256.c index c8065c70337b..4ad3ffb8e0a9 100644 --- a/arch/riscv/lib/crypto/sha256.c +++ b/arch/riscv/lib/crypto/sha256.c @@ -32,13 +32,13 @@ void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS], sha256_blocks_generic(state, data, nblocks); } } -EXPORT_SYMBOL(sha256_blocks_arch); +EXPORT_SYMBOL_GPL(sha256_blocks_arch); bool sha256_is_arch_optimized(void) { return static_key_enabled(&have_extensions); } -EXPORT_SYMBOL(sha256_is_arch_optimized); +EXPORT_SYMBOL_GPL(sha256_is_arch_optimized); static int __init riscv64_sha256_mod_init(void) { diff --git a/arch/s390/lib/crypto/sha256.c b/arch/s390/lib/crypto/sha256.c index f1d73ea1eef6..7dfe120fafab 100644 --- a/arch/s390/lib/crypto/sha256.c +++ b/arch/s390/lib/crypto/sha256.c @@ -21,13 +21,13 @@ void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS], else sha256_blocks_generic(state, data, nblocks); } -EXPORT_SYMBOL(sha256_blocks_arch); +EXPORT_SYMBOL_GPL(sha256_blocks_arch); bool sha256_is_arch_optimized(void) { return static_key_enabled(&have_cpacf_sha256); } -EXPORT_SYMBOL(sha256_is_arch_optimized); +EXPORT_SYMBOL_GPL(sha256_is_arch_optimized); static int __init sha256_s390_mod_init(void) { diff --git a/arch/sparc/lib/crypto/sha256.c b/arch/sparc/lib/crypto/sha256.c index 0394c2b72f9e..8bdec2db08b3 100644 --- a/arch/sparc/lib/crypto/sha256.c +++ b/arch/sparc/lib/crypto/sha256.c @@ -30,13 +30,13 @@ void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS], else sha256_blocks_generic(state, data, nblocks); } -EXPORT_SYMBOL(sha256_blocks_arch); +EXPORT_SYMBOL_GPL(sha256_blocks_arch); bool sha256_is_arch_optimized(void) { return static_key_enabled(&have_sha256_opcodes); } -EXPORT_SYMBOL(sha256_is_arch_optimized); +EXPORT_SYMBOL_GPL(sha256_is_arch_optimized); static int __init sha256_sparc64_mod_init(void) { diff --git a/arch/x86/lib/crypto/sha256.c b/arch/x86/lib/crypto/sha256.c index ea17514cb0f3..baba74d7d26f 100644 --- a/arch/x86/lib/crypto/sha256.c +++ b/arch/x86/lib/crypto/sha256.c @@ -35,13 +35,13 @@ void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS], sha256_blocks_generic(state, data, nblocks); } } -EXPORT_SYMBOL(sha256_blocks_arch); +EXPORT_SYMBOL_GPL(sha256_blocks_arch); bool sha256_is_arch_optimized(void) { return static_key_enabled(&have_sha256_x86); } -EXPORT_SYMBOL(sha256_is_arch_optimized); +EXPORT_SYMBOL_GPL(sha256_is_arch_optimized); static int __init sha256_x86_mod_init(void) { From 2e43fc6d79cb24e0fe34aa4c2229ab6c414d852a Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 2 May 2025 13:31:00 +0800 Subject: [PATCH 300/382] crypto: arm/sha256 - Add simd block function Add CRYPTO_ARCH_HAVE_LIB_SHA256_SIMD and a SIMD block function so that the caller can decide whether to use SIMD. Signed-off-by: Herbert Xu --- arch/arm/lib/crypto/Kconfig | 1 + arch/arm/lib/crypto/sha256-armv4.pl | 20 ++++++++++---------- arch/arm/lib/crypto/sha256.c | 14 +++++++------- 3 files changed, 18 insertions(+), 17 deletions(-) diff --git a/arch/arm/lib/crypto/Kconfig b/arch/arm/lib/crypto/Kconfig index 9f3ff30f4032..d1ad664f0c67 100644 --- a/arch/arm/lib/crypto/Kconfig +++ b/arch/arm/lib/crypto/Kconfig @@ -28,3 +28,4 @@ config CRYPTO_SHA256_ARM depends on !CPU_V7M default CRYPTO_LIB_SHA256 select CRYPTO_ARCH_HAVE_LIB_SHA256 + select CRYPTO_ARCH_HAVE_LIB_SHA256_SIMD diff --git a/arch/arm/lib/crypto/sha256-armv4.pl b/arch/arm/lib/crypto/sha256-armv4.pl index f3a2b54efd4e..8122db7fd599 100644 --- a/arch/arm/lib/crypto/sha256-armv4.pl +++ b/arch/arm/lib/crypto/sha256-armv4.pl @@ -204,18 +204,18 @@ K256: .word 0 @ terminator #if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__) .LOPENSSL_armcap: -.word OPENSSL_armcap_P-sha256_block_data_order +.word OPENSSL_armcap_P-sha256_blocks_arch #endif .align 5 -.global sha256_block_data_order -.type sha256_block_data_order,%function -sha256_block_data_order: -.Lsha256_block_data_order: +.global sha256_blocks_arch +.type sha256_blocks_arch,%function +sha256_blocks_arch: +.Lsha256_blocks_arch: #if __ARM_ARCH__<7 - sub r3,pc,#8 @ sha256_block_data_order + sub r3,pc,#8 @ sha256_blocks_arch #else - adr r3,.Lsha256_block_data_order + adr r3,.Lsha256_blocks_arch #endif #if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__) ldr r12,.LOPENSSL_armcap @@ -282,7 +282,7 @@ $code.=<<___; moveq pc,lr @ be binary compatible with V4, yet bx lr @ interoperable with Thumb ISA:-) #endif -.size sha256_block_data_order,.-sha256_block_data_order +.size sha256_blocks_arch,.-sha256_blocks_arch ___ ###################################################################### # NEON stuff @@ -470,8 +470,8 @@ sha256_block_data_order_neon: stmdb sp!,{r4-r12,lr} sub $H,sp,#16*4+16 - adr $Ktbl,.Lsha256_block_data_order - sub $Ktbl,$Ktbl,#.Lsha256_block_data_order-K256 + adr $Ktbl,.Lsha256_blocks_arch + sub $Ktbl,$Ktbl,#.Lsha256_blocks_arch-K256 bic $H,$H,#15 @ align for 128-bit stores mov $t2,sp mov sp,$H @ alloca diff --git a/arch/arm/lib/crypto/sha256.c b/arch/arm/lib/crypto/sha256.c index 2c9cfdaaa069..109192e54b0f 100644 --- a/arch/arm/lib/crypto/sha256.c +++ b/arch/arm/lib/crypto/sha256.c @@ -6,12 +6,12 @@ */ #include #include -#include #include #include -asmlinkage void sha256_block_data_order(u32 state[SHA256_STATE_WORDS], - const u8 *data, size_t nblocks); +asmlinkage void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS], + const u8 *data, size_t nblocks); +EXPORT_SYMBOL_GPL(sha256_blocks_arch); asmlinkage void sha256_block_data_order_neon(u32 state[SHA256_STATE_WORDS], const u8 *data, size_t nblocks); asmlinkage void sha256_ce_transform(u32 state[SHA256_STATE_WORDS], @@ -20,11 +20,11 @@ asmlinkage void sha256_ce_transform(u32 state[SHA256_STATE_WORDS], static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon); static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_ce); -void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS], +void sha256_blocks_simd(u32 state[SHA256_STATE_WORDS], const u8 *data, size_t nblocks) { if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && - static_branch_likely(&have_neon) && crypto_simd_usable()) { + static_branch_likely(&have_neon)) { kernel_neon_begin(); if (static_branch_likely(&have_ce)) sha256_ce_transform(state, data, nblocks); @@ -32,10 +32,10 @@ void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS], sha256_block_data_order_neon(state, data, nblocks); kernel_neon_end(); } else { - sha256_block_data_order(state, data, nblocks); + sha256_blocks_arch(state, data, nblocks); } } -EXPORT_SYMBOL_GPL(sha256_blocks_arch); +EXPORT_SYMBOL_GPL(sha256_blocks_simd); bool sha256_is_arch_optimized(void) { From adcb9e32e5e28935ec1148e1a314282a7367428d Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 2 May 2025 13:31:03 +0800 Subject: [PATCH 301/382] crypto: arm64/sha256 - Add simd block function Add CRYPTO_ARCH_HAVE_LIB_SHA256_SIMD and a SIMD block function so that the caller can decide whether to use SIMD. Signed-off-by: Herbert Xu --- arch/arm64/crypto/sha512-glue.c | 6 +++--- arch/arm64/lib/crypto/Kconfig | 1 + arch/arm64/lib/crypto/sha2-armv8.pl | 2 +- arch/arm64/lib/crypto/sha256.c | 14 +++++++------- 4 files changed, 12 insertions(+), 11 deletions(-) diff --git a/arch/arm64/crypto/sha512-glue.c b/arch/arm64/crypto/sha512-glue.c index ab2e1c13dfad..15aa9d8b7b2c 100644 --- a/arch/arm64/crypto/sha512-glue.c +++ b/arch/arm64/crypto/sha512-glue.c @@ -18,13 +18,13 @@ MODULE_LICENSE("GPL v2"); MODULE_ALIAS_CRYPTO("sha384"); MODULE_ALIAS_CRYPTO("sha512"); -asmlinkage void sha512_block_data_order(u64 *digest, const void *data, - unsigned int num_blks); +asmlinkage void sha512_blocks_arch(u64 *digest, const void *data, + unsigned int num_blks); static void sha512_arm64_transform(struct sha512_state *sst, u8 const *src, int blocks) { - sha512_block_data_order(sst->state, src, blocks); + sha512_blocks_arch(sst->state, src, blocks); } static int sha512_update(struct shash_desc *desc, const u8 *data, diff --git a/arch/arm64/lib/crypto/Kconfig b/arch/arm64/lib/crypto/Kconfig index 49e57bfdb5b5..129a7685cb4c 100644 --- a/arch/arm64/lib/crypto/Kconfig +++ b/arch/arm64/lib/crypto/Kconfig @@ -17,3 +17,4 @@ config CRYPTO_SHA256_ARM64 tristate default CRYPTO_LIB_SHA256 select CRYPTO_ARCH_HAVE_LIB_SHA256 + select CRYPTO_ARCH_HAVE_LIB_SHA256_SIMD diff --git a/arch/arm64/lib/crypto/sha2-armv8.pl b/arch/arm64/lib/crypto/sha2-armv8.pl index 35ec9ae99fe1..4aebd20c498b 100644 --- a/arch/arm64/lib/crypto/sha2-armv8.pl +++ b/arch/arm64/lib/crypto/sha2-armv8.pl @@ -95,7 +95,7 @@ if ($output =~ /512/) { $reg_t="w"; } -$func="sha${BITS}_block_data_order"; +$func="sha${BITS}_blocks_arch"; ($ctx,$inp,$num,$Ktbl)=map("x$_",(0..2,30)); diff --git a/arch/arm64/lib/crypto/sha256.c b/arch/arm64/lib/crypto/sha256.c index fb9bff40357b..bcf7a3adc0c4 100644 --- a/arch/arm64/lib/crypto/sha256.c +++ b/arch/arm64/lib/crypto/sha256.c @@ -6,12 +6,12 @@ */ #include #include -#include #include #include -asmlinkage void sha256_block_data_order(u32 state[SHA256_STATE_WORDS], - const u8 *data, size_t nblocks); +asmlinkage void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS], + const u8 *data, size_t nblocks); +EXPORT_SYMBOL_GPL(sha256_blocks_arch); asmlinkage void sha256_block_neon(u32 state[SHA256_STATE_WORDS], const u8 *data, size_t nblocks); asmlinkage size_t __sha256_ce_transform(u32 state[SHA256_STATE_WORDS], @@ -20,11 +20,11 @@ asmlinkage size_t __sha256_ce_transform(u32 state[SHA256_STATE_WORDS], static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon); static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_ce); -void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS], +void sha256_blocks_simd(u32 state[SHA256_STATE_WORDS], const u8 *data, size_t nblocks) { if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && - static_branch_likely(&have_neon) && crypto_simd_usable()) { + static_branch_likely(&have_neon)) { if (static_branch_likely(&have_ce)) { do { size_t rem; @@ -42,10 +42,10 @@ void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS], kernel_neon_end(); } } else { - sha256_block_data_order(state, data, nblocks); + sha256_blocks_arch(state, data, nblocks); } } -EXPORT_SYMBOL_GPL(sha256_blocks_arch); +EXPORT_SYMBOL_GPL(sha256_blocks_simd); bool sha256_is_arch_optimized(void) { From 491d6024f2820c78216b07cec1cb47c87dcae077 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 2 May 2025 13:31:05 +0800 Subject: [PATCH 302/382] crypto: riscv/sha256 - Add simd block function Add CRYPTO_ARCH_HAVE_LIB_SHA256_SIMD and a SIMD block function so that the caller can decide whether to use SIMD. Signed-off-by: Herbert Xu --- arch/riscv/lib/crypto/Kconfig | 1 + arch/riscv/lib/crypto/sha256.c | 13 +++++++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/arch/riscv/lib/crypto/Kconfig b/arch/riscv/lib/crypto/Kconfig index c100571feb7e..47c99ea97ce2 100644 --- a/arch/riscv/lib/crypto/Kconfig +++ b/arch/riscv/lib/crypto/Kconfig @@ -12,4 +12,5 @@ config CRYPTO_SHA256_RISCV64 depends on 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO default CRYPTO_LIB_SHA256 select CRYPTO_ARCH_HAVE_LIB_SHA256 + select CRYPTO_ARCH_HAVE_LIB_SHA256_SIMD select CRYPTO_LIB_SHA256_GENERIC diff --git a/arch/riscv/lib/crypto/sha256.c b/arch/riscv/lib/crypto/sha256.c index 4ad3ffb8e0a9..71808397dff4 100644 --- a/arch/riscv/lib/crypto/sha256.c +++ b/arch/riscv/lib/crypto/sha256.c @@ -9,10 +9,8 @@ * Author: Jerry Shih */ -#include #include #include -#include #include #include @@ -21,10 +19,10 @@ asmlinkage void sha256_transform_zvknha_or_zvknhb_zvkb( static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_extensions); -void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS], +void sha256_blocks_simd(u32 state[SHA256_STATE_WORDS], const u8 *data, size_t nblocks) { - if (static_branch_likely(&have_extensions) && crypto_simd_usable()) { + if (static_branch_likely(&have_extensions)) { kernel_vector_begin(); sha256_transform_zvknha_or_zvknhb_zvkb(state, data, nblocks); kernel_vector_end(); @@ -32,6 +30,13 @@ void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS], sha256_blocks_generic(state, data, nblocks); } } +EXPORT_SYMBOL_GPL(sha256_blocks_simd); + +void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS], + const u8 *data, size_t nblocks) +{ + sha256_blocks_generic(state, data, nblocks); +} EXPORT_SYMBOL_GPL(sha256_blocks_arch); bool sha256_is_arch_optimized(void) From ee8a720e39ceb7495ab639c1eb6d4987fb6a52bf Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 2 May 2025 13:31:07 +0800 Subject: [PATCH 303/382] crypto: x86/sha256 - Add simd block function Add CRYPTO_ARCH_HAVE_LIB_SHA256_SIMD and a SIMD block function so that the caller can decide whether to use SIMD. Signed-off-by: Herbert Xu --- arch/x86/lib/crypto/Kconfig | 1 + arch/x86/lib/crypto/sha256.c | 12 +++++++++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/arch/x86/lib/crypto/Kconfig b/arch/x86/lib/crypto/Kconfig index e344579db3d8..5e94cdee492c 100644 --- a/arch/x86/lib/crypto/Kconfig +++ b/arch/x86/lib/crypto/Kconfig @@ -30,4 +30,5 @@ config CRYPTO_SHA256_X86_64 depends on 64BIT default CRYPTO_LIB_SHA256 select CRYPTO_ARCH_HAVE_LIB_SHA256 + select CRYPTO_ARCH_HAVE_LIB_SHA256_SIMD select CRYPTO_LIB_SHA256_GENERIC diff --git a/arch/x86/lib/crypto/sha256.c b/arch/x86/lib/crypto/sha256.c index baba74d7d26f..80380f8fdcee 100644 --- a/arch/x86/lib/crypto/sha256.c +++ b/arch/x86/lib/crypto/sha256.c @@ -6,7 +6,6 @@ */ #include #include -#include #include #include #include @@ -24,10 +23,10 @@ static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_sha256_x86); DEFINE_STATIC_CALL(sha256_blocks_x86, sha256_transform_ssse3); -void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS], +void sha256_blocks_simd(u32 state[SHA256_STATE_WORDS], const u8 *data, size_t nblocks) { - if (static_branch_likely(&have_sha256_x86) && crypto_simd_usable()) { + if (static_branch_likely(&have_sha256_x86)) { kernel_fpu_begin(); static_call(sha256_blocks_x86)(state, data, nblocks); kernel_fpu_end(); @@ -35,6 +34,13 @@ void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS], sha256_blocks_generic(state, data, nblocks); } } +EXPORT_SYMBOL_GPL(sha256_blocks_simd); + +void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS], + const u8 *data, size_t nblocks) +{ + sha256_blocks_generic(state, data, nblocks); +} EXPORT_SYMBOL_GPL(sha256_blocks_arch); bool sha256_is_arch_optimized(void) From 3007e90572d0c5fd409c3d2fa8cedcbd5cb06d4b Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 2 May 2025 13:31:09 +0800 Subject: [PATCH 304/382] crypto: lib/sha256 - Use generic block helper Use the BLOCK_HASH_UPDATE_BLOCKS helper instead of duplicating partial block handling. Also remove the unused lib/sha256 force-generic interface. Signed-off-by: Herbert Xu --- include/crypto/internal/sha2.h | 7 ---- lib/crypto/sha256.c | 75 ++++++---------------------------- 2 files changed, 12 insertions(+), 70 deletions(-) diff --git a/include/crypto/internal/sha2.h b/include/crypto/internal/sha2.h index fff156f66edc..b9bccd3ff57f 100644 --- a/include/crypto/internal/sha2.h +++ b/include/crypto/internal/sha2.h @@ -10,13 +10,6 @@ #include #include -void sha256_update_generic(struct sha256_state *sctx, - const u8 *data, size_t len); -void sha256_final_generic(struct sha256_state *sctx, - u8 out[SHA256_DIGEST_SIZE]); -void sha224_final_generic(struct sha256_state *sctx, - u8 out[SHA224_DIGEST_SIZE]); - #if IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_SHA256) bool sha256_is_arch_optimized(void); #else diff --git a/lib/crypto/sha256.c b/lib/crypto/sha256.c index 2ced29efa181..107e5162507a 100644 --- a/lib/crypto/sha256.c +++ b/lib/crypto/sha256.c @@ -11,6 +11,7 @@ * Copyright (c) 2014 Red Hat Inc. */ +#include #include #include #include @@ -31,71 +32,40 @@ static inline bool sha256_purgatory(void) } static inline void sha256_blocks(u32 state[SHA256_STATE_WORDS], const u8 *data, - size_t nblocks, bool force_generic) + size_t nblocks) { - sha256_choose_blocks(state, data, nblocks, - force_generic || sha256_purgatory(), false); -} - -static inline void __sha256_update(struct sha256_state *sctx, const u8 *data, - size_t len, bool force_generic) -{ - size_t partial = sctx->count % SHA256_BLOCK_SIZE; - - sctx->count += len; - - if (partial + len >= SHA256_BLOCK_SIZE) { - size_t nblocks; - - if (partial) { - size_t l = SHA256_BLOCK_SIZE - partial; - - memcpy(&sctx->buf[partial], data, l); - data += l; - len -= l; - - sha256_blocks(sctx->state, sctx->buf, 1, force_generic); - } - - nblocks = len / SHA256_BLOCK_SIZE; - len %= SHA256_BLOCK_SIZE; - - if (nblocks) { - sha256_blocks(sctx->state, data, nblocks, - force_generic); - data += nblocks * SHA256_BLOCK_SIZE; - } - partial = 0; - } - if (len) - memcpy(&sctx->buf[partial], data, len); + sha256_choose_blocks(state, data, nblocks, sha256_purgatory(), false); } void sha256_update(struct sha256_state *sctx, const u8 *data, size_t len) { - __sha256_update(sctx, data, len, false); + size_t partial = sctx->count % SHA256_BLOCK_SIZE; + + sctx->count += len; + BLOCK_HASH_UPDATE_BLOCKS(sha256_blocks, sctx->ctx.state, data, len, + SHA256_BLOCK_SIZE, sctx->buf, partial); } EXPORT_SYMBOL(sha256_update); static inline void __sha256_final(struct sha256_state *sctx, u8 *out, - size_t digest_size, bool force_generic) + size_t digest_size) { size_t partial = sctx->count % SHA256_BLOCK_SIZE; sha256_finup(&sctx->ctx, sctx->buf, partial, out, digest_size, - force_generic || sha256_purgatory(), false); + sha256_purgatory(), false); memzero_explicit(sctx, sizeof(*sctx)); } void sha256_final(struct sha256_state *sctx, u8 out[SHA256_DIGEST_SIZE]) { - __sha256_final(sctx, out, SHA256_DIGEST_SIZE, false); + __sha256_final(sctx, out, SHA256_DIGEST_SIZE); } EXPORT_SYMBOL(sha256_final); void sha224_final(struct sha256_state *sctx, u8 out[SHA224_DIGEST_SIZE]) { - __sha256_final(sctx, out, SHA224_DIGEST_SIZE, false); + __sha256_final(sctx, out, SHA224_DIGEST_SIZE); } EXPORT_SYMBOL(sha224_final); @@ -109,26 +79,5 @@ void sha256(const u8 *data, size_t len, u8 out[SHA256_DIGEST_SIZE]) } EXPORT_SYMBOL(sha256); -#if IS_ENABLED(CONFIG_CRYPTO_SHA256) && !defined(__DISABLE_EXPORTS) -void sha256_update_generic(struct sha256_state *sctx, - const u8 *data, size_t len) -{ - __sha256_update(sctx, data, len, true); -} -EXPORT_SYMBOL(sha256_update_generic); - -void sha256_final_generic(struct sha256_state *sctx, u8 out[SHA256_DIGEST_SIZE]) -{ - __sha256_final(sctx, out, SHA256_DIGEST_SIZE, true); -} -EXPORT_SYMBOL(sha256_final_generic); - -void sha224_final_generic(struct sha256_state *sctx, u8 out[SHA224_DIGEST_SIZE]) -{ - __sha256_final(sctx, out, SHA224_DIGEST_SIZE, true); -} -EXPORT_SYMBOL(sha224_final_generic); -#endif - MODULE_DESCRIPTION("SHA-256 Algorithm"); MODULE_LICENSE("GPL"); From 3bf5337879101166dfacfbc2a780d1a379c288ba Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 2 May 2025 13:31:12 +0800 Subject: [PATCH 305/382] crypto: sha256 - Use the partial block API Use the shash partial block API by default. Add a separate set of lib shash algorithms to preserve testing coverage until lib/sha256 has its own tests. Signed-off-by: Herbert Xu --- crypto/sha256.c | 81 +++++++++++++++++++++++++++++++++++-------------- 1 file changed, 58 insertions(+), 23 deletions(-) diff --git a/crypto/sha256.c b/crypto/sha256.c index cf190114574e..4aeb213bab11 100644 --- a/crypto/sha256.c +++ b/crypto/sha256.c @@ -52,14 +52,20 @@ static int crypto_sha256_update_generic(struct shash_desc *desc, const u8 *data, return crypto_sha256_update(desc, data, len, true); } -static int crypto_sha256_update_arch(struct shash_desc *desc, const u8 *data, - unsigned int len) +static int crypto_sha256_update_lib(struct shash_desc *desc, const u8 *data, + unsigned int len) { sha256_update(shash_desc_ctx(desc), data, len); return 0; } -static int crypto_sha256_final_arch(struct shash_desc *desc, u8 *out) +static int crypto_sha256_update_arch(struct shash_desc *desc, const u8 *data, + unsigned int len) +{ + return crypto_sha256_update(desc, data, len, false); +} + +static int crypto_sha256_final_lib(struct shash_desc *desc, u8 *out) { sha256_final(shash_desc_ctx(desc), out); return 0; @@ -93,11 +99,7 @@ static int crypto_sha256_finup_generic(struct shash_desc *desc, const u8 *data, static int crypto_sha256_finup_arch(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) { - struct sha256_state *sctx = shash_desc_ctx(desc); - - sha256_update(sctx, data, len); - sha256_final(sctx, out); - return 0; + return crypto_sha256_finup(desc, data, len, out, false); } static int crypto_sha256_digest_generic(struct shash_desc *desc, const u8 *data, @@ -107,20 +109,27 @@ static int crypto_sha256_digest_generic(struct shash_desc *desc, const u8 *data, return crypto_sha256_finup_generic(desc, data, len, out); } -static int crypto_sha256_digest_arch(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) +static int crypto_sha256_digest_lib(struct shash_desc *desc, const u8 *data, + unsigned int len, u8 *out) { sha256(data, len, out); return 0; } +static int crypto_sha256_digest_arch(struct shash_desc *desc, const u8 *data, + unsigned int len, u8 *out) +{ + crypto_sha256_init(desc); + return crypto_sha256_finup_arch(desc, data, len, out); +} + static int crypto_sha224_init(struct shash_desc *desc) { sha224_block_init(shash_desc_ctx(desc)); return 0; } -static int crypto_sha224_final_arch(struct shash_desc *desc, u8 *out) +static int crypto_sha224_final_lib(struct shash_desc *desc, u8 *out) { sha224_final(shash_desc_ctx(desc), out); return 0; @@ -184,16 +193,14 @@ static struct shash_alg algs[] = { }, { .base.cra_name = "sha256", - .base.cra_driver_name = "sha256-" __stringify(ARCH), - .base.cra_priority = 300, + .base.cra_driver_name = "sha256-lib", .base.cra_blocksize = SHA256_BLOCK_SIZE, .base.cra_module = THIS_MODULE, .digestsize = SHA256_DIGEST_SIZE, .init = crypto_sha256_init, - .update = crypto_sha256_update_arch, - .final = crypto_sha256_final_arch, - .finup = crypto_sha256_finup_arch, - .digest = crypto_sha256_digest_arch, + .update = crypto_sha256_update_lib, + .final = crypto_sha256_final_lib, + .digest = crypto_sha256_digest_lib, .descsize = sizeof(struct sha256_state), .statesize = sizeof(struct crypto_sha256_state) + SHA256_BLOCK_SIZE + 1, @@ -202,20 +209,48 @@ static struct shash_alg algs[] = { }, { .base.cra_name = "sha224", - .base.cra_driver_name = "sha224-" __stringify(ARCH), - .base.cra_priority = 300, + .base.cra_driver_name = "sha224-lib", .base.cra_blocksize = SHA224_BLOCK_SIZE, .base.cra_module = THIS_MODULE, .digestsize = SHA224_DIGEST_SIZE, .init = crypto_sha224_init, - .update = crypto_sha256_update_arch, - .final = crypto_sha224_final_arch, + .update = crypto_sha256_update_lib, + .final = crypto_sha224_final_lib, .descsize = sizeof(struct sha256_state), .statesize = sizeof(struct crypto_sha256_state) + SHA256_BLOCK_SIZE + 1, .import = crypto_sha256_import_lib, .export = crypto_sha256_export_lib, }, + { + .base.cra_name = "sha256", + .base.cra_driver_name = "sha256-" __stringify(ARCH), + .base.cra_priority = 300, + .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, + .base.cra_blocksize = SHA256_BLOCK_SIZE, + .base.cra_module = THIS_MODULE, + .digestsize = SHA256_DIGEST_SIZE, + .init = crypto_sha256_init, + .update = crypto_sha256_update_arch, + .finup = crypto_sha256_finup_arch, + .digest = crypto_sha256_digest_arch, + .descsize = sizeof(struct crypto_sha256_state), + }, + { + .base.cra_name = "sha224", + .base.cra_driver_name = "sha224-" __stringify(ARCH), + .base.cra_priority = 300, + .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, + .base.cra_blocksize = SHA224_BLOCK_SIZE, + .base.cra_module = THIS_MODULE, + .digestsize = SHA224_DIGEST_SIZE, + .init = crypto_sha224_init, + .update = crypto_sha256_update_arch, + .finup = crypto_sha256_finup_arch, + .descsize = sizeof(struct crypto_sha256_state), + }, }; static unsigned int num_algs; @@ -224,9 +259,9 @@ static int __init crypto_sha256_mod_init(void) { /* register the arch flavours only if they differ from generic */ num_algs = ARRAY_SIZE(algs); - BUILD_BUG_ON(ARRAY_SIZE(algs) % 2 != 0); + BUILD_BUG_ON(ARRAY_SIZE(algs) <= 2); if (!sha256_is_arch_optimized()) - num_algs /= 2; + num_algs -= 2; return crypto_register_shashes(algs, ARRAY_SIZE(algs)); } module_init(crypto_sha256_mod_init); From 08811169ac016a234765e23deb45a5c8dd8aee6b Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 2 May 2025 17:00:43 +0800 Subject: [PATCH 306/382] crypto: s390/hmac - Use API partial block handling Use the Crypto API partial block handling. Also switch to the generic export format. Signed-off-by: Herbert Xu --- arch/s390/crypto/hmac_s390.c | 153 ++++++++++++++++++++++++----------- 1 file changed, 107 insertions(+), 46 deletions(-) diff --git a/arch/s390/crypto/hmac_s390.c b/arch/s390/crypto/hmac_s390.c index e6edf1013228..93a1098d9f8d 100644 --- a/arch/s390/crypto/hmac_s390.c +++ b/arch/s390/crypto/hmac_s390.c @@ -9,10 +9,14 @@ #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt #include -#include #include +#include +#include #include +#include +#include #include +#include /* * KMAC param block layout for sha2 function codes: @@ -71,7 +75,6 @@ union s390_kmac_gr0 { struct s390_kmac_sha2_ctx { u8 param[MAX_DIGEST_SIZE + MAX_IMBL_SIZE + MAX_BLOCK_SIZE]; union s390_kmac_gr0 gr0; - u8 buf[MAX_BLOCK_SIZE]; u64 buflen[2]; }; @@ -95,8 +98,8 @@ static inline void kmac_sha2_set_imbl(u8 *param, u64 buflen_lo, } } -static int hash_key(const u8 *in, unsigned int inlen, - u8 *digest, unsigned int digestsize) +static int hash_data(const u8 *in, unsigned int inlen, + u8 *digest, unsigned int digestsize, bool final) { unsigned long func; union { @@ -123,19 +126,23 @@ static int hash_key(const u8 *in, unsigned int inlen, switch (digestsize) { case SHA224_DIGEST_SIZE: - func = CPACF_KLMD_SHA_256; + func = final ? CPACF_KLMD_SHA_256 : CPACF_KIMD_SHA_256; PARAM_INIT(256, 224, inlen * 8); + if (!final) + digestsize = SHA256_DIGEST_SIZE; break; case SHA256_DIGEST_SIZE: - func = CPACF_KLMD_SHA_256; + func = final ? CPACF_KLMD_SHA_256 : CPACF_KIMD_SHA_256; PARAM_INIT(256, 256, inlen * 8); break; case SHA384_DIGEST_SIZE: - func = CPACF_KLMD_SHA_512; + func = final ? CPACF_KLMD_SHA_512 : CPACF_KIMD_SHA_512; PARAM_INIT(512, 384, inlen * 8); + if (!final) + digestsize = SHA512_DIGEST_SIZE; break; case SHA512_DIGEST_SIZE: - func = CPACF_KLMD_SHA_512; + func = final ? CPACF_KLMD_SHA_512 : CPACF_KIMD_SHA_512; PARAM_INIT(512, 512, inlen * 8); break; default: @@ -151,6 +158,12 @@ static int hash_key(const u8 *in, unsigned int inlen, return 0; } +static int hash_key(const u8 *in, unsigned int inlen, + u8 *digest, unsigned int digestsize) +{ + return hash_data(in, inlen, digest, digestsize, true); +} + static int s390_hmac_sha2_setkey(struct crypto_shash *tfm, const u8 *key, unsigned int keylen) { @@ -204,50 +217,31 @@ static int s390_hmac_sha2_update(struct shash_desc *desc, { struct s390_kmac_sha2_ctx *ctx = shash_desc_ctx(desc); unsigned int bs = crypto_shash_blocksize(desc->tfm); - unsigned int offset, n; + unsigned int n = round_down(len, bs); - /* check current buffer */ - offset = ctx->buflen[0] % bs; - ctx->buflen[0] += len; - if (ctx->buflen[0] < len) + ctx->buflen[0] += n; + if (ctx->buflen[0] < n) ctx->buflen[1]++; - if (offset + len < bs) - goto store; - /* process one stored block */ - if (offset) { - n = bs - offset; - memcpy(ctx->buf + offset, data, n); - ctx->gr0.iimp = 1; - _cpacf_kmac(&ctx->gr0.reg, ctx->param, ctx->buf, bs); - data += n; - len -= n; - offset = 0; - } /* process as many blocks as possible */ - if (len >= bs) { - n = (len / bs) * bs; - ctx->gr0.iimp = 1; - _cpacf_kmac(&ctx->gr0.reg, ctx->param, data, n); - data += n; - len -= n; - } -store: - /* store incomplete block in buffer */ - if (len) - memcpy(ctx->buf + offset, data, len); - - return 0; + ctx->gr0.iimp = 1; + _cpacf_kmac(&ctx->gr0.reg, ctx->param, data, n); + return len - n; } -static int s390_hmac_sha2_final(struct shash_desc *desc, u8 *out) +static int s390_hmac_sha2_finup(struct shash_desc *desc, const u8 *src, + unsigned int len, u8 *out) { struct s390_kmac_sha2_ctx *ctx = shash_desc_ctx(desc); unsigned int bs = crypto_shash_blocksize(desc->tfm); + ctx->buflen[0] += len; + if (ctx->buflen[0] < len) + ctx->buflen[1]++; + ctx->gr0.iimp = 0; kmac_sha2_set_imbl(ctx->param, ctx->buflen[0], ctx->buflen[1], bs); - _cpacf_kmac(&ctx->gr0.reg, ctx->param, ctx->buf, ctx->buflen[0] % bs); + _cpacf_kmac(&ctx->gr0.reg, ctx->param, src, len); memcpy(out, ctx->param, crypto_shash_digestsize(desc->tfm)); return 0; @@ -273,22 +267,89 @@ static int s390_hmac_sha2_digest(struct shash_desc *desc, return 0; } -#define S390_HMAC_SHA2_ALG(x) { \ +static int s390_hmac_export_zero(struct shash_desc *desc, void *out) +{ + struct crypto_shash *tfm = desc->tfm; + u8 ipad[SHA512_BLOCK_SIZE]; + struct s390_hmac_ctx *ctx; + unsigned int bs; + int err, i; + + ctx = crypto_shash_ctx(tfm); + bs = crypto_shash_blocksize(tfm); + for (i = 0; i < bs; i++) + ipad[i] = ctx->key[i] ^ HMAC_IPAD_VALUE; + + err = hash_data(ipad, bs, out, crypto_shash_digestsize(tfm), false); + memzero_explicit(ipad, sizeof(ipad)); + return err; +} + +static int s390_hmac_export(struct shash_desc *desc, void *out) +{ + struct s390_kmac_sha2_ctx *ctx = shash_desc_ctx(desc); + unsigned int bs = crypto_shash_blocksize(desc->tfm); + unsigned int ds = bs / 2; + union { + u8 *u8; + u64 *u64; + } p = { .u8 = out }; + int err = 0; + + if (!ctx->gr0.ikp) + err = s390_hmac_export_zero(desc, out); + else + memcpy(p.u8, ctx->param, ds); + p.u8 += ds; + put_unaligned(ctx->buflen[0], p.u64++); + if (ds == SHA512_DIGEST_SIZE) + put_unaligned(ctx->buflen[1], p.u64); + return err; +} + +static int s390_hmac_import(struct shash_desc *desc, const void *in) +{ + struct s390_kmac_sha2_ctx *ctx = shash_desc_ctx(desc); + unsigned int bs = crypto_shash_blocksize(desc->tfm); + unsigned int ds = bs / 2; + union { + const u8 *u8; + const u64 *u64; + } p = { .u8 = in }; + int err; + + err = s390_hmac_sha2_init(desc); + memcpy(ctx->param, p.u8, ds); + p.u8 += ds; + ctx->buflen[0] = get_unaligned(p.u64++); + if (ds == SHA512_DIGEST_SIZE) + ctx->buflen[1] = get_unaligned(p.u64); + if (ctx->buflen[0] | ctx->buflen[1]) + ctx->gr0.ikp = 1; + return err; +} + +#define S390_HMAC_SHA2_ALG(x, ss) { \ .fc = CPACF_KMAC_HMAC_SHA_##x, \ .alg = { \ .init = s390_hmac_sha2_init, \ .update = s390_hmac_sha2_update, \ - .final = s390_hmac_sha2_final, \ + .finup = s390_hmac_sha2_finup, \ .digest = s390_hmac_sha2_digest, \ .setkey = s390_hmac_sha2_setkey, \ + .export = s390_hmac_export, \ + .import = s390_hmac_import, \ .descsize = sizeof(struct s390_kmac_sha2_ctx), \ .halg = { \ + .statesize = ss, \ .digestsize = SHA##x##_DIGEST_SIZE, \ .base = { \ .cra_name = "hmac(sha" #x ")", \ .cra_driver_name = "hmac_s390_sha" #x, \ .cra_blocksize = SHA##x##_BLOCK_SIZE, \ .cra_priority = 400, \ + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | \ + CRYPTO_AHASH_ALG_FINUP_MAX, \ .cra_ctxsize = sizeof(struct s390_hmac_ctx), \ .cra_module = THIS_MODULE, \ }, \ @@ -301,10 +362,10 @@ static struct s390_hmac_alg { unsigned int fc; struct shash_alg alg; } s390_hmac_algs[] = { - S390_HMAC_SHA2_ALG(224), - S390_HMAC_SHA2_ALG(256), - S390_HMAC_SHA2_ALG(384), - S390_HMAC_SHA2_ALG(512), + S390_HMAC_SHA2_ALG(224, sizeof(struct crypto_sha256_state)), + S390_HMAC_SHA2_ALG(256, sizeof(struct crypto_sha256_state)), + S390_HMAC_SHA2_ALG(384, SHA512_STATE_SIZE), + S390_HMAC_SHA2_ALG(512, SHA512_STATE_SIZE), }; static __always_inline void _s390_hmac_algs_unregister(void) From f1fb16efe50bb7953ef3c224e1b3802559d26cc3 Mon Sep 17 00:00:00 2001 From: Ethan Carter Edwards Date: Sat, 3 May 2025 16:21:27 -0400 Subject: [PATCH 307/382] crypto: hisilicon/qm - remove sizeof(char) `sizeof(char)` evaluates to 1. Remove the churn. Signed-off-by: Ethan Carter Edwards Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/qm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index d3f5d108b898..80d57f0dbf26 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -862,7 +862,7 @@ int hisi_qm_set_algs(struct hisi_qm *qm, u64 alg_msk, const struct qm_dev_alg *d return -EINVAL; } - algs = devm_kzalloc(dev, QM_DEV_ALG_MAX_LEN * sizeof(char), GFP_KERNEL); + algs = devm_kzalloc(dev, QM_DEV_ALG_MAX_LEN, GFP_KERNEL); if (!algs) return -ENOMEM; From 7c234e138c67c1157b17f1fab8cc95d62f8f4444 Mon Sep 17 00:00:00 2001 From: Ethan Carter Edwards Date: Sat, 3 May 2025 16:21:28 -0400 Subject: [PATCH 308/382] crypto: hisilicon/qm - replace devm_kzalloc with devm_kcalloc Replace devm_kzalloc that has an internal multiplication with devm_kcalloc to improve code readability and safety from overflows. Signed-off-by: Ethan Carter Edwards Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/qm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index 80d57f0dbf26..7c41f9593d03 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -5224,7 +5224,7 @@ static int qm_pre_store_caps(struct hisi_qm *qm) size_t i, size; size = ARRAY_SIZE(qm_cap_query_info); - qm_cap = devm_kzalloc(&pdev->dev, sizeof(*qm_cap) * size, GFP_KERNEL); + qm_cap = devm_kcalloc(&pdev->dev, sizeof(*qm_cap), size, GFP_KERNEL); if (!qm_cap) return -ENOMEM; From ecd71c95a60e7298acfabe81189439f350bd0e18 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sun, 4 May 2025 14:13:50 +0800 Subject: [PATCH 309/382] crypto: zynqmp-sha - Fix partial block implementation The zynqmp-sha partial block was based on an old design of the partial block API where the leftover calculation was done in the Crypto API. As the leftover calculation is now done by the algorithm, fix this by passing the partial blocks to the fallback. Also zero the stack descriptors. Fixes: 201e9ec3b621 ("crypto: zynqmp-sha - Use API partial block handling") Signed-off-by: Herbert Xu --- drivers/crypto/xilinx/zynqmp-sha.c | 30 +++++++++++++++++++----------- include/crypto/sha3.h | 4 ++++ 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/drivers/crypto/xilinx/zynqmp-sha.c b/drivers/crypto/xilinx/zynqmp-sha.c index 67cf8d990a1d..5813017b6b79 100644 --- a/drivers/crypto/xilinx/zynqmp-sha.c +++ b/drivers/crypto/xilinx/zynqmp-sha.c @@ -59,7 +59,7 @@ static int zynqmp_sha_init_tfm(struct crypto_shash *hash) return PTR_ERR(fallback_tfm); if (crypto_shash_descsize(hash) < - crypto_shash_descsize(tfm_ctx->fbk_tfm)) { + crypto_shash_statesize(tfm_ctx->fbk_tfm)) { crypto_free_shash(fallback_tfm); return -EINVAL; } @@ -76,15 +76,24 @@ static void zynqmp_sha_exit_tfm(struct crypto_shash *hash) crypto_free_shash(tfm_ctx->fbk_tfm); } +static int zynqmp_sha_continue(struct shash_desc *desc, + struct shash_desc *fbdesc, int err) +{ + err = err ?: crypto_shash_export(fbdesc, shash_desc_ctx(desc)); + shash_desc_zero(fbdesc); + return err; +} + static int zynqmp_sha_init(struct shash_desc *desc) { struct zynqmp_sha_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); struct crypto_shash *fbtfm = tctx->fbk_tfm; SHASH_DESC_ON_STACK(fbdesc, fbtfm); + int err; fbdesc->tfm = fbtfm; - return crypto_shash_init(fbdesc) ?: - crypto_shash_export_core(fbdesc, shash_desc_ctx(desc)); + err = crypto_shash_init(fbdesc); + return zynqmp_sha_continue(desc, fbdesc, err); } static int zynqmp_sha_update(struct shash_desc *desc, const u8 *data, unsigned int length) @@ -92,11 +101,12 @@ static int zynqmp_sha_update(struct shash_desc *desc, const u8 *data, unsigned i struct zynqmp_sha_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); struct crypto_shash *fbtfm = tctx->fbk_tfm; SHASH_DESC_ON_STACK(fbdesc, fbtfm); + int err; fbdesc->tfm = fbtfm; - return crypto_shash_import_core(fbdesc, shash_desc_ctx(desc)) ?: - crypto_shash_update(fbdesc, data, length) ?: - crypto_shash_export_core(fbdesc, shash_desc_ctx(desc)); + err = crypto_shash_import(fbdesc, shash_desc_ctx(desc)) ?: + crypto_shash_update(fbdesc, data, length); + return zynqmp_sha_continue(desc, fbdesc, err); } static int zynqmp_sha_finup(struct shash_desc *desc, const u8 *data, unsigned int length, u8 *out) @@ -106,7 +116,7 @@ static int zynqmp_sha_finup(struct shash_desc *desc, const u8 *data, unsigned in SHASH_DESC_ON_STACK(fbdesc, fbtfm); fbdesc->tfm = fbtfm; - return crypto_shash_import_core(fbdesc, shash_desc_ctx(desc)) ?: + return crypto_shash_import(fbdesc, shash_desc_ctx(desc)) ?: crypto_shash_finup(fbdesc, data, length, out); } @@ -160,16 +170,14 @@ static struct zynqmp_sha_drv_ctx sha3_drv_ctx = { .digest = zynqmp_sha_digest, .init_tfm = zynqmp_sha_init_tfm, .exit_tfm = zynqmp_sha_exit_tfm, - .descsize = sizeof(struct sha3_state), + .descsize = SHA3_384_EXPORT_SIZE, .digestsize = SHA3_384_DIGEST_SIZE, .base = { .cra_name = "sha3-384", .cra_driver_name = "zynqmp-sha3-384", .cra_priority = 300, .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | - CRYPTO_ALG_NEED_FALLBACK | - CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, + CRYPTO_ALG_NEED_FALLBACK, .cra_blocksize = SHA3_384_BLOCK_SIZE, .cra_ctxsize = sizeof(struct zynqmp_sha_tfm_ctx), .cra_module = THIS_MODULE, diff --git a/include/crypto/sha3.h b/include/crypto/sha3.h index 3c2559f51ada..41e1b83a6d91 100644 --- a/include/crypto/sha3.h +++ b/include/crypto/sha3.h @@ -9,15 +9,19 @@ #define SHA3_224_DIGEST_SIZE (224 / 8) #define SHA3_224_BLOCK_SIZE (200 - 2 * SHA3_224_DIGEST_SIZE) +#define SHA3_224_EXPORT_SIZE SHA3_STATE_SIZE + SHA3_224_BLOCK_SIZE + 1 #define SHA3_256_DIGEST_SIZE (256 / 8) #define SHA3_256_BLOCK_SIZE (200 - 2 * SHA3_256_DIGEST_SIZE) +#define SHA3_256_EXPORT_SIZE SHA3_STATE_SIZE + SHA3_256_BLOCK_SIZE + 1 #define SHA3_384_DIGEST_SIZE (384 / 8) #define SHA3_384_BLOCK_SIZE (200 - 2 * SHA3_384_DIGEST_SIZE) +#define SHA3_384_EXPORT_SIZE SHA3_STATE_SIZE + SHA3_384_BLOCK_SIZE + 1 #define SHA3_512_DIGEST_SIZE (512 / 8) #define SHA3_512_BLOCK_SIZE (200 - 2 * SHA3_512_DIGEST_SIZE) +#define SHA3_512_EXPORT_SIZE SHA3_STATE_SIZE + SHA3_512_BLOCK_SIZE + 1 #define SHA3_STATE_SIZE 200 From 2b1a29ce3360570aeff053d1315cd504d94eab31 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sun, 4 May 2025 21:33:14 +0800 Subject: [PATCH 310/382] crypto: shash - Cap state size to HASH_MAX_STATESIZE Now that all shash algorithms have converted over to the generic export format, limit the shash state size to HASH_MAX_STATESIZE. Signed-off-by: Herbert Xu --- crypto/shash.c | 2 ++ include/crypto/hash.h | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/crypto/shash.c b/crypto/shash.c index c4a724e55d7a..44a6df3132ad 100644 --- a/crypto/shash.c +++ b/crypto/shash.c @@ -479,6 +479,8 @@ static int shash_prepare_alg(struct shash_alg *alg) if (alg->descsize > HASH_MAX_DESCSIZE) return -EINVAL; + if (alg->statesize > HASH_MAX_STATESIZE) + return -EINVAL; return 0; } diff --git a/include/crypto/hash.h b/include/crypto/hash.h index c2497c300a28..e0321b5ec363 100644 --- a/include/crypto/hash.h +++ b/include/crypto/hash.h @@ -167,8 +167,11 @@ struct shash_desc { #define HASH_MAX_DIGESTSIZE 64 +/* Worst case is sha3-224. */ +#define HASH_MAX_STATESIZE 200 + 144 + 1 + /* - * Worst case is hmac(sha-224-s390). Its context is a nested 'shash_desc' + * Worst case is hmac(sha3-224-s390). Its context is a nested 'shash_desc' * containing a 'struct s390_sha_ctx'. */ #define HASH_MAX_DESCSIZE (sizeof(struct shash_desc) + 360) From f4e365d5ca38941708fbe8356719e3436cef7627 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sun, 4 May 2025 21:33:16 +0800 Subject: [PATCH 311/382] crypto: shash - Mark shash algorithms as REQ_VIRT Mark shash algorithms with the REQ_VIRT bit as they can handle virtual addresses as is. Signed-off-by: Herbert Xu --- crypto/shash.c | 1 + 1 file changed, 1 insertion(+) diff --git a/crypto/shash.c b/crypto/shash.c index 44a6df3132ad..dee391d47f51 100644 --- a/crypto/shash.c +++ b/crypto/shash.c @@ -450,6 +450,7 @@ static int shash_prepare_alg(struct shash_alg *alg) base->cra_type = &crypto_shash_type; base->cra_flags |= CRYPTO_ALG_TYPE_SHASH; + base->cra_flags |= CRYPTO_ALG_REQ_VIRT; /* * Handle missing optional functions. For each one we can either From fd66f2ab09b8305006764887cc47eeeb1ca5704b Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sun, 4 May 2025 21:33:18 +0800 Subject: [PATCH 312/382] crypto: ahash - Enforce MAX_SYNC_HASH_REQSIZE for sync ahash As sync ahash algorithms (currently there are none) are used without a fallback, ensure that they obey the MAX_SYNC_HASH_REQSIZE rule just like shash algorithms. Signed-off-by: Herbert Xu --- crypto/ahash.c | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/crypto/ahash.c b/crypto/ahash.c index 57c131a13067..736e9fb5d0a4 100644 --- a/crypto/ahash.c +++ b/crypto/ahash.c @@ -760,23 +760,28 @@ static int crypto_ahash_init_tfm(struct crypto_tfm *tfm) tfm->exit = crypto_ahash_exit_tfm; - if (!alg->init_tfm) { - if (!tfm->__crt_alg->cra_init) - return 0; - + if (alg->init_tfm) + err = alg->init_tfm(hash); + else if (tfm->__crt_alg->cra_init) err = tfm->__crt_alg->cra_init(tfm); - if (err) - goto out_free_sync_hash; - + else return 0; - } - err = alg->init_tfm(hash); if (err) goto out_free_sync_hash; + if (!ahash_is_async(hash) && crypto_ahash_reqsize(hash) > + MAX_SYNC_HASH_REQSIZE) + goto out_exit_tfm; + return 0; +out_exit_tfm: + if (alg->exit_tfm) + alg->exit_tfm(hash); + else if (tfm->__crt_alg->cra_exit) + tfm->__crt_alg->cra_exit(tfm); + err = -EINVAL; out_free_sync_hash: crypto_free_ahash(fb); return err; @@ -954,6 +959,10 @@ static int ahash_prepare_alg(struct ahash_alg *alg) if (base->cra_reqsize && base->cra_reqsize < alg->halg.statesize) return -EINVAL; + if (!(base->cra_flags & CRYPTO_ALG_ASYNC) && + base->cra_reqsize > MAX_SYNC_HASH_REQSIZE) + return -EINVAL; + err = hash_prepare_alg(&alg->halg); if (err) return err; From 1052671ca118b79fa3f5de281bba850aaf20bbf5 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sun, 4 May 2025 21:33:21 +0800 Subject: [PATCH 313/382] crypto: ahash - Add core export and import Add crypto_ahash_export_core and crypto_ahash_import_core. For now they only differ from the normal export/import functions when going through shash. Signed-off-by: Herbert Xu --- crypto/ahash.c | 25 ++++++++++++++++++++++++- include/crypto/hash.h | 24 ++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/crypto/ahash.c b/crypto/ahash.c index 736e9fb5d0a4..344bf1b43e71 100644 --- a/crypto/ahash.c +++ b/crypto/ahash.c @@ -698,6 +698,16 @@ static int ahash_def_finup(struct ahash_request *req) return ahash_def_finup_finish1(req, err); } +int crypto_ahash_export_core(struct ahash_request *req, void *out) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + + if (likely(tfm->using_shash)) + return crypto_shash_export_core(ahash_request_ctx(req), out); + return crypto_ahash_alg(tfm)->export(req, out); +} +EXPORT_SYMBOL_GPL(crypto_ahash_export_core); + int crypto_ahash_export(struct ahash_request *req, void *out) { struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); @@ -708,6 +718,19 @@ int crypto_ahash_export(struct ahash_request *req, void *out) } EXPORT_SYMBOL_GPL(crypto_ahash_export); +int crypto_ahash_import_core(struct ahash_request *req, const void *in) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + + if (likely(tfm->using_shash)) + return crypto_shash_import_core(prepare_shash_desc(req, tfm), + in); + if (crypto_ahash_get_flags(tfm) & CRYPTO_TFM_NEED_KEY) + return -ENOKEY; + return crypto_ahash_alg(tfm)->import(req, in); +} +EXPORT_SYMBOL_GPL(crypto_ahash_import_core); + int crypto_ahash_import(struct ahash_request *req, const void *in) { struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); @@ -716,7 +739,7 @@ int crypto_ahash_import(struct ahash_request *req, const void *in) return crypto_shash_import(prepare_shash_desc(req, tfm), in); if (crypto_ahash_get_flags(tfm) & CRYPTO_TFM_NEED_KEY) return -ENOKEY; - return crypto_ahash_alg(tfm)->import(req, in); + return crypto_ahash_import_core(req, in); } EXPORT_SYMBOL_GPL(crypto_ahash_import); diff --git a/include/crypto/hash.h b/include/crypto/hash.h index e0321b5ec363..1760662ad70a 100644 --- a/include/crypto/hash.h +++ b/include/crypto/hash.h @@ -506,6 +506,18 @@ int crypto_ahash_digest(struct ahash_request *req); */ int crypto_ahash_export(struct ahash_request *req, void *out); +/** + * crypto_ahash_export_core() - extract core state for message digest + * @req: reference to the ahash_request handle whose state is exported + * @out: output buffer of sufficient size that can hold the hash state + * + * Export the hash state without the partial block buffer. + * + * Context: Softirq or process context. + * Return: 0 if the export creation was successful; < 0 if an error occurred + */ +int crypto_ahash_export_core(struct ahash_request *req, void *out); + /** * crypto_ahash_import() - import message digest state * @req: reference to ahash_request handle the state is imported into @@ -519,6 +531,18 @@ int crypto_ahash_export(struct ahash_request *req, void *out); */ int crypto_ahash_import(struct ahash_request *req, const void *in); +/** + * crypto_ahash_import_core() - import core state + * @req: reference to ahash_request handle the state is imported into + * @in: buffer holding the state + * + * Import the hash state without the partial block buffer. + * + * Context: Softirq or process context. + * Return: 0 if the import was successful; < 0 if an error occurred + */ +int crypto_ahash_import_core(struct ahash_request *req, const void *in); + /** * crypto_ahash_init() - (re)initialize message digest handle * @req: ahash_request handle that already is initialized with all necessary From 88bca957e87e9a8a274213db257a744170b7248a Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sun, 4 May 2025 21:33:23 +0800 Subject: [PATCH 314/382] crypto: ahash - Add HASH_REQUEST_ZERO Add a helper to zero hash stack requests that were never cloned off the stack. Signed-off-by: Herbert Xu --- include/crypto/internal/hash.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/include/crypto/internal/hash.h b/include/crypto/internal/hash.h index e911f32f46dc..f2bbdb74e11a 100644 --- a/include/crypto/internal/hash.h +++ b/include/crypto/internal/hash.h @@ -301,5 +301,9 @@ static inline unsigned int crypto_shash_coresize(struct crypto_shash *tfm) return crypto_shash_statesize(tfm) - crypto_shash_blocksize(tfm) - 1; } +/* This can only be used if the request was never cloned. */ +#define HASH_REQUEST_ZERO(name) \ + memzero_explicit(__##name##_req, sizeof(__##name##_req)) + #endif /* _CRYPTO_INTERNAL_HASH_H */ From 870c1f0dc2a5e67eb0697ae14365b930684ddfa5 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sun, 4 May 2025 21:33:25 +0800 Subject: [PATCH 315/382] crypto: padlock-sha - Use core import and export for fallback As padlock-sha is block-only, it needs to use core import and export on the fallback. Also call sha256_block_init instead of sha256_init although this is harmless as sha256_init doesn't write into the partial block area. Fixes: 63dc06cd12f9 ("crypto: padlock-sha - Use API partial block handling") Signed-off-by: Herbert Xu --- drivers/crypto/padlock-sha.c | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/drivers/crypto/padlock-sha.c b/drivers/crypto/padlock-sha.c index c89b9c6b5f4c..329f60ad422e 100644 --- a/drivers/crypto/padlock-sha.c +++ b/drivers/crypto/padlock-sha.c @@ -42,27 +42,33 @@ static int padlock_sha1_init(struct shash_desc *desc) static int padlock_sha256_init(struct shash_desc *desc) { - struct sha256_state *sctx = padlock_shash_desc_ctx(desc); + struct crypto_sha256_state *sctx = padlock_shash_desc_ctx(desc); - sha256_init(sctx); + sha256_block_init(sctx); return 0; } static int padlock_sha_update(struct shash_desc *desc, const u8 *data, unsigned int length) { - struct padlock_sha_ctx *ctx = crypto_shash_ctx(desc->tfm); u8 *state = padlock_shash_desc_ctx(desc); - HASH_REQUEST_ON_STACK(req, ctx->fallback); - int remain; + struct crypto_shash *tfm = desc->tfm; + int err, remain; - ahash_request_set_callback(req, 0, NULL, NULL); - ahash_request_set_virt(req, data, NULL, length); - remain = crypto_ahash_import(req, state) ?: - crypto_ahash_update(req); - if (remain < 0) - return remain; - return crypto_ahash_export(req, state) ?: remain; + remain = length - round_down(length, crypto_shash_blocksize(tfm)); + { + struct padlock_sha_ctx *ctx = crypto_shash_ctx(tfm); + HASH_REQUEST_ON_STACK(req, ctx->fallback); + + ahash_request_set_callback(req, 0, NULL, NULL); + ahash_request_set_virt(req, data, NULL, length - remain); + err = crypto_ahash_import_core(req, state) ?: + crypto_ahash_update(req) ?: + crypto_ahash_export_core(req, state); + HASH_REQUEST_ZERO(req); + } + + return err ?: remain; } static int padlock_sha_export(struct shash_desc *desc, void *out) @@ -101,7 +107,7 @@ static int padlock_sha_finup(struct shash_desc *desc, const u8 *in, ahash_request_set_callback(req, 0, NULL, NULL); ahash_request_set_virt(req, in, out, count); - return crypto_ahash_import(req, padlock_shash_desc_ctx(desc)) ?: + return crypto_ahash_import_core(req, padlock_shash_desc_ctx(desc)) ?: crypto_ahash_finup(req); } @@ -165,7 +171,7 @@ static int padlock_init_tfm(struct crypto_shash *hash) return PTR_ERR(fallback_tfm); } - if (crypto_shash_statesize(hash) < + if (crypto_shash_statesize(hash) != crypto_ahash_statesize(fallback_tfm)) { crypto_free_ahash(fallback_tfm); return -EINVAL; From 64745a9ca890ed60d78162ec511e1983e1946d73 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Mon, 5 May 2025 20:54:41 +0800 Subject: [PATCH 316/382] crypto: s390/sha512 - Initialise upper counter to zero for sha384 Initialise the high bit counter to zero in sha384_init. Also change the state initialisation to use ctx->sha512.state instead of ctx->state for consistency. Fixes: 572b5c4682c7 ("crypto: s390/sha512 - Use API partial block handling") Reported-by: Ingo Franzki Reported-by: Harald Freudenberger Signed-off-by: Herbert Xu --- arch/s390/crypto/sha512_s390.c | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/arch/s390/crypto/sha512_s390.c b/arch/s390/crypto/sha512_s390.c index 3c5175e6dda6..33711a29618c 100644 --- a/arch/s390/crypto/sha512_s390.c +++ b/arch/s390/crypto/sha512_s390.c @@ -86,15 +86,16 @@ static int sha384_init(struct shash_desc *desc) { struct s390_sha_ctx *ctx = shash_desc_ctx(desc); - *(__u64 *)&ctx->state[0] = SHA384_H0; - *(__u64 *)&ctx->state[2] = SHA384_H1; - *(__u64 *)&ctx->state[4] = SHA384_H2; - *(__u64 *)&ctx->state[6] = SHA384_H3; - *(__u64 *)&ctx->state[8] = SHA384_H4; - *(__u64 *)&ctx->state[10] = SHA384_H5; - *(__u64 *)&ctx->state[12] = SHA384_H6; - *(__u64 *)&ctx->state[14] = SHA384_H7; + ctx->sha512.state[0] = SHA384_H0; + ctx->sha512.state[1] = SHA384_H1; + ctx->sha512.state[2] = SHA384_H2; + ctx->sha512.state[3] = SHA384_H3; + ctx->sha512.state[4] = SHA384_H4; + ctx->sha512.state[5] = SHA384_H5; + ctx->sha512.state[6] = SHA384_H6; + ctx->sha512.state[7] = SHA384_H7; ctx->count = 0; + ctx->sha512.count_hi = 0; ctx->func = CPACF_KIMD_SHA_512; return 0; From 9b9d4ef0cf750c514735bfd77745387b95cbddda Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Tue, 6 May 2025 10:05:08 +0800 Subject: [PATCH 317/382] crypto: lib/poly1305 - Build main library on LIB_POLY1305 and split generic code out Split the lib poly1305 code just as was done with sha256. Make the main library code conditional on LIB_POLY1305 instead of LIB_POLY1305_GENERIC. Reported-by: Giovanni Cabiddu Fixes: 10a6d72ea355 ("crypto: lib/poly1305 - Use block-only interface") Signed-off-by: Herbert Xu --- lib/crypto/Makefile | 9 ++++++--- lib/crypto/poly1305-generic.c | 24 ++++++++++++++++++++++++ lib/crypto/poly1305.c | 8 -------- 3 files changed, 30 insertions(+), 11 deletions(-) create mode 100644 lib/crypto/poly1305-generic.c diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index 71d3d05d666a..ff4aa22e5ccc 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -40,11 +40,14 @@ libcurve25519-y += curve25519.o obj-$(CONFIG_CRYPTO_LIB_DES) += libdes.o libdes-y := des.o -obj-$(CONFIG_CRYPTO_LIB_POLY1305_GENERIC) += libpoly1305.o -libpoly1305-y := poly1305-donna32.o -libpoly1305-$(CONFIG_ARCH_SUPPORTS_INT128) := poly1305-donna64.o +obj-$(CONFIG_CRYPTO_LIB_POLY1305) += libpoly1305.o libpoly1305-y += poly1305.o +obj-$(CONFIG_CRYPTO_LIB_POLY1305_GENERIC) += libpoly1305-generic.o +libpoly1305-generic-y := poly1305-donna32.o +libpoly1305-generic-$(CONFIG_ARCH_SUPPORTS_INT128) := poly1305-donna64.o +libpoly1305-generic-y += poly1305-generic.o + obj-$(CONFIG_CRYPTO_LIB_SHA1) += libsha1.o libsha1-y := sha1.o diff --git a/lib/crypto/poly1305-generic.c b/lib/crypto/poly1305-generic.c new file mode 100644 index 000000000000..a73f700fa1fb --- /dev/null +++ b/lib/crypto/poly1305-generic.c @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Poly1305 authenticator algorithm, RFC7539 + * + * Copyright (C) 2015 Martin Willi + * + * Based on public domain code by Andrew Moon and Daniel J. Bernstein. + */ + +#include +#include +#include + +void poly1305_block_init_generic(struct poly1305_block_state *desc, + const u8 raw_key[POLY1305_BLOCK_SIZE]) +{ + poly1305_core_init(&desc->h); + poly1305_core_setkey(&desc->core_r, raw_key); +} +EXPORT_SYMBOL_GPL(poly1305_block_init_generic); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Martin Willi "); +MODULE_DESCRIPTION("Poly1305 algorithm (generic implementation)"); diff --git a/lib/crypto/poly1305.c b/lib/crypto/poly1305.c index 4c9996864090..5f2f2af3b59f 100644 --- a/lib/crypto/poly1305.c +++ b/lib/crypto/poly1305.c @@ -14,14 +14,6 @@ #include #include -void poly1305_block_init_generic(struct poly1305_block_state *desc, - const u8 raw_key[POLY1305_BLOCK_SIZE]) -{ - poly1305_core_init(&desc->h); - poly1305_core_setkey(&desc->core_r, raw_key); -} -EXPORT_SYMBOL_GPL(poly1305_block_init_generic); - void poly1305_init(struct poly1305_desc_ctx *desc, const u8 key[POLY1305_KEY_SIZE]) { From 20e9579f11b6cbdf0556d9cd85a0aa7653caf341 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Wed, 7 May 2025 09:41:51 +0800 Subject: [PATCH 318/382] um: Include linux/types.h in asm/fpu/api.h Include linux/types.h before using bool. Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202505070045.vWc04ygs-lkp@intel.com/ Signed-off-by: Herbert Xu Acked-by: Johannes Berg Signed-off-by: Herbert Xu --- arch/um/include/asm/fpu/api.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/um/include/asm/fpu/api.h b/arch/um/include/asm/fpu/api.h index 71bfd9ef3938..3abf67c83c40 100644 --- a/arch/um/include/asm/fpu/api.h +++ b/arch/um/include/asm/fpu/api.h @@ -2,6 +2,8 @@ #ifndef _ASM_UM_FPU_API_H #define _ASM_UM_FPU_API_H +#include + /* Copyright (c) 2020 Cambridge Greys Ltd * Copyright (c) 2020 Red Hat Inc. * A set of "dummy" defines to allow the direct inclusion From 3bbc902ece47198825d682c8579214306fd0748c Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Wed, 7 May 2025 19:36:58 +0800 Subject: [PATCH 319/382] crypto: powerpc/poly1305 - Add missing poly1305_emit_arch Rename poly1305_emit_64 to poly1305_emit_arch to conform with the expectation of the poly1305 library. Reported-by: Thorsten Leemhuis Fixes: 14d31979145d ("crypto: powerpc/poly1305 - Add block-only interface") Signed-off-by: Herbert Xu Tested-by: Thorsten Leemhuis Signed-off-by: Herbert Xu --- arch/powerpc/lib/crypto/poly1305-p10-glue.c | 1 + arch/powerpc/lib/crypto/poly1305-p10le_64.S | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/arch/powerpc/lib/crypto/poly1305-p10-glue.c b/arch/powerpc/lib/crypto/poly1305-p10-glue.c index 16c2a8316696..7cea0ebcc6bc 100644 --- a/arch/powerpc/lib/crypto/poly1305-p10-glue.c +++ b/arch/powerpc/lib/crypto/poly1305-p10-glue.c @@ -17,6 +17,7 @@ asmlinkage void poly1305_64s(struct poly1305_block_state *state, const u8 *m, u3 asmlinkage void poly1305_emit_arch(const struct poly1305_state *state, u8 digest[POLY1305_DIGEST_SIZE], const u32 nonce[4]); +EXPORT_SYMBOL_GPL(poly1305_emit_arch); static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_p10); diff --git a/arch/powerpc/lib/crypto/poly1305-p10le_64.S b/arch/powerpc/lib/crypto/poly1305-p10le_64.S index a3c1987f1ecd..2ba2911b8038 100644 --- a/arch/powerpc/lib/crypto/poly1305-p10le_64.S +++ b/arch/powerpc/lib/crypto/poly1305-p10le_64.S @@ -1030,7 +1030,7 @@ SYM_FUNC_END(poly1305_64s) # Input: r3 = h, r4 = s, r5 = mac # mac = h + s # -SYM_FUNC_START(poly1305_emit_64) +SYM_FUNC_START(poly1305_emit_arch) ld 10, 0(3) ld 11, 8(3) ld 12, 16(3) @@ -1060,7 +1060,7 @@ Skip_h64: std 10, 0(5) std 11, 8(5) blr -SYM_FUNC_END(poly1305_emit_64) +SYM_FUNC_END(poly1305_emit_arch) SYM_DATA_START_LOCAL(RMASK) .align 5 From 4e0146a94c516e0ce119cd34743108c7652497bd Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Wed, 7 May 2025 10:09:01 -0700 Subject: [PATCH 320/382] crypto: arm64/sha256 - fix build when CONFIG_PREEMPT_VOLUNTARY=y Fix the build of sha256-ce.S when CONFIG_PREEMPT_VOLUNTARY=y by passing the correct label to the cond_yield macro. Also adjust the code to execute only one branch instruction when CONFIG_PREEMPT_VOLUNTARY=n. Fixes: 6e36be511d28 ("crypto: arm64/sha256 - implement library instead of shash") Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202505071811.yYpLUbav-lkp@intel.com/ Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/arm64/lib/crypto/sha256-ce.S | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/arch/arm64/lib/crypto/sha256-ce.S b/arch/arm64/lib/crypto/sha256-ce.S index a8461d6dad63..f3e21c6d87d2 100644 --- a/arch/arm64/lib/crypto/sha256-ce.S +++ b/arch/arm64/lib/crypto/sha256-ce.S @@ -123,10 +123,11 @@ CPU_LE( rev32 v19.16b, v19.16b ) add dgav.4s, dgav.4s, dg0v.4s add dgbv.4s, dgbv.4s, dg1v.4s + /* return early if voluntary preemption is needed */ + cond_yield 1f, x5, x6 + /* handled all input blocks? */ - cbz x2, 1f - cond_yield 3f, x5, x6 - b 0b + cbnz x2, 0b /* store new state */ 1: st1 {dgav.4s, dgbv.4s}, [x0] From 7916eddc533248ec64f5b68657c11b4fca057fab Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sat, 10 May 2025 13:10:22 +0800 Subject: [PATCH 321/382] crypto: powerpc/poly1305 - Add poly1305_emit_arch wrapper Add poly1305_emit_arch with fallback instead of calling assembly directly. This is because the state format differs between p10 and that of the generic implementation. Reported-by: Venkat Rao Bagalkote Reported-by: Eric Biggers Fixes: 14d31979145d ("crypto: powerpc/poly1305 - Add block-only interface") Signed-off-by: Herbert Xu --- arch/powerpc/lib/crypto/poly1305-p10-glue.c | 15 +++++++++++---- arch/powerpc/lib/crypto/poly1305-p10le_64.S | 4 ++-- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/arch/powerpc/lib/crypto/poly1305-p10-glue.c b/arch/powerpc/lib/crypto/poly1305-p10-glue.c index 7cea0ebcc6bc..3f1664a724b6 100644 --- a/arch/powerpc/lib/crypto/poly1305-p10-glue.c +++ b/arch/powerpc/lib/crypto/poly1305-p10-glue.c @@ -14,10 +14,7 @@ asmlinkage void poly1305_p10le_4blocks(struct poly1305_block_state *state, const u8 *m, u32 mlen); asmlinkage void poly1305_64s(struct poly1305_block_state *state, const u8 *m, u32 mlen, int highbit); -asmlinkage void poly1305_emit_arch(const struct poly1305_state *state, - u8 digest[POLY1305_DIGEST_SIZE], - const u32 nonce[4]); -EXPORT_SYMBOL_GPL(poly1305_emit_arch); +asmlinkage void poly1305_emit_64(const struct poly1305_state *state, const u32 nonce[4], u8 digest[POLY1305_DIGEST_SIZE]); static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_p10); @@ -65,6 +62,16 @@ void poly1305_blocks_arch(struct poly1305_block_state *state, const u8 *src, } EXPORT_SYMBOL_GPL(poly1305_blocks_arch); +void poly1305_emit_arch(const struct poly1305_state *state, + u8 digest[POLY1305_DIGEST_SIZE], + const u32 nonce[4]) +{ + if (!static_key_enabled(&have_p10)) + return poly1305_emit_generic(state, digest, nonce); + poly1305_emit_64(state, nonce, digest); +} +EXPORT_SYMBOL_GPL(poly1305_emit_arch); + bool poly1305_is_arch_optimized(void) { return static_key_enabled(&have_p10); diff --git a/arch/powerpc/lib/crypto/poly1305-p10le_64.S b/arch/powerpc/lib/crypto/poly1305-p10le_64.S index 2ba2911b8038..a3c1987f1ecd 100644 --- a/arch/powerpc/lib/crypto/poly1305-p10le_64.S +++ b/arch/powerpc/lib/crypto/poly1305-p10le_64.S @@ -1030,7 +1030,7 @@ SYM_FUNC_END(poly1305_64s) # Input: r3 = h, r4 = s, r5 = mac # mac = h + s # -SYM_FUNC_START(poly1305_emit_arch) +SYM_FUNC_START(poly1305_emit_64) ld 10, 0(3) ld 11, 8(3) ld 12, 16(3) @@ -1060,7 +1060,7 @@ Skip_h64: std 10, 0(5) std 11, 8(5) blr -SYM_FUNC_END(poly1305_emit_arch) +SYM_FUNC_END(poly1305_emit_64) SYM_DATA_START_LOCAL(RMASK) .align 5 From 97855e7f1ccf4917f305baab199edb9f2595ff5b Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Mon, 5 May 2025 15:15:13 +0800 Subject: [PATCH 322/382] crypto: crypto4xx - Remove ahash-related code The hash implementation in crypto4xx has been disabled since 2009. As nobody has tried to fix this remove all the dead code. Signed-off-by: Herbert Xu --- drivers/crypto/amcc/crypto4xx_alg.c | 106 --------------------------- drivers/crypto/amcc/crypto4xx_core.c | 43 +---------- drivers/crypto/amcc/crypto4xx_core.h | 7 -- 3 files changed, 1 insertion(+), 155 deletions(-) diff --git a/drivers/crypto/amcc/crypto4xx_alg.c b/drivers/crypto/amcc/crypto4xx_alg.c index 289750f34ccf..38e8a61e9166 100644 --- a/drivers/crypto/amcc/crypto4xx_alg.c +++ b/drivers/crypto/amcc/crypto4xx_alg.c @@ -12,9 +12,6 @@ #include #include #include -#include -#include -#include #include #include #include @@ -602,106 +599,3 @@ int crypto4xx_decrypt_aes_gcm(struct aead_request *req) { return crypto4xx_crypt_aes_gcm(req, true); } - -/* - * HASH SHA1 Functions - */ -static int crypto4xx_hash_alg_init(struct crypto_tfm *tfm, - unsigned int sa_len, - unsigned char ha, - unsigned char hm) -{ - struct crypto_alg *alg = tfm->__crt_alg; - struct crypto4xx_alg *my_alg; - struct crypto4xx_ctx *ctx = crypto_tfm_ctx(tfm); - struct dynamic_sa_hash160 *sa; - int rc; - - my_alg = container_of(__crypto_ahash_alg(alg), struct crypto4xx_alg, - alg.u.hash); - ctx->dev = my_alg->dev; - - /* Create SA */ - if (ctx->sa_in || ctx->sa_out) - crypto4xx_free_sa(ctx); - - rc = crypto4xx_alloc_sa(ctx, sa_len); - if (rc) - return rc; - - crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), - sizeof(struct crypto4xx_ctx)); - sa = (struct dynamic_sa_hash160 *)ctx->sa_in; - set_dynamic_sa_command_0(&sa->ctrl, SA_SAVE_HASH, SA_NOT_SAVE_IV, - SA_NOT_LOAD_HASH, SA_LOAD_IV_FROM_SA, - SA_NO_HEADER_PROC, ha, SA_CIPHER_ALG_NULL, - SA_PAD_TYPE_ZERO, SA_OP_GROUP_BASIC, - SA_OPCODE_HASH, DIR_INBOUND); - set_dynamic_sa_command_1(&sa->ctrl, 0, SA_HASH_MODE_HASH, - CRYPTO_FEEDBACK_MODE_NO_FB, SA_EXTENDED_SN_OFF, - SA_SEQ_MASK_OFF, SA_MC_ENABLE, - SA_NOT_COPY_PAD, SA_NOT_COPY_PAYLOAD, - SA_NOT_COPY_HDR); - /* Need to zero hash digest in SA */ - memset(sa->inner_digest, 0, sizeof(sa->inner_digest)); - memset(sa->outer_digest, 0, sizeof(sa->outer_digest)); - - return 0; -} - -int crypto4xx_hash_init(struct ahash_request *req) -{ - struct crypto4xx_ctx *ctx = crypto_tfm_ctx(req->base.tfm); - int ds; - struct dynamic_sa_ctl *sa; - - sa = ctx->sa_in; - ds = crypto_ahash_digestsize( - __crypto_ahash_cast(req->base.tfm)); - sa->sa_command_0.bf.digest_len = ds >> 2; - sa->sa_command_0.bf.load_hash_state = SA_LOAD_HASH_FROM_SA; - - return 0; -} - -int crypto4xx_hash_update(struct ahash_request *req) -{ - struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); - struct crypto4xx_ctx *ctx = crypto_tfm_ctx(req->base.tfm); - struct scatterlist dst; - unsigned int ds = crypto_ahash_digestsize(ahash); - - sg_init_one(&dst, req->result, ds); - - return crypto4xx_build_pd(&req->base, ctx, req->src, &dst, - req->nbytes, NULL, 0, ctx->sa_in, - ctx->sa_len, 0, NULL); -} - -int crypto4xx_hash_final(struct ahash_request *req) -{ - return 0; -} - -int crypto4xx_hash_digest(struct ahash_request *req) -{ - struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); - struct crypto4xx_ctx *ctx = crypto_tfm_ctx(req->base.tfm); - struct scatterlist dst; - unsigned int ds = crypto_ahash_digestsize(ahash); - - sg_init_one(&dst, req->result, ds); - - return crypto4xx_build_pd(&req->base, ctx, req->src, &dst, - req->nbytes, NULL, 0, ctx->sa_in, - ctx->sa_len, 0, NULL); -} - -/* - * SHA1 Algorithm - */ -int crypto4xx_sha1_alg_init(struct crypto_tfm *tfm) -{ - return crypto4xx_hash_alg_init(tfm, SA_HASH160_LEN, SA_HASH_ALG_SHA1, - SA_HASH_MODE_HASH); -} diff --git a/drivers/crypto/amcc/crypto4xx_core.c b/drivers/crypto/amcc/crypto4xx_core.c index c77d06ddb1ec..8cdc66d520c9 100644 --- a/drivers/crypto/amcc/crypto4xx_core.c +++ b/drivers/crypto/amcc/crypto4xx_core.c @@ -485,18 +485,6 @@ static void crypto4xx_copy_pkt_to_dst(struct crypto4xx_device *dev, } } -static void crypto4xx_copy_digest_to_dst(void *dst, - struct pd_uinfo *pd_uinfo, - struct crypto4xx_ctx *ctx) -{ - struct dynamic_sa_ctl *sa = (struct dynamic_sa_ctl *) ctx->sa_in; - - if (sa->sa_command_0.bf.hash_alg == SA_HASH_ALG_SHA1) { - memcpy(dst, pd_uinfo->sr_va->save_digest, - SA_HASH_ALG_SHA1_DIGEST_SIZE); - } -} - static void crypto4xx_ret_sg_desc(struct crypto4xx_device *dev, struct pd_uinfo *pd_uinfo) { @@ -549,23 +537,6 @@ static void crypto4xx_cipher_done(struct crypto4xx_device *dev, skcipher_request_complete(req, 0); } -static void crypto4xx_ahash_done(struct crypto4xx_device *dev, - struct pd_uinfo *pd_uinfo) -{ - struct crypto4xx_ctx *ctx; - struct ahash_request *ahash_req; - - ahash_req = ahash_request_cast(pd_uinfo->async_req); - ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(ahash_req)); - - crypto4xx_copy_digest_to_dst(ahash_req->result, pd_uinfo, ctx); - crypto4xx_ret_sg_desc(dev, pd_uinfo); - - if (pd_uinfo->state & PD_ENTRY_BUSY) - ahash_request_complete(ahash_req, -EINPROGRESS); - ahash_request_complete(ahash_req, 0); -} - static void crypto4xx_aead_done(struct crypto4xx_device *dev, struct pd_uinfo *pd_uinfo, struct ce_pd *pd) @@ -642,9 +613,6 @@ static void crypto4xx_pd_done(struct crypto4xx_device *dev, u32 idx) case CRYPTO_ALG_TYPE_AEAD: crypto4xx_aead_done(dev, pd_uinfo, pd); break; - case CRYPTO_ALG_TYPE_AHASH: - crypto4xx_ahash_done(dev, pd_uinfo); - break; } } @@ -912,8 +880,7 @@ int crypto4xx_build_pd(struct crypto_async_request *req, } pd->pd_ctl.w = PD_CTL_HOST_READY | - ((crypto_tfm_alg_type(req->tfm) == CRYPTO_ALG_TYPE_AHASH) || - (crypto_tfm_alg_type(req->tfm) == CRYPTO_ALG_TYPE_AEAD) ? + ((crypto_tfm_alg_type(req->tfm) == CRYPTO_ALG_TYPE_AEAD) ? PD_CTL_HASH_FINAL : 0); pd->pd_ctl_len.w = 0x00400000 | (assoclen + datalen); pd_uinfo->state = PD_ENTRY_INUSE | (is_busy ? PD_ENTRY_BUSY : 0); @@ -1019,10 +986,6 @@ static int crypto4xx_register_alg(struct crypto4xx_device *sec_dev, rc = crypto_register_aead(&alg->alg.u.aead); break; - case CRYPTO_ALG_TYPE_AHASH: - rc = crypto_register_ahash(&alg->alg.u.hash); - break; - case CRYPTO_ALG_TYPE_RNG: rc = crypto_register_rng(&alg->alg.u.rng); break; @@ -1048,10 +1011,6 @@ static void crypto4xx_unregister_alg(struct crypto4xx_device *sec_dev) list_for_each_entry_safe(alg, tmp, &sec_dev->alg_list, entry) { list_del(&alg->entry); switch (alg->alg.type) { - case CRYPTO_ALG_TYPE_AHASH: - crypto_unregister_ahash(&alg->alg.u.hash); - break; - case CRYPTO_ALG_TYPE_AEAD: crypto_unregister_aead(&alg->alg.u.aead); break; diff --git a/drivers/crypto/amcc/crypto4xx_core.h b/drivers/crypto/amcc/crypto4xx_core.h index 9c56c7ac6e4c..ee36630c670f 100644 --- a/drivers/crypto/amcc/crypto4xx_core.h +++ b/drivers/crypto/amcc/crypto4xx_core.h @@ -16,7 +16,6 @@ #include #include #include -#include #include #include #include @@ -135,7 +134,6 @@ struct crypto4xx_alg_common { u32 type; union { struct skcipher_alg cipher; - struct ahash_alg hash; struct aead_alg aead; struct rng_alg rng; } u; @@ -183,11 +181,6 @@ int crypto4xx_encrypt_noiv_block(struct skcipher_request *req); int crypto4xx_decrypt_noiv_block(struct skcipher_request *req); int crypto4xx_rfc3686_encrypt(struct skcipher_request *req); int crypto4xx_rfc3686_decrypt(struct skcipher_request *req); -int crypto4xx_sha1_alg_init(struct crypto_tfm *tfm); -int crypto4xx_hash_digest(struct ahash_request *req); -int crypto4xx_hash_final(struct ahash_request *req); -int crypto4xx_hash_update(struct ahash_request *req); -int crypto4xx_hash_init(struct ahash_request *req); /* * Note: Only use this function to copy items that is word aligned. From 98066f2f8901ccf72f3c5d6c391c8fff1cabd49d Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 5 May 2025 11:18:21 -0700 Subject: [PATCH 323/382] crypto: lib/chacha - strongly type the ChaCha state The ChaCha state matrix is 16 32-bit words. Currently it is represented in the code as a raw u32 array, or even just a pointer to u32. This weak typing is error-prone. Instead, introduce struct chacha_state: struct chacha_state { u32 x[16]; }; Convert all ChaCha and HChaCha functions to use struct chacha_state. No functional changes. Signed-off-by: Eric Biggers Acked-by: Kent Overstreet Signed-off-by: Herbert Xu --- arch/arm/lib/crypto/chacha-glue.c | 30 +++++---- arch/arm/lib/crypto/chacha-scalar-core.S | 5 +- arch/arm64/lib/crypto/chacha-neon-glue.c | 23 ++++--- arch/mips/lib/crypto/chacha-glue.c | 6 +- arch/powerpc/lib/crypto/chacha-p10-glue.c | 15 +++-- arch/powerpc/lib/crypto/chacha-p10le-8x.S | 6 +- arch/riscv/lib/crypto/chacha-riscv64-glue.c | 8 +-- arch/riscv/lib/crypto/chacha-riscv64-zvkb.S | 10 +-- arch/s390/lib/crypto/chacha-glue.c | 11 ++-- arch/x86/lib/crypto/chacha_glue.c | 58 ++++++++++------- crypto/chacha.c | 16 ++--- drivers/char/random.c | 41 ++++++------ fs/bcachefs/checksum.c | 18 ++--- include/crypto/chacha.h | 65 +++++++++++-------- lib/crypto/chacha.c | 35 +++++----- lib/crypto/chacha20poly1305-selftest.c | 8 +-- lib/crypto/chacha20poly1305.c | 51 ++++++++------- lib/crypto/libchacha.c | 2 +- .../crypto/chacha20-s390/test-cipher.c | 10 +-- 19 files changed, 228 insertions(+), 190 deletions(-) diff --git a/arch/arm/lib/crypto/chacha-glue.c b/arch/arm/lib/crypto/chacha-glue.c index 1e28736834a0..0c2b4c62d484 100644 --- a/arch/arm/lib/crypto/chacha-glue.c +++ b/arch/arm/lib/crypto/chacha-glue.c @@ -17,15 +17,18 @@ #include #include -asmlinkage void chacha_block_xor_neon(const u32 *state, u8 *dst, const u8 *src, - int nrounds); -asmlinkage void chacha_4block_xor_neon(const u32 *state, u8 *dst, const u8 *src, +asmlinkage void chacha_block_xor_neon(const struct chacha_state *state, + u8 *dst, const u8 *src, int nrounds); +asmlinkage void chacha_4block_xor_neon(const struct chacha_state *state, + u8 *dst, const u8 *src, int nrounds, unsigned int nbytes); -asmlinkage void hchacha_block_arm(const u32 *state, u32 *out, int nrounds); -asmlinkage void hchacha_block_neon(const u32 *state, u32 *out, int nrounds); +asmlinkage void hchacha_block_arm(const struct chacha_state *state, + u32 *out, int nrounds); +asmlinkage void hchacha_block_neon(const struct chacha_state *state, + u32 *out, int nrounds); asmlinkage void chacha_doarm(u8 *dst, const u8 *src, unsigned int bytes, - const u32 *state, int nrounds); + const struct chacha_state *state, int nrounds); static __ro_after_init DEFINE_STATIC_KEY_FALSE(use_neon); @@ -34,7 +37,7 @@ static inline bool neon_usable(void) return static_branch_likely(&use_neon) && crypto_simd_usable(); } -static void chacha_doneon(u32 *state, u8 *dst, const u8 *src, +static void chacha_doneon(struct chacha_state *state, u8 *dst, const u8 *src, unsigned int bytes, int nrounds) { u8 buf[CHACHA_BLOCK_SIZE]; @@ -46,7 +49,7 @@ static void chacha_doneon(u32 *state, u8 *dst, const u8 *src, bytes -= l; src += l; dst += l; - state[12] += DIV_ROUND_UP(l, CHACHA_BLOCK_SIZE); + state->x[12] += DIV_ROUND_UP(l, CHACHA_BLOCK_SIZE); } if (bytes) { const u8 *s = src; @@ -57,11 +60,12 @@ static void chacha_doneon(u32 *state, u8 *dst, const u8 *src, chacha_block_xor_neon(state, d, s, nrounds); if (d != dst) memcpy(dst, buf, bytes); - state[12]++; + state->x[12]++; } } -void hchacha_block_arch(const u32 *state, u32 *stream, int nrounds) +void hchacha_block_arch(const struct chacha_state *state, u32 *stream, + int nrounds) { if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) || !neon_usable()) { hchacha_block_arm(state, stream, nrounds); @@ -73,13 +77,13 @@ void hchacha_block_arch(const u32 *state, u32 *stream, int nrounds) } EXPORT_SYMBOL(hchacha_block_arch); -void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *src, unsigned int bytes, - int nrounds) +void chacha_crypt_arch(struct chacha_state *state, u8 *dst, const u8 *src, + unsigned int bytes, int nrounds) { if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) || !neon_usable() || bytes <= CHACHA_BLOCK_SIZE) { chacha_doarm(dst, src, bytes, state, nrounds); - state[12] += DIV_ROUND_UP(bytes, CHACHA_BLOCK_SIZE); + state->x[12] += DIV_ROUND_UP(bytes, CHACHA_BLOCK_SIZE); return; } diff --git a/arch/arm/lib/crypto/chacha-scalar-core.S b/arch/arm/lib/crypto/chacha-scalar-core.S index 083fe1ab96d0..d20b5de755cc 100644 --- a/arch/arm/lib/crypto/chacha-scalar-core.S +++ b/arch/arm/lib/crypto/chacha-scalar-core.S @@ -367,7 +367,7 @@ /* * void chacha_doarm(u8 *dst, const u8 *src, unsigned int bytes, - * const u32 *state, int nrounds); + * const struct chacha_state *state, int nrounds); */ ENTRY(chacha_doarm) cmp r2, #0 // len == 0? @@ -407,7 +407,8 @@ ENTRY(chacha_doarm) ENDPROC(chacha_doarm) /* - * void hchacha_block_arm(const u32 state[16], u32 out[8], int nrounds); + * void hchacha_block_arm(const struct chacha_state *state, + * u32 out[8], int nrounds); */ ENTRY(hchacha_block_arm) push {r1,r4-r11,lr} diff --git a/arch/arm64/lib/crypto/chacha-neon-glue.c b/arch/arm64/lib/crypto/chacha-neon-glue.c index 2b0de97a6daf..7b451b3c7240 100644 --- a/arch/arm64/lib/crypto/chacha-neon-glue.c +++ b/arch/arm64/lib/crypto/chacha-neon-glue.c @@ -28,15 +28,17 @@ #include #include -asmlinkage void chacha_block_xor_neon(u32 *state, u8 *dst, const u8 *src, - int nrounds); -asmlinkage void chacha_4block_xor_neon(u32 *state, u8 *dst, const u8 *src, +asmlinkage void chacha_block_xor_neon(const struct chacha_state *state, + u8 *dst, const u8 *src, int nrounds); +asmlinkage void chacha_4block_xor_neon(const struct chacha_state *state, + u8 *dst, const u8 *src, int nrounds, int bytes); -asmlinkage void hchacha_block_neon(const u32 *state, u32 *out, int nrounds); +asmlinkage void hchacha_block_neon(const struct chacha_state *state, + u32 *out, int nrounds); static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon); -static void chacha_doneon(u32 *state, u8 *dst, const u8 *src, +static void chacha_doneon(struct chacha_state *state, u8 *dst, const u8 *src, int bytes, int nrounds) { while (bytes > 0) { @@ -48,18 +50,19 @@ static void chacha_doneon(u32 *state, u8 *dst, const u8 *src, memcpy(buf, src, l); chacha_block_xor_neon(state, buf, buf, nrounds); memcpy(dst, buf, l); - state[12] += 1; + state->x[12] += 1; break; } chacha_4block_xor_neon(state, dst, src, nrounds, l); bytes -= l; src += l; dst += l; - state[12] += DIV_ROUND_UP(l, CHACHA_BLOCK_SIZE); + state->x[12] += DIV_ROUND_UP(l, CHACHA_BLOCK_SIZE); } } -void hchacha_block_arch(const u32 *state, u32 *stream, int nrounds) +void hchacha_block_arch(const struct chacha_state *state, u32 *stream, + int nrounds) { if (!static_branch_likely(&have_neon) || !crypto_simd_usable()) { hchacha_block_generic(state, stream, nrounds); @@ -71,8 +74,8 @@ void hchacha_block_arch(const u32 *state, u32 *stream, int nrounds) } EXPORT_SYMBOL(hchacha_block_arch); -void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *src, unsigned int bytes, - int nrounds) +void chacha_crypt_arch(struct chacha_state *state, u8 *dst, const u8 *src, + unsigned int bytes, int nrounds) { if (!static_branch_likely(&have_neon) || bytes <= CHACHA_BLOCK_SIZE || !crypto_simd_usable()) diff --git a/arch/mips/lib/crypto/chacha-glue.c b/arch/mips/lib/crypto/chacha-glue.c index 334ecb29fb8f..75df4040cded 100644 --- a/arch/mips/lib/crypto/chacha-glue.c +++ b/arch/mips/lib/crypto/chacha-glue.c @@ -9,11 +9,13 @@ #include #include -asmlinkage void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *src, +asmlinkage void chacha_crypt_arch(struct chacha_state *state, + u8 *dst, const u8 *src, unsigned int bytes, int nrounds); EXPORT_SYMBOL(chacha_crypt_arch); -asmlinkage void hchacha_block_arch(const u32 *state, u32 *stream, int nrounds); +asmlinkage void hchacha_block_arch(const struct chacha_state *state, + u32 *stream, int nrounds); EXPORT_SYMBOL(hchacha_block_arch); bool chacha_is_arch_optimized(void) diff --git a/arch/powerpc/lib/crypto/chacha-p10-glue.c b/arch/powerpc/lib/crypto/chacha-p10-glue.c index 51daeaf5d26e..a6e6a8da1b8b 100644 --- a/arch/powerpc/lib/crypto/chacha-p10-glue.c +++ b/arch/powerpc/lib/crypto/chacha-p10-glue.c @@ -14,8 +14,8 @@ #include #include -asmlinkage void chacha_p10le_8x(u32 *state, u8 *dst, const u8 *src, - unsigned int len, int nrounds); +asmlinkage void chacha_p10le_8x(const struct chacha_state *state, u8 *dst, + const u8 *src, unsigned int len, int nrounds); static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_p10); @@ -31,7 +31,7 @@ static void vsx_end(void) preempt_enable(); } -static void chacha_p10_do_8x(u32 *state, u8 *dst, const u8 *src, +static void chacha_p10_do_8x(struct chacha_state *state, u8 *dst, const u8 *src, unsigned int bytes, int nrounds) { unsigned int l = bytes & ~0x0FF; @@ -41,21 +41,22 @@ static void chacha_p10_do_8x(u32 *state, u8 *dst, const u8 *src, bytes -= l; src += l; dst += l; - state[12] += l / CHACHA_BLOCK_SIZE; + state->x[12] += l / CHACHA_BLOCK_SIZE; } if (bytes > 0) chacha_crypt_generic(state, dst, src, bytes, nrounds); } -void hchacha_block_arch(const u32 *state, u32 *stream, int nrounds) +void hchacha_block_arch(const struct chacha_state *state, + u32 *stream, int nrounds) { hchacha_block_generic(state, stream, nrounds); } EXPORT_SYMBOL(hchacha_block_arch); -void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *src, unsigned int bytes, - int nrounds) +void chacha_crypt_arch(struct chacha_state *state, u8 *dst, const u8 *src, + unsigned int bytes, int nrounds) { if (!static_branch_likely(&have_p10) || bytes <= CHACHA_BLOCK_SIZE || !crypto_simd_usable()) diff --git a/arch/powerpc/lib/crypto/chacha-p10le-8x.S b/arch/powerpc/lib/crypto/chacha-p10le-8x.S index 17bedb66b822..b29562bd5d40 100644 --- a/arch/powerpc/lib/crypto/chacha-p10le-8x.S +++ b/arch/powerpc/lib/crypto/chacha-p10le-8x.S @@ -7,9 +7,6 @@ #=================================================================================== # Written by Danny Tsen # -# chacha_p10le_8x(u32 *state, byte *dst, const byte *src, -# size_t len, int nrounds); -# # do rounds, 8 quarter rounds # 1. a += b; d ^= a; d <<<= 16; # 2. c += d; b ^= c; b <<<= 12; @@ -575,7 +572,8 @@ .endm # -# chacha20_p10le_8x(u32 *state, byte *dst, const byte *src, size_t len, int nrounds); +# void chacha_p10le_8x(const struct chacha_state *state, u8 *dst, const u8 *src, +# unsigned int len, int nrounds); # SYM_FUNC_START(chacha_p10le_8x) .align 5 diff --git a/arch/riscv/lib/crypto/chacha-riscv64-glue.c b/arch/riscv/lib/crypto/chacha-riscv64-glue.c index 1740e1ca3a94..57541621981e 100644 --- a/arch/riscv/lib/crypto/chacha-riscv64-glue.c +++ b/arch/riscv/lib/crypto/chacha-riscv64-glue.c @@ -15,17 +15,17 @@ static __ro_after_init DEFINE_STATIC_KEY_FALSE(use_zvkb); -asmlinkage void chacha_zvkb(u32 state[16], const u8 *in, u8 *out, +asmlinkage void chacha_zvkb(struct chacha_state *state, const u8 *in, u8 *out, size_t nblocks, int nrounds); -void hchacha_block_arch(const u32 *state, u32 *out, int nrounds) +void hchacha_block_arch(const struct chacha_state *state, u32 *out, int nrounds) { hchacha_block_generic(state, out, nrounds); } EXPORT_SYMBOL(hchacha_block_arch); -void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *src, unsigned int bytes, - int nrounds) +void chacha_crypt_arch(struct chacha_state *state, u8 *dst, const u8 *src, + unsigned int bytes, int nrounds) { u8 block_buffer[CHACHA_BLOCK_SIZE]; unsigned int full_blocks = bytes / CHACHA_BLOCK_SIZE; diff --git a/arch/riscv/lib/crypto/chacha-riscv64-zvkb.S b/arch/riscv/lib/crypto/chacha-riscv64-zvkb.S index ab4423b3880e..b777d0b4e379 100644 --- a/arch/riscv/lib/crypto/chacha-riscv64-zvkb.S +++ b/arch/riscv/lib/crypto/chacha-riscv64-zvkb.S @@ -132,15 +132,15 @@ vror.vi \b3, \b3, 32 - 7 .endm -// void chacha_zvkb(u32 state[16], const u8 *in, u8 *out, size_t nblocks, -// int nrounds); +// void chacha_zvkb(struct chacha_state *state, const u8 *in, u8 *out, +// size_t nblocks, int nrounds); // // |nblocks| is the number of 64-byte blocks to process, and must be nonzero. // // |state| gives the ChaCha state matrix, including the 32-bit counter in -// state[12] following the RFC7539 convention; note that this differs from the -// original Salsa20 paper which uses a 64-bit counter in state[12..13]. The -// updated 32-bit counter is written back to state[12] before returning. +// state->x[12] following the RFC7539 convention; note that this differs from +// the original Salsa20 paper which uses a 64-bit counter in state->x[12..13]. +// The updated 32-bit counter is written back to state->x[12] before returning. SYM_FUNC_START(chacha_zvkb) addi sp, sp, -96 sd s0, 0(sp) diff --git a/arch/s390/lib/crypto/chacha-glue.c b/arch/s390/lib/crypto/chacha-glue.c index b3ffaa555385..0a9fd50c1bd8 100644 --- a/arch/s390/lib/crypto/chacha-glue.c +++ b/arch/s390/lib/crypto/chacha-glue.c @@ -16,14 +16,15 @@ #include #include "chacha-s390.h" -void hchacha_block_arch(const u32 *state, u32 *stream, int nrounds) +void hchacha_block_arch(const struct chacha_state *state, + u32 *stream, int nrounds) { /* TODO: implement hchacha_block_arch() in assembly */ hchacha_block_generic(state, stream, nrounds); } EXPORT_SYMBOL(hchacha_block_arch); -void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *src, +void chacha_crypt_arch(struct chacha_state *state, u8 *dst, const u8 *src, unsigned int bytes, int nrounds) { /* s390 chacha20 implementation has 20 rounds hard-coded, @@ -36,11 +37,11 @@ void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *src, DECLARE_KERNEL_FPU_ONSTACK32(vxstate); kernel_fpu_begin(&vxstate, KERNEL_VXR); - chacha20_vx(dst, src, bytes, &state[4], &state[12]); + chacha20_vx(dst, src, bytes, &state->x[4], &state->x[12]); kernel_fpu_end(&vxstate, KERNEL_VXR); - state[12] += round_up(bytes, CHACHA_BLOCK_SIZE) / - CHACHA_BLOCK_SIZE; + state->x[12] += round_up(bytes, CHACHA_BLOCK_SIZE) / + CHACHA_BLOCK_SIZE; } } EXPORT_SYMBOL(chacha_crypt_arch); diff --git a/arch/x86/lib/crypto/chacha_glue.c b/arch/x86/lib/crypto/chacha_glue.c index 94fcefbc8827..6f00a56e3e9a 100644 --- a/arch/x86/lib/crypto/chacha_glue.c +++ b/arch/x86/lib/crypto/chacha_glue.c @@ -12,24 +12,33 @@ #include #include -asmlinkage void chacha_block_xor_ssse3(u32 *state, u8 *dst, const u8 *src, +asmlinkage void chacha_block_xor_ssse3(const struct chacha_state *state, + u8 *dst, const u8 *src, unsigned int len, int nrounds); -asmlinkage void chacha_4block_xor_ssse3(u32 *state, u8 *dst, const u8 *src, +asmlinkage void chacha_4block_xor_ssse3(const struct chacha_state *state, + u8 *dst, const u8 *src, unsigned int len, int nrounds); -asmlinkage void hchacha_block_ssse3(const u32 *state, u32 *out, int nrounds); +asmlinkage void hchacha_block_ssse3(const struct chacha_state *state, + u32 *out, int nrounds); -asmlinkage void chacha_2block_xor_avx2(u32 *state, u8 *dst, const u8 *src, +asmlinkage void chacha_2block_xor_avx2(const struct chacha_state *state, + u8 *dst, const u8 *src, unsigned int len, int nrounds); -asmlinkage void chacha_4block_xor_avx2(u32 *state, u8 *dst, const u8 *src, +asmlinkage void chacha_4block_xor_avx2(const struct chacha_state *state, + u8 *dst, const u8 *src, unsigned int len, int nrounds); -asmlinkage void chacha_8block_xor_avx2(u32 *state, u8 *dst, const u8 *src, +asmlinkage void chacha_8block_xor_avx2(const struct chacha_state *state, + u8 *dst, const u8 *src, unsigned int len, int nrounds); -asmlinkage void chacha_2block_xor_avx512vl(u32 *state, u8 *dst, const u8 *src, +asmlinkage void chacha_2block_xor_avx512vl(const struct chacha_state *state, + u8 *dst, const u8 *src, unsigned int len, int nrounds); -asmlinkage void chacha_4block_xor_avx512vl(u32 *state, u8 *dst, const u8 *src, +asmlinkage void chacha_4block_xor_avx512vl(const struct chacha_state *state, + u8 *dst, const u8 *src, unsigned int len, int nrounds); -asmlinkage void chacha_8block_xor_avx512vl(u32 *state, u8 *dst, const u8 *src, +asmlinkage void chacha_8block_xor_avx512vl(const struct chacha_state *state, + u8 *dst, const u8 *src, unsigned int len, int nrounds); static __ro_after_init DEFINE_STATIC_KEY_FALSE(chacha_use_simd); @@ -42,7 +51,7 @@ static unsigned int chacha_advance(unsigned int len, unsigned int maxblocks) return round_up(len, CHACHA_BLOCK_SIZE) / CHACHA_BLOCK_SIZE; } -static void chacha_dosimd(u32 *state, u8 *dst, const u8 *src, +static void chacha_dosimd(struct chacha_state *state, u8 *dst, const u8 *src, unsigned int bytes, int nrounds) { if (static_branch_likely(&chacha_use_avx512vl)) { @@ -52,24 +61,24 @@ static void chacha_dosimd(u32 *state, u8 *dst, const u8 *src, bytes -= CHACHA_BLOCK_SIZE * 8; src += CHACHA_BLOCK_SIZE * 8; dst += CHACHA_BLOCK_SIZE * 8; - state[12] += 8; + state->x[12] += 8; } if (bytes > CHACHA_BLOCK_SIZE * 4) { chacha_8block_xor_avx512vl(state, dst, src, bytes, nrounds); - state[12] += chacha_advance(bytes, 8); + state->x[12] += chacha_advance(bytes, 8); return; } if (bytes > CHACHA_BLOCK_SIZE * 2) { chacha_4block_xor_avx512vl(state, dst, src, bytes, nrounds); - state[12] += chacha_advance(bytes, 4); + state->x[12] += chacha_advance(bytes, 4); return; } if (bytes) { chacha_2block_xor_avx512vl(state, dst, src, bytes, nrounds); - state[12] += chacha_advance(bytes, 2); + state->x[12] += chacha_advance(bytes, 2); return; } } @@ -80,21 +89,21 @@ static void chacha_dosimd(u32 *state, u8 *dst, const u8 *src, bytes -= CHACHA_BLOCK_SIZE * 8; src += CHACHA_BLOCK_SIZE * 8; dst += CHACHA_BLOCK_SIZE * 8; - state[12] += 8; + state->x[12] += 8; } if (bytes > CHACHA_BLOCK_SIZE * 4) { chacha_8block_xor_avx2(state, dst, src, bytes, nrounds); - state[12] += chacha_advance(bytes, 8); + state->x[12] += chacha_advance(bytes, 8); return; } if (bytes > CHACHA_BLOCK_SIZE * 2) { chacha_4block_xor_avx2(state, dst, src, bytes, nrounds); - state[12] += chacha_advance(bytes, 4); + state->x[12] += chacha_advance(bytes, 4); return; } if (bytes > CHACHA_BLOCK_SIZE) { chacha_2block_xor_avx2(state, dst, src, bytes, nrounds); - state[12] += chacha_advance(bytes, 2); + state->x[12] += chacha_advance(bytes, 2); return; } } @@ -104,20 +113,21 @@ static void chacha_dosimd(u32 *state, u8 *dst, const u8 *src, bytes -= CHACHA_BLOCK_SIZE * 4; src += CHACHA_BLOCK_SIZE * 4; dst += CHACHA_BLOCK_SIZE * 4; - state[12] += 4; + state->x[12] += 4; } if (bytes > CHACHA_BLOCK_SIZE) { chacha_4block_xor_ssse3(state, dst, src, bytes, nrounds); - state[12] += chacha_advance(bytes, 4); + state->x[12] += chacha_advance(bytes, 4); return; } if (bytes) { chacha_block_xor_ssse3(state, dst, src, bytes, nrounds); - state[12]++; + state->x[12]++; } } -void hchacha_block_arch(const u32 *state, u32 *stream, int nrounds) +void hchacha_block_arch(const struct chacha_state *state, + u32 *stream, int nrounds) { if (!static_branch_likely(&chacha_use_simd)) { hchacha_block_generic(state, stream, nrounds); @@ -129,8 +139,8 @@ void hchacha_block_arch(const u32 *state, u32 *stream, int nrounds) } EXPORT_SYMBOL(hchacha_block_arch); -void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *src, unsigned int bytes, - int nrounds) +void chacha_crypt_arch(struct chacha_state *state, u8 *dst, const u8 *src, + unsigned int bytes, int nrounds) { if (!static_branch_likely(&chacha_use_simd) || bytes <= CHACHA_BLOCK_SIZE) diff --git a/crypto/chacha.c b/crypto/chacha.c index 28a8ad6197ab..73ce62a9ac22 100644 --- a/crypto/chacha.c +++ b/crypto/chacha.c @@ -50,12 +50,12 @@ static int chacha_stream_xor(struct skcipher_request *req, bool arch) { struct skcipher_walk walk; - u32 state[16]; + struct chacha_state state; int err; err = skcipher_walk_virt(&walk, req, false); - chacha_init(state, ctx->key, iv); + chacha_init(&state, ctx->key, iv); while (walk.nbytes > 0) { unsigned int nbytes = walk.nbytes; @@ -64,10 +64,10 @@ static int chacha_stream_xor(struct skcipher_request *req, nbytes = round_down(nbytes, CHACHA_BLOCK_SIZE); if (arch) - chacha_crypt(state, walk.dst.virt.addr, + chacha_crypt(&state, walk.dst.virt.addr, walk.src.virt.addr, nbytes, ctx->nrounds); else - chacha_crypt_generic(state, walk.dst.virt.addr, + chacha_crypt_generic(&state, walk.dst.virt.addr, walk.src.virt.addr, nbytes, ctx->nrounds); err = skcipher_walk_done(&walk, walk.nbytes - nbytes); @@ -97,15 +97,15 @@ static int crypto_xchacha_crypt(struct skcipher_request *req, bool arch) struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); const struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm); struct chacha_ctx subctx; - u32 state[16]; + struct chacha_state state; u8 real_iv[16]; /* Compute the subkey given the original key and first 128 nonce bits */ - chacha_init(state, ctx->key, req->iv); + chacha_init(&state, ctx->key, req->iv); if (arch) - hchacha_block(state, subctx.key, ctx->nrounds); + hchacha_block(&state, subctx.key, ctx->nrounds); else - hchacha_block_generic(state, subctx.key, ctx->nrounds); + hchacha_block_generic(&state, subctx.key, ctx->nrounds); subctx.nrounds = ctx->nrounds; /* Build the real IV */ diff --git a/drivers/char/random.c b/drivers/char/random.c index 38f2fab29c56..9f876ed2655b 100644 --- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -309,11 +309,11 @@ static void crng_reseed(struct work_struct *work) * key value, at index 4, so the state should always be zeroed out * immediately after using in order to maintain forward secrecy. * If the state cannot be erased in a timely manner, then it is - * safer to set the random_data parameter to &chacha_state[4] so - * that this function overwrites it before returning. + * safer to set the random_data parameter to &chacha_state->x[4] + * so that this function overwrites it before returning. */ static void crng_fast_key_erasure(u8 key[CHACHA_KEY_SIZE], - u32 chacha_state[CHACHA_STATE_WORDS], + struct chacha_state *chacha_state, u8 *random_data, size_t random_data_len) { u8 first_block[CHACHA_BLOCK_SIZE]; @@ -321,8 +321,8 @@ static void crng_fast_key_erasure(u8 key[CHACHA_KEY_SIZE], BUG_ON(random_data_len > 32); chacha_init_consts(chacha_state); - memcpy(&chacha_state[4], key, CHACHA_KEY_SIZE); - memset(&chacha_state[12], 0, sizeof(u32) * 4); + memcpy(&chacha_state->x[4], key, CHACHA_KEY_SIZE); + memset(&chacha_state->x[12], 0, sizeof(u32) * 4); chacha20_block(chacha_state, first_block); memcpy(key, first_block, CHACHA_KEY_SIZE); @@ -335,7 +335,7 @@ static void crng_fast_key_erasure(u8 key[CHACHA_KEY_SIZE], * random data. It also returns up to 32 bytes on its own of random data * that may be used; random_data_len may not be greater than 32. */ -static void crng_make_state(u32 chacha_state[CHACHA_STATE_WORDS], +static void crng_make_state(struct chacha_state *chacha_state, u8 *random_data, size_t random_data_len) { unsigned long flags; @@ -395,7 +395,7 @@ static void crng_make_state(u32 chacha_state[CHACHA_STATE_WORDS], static void _get_random_bytes(void *buf, size_t len) { - u32 chacha_state[CHACHA_STATE_WORDS]; + struct chacha_state chacha_state; u8 tmp[CHACHA_BLOCK_SIZE]; size_t first_block_len; @@ -403,26 +403,26 @@ static void _get_random_bytes(void *buf, size_t len) return; first_block_len = min_t(size_t, 32, len); - crng_make_state(chacha_state, buf, first_block_len); + crng_make_state(&chacha_state, buf, first_block_len); len -= first_block_len; buf += first_block_len; while (len) { if (len < CHACHA_BLOCK_SIZE) { - chacha20_block(chacha_state, tmp); + chacha20_block(&chacha_state, tmp); memcpy(buf, tmp, len); memzero_explicit(tmp, sizeof(tmp)); break; } - chacha20_block(chacha_state, buf); - if (unlikely(chacha_state[12] == 0)) - ++chacha_state[13]; + chacha20_block(&chacha_state, buf); + if (unlikely(chacha_state.x[12] == 0)) + ++chacha_state.x[13]; len -= CHACHA_BLOCK_SIZE; buf += CHACHA_BLOCK_SIZE; } - memzero_explicit(chacha_state, sizeof(chacha_state)); + memzero_explicit(&chacha_state, sizeof(chacha_state)); } /* @@ -441,7 +441,7 @@ EXPORT_SYMBOL(get_random_bytes); static ssize_t get_random_bytes_user(struct iov_iter *iter) { - u32 chacha_state[CHACHA_STATE_WORDS]; + struct chacha_state chacha_state; u8 block[CHACHA_BLOCK_SIZE]; size_t ret = 0, copied; @@ -453,21 +453,22 @@ static ssize_t get_random_bytes_user(struct iov_iter *iter) * bytes, in case userspace causes copy_to_iter() below to sleep * forever, so that we still retain forward secrecy in that case. */ - crng_make_state(chacha_state, (u8 *)&chacha_state[4], CHACHA_KEY_SIZE); + crng_make_state(&chacha_state, (u8 *)&chacha_state.x[4], + CHACHA_KEY_SIZE); /* * However, if we're doing a read of len <= 32, we don't need to * use chacha_state after, so we can simply return those bytes to * the user directly. */ if (iov_iter_count(iter) <= CHACHA_KEY_SIZE) { - ret = copy_to_iter(&chacha_state[4], CHACHA_KEY_SIZE, iter); + ret = copy_to_iter(&chacha_state.x[4], CHACHA_KEY_SIZE, iter); goto out_zero_chacha; } for (;;) { - chacha20_block(chacha_state, block); - if (unlikely(chacha_state[12] == 0)) - ++chacha_state[13]; + chacha20_block(&chacha_state, block); + if (unlikely(chacha_state.x[12] == 0)) + ++chacha_state.x[13]; copied = copy_to_iter(block, sizeof(block), iter); ret += copied; @@ -484,7 +485,7 @@ static ssize_t get_random_bytes_user(struct iov_iter *iter) memzero_explicit(block, sizeof(block)); out_zero_chacha: - memzero_explicit(chacha_state, sizeof(chacha_state)); + memzero_explicit(&chacha_state, sizeof(chacha_state)); return ret ? ret : -EFAULT; } diff --git a/fs/bcachefs/checksum.c b/fs/bcachefs/checksum.c index d0a34a097b80..312fda4bb1b5 100644 --- a/fs/bcachefs/checksum.c +++ b/fs/bcachefs/checksum.c @@ -91,7 +91,7 @@ static void bch2_checksum_update(struct bch2_checksum_state *state, const void * } } -static void bch2_chacha20_init(u32 state[CHACHA_STATE_WORDS], +static void bch2_chacha20_init(struct chacha_state *state, const struct bch_key *key, struct nonce nonce) { u32 key_words[CHACHA_KEY_SIZE / sizeof(u32)]; @@ -109,11 +109,11 @@ static void bch2_chacha20_init(u32 state[CHACHA_STATE_WORDS], static void bch2_chacha20(const struct bch_key *key, struct nonce nonce, void *data, size_t len) { - u32 state[CHACHA_STATE_WORDS]; + struct chacha_state state; - bch2_chacha20_init(state, key, nonce); - chacha20_crypt(state, data, data, len); - memzero_explicit(state, sizeof(state)); + bch2_chacha20_init(&state, key, nonce); + chacha20_crypt(&state, data, data, len); + memzero_explicit(&state, sizeof(state)); } static void bch2_poly1305_init(struct poly1305_desc_ctx *desc, @@ -257,14 +257,14 @@ int __bch2_encrypt_bio(struct bch_fs *c, unsigned type, { struct bio_vec bv; struct bvec_iter iter; - u32 chacha_state[CHACHA_STATE_WORDS]; + struct chacha_state chacha_state; int ret = 0; if (bch2_fs_inconsistent_on(!c->chacha20_key_set, c, "attempting to encrypt without encryption key")) return -BCH_ERR_no_encryption_key; - bch2_chacha20_init(chacha_state, &c->chacha20_key, nonce); + bch2_chacha20_init(&chacha_state, &c->chacha20_key, nonce); bio_for_each_segment(bv, bio, iter) { void *p; @@ -280,10 +280,10 @@ int __bch2_encrypt_bio(struct bch_fs *c, unsigned type, } p = bvec_kmap_local(&bv); - chacha20_crypt(chacha_state, p, p, bv.bv_len); + chacha20_crypt(&chacha_state, p, p, bv.bv_len); kunmap_local(p); } - memzero_explicit(chacha_state, sizeof(chacha_state)); + memzero_explicit(&chacha_state, sizeof(chacha_state)); return ret; } diff --git a/include/crypto/chacha.h b/include/crypto/chacha.h index 58129e18cc31..64fb270f2bfc 100644 --- a/include/crypto/chacha.h +++ b/include/crypto/chacha.h @@ -30,16 +30,23 @@ /* 192-bit nonce, then 64-bit stream position */ #define XCHACHA_IV_SIZE 32 -void chacha_block_generic(u32 *state, u8 *stream, int nrounds); -static inline void chacha20_block(u32 *state, u8 *stream) +struct chacha_state { + u32 x[CHACHA_STATE_WORDS]; +}; + +void chacha_block_generic(struct chacha_state *state, u8 *stream, int nrounds); +static inline void chacha20_block(struct chacha_state *state, u8 *stream) { chacha_block_generic(state, stream, 20); } -void hchacha_block_arch(const u32 *state, u32 *out, int nrounds); -void hchacha_block_generic(const u32 *state, u32 *out, int nrounds); +void hchacha_block_arch(const struct chacha_state *state, u32 *out, + int nrounds); +void hchacha_block_generic(const struct chacha_state *state, u32 *out, + int nrounds); -static inline void hchacha_block(const u32 *state, u32 *out, int nrounds) +static inline void hchacha_block(const struct chacha_state *state, u32 *out, + int nrounds) { if (IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_CHACHA)) hchacha_block_arch(state, out, nrounds); @@ -54,37 +61,39 @@ enum chacha_constants { /* expand 32-byte k */ CHACHA_CONSTANT_TE_K = 0x6b206574U }; -static inline void chacha_init_consts(u32 *state) +static inline void chacha_init_consts(struct chacha_state *state) { - state[0] = CHACHA_CONSTANT_EXPA; - state[1] = CHACHA_CONSTANT_ND_3; - state[2] = CHACHA_CONSTANT_2_BY; - state[3] = CHACHA_CONSTANT_TE_K; + state->x[0] = CHACHA_CONSTANT_EXPA; + state->x[1] = CHACHA_CONSTANT_ND_3; + state->x[2] = CHACHA_CONSTANT_2_BY; + state->x[3] = CHACHA_CONSTANT_TE_K; } -static inline void chacha_init(u32 *state, const u32 *key, const u8 *iv) +static inline void chacha_init(struct chacha_state *state, + const u32 *key, const u8 *iv) { chacha_init_consts(state); - state[4] = key[0]; - state[5] = key[1]; - state[6] = key[2]; - state[7] = key[3]; - state[8] = key[4]; - state[9] = key[5]; - state[10] = key[6]; - state[11] = key[7]; - state[12] = get_unaligned_le32(iv + 0); - state[13] = get_unaligned_le32(iv + 4); - state[14] = get_unaligned_le32(iv + 8); - state[15] = get_unaligned_le32(iv + 12); + state->x[4] = key[0]; + state->x[5] = key[1]; + state->x[6] = key[2]; + state->x[7] = key[3]; + state->x[8] = key[4]; + state->x[9] = key[5]; + state->x[10] = key[6]; + state->x[11] = key[7]; + state->x[12] = get_unaligned_le32(iv + 0); + state->x[13] = get_unaligned_le32(iv + 4); + state->x[14] = get_unaligned_le32(iv + 8); + state->x[15] = get_unaligned_le32(iv + 12); } -void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *src, +void chacha_crypt_arch(struct chacha_state *state, u8 *dst, const u8 *src, unsigned int bytes, int nrounds); -void chacha_crypt_generic(u32 *state, u8 *dst, const u8 *src, +void chacha_crypt_generic(struct chacha_state *state, u8 *dst, const u8 *src, unsigned int bytes, int nrounds); -static inline void chacha_crypt(u32 *state, u8 *dst, const u8 *src, +static inline void chacha_crypt(struct chacha_state *state, + u8 *dst, const u8 *src, unsigned int bytes, int nrounds) { if (IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_CHACHA)) @@ -93,8 +102,8 @@ static inline void chacha_crypt(u32 *state, u8 *dst, const u8 *src, chacha_crypt_generic(state, dst, src, bytes, nrounds); } -static inline void chacha20_crypt(u32 *state, u8 *dst, const u8 *src, - unsigned int bytes) +static inline void chacha20_crypt(struct chacha_state *state, + u8 *dst, const u8 *src, unsigned int bytes) { chacha_crypt(state, dst, src, bytes, 20); } diff --git a/lib/crypto/chacha.c b/lib/crypto/chacha.c index 3cdda3b5ee06..a7f5eb091839 100644 --- a/lib/crypto/chacha.c +++ b/lib/crypto/chacha.c @@ -13,8 +13,9 @@ #include #include -static void chacha_permute(u32 *x, int nrounds) +static void chacha_permute(struct chacha_state *state, int nrounds) { + u32 *x = state->x; int i; /* whitelist the allowed round counts */ @@ -65,7 +66,7 @@ static void chacha_permute(u32 *x, int nrounds) /** * chacha_block_generic - generate one keystream block and increment block counter - * @state: input state matrix (16 32-bit words) + * @state: input state matrix * @stream: output keystream block (64 bytes) * @nrounds: number of rounds (20 or 12; 20 is recommended) * @@ -73,25 +74,26 @@ static void chacha_permute(u32 *x, int nrounds) * The caller has already converted the endianness of the input. This function * also handles incrementing the block counter in the input matrix. */ -void chacha_block_generic(u32 *state, u8 *stream, int nrounds) +void chacha_block_generic(struct chacha_state *state, u8 *stream, int nrounds) { - u32 x[16]; + struct chacha_state permuted_state; int i; - memcpy(x, state, 64); + memcpy(permuted_state.x, state->x, 64); - chacha_permute(x, nrounds); + chacha_permute(&permuted_state, nrounds); - for (i = 0; i < ARRAY_SIZE(x); i++) - put_unaligned_le32(x[i] + state[i], &stream[i * sizeof(u32)]); + for (i = 0; i < ARRAY_SIZE(state->x); i++) + put_unaligned_le32(permuted_state.x[i] + state->x[i], + &stream[i * sizeof(u32)]); - state[12]++; + state->x[12]++; } EXPORT_SYMBOL(chacha_block_generic); /** * hchacha_block_generic - abbreviated ChaCha core, for XChaCha - * @state: input state matrix (16 32-bit words) + * @state: input state matrix * @stream: output (8 32-bit words) * @nrounds: number of rounds (20 or 12; 20 is recommended) * @@ -100,15 +102,16 @@ EXPORT_SYMBOL(chacha_block_generic); * skips the final addition of the initial state, and outputs only certain words * of the state. It should not be used for streaming directly. */ -void hchacha_block_generic(const u32 *state, u32 *stream, int nrounds) +void hchacha_block_generic(const struct chacha_state *state, + u32 *stream, int nrounds) { - u32 x[16]; + struct chacha_state permuted_state; - memcpy(x, state, 64); + memcpy(permuted_state.x, state->x, 64); - chacha_permute(x, nrounds); + chacha_permute(&permuted_state, nrounds); - memcpy(&stream[0], &x[0], 16); - memcpy(&stream[4], &x[12], 16); + memcpy(&stream[0], &permuted_state.x[0], 16); + memcpy(&stream[4], &permuted_state.x[12], 16); } EXPORT_SYMBOL(hchacha_block_generic); diff --git a/lib/crypto/chacha20poly1305-selftest.c b/lib/crypto/chacha20poly1305-selftest.c index 2ea61c28be4f..e4c85bc5a6d7 100644 --- a/lib/crypto/chacha20poly1305-selftest.c +++ b/lib/crypto/chacha20poly1305-selftest.c @@ -8832,7 +8832,7 @@ chacha20poly1305_encrypt_bignonce(u8 *dst, const u8 *src, const size_t src_len, { const u8 *pad0 = page_address(ZERO_PAGE(0)); struct poly1305_desc_ctx poly1305_state; - u32 chacha20_state[CHACHA_STATE_WORDS]; + struct chacha_state chacha20_state; union { u8 block0[POLY1305_KEY_SIZE]; __le64 lens[2]; @@ -8844,12 +8844,12 @@ chacha20poly1305_encrypt_bignonce(u8 *dst, const u8 *src, const size_t src_len, memcpy(&bottom_row[4], nonce, 12); for (i = 0; i < 8; ++i) le_key[i] = get_unaligned_le32(key + sizeof(le_key[i]) * i); - chacha_init(chacha20_state, le_key, bottom_row); - chacha20_crypt(chacha20_state, b.block0, b.block0, sizeof(b.block0)); + chacha_init(&chacha20_state, le_key, bottom_row); + chacha20_crypt(&chacha20_state, b.block0, b.block0, sizeof(b.block0)); poly1305_init(&poly1305_state, b.block0); poly1305_update(&poly1305_state, ad, ad_len); poly1305_update(&poly1305_state, pad0, (0x10 - ad_len) & 0xf); - chacha20_crypt(chacha20_state, dst, src, src_len); + chacha20_crypt(&chacha20_state, dst, src, src_len); poly1305_update(&poly1305_state, dst, src_len); poly1305_update(&poly1305_state, pad0, (0x10 - src_len) & 0xf); b.lens[0] = cpu_to_le64(ad_len); diff --git a/lib/crypto/chacha20poly1305.c b/lib/crypto/chacha20poly1305.c index 9cfa886f1f89..ed81f0658956 100644 --- a/lib/crypto/chacha20poly1305.c +++ b/lib/crypto/chacha20poly1305.c @@ -32,7 +32,8 @@ static void chacha_load_key(u32 *k, const u8 *in) k[7] = get_unaligned_le32(in + 28); } -static void xchacha_init(u32 *chacha_state, const u8 *key, const u8 *nonce) +static void xchacha_init(struct chacha_state *chacha_state, + const u8 *key, const u8 *nonce) { u32 k[CHACHA_KEY_WORDS]; u8 iv[CHACHA_IV_SIZE]; @@ -54,7 +55,8 @@ static void xchacha_init(u32 *chacha_state, const u8 *key, const u8 *nonce) static void __chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len, - const u8 *ad, const size_t ad_len, u32 *chacha_state) + const u8 *ad, const size_t ad_len, + struct chacha_state *chacha_state) { const u8 *pad0 = page_address(ZERO_PAGE(0)); struct poly1305_desc_ctx poly1305_state; @@ -82,7 +84,7 @@ __chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len, poly1305_final(&poly1305_state, dst + src_len); - memzero_explicit(chacha_state, CHACHA_STATE_WORDS * sizeof(u32)); + memzero_explicit(chacha_state, sizeof(*chacha_state)); memzero_explicit(&b, sizeof(b)); } @@ -91,7 +93,7 @@ void chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len, const u64 nonce, const u8 key[CHACHA20POLY1305_KEY_SIZE]) { - u32 chacha_state[CHACHA_STATE_WORDS]; + struct chacha_state chacha_state; u32 k[CHACHA_KEY_WORDS]; __le64 iv[2]; @@ -100,8 +102,9 @@ void chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len, iv[0] = 0; iv[1] = cpu_to_le64(nonce); - chacha_init(chacha_state, k, (u8 *)iv); - __chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, chacha_state); + chacha_init(&chacha_state, k, (u8 *)iv); + __chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, + &chacha_state); memzero_explicit(iv, sizeof(iv)); memzero_explicit(k, sizeof(k)); @@ -113,16 +116,18 @@ void xchacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len, const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE], const u8 key[CHACHA20POLY1305_KEY_SIZE]) { - u32 chacha_state[CHACHA_STATE_WORDS]; + struct chacha_state chacha_state; - xchacha_init(chacha_state, key, nonce); - __chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, chacha_state); + xchacha_init(&chacha_state, key, nonce); + __chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, + &chacha_state); } EXPORT_SYMBOL(xchacha20poly1305_encrypt); static bool __chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len, - const u8 *ad, const size_t ad_len, u32 *chacha_state) + const u8 *ad, const size_t ad_len, + struct chacha_state *chacha_state) { const u8 *pad0 = page_address(ZERO_PAGE(0)); struct poly1305_desc_ctx poly1305_state; @@ -169,7 +174,7 @@ bool chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len, const u64 nonce, const u8 key[CHACHA20POLY1305_KEY_SIZE]) { - u32 chacha_state[CHACHA_STATE_WORDS]; + struct chacha_state chacha_state; u32 k[CHACHA_KEY_WORDS]; __le64 iv[2]; bool ret; @@ -179,11 +184,11 @@ bool chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len, iv[0] = 0; iv[1] = cpu_to_le64(nonce); - chacha_init(chacha_state, k, (u8 *)iv); + chacha_init(&chacha_state, k, (u8 *)iv); ret = __chacha20poly1305_decrypt(dst, src, src_len, ad, ad_len, - chacha_state); + &chacha_state); - memzero_explicit(chacha_state, sizeof(chacha_state)); + memzero_explicit(&chacha_state, sizeof(chacha_state)); memzero_explicit(iv, sizeof(iv)); memzero_explicit(k, sizeof(k)); return ret; @@ -195,11 +200,11 @@ bool xchacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len, const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE], const u8 key[CHACHA20POLY1305_KEY_SIZE]) { - u32 chacha_state[CHACHA_STATE_WORDS]; + struct chacha_state chacha_state; - xchacha_init(chacha_state, key, nonce); + xchacha_init(&chacha_state, key, nonce); return __chacha20poly1305_decrypt(dst, src, src_len, ad, ad_len, - chacha_state); + &chacha_state); } EXPORT_SYMBOL(xchacha20poly1305_decrypt); @@ -213,7 +218,7 @@ bool chacha20poly1305_crypt_sg_inplace(struct scatterlist *src, { const u8 *pad0 = page_address(ZERO_PAGE(0)); struct poly1305_desc_ctx poly1305_state; - u32 chacha_state[CHACHA_STATE_WORDS]; + struct chacha_state chacha_state; struct sg_mapping_iter miter; size_t partial = 0; unsigned int flags; @@ -240,8 +245,8 @@ bool chacha20poly1305_crypt_sg_inplace(struct scatterlist *src, b.iv[0] = 0; b.iv[1] = cpu_to_le64(nonce); - chacha_init(chacha_state, b.k, (u8 *)b.iv); - chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0)); + chacha_init(&chacha_state, b.k, (u8 *)b.iv); + chacha20_crypt(&chacha_state, b.block0, pad0, sizeof(b.block0)); poly1305_init(&poly1305_state, b.block0); if (unlikely(ad_len)) { @@ -276,13 +281,13 @@ bool chacha20poly1305_crypt_sg_inplace(struct scatterlist *src, if (unlikely(length < sl)) l &= ~(CHACHA_BLOCK_SIZE - 1); - chacha20_crypt(chacha_state, addr, addr, l); + chacha20_crypt(&chacha_state, addr, addr, l); addr += l; length -= l; } if (unlikely(length > 0)) { - chacha20_crypt(chacha_state, b.chacha_stream, pad0, + chacha20_crypt(&chacha_state, b.chacha_stream, pad0, CHACHA_BLOCK_SIZE); crypto_xor(addr, b.chacha_stream, length); partial = length; @@ -323,7 +328,7 @@ bool chacha20poly1305_crypt_sg_inplace(struct scatterlist *src, !crypto_memneq(b.mac[0], b.mac[1], POLY1305_DIGEST_SIZE); } - memzero_explicit(chacha_state, sizeof(chacha_state)); + memzero_explicit(&chacha_state, sizeof(chacha_state)); memzero_explicit(&b, sizeof(b)); return ret; diff --git a/lib/crypto/libchacha.c b/lib/crypto/libchacha.c index cc1be0496eb9..ebcca381e248 100644 --- a/lib/crypto/libchacha.c +++ b/lib/crypto/libchacha.c @@ -12,7 +12,7 @@ #include // for crypto_xor_cpy #include -void chacha_crypt_generic(u32 *state, u8 *dst, const u8 *src, +void chacha_crypt_generic(struct chacha_state *state, u8 *dst, const u8 *src, unsigned int bytes, int nrounds) { /* aligned to potentially speed up crypto_xor() */ diff --git a/tools/testing/crypto/chacha20-s390/test-cipher.c b/tools/testing/crypto/chacha20-s390/test-cipher.c index 35ea65c54ffa..827507844e8f 100644 --- a/tools/testing/crypto/chacha20-s390/test-cipher.c +++ b/tools/testing/crypto/chacha20-s390/test-cipher.c @@ -50,7 +50,7 @@ struct skcipher_def { /* Perform cipher operations with the chacha lib */ static int test_lib_chacha(u8 *revert, u8 *cipher, u8 *plain) { - u32 chacha_state[CHACHA_STATE_WORDS]; + struct chacha_state chacha_state; u8 iv[16], key[32]; u64 start, end; @@ -66,10 +66,10 @@ static int test_lib_chacha(u8 *revert, u8 *cipher, u8 *plain) } /* Encrypt */ - chacha_init(chacha_state, (u32 *)key, iv); + chacha_init(&chacha_state, (u32 *)key, iv); start = ktime_get_ns(); - chacha_crypt_arch(chacha_state, cipher, plain, data_size, 20); + chacha_crypt_arch(&chacha_state, cipher, plain, data_size, 20); end = ktime_get_ns(); @@ -81,10 +81,10 @@ static int test_lib_chacha(u8 *revert, u8 *cipher, u8 *plain) pr_info("lib encryption took: %lld nsec", end - start); /* Decrypt */ - chacha_init(chacha_state, (u32 *)key, iv); + chacha_init(&chacha_state, (u32 *)key, iv); start = ktime_get_ns(); - chacha_crypt_arch(chacha_state, revert, cipher, data_size, 20); + chacha_crypt_arch(&chacha_state, revert, cipher, data_size, 20); end = ktime_get_ns(); if (debug) From 32c9541189eb31ba6b25e2ff28e42660394a62af Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 5 May 2025 11:18:22 -0700 Subject: [PATCH 324/382] crypto: lib/chacha - use struct assignment to copy state Use struct assignment instead of memcpy() in lib/crypto/chacha.c where appropriate. No functional change. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- lib/crypto/chacha.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/lib/crypto/chacha.c b/lib/crypto/chacha.c index a7f5eb091839..ae50e441f9fb 100644 --- a/lib/crypto/chacha.c +++ b/lib/crypto/chacha.c @@ -76,11 +76,9 @@ static void chacha_permute(struct chacha_state *state, int nrounds) */ void chacha_block_generic(struct chacha_state *state, u8 *stream, int nrounds) { - struct chacha_state permuted_state; + struct chacha_state permuted_state = *state; int i; - memcpy(permuted_state.x, state->x, 64); - chacha_permute(&permuted_state, nrounds); for (i = 0; i < ARRAY_SIZE(state->x); i++) @@ -105,9 +103,7 @@ EXPORT_SYMBOL(chacha_block_generic); void hchacha_block_generic(const struct chacha_state *state, u32 *stream, int nrounds) { - struct chacha_state permuted_state; - - memcpy(permuted_state.x, state->x, 64); + struct chacha_state permuted_state = *state; chacha_permute(&permuted_state, nrounds); From 607c92141cdec6e472d80de813f5251685b9ddc1 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 5 May 2025 11:18:23 -0700 Subject: [PATCH 325/382] crypto: lib/chacha - add strongly-typed state zeroization Now that the ChaCha state matrix is strongly-typed, add a helper function chacha_zeroize_state() which zeroizes it. Then convert all applicable callers to use it instead of direct memzero_explicit. No functional changes. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- drivers/char/random.c | 4 ++-- fs/bcachefs/checksum.c | 4 ++-- include/crypto/chacha.h | 6 ++++++ lib/crypto/chacha20poly1305.c | 6 +++--- 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/drivers/char/random.c b/drivers/char/random.c index 9f876ed2655b..5f22a08101f6 100644 --- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -422,7 +422,7 @@ static void _get_random_bytes(void *buf, size_t len) buf += CHACHA_BLOCK_SIZE; } - memzero_explicit(&chacha_state, sizeof(chacha_state)); + chacha_zeroize_state(&chacha_state); } /* @@ -485,7 +485,7 @@ static ssize_t get_random_bytes_user(struct iov_iter *iter) memzero_explicit(block, sizeof(block)); out_zero_chacha: - memzero_explicit(&chacha_state, sizeof(chacha_state)); + chacha_zeroize_state(&chacha_state); return ret ? ret : -EFAULT; } diff --git a/fs/bcachefs/checksum.c b/fs/bcachefs/checksum.c index 312fda4bb1b5..a4df8eba75f3 100644 --- a/fs/bcachefs/checksum.c +++ b/fs/bcachefs/checksum.c @@ -113,7 +113,7 @@ static void bch2_chacha20(const struct bch_key *key, struct nonce nonce, bch2_chacha20_init(&state, key, nonce); chacha20_crypt(&state, data, data, len); - memzero_explicit(&state, sizeof(state)); + chacha_zeroize_state(&state); } static void bch2_poly1305_init(struct poly1305_desc_ctx *desc, @@ -283,7 +283,7 @@ int __bch2_encrypt_bio(struct bch_fs *c, unsigned type, chacha20_crypt(&chacha_state, p, p, bv.bv_len); kunmap_local(p); } - memzero_explicit(&chacha_state, sizeof(chacha_state)); + chacha_zeroize_state(&chacha_state); return ret; } diff --git a/include/crypto/chacha.h b/include/crypto/chacha.h index 64fb270f2bfc..7c2e6c68919b 100644 --- a/include/crypto/chacha.h +++ b/include/crypto/chacha.h @@ -16,6 +16,7 @@ #define _CRYPTO_CHACHA_H #include +#include #include /* 32-bit stream position, then 96-bit nonce (RFC7539 convention) */ @@ -108,6 +109,11 @@ static inline void chacha20_crypt(struct chacha_state *state, chacha_crypt(state, dst, src, bytes, 20); } +static inline void chacha_zeroize_state(struct chacha_state *state) +{ + memzero_explicit(state, sizeof(*state)); +} + #if IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_CHACHA) bool chacha_is_arch_optimized(void); #else diff --git a/lib/crypto/chacha20poly1305.c b/lib/crypto/chacha20poly1305.c index ed81f0658956..2e7bbc1a67ea 100644 --- a/lib/crypto/chacha20poly1305.c +++ b/lib/crypto/chacha20poly1305.c @@ -84,7 +84,7 @@ __chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len, poly1305_final(&poly1305_state, dst + src_len); - memzero_explicit(chacha_state, sizeof(*chacha_state)); + chacha_zeroize_state(chacha_state); memzero_explicit(&b, sizeof(b)); } @@ -188,7 +188,7 @@ bool chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len, ret = __chacha20poly1305_decrypt(dst, src, src_len, ad, ad_len, &chacha_state); - memzero_explicit(&chacha_state, sizeof(chacha_state)); + chacha_zeroize_state(&chacha_state); memzero_explicit(iv, sizeof(iv)); memzero_explicit(k, sizeof(k)); return ret; @@ -328,7 +328,7 @@ bool chacha20poly1305_crypt_sg_inplace(struct scatterlist *src, !crypto_memneq(b.mac[0], b.mac[1], POLY1305_DIGEST_SIZE); } - memzero_explicit(&chacha_state, sizeof(chacha_state)); + chacha_zeroize_state(&chacha_state); memzero_explicit(&b, sizeof(b)); return ret; From bdc2a55687f123bd32aaefb81e11c7450a431eaf Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 5 May 2025 11:18:24 -0700 Subject: [PATCH 326/382] crypto: lib/chacha - add array bounds to function prototypes Add explicit array bounds to the function prototypes for the parameters that didn't already get handled by the conversion to use chacha_state: - chacha_block_*(): Change 'u8 *out' or 'u8 *stream' to u8 out[CHACHA_BLOCK_SIZE]. - hchacha_block_*(): Change 'u32 *out' or 'u32 *stream' to u32 out[HCHACHA_OUT_WORDS]. - chacha_init(): Change 'const u32 *key' to 'const u32 key[CHACHA_KEY_WORDS]'. Change 'const u8 *iv' to 'const u8 iv[CHACHA_IV_SIZE]'. No functional changes. This just makes it clear when fixed-size arrays are expected. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/arm/lib/crypto/chacha-glue.c | 12 ++++----- arch/arm/lib/crypto/chacha-scalar-core.S | 2 +- arch/arm64/lib/crypto/chacha-neon-glue.c | 10 ++++---- arch/mips/lib/crypto/chacha-glue.c | 2 +- arch/powerpc/lib/crypto/chacha-p10-glue.c | 4 +-- arch/riscv/lib/crypto/chacha-riscv64-glue.c | 3 ++- arch/s390/lib/crypto/chacha-glue.c | 4 +-- arch/x86/lib/crypto/chacha_glue.c | 8 +++--- crypto/chacha.c | 4 +-- include/crypto/chacha.h | 27 ++++++++++++--------- lib/crypto/chacha.c | 15 ++++++------ lib/crypto/chacha20poly1305.c | 2 -- 12 files changed, 49 insertions(+), 44 deletions(-) diff --git a/arch/arm/lib/crypto/chacha-glue.c b/arch/arm/lib/crypto/chacha-glue.c index 0c2b4c62d484..88ec96415283 100644 --- a/arch/arm/lib/crypto/chacha-glue.c +++ b/arch/arm/lib/crypto/chacha-glue.c @@ -23,9 +23,9 @@ asmlinkage void chacha_4block_xor_neon(const struct chacha_state *state, u8 *dst, const u8 *src, int nrounds, unsigned int nbytes); asmlinkage void hchacha_block_arm(const struct chacha_state *state, - u32 *out, int nrounds); + u32 out[HCHACHA_OUT_WORDS], int nrounds); asmlinkage void hchacha_block_neon(const struct chacha_state *state, - u32 *out, int nrounds); + u32 out[HCHACHA_OUT_WORDS], int nrounds); asmlinkage void chacha_doarm(u8 *dst, const u8 *src, unsigned int bytes, const struct chacha_state *state, int nrounds); @@ -64,14 +64,14 @@ static void chacha_doneon(struct chacha_state *state, u8 *dst, const u8 *src, } } -void hchacha_block_arch(const struct chacha_state *state, u32 *stream, - int nrounds) +void hchacha_block_arch(const struct chacha_state *state, + u32 out[HCHACHA_OUT_WORDS], int nrounds) { if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) || !neon_usable()) { - hchacha_block_arm(state, stream, nrounds); + hchacha_block_arm(state, out, nrounds); } else { kernel_neon_begin(); - hchacha_block_neon(state, stream, nrounds); + hchacha_block_neon(state, out, nrounds); kernel_neon_end(); } } diff --git a/arch/arm/lib/crypto/chacha-scalar-core.S b/arch/arm/lib/crypto/chacha-scalar-core.S index d20b5de755cc..4951df05c158 100644 --- a/arch/arm/lib/crypto/chacha-scalar-core.S +++ b/arch/arm/lib/crypto/chacha-scalar-core.S @@ -408,7 +408,7 @@ ENDPROC(chacha_doarm) /* * void hchacha_block_arm(const struct chacha_state *state, - * u32 out[8], int nrounds); + * u32 out[HCHACHA_OUT_WORDS], int nrounds); */ ENTRY(hchacha_block_arm) push {r1,r4-r11,lr} diff --git a/arch/arm64/lib/crypto/chacha-neon-glue.c b/arch/arm64/lib/crypto/chacha-neon-glue.c index 7b451b3c7240..d0188f974ca5 100644 --- a/arch/arm64/lib/crypto/chacha-neon-glue.c +++ b/arch/arm64/lib/crypto/chacha-neon-glue.c @@ -34,7 +34,7 @@ asmlinkage void chacha_4block_xor_neon(const struct chacha_state *state, u8 *dst, const u8 *src, int nrounds, int bytes); asmlinkage void hchacha_block_neon(const struct chacha_state *state, - u32 *out, int nrounds); + u32 out[HCHACHA_OUT_WORDS], int nrounds); static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon); @@ -61,14 +61,14 @@ static void chacha_doneon(struct chacha_state *state, u8 *dst, const u8 *src, } } -void hchacha_block_arch(const struct chacha_state *state, u32 *stream, - int nrounds) +void hchacha_block_arch(const struct chacha_state *state, + u32 out[HCHACHA_OUT_WORDS], int nrounds) { if (!static_branch_likely(&have_neon) || !crypto_simd_usable()) { - hchacha_block_generic(state, stream, nrounds); + hchacha_block_generic(state, out, nrounds); } else { kernel_neon_begin(); - hchacha_block_neon(state, stream, nrounds); + hchacha_block_neon(state, out, nrounds); kernel_neon_end(); } } diff --git a/arch/mips/lib/crypto/chacha-glue.c b/arch/mips/lib/crypto/chacha-glue.c index 75df4040cded..88c097594eb0 100644 --- a/arch/mips/lib/crypto/chacha-glue.c +++ b/arch/mips/lib/crypto/chacha-glue.c @@ -15,7 +15,7 @@ asmlinkage void chacha_crypt_arch(struct chacha_state *state, EXPORT_SYMBOL(chacha_crypt_arch); asmlinkage void hchacha_block_arch(const struct chacha_state *state, - u32 *stream, int nrounds); + u32 out[HCHACHA_OUT_WORDS], int nrounds); EXPORT_SYMBOL(hchacha_block_arch); bool chacha_is_arch_optimized(void) diff --git a/arch/powerpc/lib/crypto/chacha-p10-glue.c b/arch/powerpc/lib/crypto/chacha-p10-glue.c index a6e6a8da1b8b..fcd23c6f1590 100644 --- a/arch/powerpc/lib/crypto/chacha-p10-glue.c +++ b/arch/powerpc/lib/crypto/chacha-p10-glue.c @@ -49,9 +49,9 @@ static void chacha_p10_do_8x(struct chacha_state *state, u8 *dst, const u8 *src, } void hchacha_block_arch(const struct chacha_state *state, - u32 *stream, int nrounds) + u32 out[HCHACHA_OUT_WORDS], int nrounds) { - hchacha_block_generic(state, stream, nrounds); + hchacha_block_generic(state, out, nrounds); } EXPORT_SYMBOL(hchacha_block_arch); diff --git a/arch/riscv/lib/crypto/chacha-riscv64-glue.c b/arch/riscv/lib/crypto/chacha-riscv64-glue.c index 57541621981e..8c3f11d79be3 100644 --- a/arch/riscv/lib/crypto/chacha-riscv64-glue.c +++ b/arch/riscv/lib/crypto/chacha-riscv64-glue.c @@ -18,7 +18,8 @@ static __ro_after_init DEFINE_STATIC_KEY_FALSE(use_zvkb); asmlinkage void chacha_zvkb(struct chacha_state *state, const u8 *in, u8 *out, size_t nblocks, int nrounds); -void hchacha_block_arch(const struct chacha_state *state, u32 *out, int nrounds) +void hchacha_block_arch(const struct chacha_state *state, + u32 out[HCHACHA_OUT_WORDS], int nrounds) { hchacha_block_generic(state, out, nrounds); } diff --git a/arch/s390/lib/crypto/chacha-glue.c b/arch/s390/lib/crypto/chacha-glue.c index 0a9fd50c1bd8..f95ba3483bbc 100644 --- a/arch/s390/lib/crypto/chacha-glue.c +++ b/arch/s390/lib/crypto/chacha-glue.c @@ -17,10 +17,10 @@ #include "chacha-s390.h" void hchacha_block_arch(const struct chacha_state *state, - u32 *stream, int nrounds) + u32 out[HCHACHA_OUT_WORDS], int nrounds) { /* TODO: implement hchacha_block_arch() in assembly */ - hchacha_block_generic(state, stream, nrounds); + hchacha_block_generic(state, out, nrounds); } EXPORT_SYMBOL(hchacha_block_arch); diff --git a/arch/x86/lib/crypto/chacha_glue.c b/arch/x86/lib/crypto/chacha_glue.c index 6f00a56e3e9a..10b2c945f541 100644 --- a/arch/x86/lib/crypto/chacha_glue.c +++ b/arch/x86/lib/crypto/chacha_glue.c @@ -19,7 +19,7 @@ asmlinkage void chacha_4block_xor_ssse3(const struct chacha_state *state, u8 *dst, const u8 *src, unsigned int len, int nrounds); asmlinkage void hchacha_block_ssse3(const struct chacha_state *state, - u32 *out, int nrounds); + u32 out[HCHACHA_OUT_WORDS], int nrounds); asmlinkage void chacha_2block_xor_avx2(const struct chacha_state *state, u8 *dst, const u8 *src, @@ -127,13 +127,13 @@ static void chacha_dosimd(struct chacha_state *state, u8 *dst, const u8 *src, } void hchacha_block_arch(const struct chacha_state *state, - u32 *stream, int nrounds) + u32 out[HCHACHA_OUT_WORDS], int nrounds) { if (!static_branch_likely(&chacha_use_simd)) { - hchacha_block_generic(state, stream, nrounds); + hchacha_block_generic(state, out, nrounds); } else { kernel_fpu_begin(); - hchacha_block_ssse3(state, stream, nrounds); + hchacha_block_ssse3(state, out, nrounds); kernel_fpu_end(); } } diff --git a/crypto/chacha.c b/crypto/chacha.c index 73ce62a9ac22..c3a11f4e2d13 100644 --- a/crypto/chacha.c +++ b/crypto/chacha.c @@ -46,8 +46,8 @@ static int chacha12_setkey(struct crypto_skcipher *tfm, } static int chacha_stream_xor(struct skcipher_request *req, - const struct chacha_ctx *ctx, const u8 *iv, - bool arch) + const struct chacha_ctx *ctx, + const u8 iv[CHACHA_IV_SIZE], bool arch) { struct skcipher_walk walk; struct chacha_state state; diff --git a/include/crypto/chacha.h b/include/crypto/chacha.h index 7c2e6c68919b..91f6b4cf561c 100644 --- a/include/crypto/chacha.h +++ b/include/crypto/chacha.h @@ -26,7 +26,9 @@ #define CHACHA_BLOCK_SIZE 64 #define CHACHAPOLY_IV_SIZE 12 -#define CHACHA_STATE_WORDS (CHACHA_BLOCK_SIZE / sizeof(u32)) +#define CHACHA_KEY_WORDS 8 +#define CHACHA_STATE_WORDS 16 +#define HCHACHA_OUT_WORDS 8 /* 192-bit nonce, then 64-bit stream position */ #define XCHACHA_IV_SIZE 32 @@ -35,19 +37,21 @@ struct chacha_state { u32 x[CHACHA_STATE_WORDS]; }; -void chacha_block_generic(struct chacha_state *state, u8 *stream, int nrounds); -static inline void chacha20_block(struct chacha_state *state, u8 *stream) +void chacha_block_generic(struct chacha_state *state, + u8 out[CHACHA_BLOCK_SIZE], int nrounds); +static inline void chacha20_block(struct chacha_state *state, + u8 out[CHACHA_BLOCK_SIZE]) { - chacha_block_generic(state, stream, 20); + chacha_block_generic(state, out, 20); } -void hchacha_block_arch(const struct chacha_state *state, u32 *out, - int nrounds); -void hchacha_block_generic(const struct chacha_state *state, u32 *out, - int nrounds); +void hchacha_block_arch(const struct chacha_state *state, + u32 out[HCHACHA_OUT_WORDS], int nrounds); +void hchacha_block_generic(const struct chacha_state *state, + u32 out[HCHACHA_OUT_WORDS], int nrounds); -static inline void hchacha_block(const struct chacha_state *state, u32 *out, - int nrounds) +static inline void hchacha_block(const struct chacha_state *state, + u32 out[HCHACHA_OUT_WORDS], int nrounds) { if (IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_CHACHA)) hchacha_block_arch(state, out, nrounds); @@ -71,7 +75,8 @@ static inline void chacha_init_consts(struct chacha_state *state) } static inline void chacha_init(struct chacha_state *state, - const u32 *key, const u8 *iv) + const u32 key[CHACHA_KEY_WORDS], + const u8 iv[CHACHA_IV_SIZE]) { chacha_init_consts(state); state->x[4] = key[0]; diff --git a/lib/crypto/chacha.c b/lib/crypto/chacha.c index ae50e441f9fb..ced87dd31a97 100644 --- a/lib/crypto/chacha.c +++ b/lib/crypto/chacha.c @@ -67,14 +67,15 @@ static void chacha_permute(struct chacha_state *state, int nrounds) /** * chacha_block_generic - generate one keystream block and increment block counter * @state: input state matrix - * @stream: output keystream block (64 bytes) + * @out: output keystream block * @nrounds: number of rounds (20 or 12; 20 is recommended) * * This is the ChaCha core, a function from 64-byte strings to 64-byte strings. * The caller has already converted the endianness of the input. This function * also handles incrementing the block counter in the input matrix. */ -void chacha_block_generic(struct chacha_state *state, u8 *stream, int nrounds) +void chacha_block_generic(struct chacha_state *state, + u8 out[CHACHA_BLOCK_SIZE], int nrounds) { struct chacha_state permuted_state = *state; int i; @@ -83,7 +84,7 @@ void chacha_block_generic(struct chacha_state *state, u8 *stream, int nrounds) for (i = 0; i < ARRAY_SIZE(state->x); i++) put_unaligned_le32(permuted_state.x[i] + state->x[i], - &stream[i * sizeof(u32)]); + &out[i * sizeof(u32)]); state->x[12]++; } @@ -92,7 +93,7 @@ EXPORT_SYMBOL(chacha_block_generic); /** * hchacha_block_generic - abbreviated ChaCha core, for XChaCha * @state: input state matrix - * @stream: output (8 32-bit words) + * @out: the output words * @nrounds: number of rounds (20 or 12; 20 is recommended) * * HChaCha is the ChaCha equivalent of HSalsa and is an intermediate step @@ -101,13 +102,13 @@ EXPORT_SYMBOL(chacha_block_generic); * of the state. It should not be used for streaming directly. */ void hchacha_block_generic(const struct chacha_state *state, - u32 *stream, int nrounds) + u32 out[HCHACHA_OUT_WORDS], int nrounds) { struct chacha_state permuted_state = *state; chacha_permute(&permuted_state, nrounds); - memcpy(&stream[0], &permuted_state.x[0], 16); - memcpy(&stream[4], &permuted_state.x[12], 16); + memcpy(&out[0], &permuted_state.x[0], 16); + memcpy(&out[4], &permuted_state.x[12], 16); } EXPORT_SYMBOL(hchacha_block_generic); diff --git a/lib/crypto/chacha20poly1305.c b/lib/crypto/chacha20poly1305.c index 2e7bbc1a67ea..fbd3690e2531 100644 --- a/lib/crypto/chacha20poly1305.c +++ b/lib/crypto/chacha20poly1305.c @@ -18,8 +18,6 @@ #include #include -#define CHACHA_KEY_WORDS (CHACHA_KEY_SIZE / sizeof(u32)) - static void chacha_load_key(u32 *k, const u8 *in) { k[0] = get_unaligned_le32(in); From f2804d0eee8ddd57aa79d0b82872b74c21e1b69b Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 5 May 2025 12:10:38 -0700 Subject: [PATCH 327/382] crypto: algif_aead - use memcpy_sglist() instead of null skcipher For copying data between two scatterlists, just use memcpy_sglist() instead of the so-called "null skcipher". This is much simpler. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/Kconfig | 1 - crypto/algif_aead.c | 101 ++++++++------------------------------------ 2 files changed, 18 insertions(+), 84 deletions(-) diff --git a/crypto/Kconfig b/crypto/Kconfig index 7bfad077f308..551eeeab3a0a 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -1392,7 +1392,6 @@ config CRYPTO_USER_API_AEAD depends on NET select CRYPTO_AEAD select CRYPTO_SKCIPHER - select CRYPTO_NULL select CRYPTO_USER_API help Enable the userspace interface for AEAD cipher algorithms. diff --git a/crypto/algif_aead.c b/crypto/algif_aead.c index 7d58cbbce4af..79b016a899a1 100644 --- a/crypto/algif_aead.c +++ b/crypto/algif_aead.c @@ -27,7 +27,6 @@ #include #include #include -#include #include #include #include @@ -36,19 +35,13 @@ #include #include -struct aead_tfm { - struct crypto_aead *aead; - struct crypto_sync_skcipher *null_tfm; -}; - static inline bool aead_sufficient_data(struct sock *sk) { struct alg_sock *ask = alg_sk(sk); struct sock *psk = ask->parent; struct alg_sock *pask = alg_sk(psk); struct af_alg_ctx *ctx = ask->private; - struct aead_tfm *aeadc = pask->private; - struct crypto_aead *tfm = aeadc->aead; + struct crypto_aead *tfm = pask->private; unsigned int as = crypto_aead_authsize(tfm); /* @@ -64,27 +57,12 @@ static int aead_sendmsg(struct socket *sock, struct msghdr *msg, size_t size) struct alg_sock *ask = alg_sk(sk); struct sock *psk = ask->parent; struct alg_sock *pask = alg_sk(psk); - struct aead_tfm *aeadc = pask->private; - struct crypto_aead *tfm = aeadc->aead; + struct crypto_aead *tfm = pask->private; unsigned int ivsize = crypto_aead_ivsize(tfm); return af_alg_sendmsg(sock, msg, size, ivsize); } -static int crypto_aead_copy_sgl(struct crypto_sync_skcipher *null_tfm, - struct scatterlist *src, - struct scatterlist *dst, unsigned int len) -{ - SYNC_SKCIPHER_REQUEST_ON_STACK(skreq, null_tfm); - - skcipher_request_set_sync_tfm(skreq, null_tfm); - skcipher_request_set_callback(skreq, CRYPTO_TFM_REQ_MAY_SLEEP, - NULL, NULL); - skcipher_request_set_crypt(skreq, src, dst, len, NULL); - - return crypto_skcipher_encrypt(skreq); -} - static int _aead_recvmsg(struct socket *sock, struct msghdr *msg, size_t ignored, int flags) { @@ -93,9 +71,7 @@ static int _aead_recvmsg(struct socket *sock, struct msghdr *msg, struct sock *psk = ask->parent; struct alg_sock *pask = alg_sk(psk); struct af_alg_ctx *ctx = ask->private; - struct aead_tfm *aeadc = pask->private; - struct crypto_aead *tfm = aeadc->aead; - struct crypto_sync_skcipher *null_tfm = aeadc->null_tfm; + struct crypto_aead *tfm = pask->private; unsigned int i, as = crypto_aead_authsize(tfm); struct af_alg_async_req *areq; struct af_alg_tsgl *tsgl, *tmp; @@ -223,11 +199,8 @@ static int _aead_recvmsg(struct socket *sock, struct msghdr *msg, * v v * RX SGL: AAD || PT || Tag */ - err = crypto_aead_copy_sgl(null_tfm, tsgl_src, - areq->first_rsgl.sgl.sgt.sgl, - processed); - if (err) - goto free; + memcpy_sglist(areq->first_rsgl.sgl.sgt.sgl, tsgl_src, + processed); af_alg_pull_tsgl(sk, processed, NULL, 0); } else { /* @@ -241,12 +214,8 @@ static int _aead_recvmsg(struct socket *sock, struct msghdr *msg, * RX SGL: AAD || CT ----+ */ - /* Copy AAD || CT to RX SGL buffer for in-place operation. */ - err = crypto_aead_copy_sgl(null_tfm, tsgl_src, - areq->first_rsgl.sgl.sgt.sgl, - outlen); - if (err) - goto free; + /* Copy AAD || CT to RX SGL buffer for in-place operation. */ + memcpy_sglist(areq->first_rsgl.sgl.sgt.sgl, tsgl_src, outlen); /* Create TX SGL for tag and chain it to RX SGL. */ areq->tsgl_entries = af_alg_count_tsgl(sk, processed, @@ -379,7 +348,7 @@ static int aead_check_key(struct socket *sock) int err = 0; struct sock *psk; struct alg_sock *pask; - struct aead_tfm *tfm; + struct crypto_aead *tfm; struct sock *sk = sock->sk; struct alg_sock *ask = alg_sk(sk); @@ -393,7 +362,7 @@ static int aead_check_key(struct socket *sock) err = -ENOKEY; lock_sock_nested(psk, SINGLE_DEPTH_NESTING); - if (crypto_aead_get_flags(tfm->aead) & CRYPTO_TFM_NEED_KEY) + if (crypto_aead_get_flags(tfm) & CRYPTO_TFM_NEED_KEY) goto unlock; atomic_dec(&pask->nokey_refcnt); @@ -454,54 +423,22 @@ static struct proto_ops algif_aead_ops_nokey = { static void *aead_bind(const char *name, u32 type, u32 mask) { - struct aead_tfm *tfm; - struct crypto_aead *aead; - struct crypto_sync_skcipher *null_tfm; - - tfm = kzalloc(sizeof(*tfm), GFP_KERNEL); - if (!tfm) - return ERR_PTR(-ENOMEM); - - aead = crypto_alloc_aead(name, type, mask); - if (IS_ERR(aead)) { - kfree(tfm); - return ERR_CAST(aead); - } - - null_tfm = crypto_get_default_null_skcipher(); - if (IS_ERR(null_tfm)) { - crypto_free_aead(aead); - kfree(tfm); - return ERR_CAST(null_tfm); - } - - tfm->aead = aead; - tfm->null_tfm = null_tfm; - - return tfm; + return crypto_alloc_aead(name, type, mask); } static void aead_release(void *private) { - struct aead_tfm *tfm = private; - - crypto_free_aead(tfm->aead); - crypto_put_default_null_skcipher(); - kfree(tfm); + crypto_free_aead(private); } static int aead_setauthsize(void *private, unsigned int authsize) { - struct aead_tfm *tfm = private; - - return crypto_aead_setauthsize(tfm->aead, authsize); + return crypto_aead_setauthsize(private, authsize); } static int aead_setkey(void *private, const u8 *key, unsigned int keylen) { - struct aead_tfm *tfm = private; - - return crypto_aead_setkey(tfm->aead, key, keylen); + return crypto_aead_setkey(private, key, keylen); } static void aead_sock_destruct(struct sock *sk) @@ -510,8 +447,7 @@ static void aead_sock_destruct(struct sock *sk) struct af_alg_ctx *ctx = ask->private; struct sock *psk = ask->parent; struct alg_sock *pask = alg_sk(psk); - struct aead_tfm *aeadc = pask->private; - struct crypto_aead *tfm = aeadc->aead; + struct crypto_aead *tfm = pask->private; unsigned int ivlen = crypto_aead_ivsize(tfm); af_alg_pull_tsgl(sk, ctx->used, NULL, 0); @@ -524,10 +460,9 @@ static int aead_accept_parent_nokey(void *private, struct sock *sk) { struct af_alg_ctx *ctx; struct alg_sock *ask = alg_sk(sk); - struct aead_tfm *tfm = private; - struct crypto_aead *aead = tfm->aead; + struct crypto_aead *tfm = private; unsigned int len = sizeof(*ctx); - unsigned int ivlen = crypto_aead_ivsize(aead); + unsigned int ivlen = crypto_aead_ivsize(tfm); ctx = sock_kmalloc(sk, len, GFP_KERNEL); if (!ctx) @@ -554,9 +489,9 @@ static int aead_accept_parent_nokey(void *private, struct sock *sk) static int aead_accept_parent(void *private, struct sock *sk) { - struct aead_tfm *tfm = private; + struct crypto_aead *tfm = private; - if (crypto_aead_get_flags(tfm->aead) & CRYPTO_TFM_NEED_KEY) + if (crypto_aead_get_flags(tfm) & CRYPTO_TFM_NEED_KEY) return -ENOKEY; return aead_accept_parent_nokey(private, sk); From dbc4b1458e931e47198c3165ff5853bc1ad6bd7a Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 5 May 2025 12:10:39 -0700 Subject: [PATCH 328/382] crypto: authenc - use memcpy_sglist() instead of null skcipher For copying data between two scatterlists, just use memcpy_sglist() instead of the so-called "null skcipher". This is much simpler. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/Kconfig | 1 - crypto/authenc.c | 32 +------------------------------- crypto/authencesn.c | 38 +++----------------------------------- 3 files changed, 4 insertions(+), 67 deletions(-) diff --git a/crypto/Kconfig b/crypto/Kconfig index 551eeeab3a0a..537602b8e60e 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -228,7 +228,6 @@ config CRYPTO_AUTHENC select CRYPTO_SKCIPHER select CRYPTO_MANAGER select CRYPTO_HASH - select CRYPTO_NULL help Authenc: Combined mode wrapper for IPsec. diff --git a/crypto/authenc.c b/crypto/authenc.c index 9521ae2f112e..a723769c8777 100644 --- a/crypto/authenc.c +++ b/crypto/authenc.c @@ -9,7 +9,6 @@ #include #include #include -#include #include #include #include @@ -28,7 +27,6 @@ struct authenc_instance_ctx { struct crypto_authenc_ctx { struct crypto_ahash *auth; struct crypto_skcipher *enc; - struct crypto_sync_skcipher *null; }; struct authenc_request_ctx { @@ -170,21 +168,6 @@ out: authenc_request_complete(areq, err); } -static int crypto_authenc_copy_assoc(struct aead_request *req) -{ - struct crypto_aead *authenc = crypto_aead_reqtfm(req); - struct crypto_authenc_ctx *ctx = crypto_aead_ctx(authenc); - SYNC_SKCIPHER_REQUEST_ON_STACK(skreq, ctx->null); - - skcipher_request_set_sync_tfm(skreq, ctx->null); - skcipher_request_set_callback(skreq, aead_request_flags(req), - NULL, NULL); - skcipher_request_set_crypt(skreq, req->src, req->dst, req->assoclen, - NULL); - - return crypto_skcipher_encrypt(skreq); -} - static int crypto_authenc_encrypt(struct aead_request *req) { struct crypto_aead *authenc = crypto_aead_reqtfm(req); @@ -203,10 +186,7 @@ static int crypto_authenc_encrypt(struct aead_request *req) dst = src; if (req->src != req->dst) { - err = crypto_authenc_copy_assoc(req); - if (err) - return err; - + memcpy_sglist(req->dst, req->src, req->assoclen); dst = scatterwalk_ffwd(areq_ctx->dst, req->dst, req->assoclen); } @@ -303,7 +283,6 @@ static int crypto_authenc_init_tfm(struct crypto_aead *tfm) struct crypto_authenc_ctx *ctx = crypto_aead_ctx(tfm); struct crypto_ahash *auth; struct crypto_skcipher *enc; - struct crypto_sync_skcipher *null; int err; auth = crypto_spawn_ahash(&ictx->auth); @@ -315,14 +294,8 @@ static int crypto_authenc_init_tfm(struct crypto_aead *tfm) if (IS_ERR(enc)) goto err_free_ahash; - null = crypto_get_default_null_skcipher(); - err = PTR_ERR(null); - if (IS_ERR(null)) - goto err_free_skcipher; - ctx->auth = auth; ctx->enc = enc; - ctx->null = null; crypto_aead_set_reqsize( tfm, @@ -336,8 +309,6 @@ static int crypto_authenc_init_tfm(struct crypto_aead *tfm) return 0; -err_free_skcipher: - crypto_free_skcipher(enc); err_free_ahash: crypto_free_ahash(auth); return err; @@ -349,7 +320,6 @@ static void crypto_authenc_exit_tfm(struct crypto_aead *tfm) crypto_free_ahash(ctx->auth); crypto_free_skcipher(ctx->enc); - crypto_put_default_null_skcipher(); } static void crypto_authenc_free(struct aead_instance *inst) diff --git a/crypto/authencesn.c b/crypto/authencesn.c index b1c78313cbc1..d1bf0fda3f2e 100644 --- a/crypto/authencesn.c +++ b/crypto/authencesn.c @@ -12,7 +12,6 @@ #include #include #include -#include #include #include #include @@ -31,7 +30,6 @@ struct crypto_authenc_esn_ctx { unsigned int reqoff; struct crypto_ahash *auth; struct crypto_skcipher *enc; - struct crypto_sync_skcipher *null; }; struct authenc_esn_request_ctx { @@ -158,20 +156,6 @@ static void crypto_authenc_esn_encrypt_done(void *data, int err) authenc_esn_request_complete(areq, err); } -static int crypto_authenc_esn_copy(struct aead_request *req, unsigned int len) -{ - struct crypto_aead *authenc_esn = crypto_aead_reqtfm(req); - struct crypto_authenc_esn_ctx *ctx = crypto_aead_ctx(authenc_esn); - SYNC_SKCIPHER_REQUEST_ON_STACK(skreq, ctx->null); - - skcipher_request_set_sync_tfm(skreq, ctx->null); - skcipher_request_set_callback(skreq, aead_request_flags(req), - NULL, NULL); - skcipher_request_set_crypt(skreq, req->src, req->dst, len, NULL); - - return crypto_skcipher_encrypt(skreq); -} - static int crypto_authenc_esn_encrypt(struct aead_request *req) { struct crypto_aead *authenc_esn = crypto_aead_reqtfm(req); @@ -190,10 +174,7 @@ static int crypto_authenc_esn_encrypt(struct aead_request *req) dst = src; if (req->src != req->dst) { - err = crypto_authenc_esn_copy(req, assoclen); - if (err) - return err; - + memcpy_sglist(req->dst, req->src, assoclen); sg_init_table(areq_ctx->dst, 2); dst = scatterwalk_ffwd(areq_ctx->dst, req->dst, assoclen); } @@ -277,11 +258,8 @@ static int crypto_authenc_esn_decrypt(struct aead_request *req) cryptlen -= authsize; - if (req->src != dst) { - err = crypto_authenc_esn_copy(req, assoclen + cryptlen); - if (err) - return err; - } + if (req->src != dst) + memcpy_sglist(dst, req->src, assoclen + cryptlen); scatterwalk_map_and_copy(ihash, req->src, assoclen + cryptlen, authsize, 0); @@ -317,7 +295,6 @@ static int crypto_authenc_esn_init_tfm(struct crypto_aead *tfm) struct crypto_authenc_esn_ctx *ctx = crypto_aead_ctx(tfm); struct crypto_ahash *auth; struct crypto_skcipher *enc; - struct crypto_sync_skcipher *null; int err; auth = crypto_spawn_ahash(&ictx->auth); @@ -329,14 +306,8 @@ static int crypto_authenc_esn_init_tfm(struct crypto_aead *tfm) if (IS_ERR(enc)) goto err_free_ahash; - null = crypto_get_default_null_skcipher(); - err = PTR_ERR(null); - if (IS_ERR(null)) - goto err_free_skcipher; - ctx->auth = auth; ctx->enc = enc; - ctx->null = null; ctx->reqoff = 2 * crypto_ahash_digestsize(auth); @@ -352,8 +323,6 @@ static int crypto_authenc_esn_init_tfm(struct crypto_aead *tfm) return 0; -err_free_skcipher: - crypto_free_skcipher(enc); err_free_ahash: crypto_free_ahash(auth); return err; @@ -365,7 +334,6 @@ static void crypto_authenc_esn_exit_tfm(struct crypto_aead *tfm) crypto_free_ahash(ctx->auth); crypto_free_skcipher(ctx->enc); - crypto_put_default_null_skcipher(); } static void crypto_authenc_esn_free(struct aead_instance *inst) From 4e39d0712ec40079c45703b43806bb05b2cfa9b0 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 5 May 2025 12:10:40 -0700 Subject: [PATCH 329/382] crypto: gcm - use memcpy_sglist() instead of null skcipher For copying data between two scatterlists, just use memcpy_sglist() instead of the so-called "null skcipher". This is much simpler. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/Kconfig | 1 - crypto/gcm.c | 41 ++++------------------------------------- 2 files changed, 4 insertions(+), 38 deletions(-) diff --git a/crypto/Kconfig b/crypto/Kconfig index 537602b8e60e..231791703594 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -805,7 +805,6 @@ config CRYPTO_GCM select CRYPTO_CTR select CRYPTO_AEAD select CRYPTO_GHASH - select CRYPTO_NULL select CRYPTO_MANAGER help GCM (Galois/Counter Mode) authenticated encryption mode and GMAC diff --git a/crypto/gcm.c b/crypto/gcm.c index 54ca9faf0e0c..97716482bed0 100644 --- a/crypto/gcm.c +++ b/crypto/gcm.c @@ -9,7 +9,6 @@ #include #include #include -#include #include #include #include @@ -46,7 +45,6 @@ struct crypto_rfc4543_instance_ctx { struct crypto_rfc4543_ctx { struct crypto_aead *child; - struct crypto_sync_skcipher *null; u8 nonce[4]; }; @@ -79,8 +77,6 @@ static struct { struct scatterlist sg; } *gcm_zeroes; -static int crypto_rfc4543_copy_src_to_dst(struct aead_request *req, bool enc); - static inline struct crypto_gcm_req_priv_ctx *crypto_gcm_reqctx( struct aead_request *req) { @@ -930,12 +926,12 @@ static int crypto_rfc4543_crypt(struct aead_request *req, bool enc) unsigned int authsize = crypto_aead_authsize(aead); u8 *iv = PTR_ALIGN((u8 *)(rctx + 1) + crypto_aead_reqsize(ctx->child), crypto_aead_alignmask(ctx->child) + 1); - int err; if (req->src != req->dst) { - err = crypto_rfc4543_copy_src_to_dst(req, enc); - if (err) - return err; + unsigned int nbytes = req->assoclen + req->cryptlen - + (enc ? 0 : authsize); + + memcpy_sglist(req->dst, req->src, nbytes); } memcpy(iv, ctx->nonce, 4); @@ -952,22 +948,6 @@ static int crypto_rfc4543_crypt(struct aead_request *req, bool enc) return enc ? crypto_aead_encrypt(subreq) : crypto_aead_decrypt(subreq); } -static int crypto_rfc4543_copy_src_to_dst(struct aead_request *req, bool enc) -{ - struct crypto_aead *aead = crypto_aead_reqtfm(req); - struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(aead); - unsigned int authsize = crypto_aead_authsize(aead); - unsigned int nbytes = req->assoclen + req->cryptlen - - (enc ? 0 : authsize); - SYNC_SKCIPHER_REQUEST_ON_STACK(nreq, ctx->null); - - skcipher_request_set_sync_tfm(nreq, ctx->null); - skcipher_request_set_callback(nreq, req->base.flags, NULL, NULL); - skcipher_request_set_crypt(nreq, req->src, req->dst, nbytes, NULL); - - return crypto_skcipher_encrypt(nreq); -} - static int crypto_rfc4543_encrypt(struct aead_request *req) { return crypto_ipsec_check_assoclen(req->assoclen) ?: @@ -987,21 +967,13 @@ static int crypto_rfc4543_init_tfm(struct crypto_aead *tfm) struct crypto_aead_spawn *spawn = &ictx->aead; struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(tfm); struct crypto_aead *aead; - struct crypto_sync_skcipher *null; unsigned long align; - int err = 0; aead = crypto_spawn_aead(spawn); if (IS_ERR(aead)) return PTR_ERR(aead); - null = crypto_get_default_null_skcipher(); - err = PTR_ERR(null); - if (IS_ERR(null)) - goto err_free_aead; - ctx->child = aead; - ctx->null = null; align = crypto_aead_alignmask(aead); align &= ~(crypto_tfm_ctx_alignment() - 1); @@ -1012,10 +984,6 @@ static int crypto_rfc4543_init_tfm(struct crypto_aead *tfm) align + GCM_AES_IV_SIZE); return 0; - -err_free_aead: - crypto_free_aead(aead); - return err; } static void crypto_rfc4543_exit_tfm(struct crypto_aead *tfm) @@ -1023,7 +991,6 @@ static void crypto_rfc4543_exit_tfm(struct crypto_aead *tfm) struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(tfm); crypto_free_aead(ctx->child); - crypto_put_default_null_skcipher(); } static void crypto_rfc4543_free(struct aead_instance *inst) From d8aeec147affcb6cb65060ed5fe095a2d0fe8a20 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 5 May 2025 12:10:41 -0700 Subject: [PATCH 330/382] crypto: geniv - use memcpy_sglist() instead of null skcipher For copying data between two scatterlists, just use memcpy_sglist() instead of the so-called "null skcipher". This is much simpler. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/Kconfig | 1 - crypto/echainiv.c | 18 +++--------------- crypto/geniv.c | 13 +------------ crypto/seqiv.c | 17 +++-------------- include/crypto/internal/geniv.h | 1 - 5 files changed, 7 insertions(+), 43 deletions(-) diff --git a/crypto/Kconfig b/crypto/Kconfig index 231791703594..f0c8cc5e30ae 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -815,7 +815,6 @@ config CRYPTO_GCM config CRYPTO_GENIV tristate select CRYPTO_AEAD - select CRYPTO_NULL select CRYPTO_MANAGER select CRYPTO_RNG_DEFAULT diff --git a/crypto/echainiv.c b/crypto/echainiv.c index 1913be8dfbba..e0a2d3209938 100644 --- a/crypto/echainiv.c +++ b/crypto/echainiv.c @@ -32,7 +32,6 @@ static int echainiv_encrypt(struct aead_request *req) u64 seqno; u8 *info; unsigned int ivsize = crypto_aead_ivsize(geniv); - int err; if (req->cryptlen < ivsize) return -EINVAL; @@ -41,20 +40,9 @@ static int echainiv_encrypt(struct aead_request *req) info = req->iv; - if (req->src != req->dst) { - SYNC_SKCIPHER_REQUEST_ON_STACK(nreq, ctx->sknull); - - skcipher_request_set_sync_tfm(nreq, ctx->sknull); - skcipher_request_set_callback(nreq, req->base.flags, - NULL, NULL); - skcipher_request_set_crypt(nreq, req->src, req->dst, - req->assoclen + req->cryptlen, - NULL); - - err = crypto_skcipher_encrypt(nreq); - if (err) - return err; - } + if (req->src != req->dst) + memcpy_sglist(req->dst, req->src, + req->assoclen + req->cryptlen); aead_request_set_callback(subreq, req->base.flags, req->base.complete, req->base.data); diff --git a/crypto/geniv.c b/crypto/geniv.c index bee4621b4f12..42eff6a7387c 100644 --- a/crypto/geniv.c +++ b/crypto/geniv.c @@ -9,7 +9,6 @@ #include #include -#include #include #include #include @@ -125,15 +124,10 @@ int aead_init_geniv(struct crypto_aead *aead) if (err) goto out; - ctx->sknull = crypto_get_default_null_skcipher(); - err = PTR_ERR(ctx->sknull); - if (IS_ERR(ctx->sknull)) - goto out; - child = crypto_spawn_aead(aead_instance_ctx(inst)); err = PTR_ERR(child); if (IS_ERR(child)) - goto drop_null; + goto out; ctx->child = child; crypto_aead_set_reqsize(aead, crypto_aead_reqsize(child) + @@ -143,10 +137,6 @@ int aead_init_geniv(struct crypto_aead *aead) out: return err; - -drop_null: - crypto_put_default_null_skcipher(); - goto out; } EXPORT_SYMBOL_GPL(aead_init_geniv); @@ -155,7 +145,6 @@ void aead_exit_geniv(struct crypto_aead *tfm) struct aead_geniv_ctx *ctx = crypto_aead_ctx(tfm); crypto_free_aead(ctx->child); - crypto_put_default_null_skcipher(); } EXPORT_SYMBOL_GPL(aead_exit_geniv); diff --git a/crypto/seqiv.c b/crypto/seqiv.c index a17ef5184398..2bae99e33526 100644 --- a/crypto/seqiv.c +++ b/crypto/seqiv.c @@ -64,20 +64,9 @@ static int seqiv_aead_encrypt(struct aead_request *req) data = req->base.data; info = req->iv; - if (req->src != req->dst) { - SYNC_SKCIPHER_REQUEST_ON_STACK(nreq, ctx->sknull); - - skcipher_request_set_sync_tfm(nreq, ctx->sknull); - skcipher_request_set_callback(nreq, req->base.flags, - NULL, NULL); - skcipher_request_set_crypt(nreq, req->src, req->dst, - req->assoclen + req->cryptlen, - NULL); - - err = crypto_skcipher_encrypt(nreq); - if (err) - return err; - } + if (req->src != req->dst) + memcpy_sglist(req->dst, req->src, + req->assoclen + req->cryptlen); if (unlikely(!IS_ALIGNED((unsigned long)info, crypto_aead_alignmask(geniv) + 1))) { diff --git a/include/crypto/internal/geniv.h b/include/crypto/internal/geniv.h index 7fd7126f593a..012f5fb22d43 100644 --- a/include/crypto/internal/geniv.h +++ b/include/crypto/internal/geniv.h @@ -15,7 +15,6 @@ struct aead_geniv_ctx { spinlock_t lock; struct crypto_aead *child; - struct crypto_sync_skcipher *sknull; u8 salt[] __attribute__ ((aligned(__alignof__(u32)))); }; From 0c08c72980ea8218e6b3829a4bc00664e55a4912 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 5 May 2025 12:10:42 -0700 Subject: [PATCH 331/382] crypto: krb5enc - do not select CRYPTO_NULL The krb5enc code does not use any of the so-called "null algorithms", so it does not need to select CRYPTO_NULL. Presumably this unused dependency got copied from one of the other kconfig options. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/Kconfig | 1 - 1 file changed, 1 deletion(-) diff --git a/crypto/Kconfig b/crypto/Kconfig index f0c8cc5e30ae..cf5a427bb54d 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -239,7 +239,6 @@ config CRYPTO_KRB5ENC select CRYPTO_SKCIPHER select CRYPTO_MANAGER select CRYPTO_HASH - select CRYPTO_NULL help Combined hash and cipher support for Kerberos 5 RFC3961 simplified profile. This is required for Kerberos 5-style encryption, used by From c10f66b0acc4ec9db4d443779871132108e57f10 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 5 May 2025 12:10:43 -0700 Subject: [PATCH 332/382] crypto: null - remove the default null skcipher crypto_get_default_null_skcipher() and crypto_put_default_null_skcipher() are no longer used, so remove them. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/crypto_null.c | 53 ------------------------------------------- include/crypto/null.h | 3 --- 2 files changed, 56 deletions(-) diff --git a/crypto/crypto_null.c b/crypto/crypto_null.c index 5822753b0995..48c71b925f37 100644 --- a/crypto/crypto_null.c +++ b/crypto/crypto_null.c @@ -17,13 +17,8 @@ #include #include #include -#include #include -static DEFINE_SPINLOCK(crypto_default_null_skcipher_lock); -static struct crypto_sync_skcipher *crypto_default_null_skcipher; -static int crypto_default_null_skcipher_refcnt; - static int null_init(struct shash_desc *desc) { return 0; @@ -129,54 +124,6 @@ static struct crypto_alg cipher_null = { MODULE_ALIAS_CRYPTO("digest_null"); MODULE_ALIAS_CRYPTO("cipher_null"); -struct crypto_sync_skcipher *crypto_get_default_null_skcipher(void) -{ - struct crypto_sync_skcipher *ntfm = NULL; - struct crypto_sync_skcipher *tfm; - - spin_lock_bh(&crypto_default_null_skcipher_lock); - tfm = crypto_default_null_skcipher; - - if (!tfm) { - spin_unlock_bh(&crypto_default_null_skcipher_lock); - - ntfm = crypto_alloc_sync_skcipher("ecb(cipher_null)", 0, 0); - if (IS_ERR(ntfm)) - return ntfm; - - spin_lock_bh(&crypto_default_null_skcipher_lock); - tfm = crypto_default_null_skcipher; - if (!tfm) { - tfm = ntfm; - ntfm = NULL; - crypto_default_null_skcipher = tfm; - } - } - - crypto_default_null_skcipher_refcnt++; - spin_unlock_bh(&crypto_default_null_skcipher_lock); - - crypto_free_sync_skcipher(ntfm); - - return tfm; -} -EXPORT_SYMBOL_GPL(crypto_get_default_null_skcipher); - -void crypto_put_default_null_skcipher(void) -{ - struct crypto_sync_skcipher *tfm = NULL; - - spin_lock_bh(&crypto_default_null_skcipher_lock); - if (!--crypto_default_null_skcipher_refcnt) { - tfm = crypto_default_null_skcipher; - crypto_default_null_skcipher = NULL; - } - spin_unlock_bh(&crypto_default_null_skcipher_lock); - - crypto_free_sync_skcipher(tfm); -} -EXPORT_SYMBOL_GPL(crypto_put_default_null_skcipher); - static int __init crypto_null_mod_init(void) { int ret = 0; diff --git a/include/crypto/null.h b/include/crypto/null.h index 0ef577cc00e3..1c66abf9de3b 100644 --- a/include/crypto/null.h +++ b/include/crypto/null.h @@ -9,7 +9,4 @@ #define NULL_DIGEST_SIZE 0 #define NULL_IV_SIZE 0 -struct crypto_sync_skcipher *crypto_get_default_null_skcipher(void); -void crypto_put_default_null_skcipher(void); - #endif From bde393057bbc452731a521bafa7441932da5f564 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 5 May 2025 12:10:44 -0700 Subject: [PATCH 333/382] crypto: null - merge CRYPTO_NULL2 into CRYPTO_NULL There is no reason to have separate CRYPTO_NULL2 and CRYPTO_NULL options. Just merge them into CRYPTO_NULL. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/Kconfig | 10 +++------- crypto/Makefile | 2 +- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/crypto/Kconfig b/crypto/Kconfig index cf5a427bb54d..7347277bedf3 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -192,16 +192,12 @@ config CRYPTO_MANAGER_EXTRA_TESTS config CRYPTO_NULL tristate "Null algorithms" - select CRYPTO_NULL2 + select CRYPTO_ALGAPI + select CRYPTO_SKCIPHER + select CRYPTO_HASH help These are 'Null' algorithms, used by IPsec, which do nothing. -config CRYPTO_NULL2 - tristate - select CRYPTO_ALGAPI2 - select CRYPTO_SKCIPHER2 - select CRYPTO_HASH2 - config CRYPTO_PCRYPT tristate "Parallel crypto engine" depends on SMP diff --git a/crypto/Makefile b/crypto/Makefile index 84f6911dc9ba..0f77e093512c 100644 --- a/crypto/Makefile +++ b/crypto/Makefile @@ -71,7 +71,7 @@ obj-$(CONFIG_CRYPTO_USER) += crypto_user.o obj-$(CONFIG_CRYPTO_CMAC) += cmac.o obj-$(CONFIG_CRYPTO_HMAC) += hmac.o obj-$(CONFIG_CRYPTO_XCBC) += xcbc.o -obj-$(CONFIG_CRYPTO_NULL2) += crypto_null.o +obj-$(CONFIG_CRYPTO_NULL) += crypto_null.o obj-$(CONFIG_CRYPTO_MD4) += md4.o obj-$(CONFIG_CRYPTO_MD5) += md5.o obj-$(CONFIG_CRYPTO_RMD160) += rmd160.o From aeaad5bfb18890cc73ca32d63f1f02feb5f3f651 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 5 May 2025 12:10:45 -0700 Subject: [PATCH 334/382] crypto: null - use memcpy_sglist() Make null_skcipher_crypt() use memcpy_sglist() instead of the skcipher_walk API, as this is simpler. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/crypto_null.c | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/crypto/crypto_null.c b/crypto/crypto_null.c index 48c71b925f37..34588f39fdfc 100644 --- a/crypto/crypto_null.c +++ b/crypto/crypto_null.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -60,19 +61,9 @@ static void null_crypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) static int null_skcipher_crypt(struct skcipher_request *req) { - struct skcipher_walk walk; - int err; - - err = skcipher_walk_virt(&walk, req, false); - - while (walk.nbytes) { - if (walk.src.virt.addr != walk.dst.virt.addr) - memcpy(walk.dst.virt.addr, walk.src.virt.addr, - walk.nbytes); - err = skcipher_walk_done(&walk, 0); - } - - return err; + if (req->src != req->dst) + memcpy_sglist(req->dst, req->src, req->cryptlen); + return 0; } static struct shash_alg digest_null = { From 3357b6c94569095f87a350bffa5a0a6e0c19c962 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 5 May 2025 13:33:39 -0700 Subject: [PATCH 335/382] crypto: tcrypt - rename CRYPTO_TEST to CRYPTO_BENCHMARK tcrypt is actually a benchmarking module and not the actual tests. This regularly causes confusion. Update the kconfig option name and help text accordingly. Signed-off-by: Eric Biggers Acked-by: Geert Uytterhoeven # m68k Signed-off-by: Herbert Xu --- arch/arm/configs/exynos_defconfig | 2 +- arch/arm/configs/pxa_defconfig | 2 +- arch/arm/configs/spitz_defconfig | 2 +- arch/arm64/configs/defconfig | 2 +- arch/m68k/configs/amiga_defconfig | 2 +- arch/m68k/configs/apollo_defconfig | 2 +- arch/m68k/configs/atari_defconfig | 2 +- arch/m68k/configs/bvme6000_defconfig | 2 +- arch/m68k/configs/hp300_defconfig | 2 +- arch/m68k/configs/mac_defconfig | 2 +- arch/m68k/configs/multi_defconfig | 2 +- arch/m68k/configs/mvme147_defconfig | 2 +- arch/m68k/configs/mvme16x_defconfig | 2 +- arch/m68k/configs/q40_defconfig | 2 +- arch/m68k/configs/sun3_defconfig | 2 +- arch/m68k/configs/sun3x_defconfig | 2 +- arch/mips/configs/gpr_defconfig | 2 +- arch/mips/configs/lemote2f_defconfig | 2 +- arch/mips/configs/mtx1_defconfig | 2 +- arch/mips/configs/rb532_defconfig | 2 +- arch/parisc/configs/generic-32bit_defconfig | 2 +- arch/powerpc/configs/g5_defconfig | 2 +- arch/powerpc/configs/powernv_defconfig | 2 +- arch/powerpc/configs/ppc64_defconfig | 2 +- arch/powerpc/configs/ppc64e_defconfig | 2 +- arch/powerpc/configs/ppc6xx_defconfig | 2 +- arch/s390/configs/debug_defconfig | 2 +- arch/s390/configs/defconfig | 2 +- arch/sparc/configs/sparc64_defconfig | 2 +- crypto/Kconfig | 10 +++++++--- crypto/Makefile | 2 +- crypto/tcrypt.c | 8 ++++---- crypto/tcrypt.h | 4 ++-- 33 files changed, 43 insertions(+), 39 deletions(-) diff --git a/arch/arm/configs/exynos_defconfig b/arch/arm/configs/exynos_defconfig index c6792c0256a6..e81964cce516 100644 --- a/arch/arm/configs/exynos_defconfig +++ b/arch/arm/configs/exynos_defconfig @@ -349,7 +349,7 @@ CONFIG_NLS_ASCII=y CONFIG_NLS_ISO8859_1=y CONFIG_NLS_UTF8=y CONFIG_CRYPTO_USER=m -CONFIG_CRYPTO_TEST=m +CONFIG_CRYPTO_BENCHMARK=m CONFIG_CRYPTO_DH=m CONFIG_CRYPTO_LRW=m CONFIG_CRYPTO_XTS=m diff --git a/arch/arm/configs/pxa_defconfig b/arch/arm/configs/pxa_defconfig index 56be85752909..edc5b4d17f77 100644 --- a/arch/arm/configs/pxa_defconfig +++ b/arch/arm/configs/pxa_defconfig @@ -639,7 +639,7 @@ CONFIG_SECURITY=y CONFIG_CRYPTO_MANAGER=y CONFIG_CRYPTO_CRYPTD=m CONFIG_CRYPTO_AUTHENC=m -CONFIG_CRYPTO_TEST=m +CONFIG_CRYPTO_BENCHMARK=m CONFIG_CRYPTO_BLOWFISH=m CONFIG_CRYPTO_CAST5=m CONFIG_CRYPTO_CAST6=m diff --git a/arch/arm/configs/spitz_defconfig b/arch/arm/configs/spitz_defconfig index ffec59e3f49c..ac2a0f998c73 100644 --- a/arch/arm/configs/spitz_defconfig +++ b/arch/arm/configs/spitz_defconfig @@ -215,7 +215,7 @@ CONFIG_NLS_ISO8859_1=y CONFIG_NLS_UTF8=y CONFIG_DEBUG_KERNEL=y CONFIG_CRYPTO_NULL=m -CONFIG_CRYPTO_TEST=m +CONFIG_CRYPTO_BENCHMARK=m CONFIG_CRYPTO_ECB=m CONFIG_CRYPTO_SHA256=m CONFIG_CRYPTO_AES=m diff --git a/arch/arm64/configs/defconfig b/arch/arm64/configs/defconfig index b0d4c7d173ea..4166bc4d3cec 100644 --- a/arch/arm64/configs/defconfig +++ b/arch/arm64/configs/defconfig @@ -1729,7 +1729,7 @@ CONFIG_NLS_CODEPAGE_437=y CONFIG_NLS_ISO8859_1=y CONFIG_SECURITY=y CONFIG_CRYPTO_USER=y -CONFIG_CRYPTO_TEST=m +CONFIG_CRYPTO_BENCHMARK=m CONFIG_CRYPTO_ECHAINIV=y CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_ANSI_CPRNG=y diff --git a/arch/m68k/configs/amiga_defconfig b/arch/m68k/configs/amiga_defconfig index 31ecb8b7b9f1..77f78d326a32 100644 --- a/arch/m68k/configs/amiga_defconfig +++ b/arch/m68k/configs/amiga_defconfig @@ -551,7 +551,7 @@ CONFIG_ENCRYPTED_KEYS=m CONFIG_HARDENED_USERCOPY=y CONFIG_CRYPTO_USER=m CONFIG_CRYPTO_CRYPTD=m -CONFIG_CRYPTO_TEST=m +CONFIG_CRYPTO_BENCHMARK=m CONFIG_CRYPTO_RSA=m CONFIG_CRYPTO_DH=m CONFIG_CRYPTO_ECDH=m diff --git a/arch/m68k/configs/apollo_defconfig b/arch/m68k/configs/apollo_defconfig index 1f57514624d5..f4031aa5d37f 100644 --- a/arch/m68k/configs/apollo_defconfig +++ b/arch/m68k/configs/apollo_defconfig @@ -508,7 +508,7 @@ CONFIG_ENCRYPTED_KEYS=m CONFIG_HARDENED_USERCOPY=y CONFIG_CRYPTO_USER=m CONFIG_CRYPTO_CRYPTD=m -CONFIG_CRYPTO_TEST=m +CONFIG_CRYPTO_BENCHMARK=m CONFIG_CRYPTO_RSA=m CONFIG_CRYPTO_DH=m CONFIG_CRYPTO_ECDH=m diff --git a/arch/m68k/configs/atari_defconfig b/arch/m68k/configs/atari_defconfig index 02db7a48e57e..fa92131cf4b3 100644 --- a/arch/m68k/configs/atari_defconfig +++ b/arch/m68k/configs/atari_defconfig @@ -528,7 +528,7 @@ CONFIG_ENCRYPTED_KEYS=m CONFIG_HARDENED_USERCOPY=y CONFIG_CRYPTO_USER=m CONFIG_CRYPTO_CRYPTD=m -CONFIG_CRYPTO_TEST=m +CONFIG_CRYPTO_BENCHMARK=m CONFIG_CRYPTO_RSA=m CONFIG_CRYPTO_DH=m CONFIG_CRYPTO_ECDH=m diff --git a/arch/m68k/configs/bvme6000_defconfig b/arch/m68k/configs/bvme6000_defconfig index f0e673cb17eb..9c2afc477061 100644 --- a/arch/m68k/configs/bvme6000_defconfig +++ b/arch/m68k/configs/bvme6000_defconfig @@ -500,7 +500,7 @@ CONFIG_ENCRYPTED_KEYS=m CONFIG_HARDENED_USERCOPY=y CONFIG_CRYPTO_USER=m CONFIG_CRYPTO_CRYPTD=m -CONFIG_CRYPTO_TEST=m +CONFIG_CRYPTO_BENCHMARK=m CONFIG_CRYPTO_RSA=m CONFIG_CRYPTO_DH=m CONFIG_CRYPTO_ECDH=m diff --git a/arch/m68k/configs/hp300_defconfig b/arch/m68k/configs/hp300_defconfig index e8ca5a50b86d..e7cdab059d96 100644 --- a/arch/m68k/configs/hp300_defconfig +++ b/arch/m68k/configs/hp300_defconfig @@ -510,7 +510,7 @@ CONFIG_ENCRYPTED_KEYS=m CONFIG_HARDENED_USERCOPY=y CONFIG_CRYPTO_USER=m CONFIG_CRYPTO_CRYPTD=m -CONFIG_CRYPTO_TEST=m +CONFIG_CRYPTO_BENCHMARK=m CONFIG_CRYPTO_RSA=m CONFIG_CRYPTO_DH=m CONFIG_CRYPTO_ECDH=m diff --git a/arch/m68k/configs/mac_defconfig b/arch/m68k/configs/mac_defconfig index b3a270441bb1..0a79751c20a5 100644 --- a/arch/m68k/configs/mac_defconfig +++ b/arch/m68k/configs/mac_defconfig @@ -527,7 +527,7 @@ CONFIG_ENCRYPTED_KEYS=m CONFIG_HARDENED_USERCOPY=y CONFIG_CRYPTO_USER=m CONFIG_CRYPTO_CRYPTD=m -CONFIG_CRYPTO_TEST=m +CONFIG_CRYPTO_BENCHMARK=m CONFIG_CRYPTO_RSA=m CONFIG_CRYPTO_DH=m CONFIG_CRYPTO_ECDH=m diff --git a/arch/m68k/configs/multi_defconfig b/arch/m68k/configs/multi_defconfig index d215dba006ce..f8ca490ee65a 100644 --- a/arch/m68k/configs/multi_defconfig +++ b/arch/m68k/configs/multi_defconfig @@ -614,7 +614,7 @@ CONFIG_ENCRYPTED_KEYS=m CONFIG_HARDENED_USERCOPY=y CONFIG_CRYPTO_USER=m CONFIG_CRYPTO_CRYPTD=m -CONFIG_CRYPTO_TEST=m +CONFIG_CRYPTO_BENCHMARK=m CONFIG_CRYPTO_RSA=m CONFIG_CRYPTO_DH=m CONFIG_CRYPTO_ECDH=m diff --git a/arch/m68k/configs/mvme147_defconfig b/arch/m68k/configs/mvme147_defconfig index a888ed93ff82..88fdcea906f3 100644 --- a/arch/m68k/configs/mvme147_defconfig +++ b/arch/m68k/configs/mvme147_defconfig @@ -500,7 +500,7 @@ CONFIG_ENCRYPTED_KEYS=m CONFIG_HARDENED_USERCOPY=y CONFIG_CRYPTO_USER=m CONFIG_CRYPTO_CRYPTD=m -CONFIG_CRYPTO_TEST=m +CONFIG_CRYPTO_BENCHMARK=m CONFIG_CRYPTO_RSA=m CONFIG_CRYPTO_DH=m CONFIG_CRYPTO_ECDH=m diff --git a/arch/m68k/configs/mvme16x_defconfig b/arch/m68k/configs/mvme16x_defconfig index b481782375f6..8acbe83dac72 100644 --- a/arch/m68k/configs/mvme16x_defconfig +++ b/arch/m68k/configs/mvme16x_defconfig @@ -501,7 +501,7 @@ CONFIG_ENCRYPTED_KEYS=m CONFIG_HARDENED_USERCOPY=y CONFIG_CRYPTO_USER=m CONFIG_CRYPTO_CRYPTD=m -CONFIG_CRYPTO_TEST=m +CONFIG_CRYPTO_BENCHMARK=m CONFIG_CRYPTO_RSA=m CONFIG_CRYPTO_DH=m CONFIG_CRYPTO_ECDH=m diff --git a/arch/m68k/configs/q40_defconfig b/arch/m68k/configs/q40_defconfig index 6eba743d8eb5..e3095301f3c5 100644 --- a/arch/m68k/configs/q40_defconfig +++ b/arch/m68k/configs/q40_defconfig @@ -517,7 +517,7 @@ CONFIG_ENCRYPTED_KEYS=m CONFIG_HARDENED_USERCOPY=y CONFIG_CRYPTO_USER=m CONFIG_CRYPTO_CRYPTD=m -CONFIG_CRYPTO_TEST=m +CONFIG_CRYPTO_BENCHMARK=m CONFIG_CRYPTO_RSA=m CONFIG_CRYPTO_DH=m CONFIG_CRYPTO_ECDH=m diff --git a/arch/m68k/configs/sun3_defconfig b/arch/m68k/configs/sun3_defconfig index 9bdbb418ffa8..948e48ddd128 100644 --- a/arch/m68k/configs/sun3_defconfig +++ b/arch/m68k/configs/sun3_defconfig @@ -498,7 +498,7 @@ CONFIG_ENCRYPTED_KEYS=m CONFIG_HARDENED_USERCOPY=y CONFIG_CRYPTO_USER=m CONFIG_CRYPTO_CRYPTD=m -CONFIG_CRYPTO_TEST=m +CONFIG_CRYPTO_BENCHMARK=m CONFIG_CRYPTO_RSA=m CONFIG_CRYPTO_DH=m CONFIG_CRYPTO_ECDH=m diff --git a/arch/m68k/configs/sun3x_defconfig b/arch/m68k/configs/sun3x_defconfig index e1cf20fa5343..5bcf9181c37c 100644 --- a/arch/m68k/configs/sun3x_defconfig +++ b/arch/m68k/configs/sun3x_defconfig @@ -498,7 +498,7 @@ CONFIG_ENCRYPTED_KEYS=m CONFIG_HARDENED_USERCOPY=y CONFIG_CRYPTO_USER=m CONFIG_CRYPTO_CRYPTD=m -CONFIG_CRYPTO_TEST=m +CONFIG_CRYPTO_BENCHMARK=m CONFIG_CRYPTO_RSA=m CONFIG_CRYPTO_DH=m CONFIG_CRYPTO_ECDH=m diff --git a/arch/mips/configs/gpr_defconfig b/arch/mips/configs/gpr_defconfig index 12f3eed8a946..48c8feec958f 100644 --- a/arch/mips/configs/gpr_defconfig +++ b/arch/mips/configs/gpr_defconfig @@ -273,7 +273,7 @@ CONFIG_NLS_CODEPAGE_437=y CONFIG_NLS_CODEPAGE_850=y CONFIG_NLS_ISO8859_1=y CONFIG_CRYPTO_AUTHENC=m -CONFIG_CRYPTO_TEST=m +CONFIG_CRYPTO_BENCHMARK=m CONFIG_CRYPTO_PCBC=m CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_MICHAEL_MIC=m diff --git a/arch/mips/configs/lemote2f_defconfig b/arch/mips/configs/lemote2f_defconfig index 71d6340497c9..5038a27d035f 100644 --- a/arch/mips/configs/lemote2f_defconfig +++ b/arch/mips/configs/lemote2f_defconfig @@ -297,7 +297,7 @@ CONFIG_NLS_KOI8_R=m CONFIG_NLS_KOI8_U=m CONFIG_NLS_UTF8=y CONFIG_CRYPTO_CRYPTD=m -CONFIG_CRYPTO_TEST=m +CONFIG_CRYPTO_BENCHMARK=m CONFIG_CRYPTO_BLOWFISH=m CONFIG_CRYPTO_CAMELLIA=m CONFIG_CRYPTO_CAST5=m diff --git a/arch/mips/configs/mtx1_defconfig b/arch/mips/configs/mtx1_defconfig index 06b7a0b97eca..cbf9c35a6177 100644 --- a/arch/mips/configs/mtx1_defconfig +++ b/arch/mips/configs/mtx1_defconfig @@ -662,7 +662,7 @@ CONFIG_NLS_ISO8859_15=m CONFIG_NLS_KOI8_R=m CONFIG_NLS_KOI8_U=m CONFIG_NLS_UTF8=m -CONFIG_CRYPTO_TEST=m +CONFIG_CRYPTO_BENCHMARK=m CONFIG_CRYPTO_PCBC=m CONFIG_CRYPTO_HMAC=y CONFIG_CRYPTO_MD5=y diff --git a/arch/mips/configs/rb532_defconfig b/arch/mips/configs/rb532_defconfig index 42b161d587c7..9fb114ef5e2d 100644 --- a/arch/mips/configs/rb532_defconfig +++ b/arch/mips/configs/rb532_defconfig @@ -153,6 +153,6 @@ CONFIG_JFFS2_FS=y CONFIG_JFFS2_SUMMARY=y CONFIG_JFFS2_COMPRESSION_OPTIONS=y CONFIG_SQUASHFS=y -CONFIG_CRYPTO_TEST=m +CONFIG_CRYPTO_BENCHMARK=m # CONFIG_CRYPTO_HW is not set CONFIG_STRIP_ASM_SYMS=y diff --git a/arch/parisc/configs/generic-32bit_defconfig b/arch/parisc/configs/generic-32bit_defconfig index 5b65c9859613..94928d114d4c 100644 --- a/arch/parisc/configs/generic-32bit_defconfig +++ b/arch/parisc/configs/generic-32bit_defconfig @@ -251,7 +251,7 @@ CONFIG_CIFS=m CONFIG_CIFS_XATTR=y CONFIG_CIFS_POSIX=y # CONFIG_CIFS_DEBUG is not set -CONFIG_CRYPTO_TEST=m +CONFIG_CRYPTO_BENCHMARK=m CONFIG_CRYPTO_BLOWFISH=m CONFIG_CRYPTO_CAST5=m CONFIG_CRYPTO_CAST6=m diff --git a/arch/powerpc/configs/g5_defconfig b/arch/powerpc/configs/g5_defconfig index 7e58f3e6c987..428f17b45513 100644 --- a/arch/powerpc/configs/g5_defconfig +++ b/arch/powerpc/configs/g5_defconfig @@ -235,7 +235,7 @@ CONFIG_MAGIC_SYSRQ=y CONFIG_DEBUG_KERNEL=y CONFIG_DEBUG_MUTEXES=y CONFIG_BOOTX_TEXT=y -CONFIG_CRYPTO_TEST=m +CONFIG_CRYPTO_BENCHMARK=m CONFIG_CRYPTO_PCBC=m CONFIG_CRYPTO_HMAC=y CONFIG_CRYPTO_MICHAEL_MIC=m diff --git a/arch/powerpc/configs/powernv_defconfig b/arch/powerpc/configs/powernv_defconfig index 6b6d7467fecf..379229c982a4 100644 --- a/arch/powerpc/configs/powernv_defconfig +++ b/arch/powerpc/configs/powernv_defconfig @@ -317,7 +317,7 @@ CONFIG_CODE_PATCHING_SELFTEST=y CONFIG_FTR_FIXUP_SELFTEST=y CONFIG_MSI_BITMAP_SELFTEST=y CONFIG_XMON=y -CONFIG_CRYPTO_TEST=m +CONFIG_CRYPTO_BENCHMARK=m CONFIG_CRYPTO_PCBC=m CONFIG_CRYPTO_HMAC=y CONFIG_CRYPTO_MD5_PPC=m diff --git a/arch/powerpc/configs/ppc64_defconfig b/arch/powerpc/configs/ppc64_defconfig index 5fa154185efa..3423c405cad4 100644 --- a/arch/powerpc/configs/ppc64_defconfig +++ b/arch/powerpc/configs/ppc64_defconfig @@ -377,7 +377,7 @@ CONFIG_IMA_WRITE_POLICY=y CONFIG_IMA_APPRAISE=y CONFIG_IMA_ARCH_POLICY=y CONFIG_IMA_APPRAISE_MODSIG=y -CONFIG_CRYPTO_TEST=m +CONFIG_CRYPTO_BENCHMARK=m CONFIG_CRYPTO_BLOWFISH=m CONFIG_CRYPTO_CAST6=m CONFIG_CRYPTO_SERPENT=m diff --git a/arch/powerpc/configs/ppc64e_defconfig b/arch/powerpc/configs/ppc64e_defconfig index d2e659a2d8cb..90247b2a0ab0 100644 --- a/arch/powerpc/configs/ppc64e_defconfig +++ b/arch/powerpc/configs/ppc64e_defconfig @@ -220,7 +220,7 @@ CONFIG_CODE_PATCHING_SELFTEST=y CONFIG_FTR_FIXUP_SELFTEST=y CONFIG_MSI_BITMAP_SELFTEST=y CONFIG_XMON=y -CONFIG_CRYPTO_TEST=m +CONFIG_CRYPTO_BENCHMARK=m CONFIG_CRYPTO_CCM=m CONFIG_CRYPTO_GCM=m CONFIG_CRYPTO_PCBC=m diff --git a/arch/powerpc/configs/ppc6xx_defconfig b/arch/powerpc/configs/ppc6xx_defconfig index a91a766b71a4..242c1fab9d46 100644 --- a/arch/powerpc/configs/ppc6xx_defconfig +++ b/arch/powerpc/configs/ppc6xx_defconfig @@ -1073,7 +1073,7 @@ CONFIG_SECURITY_NETWORK_XFRM=y CONFIG_SECURITY_SELINUX=y CONFIG_SECURITY_SELINUX_BOOTPARAM=y CONFIG_SECURITY_SELINUX_DISABLE=y -CONFIG_CRYPTO_TEST=m +CONFIG_CRYPTO_BENCHMARK=m CONFIG_CRYPTO_CTS=m CONFIG_CRYPTO_LRW=m CONFIG_CRYPTO_PCBC=m diff --git a/arch/s390/configs/debug_defconfig b/arch/s390/configs/debug_defconfig index de69faa4d94f..e6f7c8f4ee8b 100644 --- a/arch/s390/configs/debug_defconfig +++ b/arch/s390/configs/debug_defconfig @@ -746,7 +746,7 @@ CONFIG_CRYPTO_USER=m # CONFIG_CRYPTO_MANAGER_DISABLE_TESTS is not set CONFIG_CRYPTO_PCRYPT=m CONFIG_CRYPTO_CRYPTD=m -CONFIG_CRYPTO_TEST=m +CONFIG_CRYPTO_BENCHMARK=m CONFIG_CRYPTO_DH=m CONFIG_CRYPTO_ECDH=m CONFIG_CRYPTO_ECDSA=m diff --git a/arch/s390/configs/defconfig b/arch/s390/configs/defconfig index f12679448e97..7495bf8e483c 100644 --- a/arch/s390/configs/defconfig +++ b/arch/s390/configs/defconfig @@ -732,7 +732,7 @@ CONFIG_CRYPTO_USER=m # CONFIG_CRYPTO_MANAGER_DISABLE_TESTS is not set CONFIG_CRYPTO_PCRYPT=m CONFIG_CRYPTO_CRYPTD=m -CONFIG_CRYPTO_TEST=m +CONFIG_CRYPTO_BENCHMARK=m CONFIG_CRYPTO_DH=m CONFIG_CRYPTO_ECDH=m CONFIG_CRYPTO_ECDSA=m diff --git a/arch/sparc/configs/sparc64_defconfig b/arch/sparc/configs/sparc64_defconfig index f1ba0fefe1f9..7a7c4dec2925 100644 --- a/arch/sparc/configs/sparc64_defconfig +++ b/arch/sparc/configs/sparc64_defconfig @@ -205,7 +205,7 @@ CONFIG_BLK_DEV_IO_TRACE=y CONFIG_UPROBE_EVENTS=y CONFIG_KEYS=y CONFIG_CRYPTO_NULL=m -CONFIG_CRYPTO_TEST=m +CONFIG_CRYPTO_BENCHMARK=m CONFIG_CRYPTO_LRW=m CONFIG_CRYPTO_PCBC=m CONFIG_CRYPTO_XTS=m diff --git a/crypto/Kconfig b/crypto/Kconfig index 7347277bedf3..08af9de30ff0 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -240,12 +240,16 @@ config CRYPTO_KRB5ENC profile. This is required for Kerberos 5-style encryption, used by sunrpc/NFS and rxrpc/AFS. -config CRYPTO_TEST - tristate "Testing module" +config CRYPTO_BENCHMARK + tristate "Crypto benchmarking module" depends on m || EXPERT select CRYPTO_MANAGER help - Quick & dirty crypto test module. + Quick & dirty crypto benchmarking module. + + This is mainly intended for use by people developing cryptographic + algorithms in the kernel. It should not be enabled in production + kernels. config CRYPTO_SIMD tristate diff --git a/crypto/Makefile b/crypto/Makefile index 0f77e093512c..0b4641c5bbe1 100644 --- a/crypto/Makefile +++ b/crypto/Makefile @@ -172,7 +172,7 @@ KASAN_SANITIZE_jitterentropy.o = n UBSAN_SANITIZE_jitterentropy.o = n jitterentropy_rng-y := jitterentropy.o jitterentropy-kcapi.o obj-$(CONFIG_CRYPTO_JITTERENTROPY_TESTINTERFACE) += jitterentropy-testing.o -obj-$(CONFIG_CRYPTO_TEST) += tcrypt.o +obj-$(CONFIG_CRYPTO_BENCHMARK) += tcrypt.o obj-$(CONFIG_CRYPTO_GHASH) += ghash-generic.o obj-$(CONFIG_CRYPTO_POLYVAL) += polyval-generic.o obj-$(CONFIG_CRYPTO_USER_API) += af_alg.o diff --git a/crypto/tcrypt.c b/crypto/tcrypt.c index 879fc21dcc16..d1d88debbd71 100644 --- a/crypto/tcrypt.c +++ b/crypto/tcrypt.c @@ -1,8 +1,8 @@ // SPDX-License-Identifier: GPL-2.0-or-later /* - * Quick & dirty crypto testing module. + * Quick & dirty crypto benchmarking module. * - * This will only exist until we have a better testing mechanism + * This will only exist until we have a better benchmarking mechanism * (e.g. a char device). * * Copyright (c) 2002 James Morris @@ -39,7 +39,7 @@ #include "tcrypt.h" /* - * Need slab memory for testing (size in number of pages). + * Need slab memory for benchmarking (size in number of pages). */ #define TVMEMSIZE 4 @@ -2868,5 +2868,5 @@ module_param(klen, uint, 0); MODULE_PARM_DESC(klen, "Key length (defaults to 0)"); MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("Quick & dirty crypto testing module"); +MODULE_DESCRIPTION("Quick & dirty crypto benchmarking module"); MODULE_AUTHOR("James Morris "); diff --git a/crypto/tcrypt.h b/crypto/tcrypt.h index 96c843a24607..7f938ac93e58 100644 --- a/crypto/tcrypt.h +++ b/crypto/tcrypt.h @@ -1,8 +1,8 @@ /* SPDX-License-Identifier: GPL-2.0-or-later */ /* - * Quick & dirty crypto testing module. + * Quick & dirty crypto benchmarking module. * - * This will only exist until we have a better testing mechanism + * This will only exist until we have a better benchmarking mechanism * (e.g. a char device). * * Copyright (c) 2002 James Morris From d469eaed223fa485eabebd3bcd05ddd3c891f54e Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 5 May 2025 13:33:40 -0700 Subject: [PATCH 336/382] crypto: testmgr - remove panic_on_fail The cryptomgr.panic_on_fail=1 kernel command-line parameter is not very useful now that the tests have been fixed to WARN on failure, since developers can just use panic_on_warn=1 instead. There's no need for a special option just for the crypto self-tests. Remove it. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/testmgr.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/crypto/testmgr.c b/crypto/testmgr.c index f100be516f52..1c71616ae2cf 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -45,9 +45,6 @@ static bool notests; module_param(notests, bool, 0644); MODULE_PARM_DESC(notests, "disable crypto self-tests"); -static bool panic_on_fail; -module_param(panic_on_fail, bool, 0444); - #ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS static bool noextratests; module_param(noextratests, bool, 0644); @@ -5854,11 +5851,10 @@ int alg_test(const char *driver, const char *alg, u32 type, u32 mask) test_done: if (rc) { - if (fips_enabled || panic_on_fail) { + if (fips_enabled) { fips_fail_notify(); - panic("alg: self-tests for %s (%s) failed in %s mode!\n", - driver, alg, - fips_enabled ? "fips" : "panic_on_fail"); + panic("alg: self-tests for %s (%s) failed in fips mode!\n", + driver, alg); } pr_warn("alg: self-tests for %s using %s failed (rc=%d)", alg, driver, rc); From 40b9969796bfa49ed1b0f7ddc254f48cb2ac6d2c Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 5 May 2025 13:33:41 -0700 Subject: [PATCH 337/382] crypto: testmgr - replace CRYPTO_MANAGER_DISABLE_TESTS with CRYPTO_SELFTESTS The negative-sense of CRYPTO_MANAGER_DISABLE_TESTS is a longstanding mistake that regularly causes confusion. Especially bad is that you can have CRYPTO=n && CRYPTO_MANAGER_DISABLE_TESTS=n, which is ambiguous. Replace CRYPTO_MANAGER_DISABLE_TESTS with CRYPTO_SELFTESTS which has the expected behavior. The tests continue to be disabled by default. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/arm/configs/milbeaut_m10v_defconfig | 2 +- arch/loongarch/configs/loongson3_defconfig | 2 +- arch/s390/configs/debug_defconfig | 2 +- arch/s390/configs/defconfig | 2 +- crypto/Kconfig | 24 ++++++++++++++-------- crypto/algapi.c | 4 ++-- crypto/algboss.c | 2 +- crypto/api.c | 3 +-- crypto/hkdf.c | 2 +- crypto/internal.h | 5 ++--- crypto/kdf_sp800108.c | 2 +- crypto/testmgr.c | 12 +++++------ lib/crypto/Makefile | 9 +++----- lib/crypto/aescfb.c | 2 +- lib/crypto/aesgcm.c | 2 +- lib/crypto/blake2s.c | 2 +- lib/crypto/chacha20poly1305.c | 2 +- lib/crypto/curve25519.c | 2 +- 18 files changed, 41 insertions(+), 40 deletions(-) diff --git a/arch/arm/configs/milbeaut_m10v_defconfig b/arch/arm/configs/milbeaut_m10v_defconfig index 4ec21f477c63..9dd47e1d90bb 100644 --- a/arch/arm/configs/milbeaut_m10v_defconfig +++ b/arch/arm/configs/milbeaut_m10v_defconfig @@ -94,7 +94,7 @@ CONFIG_NLS_ISO8859_1=y CONFIG_NLS_UTF8=y CONFIG_KEYS=y CONFIG_CRYPTO_MANAGER=y -# CONFIG_CRYPTO_MANAGER_DISABLE_TESTS is not set +CONFIG_CRYPTO_SELFTESTS=y # CONFIG_CRYPTO_ECHAINIV is not set CONFIG_CRYPTO_AES=y CONFIG_CRYPTO_SEQIV=m diff --git a/arch/loongarch/configs/loongson3_defconfig b/arch/loongarch/configs/loongson3_defconfig index 90f21dfe22b1..0d59af6007b7 100644 --- a/arch/loongarch/configs/loongson3_defconfig +++ b/arch/loongarch/configs/loongson3_defconfig @@ -1026,7 +1026,7 @@ CONFIG_SECURITY_APPARMOR=y CONFIG_SECURITY_YAMA=y CONFIG_DEFAULT_SECURITY_DAC=y CONFIG_CRYPTO_USER=m -# CONFIG_CRYPTO_MANAGER_DISABLE_TESTS is not set +CONFIG_CRYPTO_SELFTESTS=y CONFIG_CRYPTO_PCRYPT=m CONFIG_CRYPTO_CRYPTD=m CONFIG_CRYPTO_ANUBIS=m diff --git a/arch/s390/configs/debug_defconfig b/arch/s390/configs/debug_defconfig index e6f7c8f4ee8b..bc960b1eabae 100644 --- a/arch/s390/configs/debug_defconfig +++ b/arch/s390/configs/debug_defconfig @@ -743,7 +743,7 @@ CONFIG_IMA_WRITE_POLICY=y CONFIG_IMA_APPRAISE=y CONFIG_BUG_ON_DATA_CORRUPTION=y CONFIG_CRYPTO_USER=m -# CONFIG_CRYPTO_MANAGER_DISABLE_TESTS is not set +CONFIG_CRYPTO_SELFTESTS=y CONFIG_CRYPTO_PCRYPT=m CONFIG_CRYPTO_CRYPTD=m CONFIG_CRYPTO_BENCHMARK=m diff --git a/arch/s390/configs/defconfig b/arch/s390/configs/defconfig index 7495bf8e483c..fda2f264e7d3 100644 --- a/arch/s390/configs/defconfig +++ b/arch/s390/configs/defconfig @@ -729,7 +729,7 @@ CONFIG_IMA_APPRAISE=y CONFIG_BUG_ON_DATA_CORRUPTION=y CONFIG_CRYPTO_FIPS=y CONFIG_CRYPTO_USER=m -# CONFIG_CRYPTO_MANAGER_DISABLE_TESTS is not set +CONFIG_CRYPTO_SELFTESTS=y CONFIG_CRYPTO_PCRYPT=m CONFIG_CRYPTO_CRYPTD=m CONFIG_CRYPTO_BENCHMARK=m diff --git a/crypto/Kconfig b/crypto/Kconfig index 08af9de30ff0..d326d3c5544f 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -25,7 +25,7 @@ menu "Crypto core or helper" config CRYPTO_FIPS bool "FIPS 200 compliance" - depends on (CRYPTO_ANSI_CPRNG || CRYPTO_DRBG) && !CRYPTO_MANAGER_DISABLE_TESTS + depends on (CRYPTO_ANSI_CPRNG || CRYPTO_DRBG) && CRYPTO_SELFTESTS depends on (MODULE_SIG || !MODULES) help This option enables the fips boot option which is @@ -143,8 +143,8 @@ config CRYPTO_ACOMP config CRYPTO_HKDF tristate - select CRYPTO_SHA256 if !CONFIG_CRYPTO_MANAGER_DISABLE_TESTS - select CRYPTO_SHA512 if !CONFIG_CRYPTO_MANAGER_DISABLE_TESTS + select CRYPTO_SHA256 if CRYPTO_SELFTESTS + select CRYPTO_SHA512 if CRYPTO_SELFTESTS select CRYPTO_HASH2 config CRYPTO_MANAGER @@ -173,16 +173,22 @@ config CRYPTO_USER Userspace configuration for cryptographic instantiations such as cbc(aes). -config CRYPTO_MANAGER_DISABLE_TESTS - bool "Disable run-time self tests" - default y +config CRYPTO_SELFTESTS + bool "Enable cryptographic self-tests" + depends on DEBUG_KERNEL help - Disable run-time self tests that normally take place at - algorithm registration. + Enable the cryptographic self-tests. + + The cryptographic self-tests run at boot time, or at algorithm + registration time if algorithms are dynamically loaded later. + + This is primarily intended for developer use. It should not be + enabled in production kernels, unless you are trying to use these + tests to fulfill a FIPS testing requirement. config CRYPTO_MANAGER_EXTRA_TESTS bool "Enable extra run-time crypto self tests" - depends on DEBUG_KERNEL && !CRYPTO_MANAGER_DISABLE_TESTS && CRYPTO_MANAGER + depends on DEBUG_KERNEL && CRYPTO_SELFTESTS && CRYPTO_MANAGER help Enable extra run-time self tests of registered crypto algorithms, including randomized fuzz tests. diff --git a/crypto/algapi.c b/crypto/algapi.c index 532d3efc3c7d..9030a30c89e8 100644 --- a/crypto/algapi.c +++ b/crypto/algapi.c @@ -275,7 +275,7 @@ static struct crypto_larval *crypto_alloc_test_larval(struct crypto_alg *alg) struct crypto_larval *larval; if (!IS_ENABLED(CONFIG_CRYPTO_MANAGER) || - IS_ENABLED(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS) || + !IS_ENABLED(CONFIG_CRYPTO_SELFTESTS) || (alg->cra_flags & CRYPTO_ALG_INTERNAL)) return NULL; /* No self-test needed */ @@ -1059,7 +1059,7 @@ static void __init crypto_start_tests(void) if (!IS_BUILTIN(CONFIG_CRYPTO_ALGAPI)) return; - if (IS_ENABLED(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS)) + if (!IS_ENABLED(CONFIG_CRYPTO_SELFTESTS)) return; set_crypto_boot_test_finished(); diff --git a/crypto/algboss.c b/crypto/algboss.c index ef5c73780fc7..846f586889ee 100644 --- a/crypto/algboss.c +++ b/crypto/algboss.c @@ -189,7 +189,7 @@ static int cryptomgr_schedule_test(struct crypto_alg *alg) struct task_struct *thread; struct crypto_test_param *param; - if (IS_ENABLED(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS)) + if (!IS_ENABLED(CONFIG_CRYPTO_SELFTESTS)) return NOTIFY_DONE; if (!try_module_get(THIS_MODULE)) diff --git a/crypto/api.c b/crypto/api.c index 5cd5ec105bb1..133d9b626922 100644 --- a/crypto/api.c +++ b/crypto/api.c @@ -31,8 +31,7 @@ EXPORT_SYMBOL_GPL(crypto_alg_sem); BLOCKING_NOTIFIER_HEAD(crypto_chain); EXPORT_SYMBOL_GPL(crypto_chain); -#if IS_BUILTIN(CONFIG_CRYPTO_ALGAPI) && \ - !IS_ENABLED(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS) +#if IS_BUILTIN(CONFIG_CRYPTO_ALGAPI) && IS_ENABLED(CONFIG_CRYPTO_SELFTESTS) DEFINE_STATIC_KEY_FALSE(__crypto_boot_test_finished); #endif diff --git a/crypto/hkdf.c b/crypto/hkdf.c index 2434c5c42545..f24c2a8d4df9 100644 --- a/crypto/hkdf.c +++ b/crypto/hkdf.c @@ -543,7 +543,7 @@ static int __init crypto_hkdf_module_init(void) { int ret = 0, i; - if (IS_ENABLED(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS)) + if (!IS_ENABLED(CONFIG_CRYPTO_SELFTESTS)) return 0; for (i = 0; i < ARRAY_SIZE(hkdf_sha256_tv); i++) { diff --git a/crypto/internal.h b/crypto/internal.h index 2ed79bf208ca..b9afd68767c1 100644 --- a/crypto/internal.h +++ b/crypto/internal.h @@ -67,8 +67,7 @@ extern struct blocking_notifier_head crypto_chain; int alg_test(const char *driver, const char *alg, u32 type, u32 mask); -#if !IS_BUILTIN(CONFIG_CRYPTO_ALGAPI) || \ - IS_ENABLED(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS) +#if !IS_BUILTIN(CONFIG_CRYPTO_ALGAPI) || !IS_ENABLED(CONFIG_CRYPTO_SELFTESTS) static inline bool crypto_boot_test_finished(void) { return true; @@ -87,7 +86,7 @@ static inline void set_crypto_boot_test_finished(void) static_branch_enable(&__crypto_boot_test_finished); } #endif /* !IS_BUILTIN(CONFIG_CRYPTO_ALGAPI) || - * IS_ENABLED(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS) + * !IS_ENABLED(CONFIG_CRYPTO_SELFTESTS) */ #ifdef CONFIG_PROC_FS diff --git a/crypto/kdf_sp800108.c b/crypto/kdf_sp800108.c index c3f9938e1ad2..b7a6bf9da773 100644 --- a/crypto/kdf_sp800108.c +++ b/crypto/kdf_sp800108.c @@ -127,7 +127,7 @@ static int __init crypto_kdf108_init(void) { int ret; - if (IS_ENABLED(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS)) + if (!IS_ENABLED(CONFIG_CRYPTO_SELFTESTS)) return 0; ret = kdf_test(&kdf_ctr_hmac_sha256_tv_template[0], "hmac(sha256)", diff --git a/crypto/testmgr.c b/crypto/testmgr.c index 1c71616ae2cf..3a624c9b5525 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -55,7 +55,7 @@ module_param(fuzz_iterations, uint, 0644); MODULE_PARM_DESC(fuzz_iterations, "number of fuzz test iterations"); #endif -#ifdef CONFIG_CRYPTO_MANAGER_DISABLE_TESTS +#ifndef CONFIG_CRYPTO_SELFTESTS /* a perfect nop */ int alg_test(const char *driver, const char *alg, u32 type, u32 mask) @@ -321,10 +321,10 @@ struct testvec_config { /* * The following are the lists of testvec_configs to test for each algorithm - * type when the basic crypto self-tests are enabled, i.e. when - * CONFIG_CRYPTO_MANAGER_DISABLE_TESTS is unset. They aim to provide good test - * coverage, while keeping the test time much shorter than the full fuzz tests - * so that the basic tests can be enabled in a wider range of circumstances. + * type when the basic crypto self-tests are enabled. They aim to provide good + * test coverage, while keeping the test time much shorter than the full fuzz + * tests so that the basic tests can be enabled in a wider range of + * circumstances. */ /* Configs for skciphers and aeads */ @@ -5899,6 +5899,6 @@ non_fips_alg: return alg_fips_disabled(driver, alg); } -#endif /* CONFIG_CRYPTO_MANAGER_DISABLE_TESTS */ +#endif /* CONFIG_CRYPTO_SELFTESTS */ EXPORT_SYMBOL_GPL(alg_test); diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index ff4aa22e5ccc..599e61375938 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -25,9 +25,11 @@ obj-$(CONFIG_CRYPTO_LIB_GF128MUL) += gf128mul.o obj-y += libblake2s.o libblake2s-y := blake2s.o libblake2s-$(CONFIG_CRYPTO_LIB_BLAKE2S_GENERIC) += blake2s-generic.o +libblake2s-$(CONFIG_CRYPTO_SELFTESTS) += blake2s-selftest.o obj-$(CONFIG_CRYPTO_LIB_CHACHA20POLY1305) += libchacha20poly1305.o libchacha20poly1305-y += chacha20poly1305.o +libchacha20poly1305-$(CONFIG_CRYPTO_SELFTESTS) += chacha20poly1305-selftest.o obj-$(CONFIG_CRYPTO_LIB_CURVE25519_GENERIC) += libcurve25519-generic.o libcurve25519-generic-y := curve25519-fiat32.o @@ -36,6 +38,7 @@ libcurve25519-generic-y += curve25519-generic.o obj-$(CONFIG_CRYPTO_LIB_CURVE25519) += libcurve25519.o libcurve25519-y += curve25519.o +libcurve25519-$(CONFIG_CRYPTO_SELFTESTS) += curve25519-selftest.o obj-$(CONFIG_CRYPTO_LIB_DES) += libdes.o libdes-y := des.o @@ -57,12 +60,6 @@ libsha256-y := sha256.o obj-$(CONFIG_CRYPTO_LIB_SHA256_GENERIC) += libsha256-generic.o libsha256-generic-y := sha256-generic.o -ifneq ($(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS),y) -libblake2s-y += blake2s-selftest.o -libchacha20poly1305-y += chacha20poly1305-selftest.o -libcurve25519-y += curve25519-selftest.o -endif - obj-$(CONFIG_MPILIB) += mpi/ obj-$(CONFIG_CRYPTO_MANAGER_EXTRA_TESTS) += simd.o diff --git a/lib/crypto/aescfb.c b/lib/crypto/aescfb.c index 749dc1258a44..437613265e14 100644 --- a/lib/crypto/aescfb.c +++ b/lib/crypto/aescfb.c @@ -99,7 +99,7 @@ MODULE_DESCRIPTION("Generic AES-CFB library"); MODULE_AUTHOR("Ard Biesheuvel "); MODULE_LICENSE("GPL"); -#ifndef CONFIG_CRYPTO_MANAGER_DISABLE_TESTS +#ifdef CONFIG_CRYPTO_SELFTESTS /* * Test code below. Vectors taken from crypto/testmgr.h diff --git a/lib/crypto/aesgcm.c b/lib/crypto/aesgcm.c index 902e49410aaf..277824d6b4af 100644 --- a/lib/crypto/aesgcm.c +++ b/lib/crypto/aesgcm.c @@ -199,7 +199,7 @@ MODULE_DESCRIPTION("Generic AES-GCM library"); MODULE_AUTHOR("Ard Biesheuvel "); MODULE_LICENSE("GPL"); -#ifndef CONFIG_CRYPTO_MANAGER_DISABLE_TESTS +#ifdef CONFIG_CRYPTO_SELFTESTS /* * Test code below. Vectors taken from crypto/testmgr.h diff --git a/lib/crypto/blake2s.c b/lib/crypto/blake2s.c index 71a316552cc5..b0f9a678300b 100644 --- a/lib/crypto/blake2s.c +++ b/lib/crypto/blake2s.c @@ -60,7 +60,7 @@ EXPORT_SYMBOL(blake2s_final); static int __init blake2s_mod_init(void) { - if (!IS_ENABLED(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS) && + if (IS_ENABLED(CONFIG_CRYPTO_SELFTESTS) && WARN_ON(!blake2s_selftest())) return -ENODEV; return 0; diff --git a/lib/crypto/chacha20poly1305.c b/lib/crypto/chacha20poly1305.c index fbd3690e2531..e29eed49a5a1 100644 --- a/lib/crypto/chacha20poly1305.c +++ b/lib/crypto/chacha20poly1305.c @@ -358,7 +358,7 @@ EXPORT_SYMBOL(chacha20poly1305_decrypt_sg_inplace); static int __init chacha20poly1305_init(void) { - if (!IS_ENABLED(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS) && + if (IS_ENABLED(CONFIG_CRYPTO_SELFTESTS) && WARN_ON(!chacha20poly1305_selftest())) return -ENODEV; return 0; diff --git a/lib/crypto/curve25519.c b/lib/crypto/curve25519.c index 064b352c6907..6850b76a80c9 100644 --- a/lib/crypto/curve25519.c +++ b/lib/crypto/curve25519.c @@ -15,7 +15,7 @@ static int __init curve25519_init(void) { - if (!IS_ENABLED(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS) && + if (IS_ENABLED(CONFIG_CRYPTO_SELFTESTS) && WARN_ON(!curve25519_selftest())) return -ENODEV; return 0; From 698de822780fbae79b11e5d749863c1aa64a0a55 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 5 May 2025 13:33:42 -0700 Subject: [PATCH 338/382] crypto: testmgr - make it easier to enable the full set of tests Currently the full set of crypto self-tests requires CONFIG_CRYPTO_MANAGER_EXTRA_TESTS=y. This is problematic in two ways. First, developers regularly overlook this option. Second, the description of the tests as "extra" sometimes gives the impression that it is not required that all algorithms pass these tests. Given that the main use case for the crypto self-tests is for developers, make enabling CONFIG_CRYPTO_SELFTESTS=y just enable the full set of crypto self-tests by default. The slow tests can still be disabled by adding the command-line parameter cryptomgr.noextratests=1, soon to be renamed to cryptomgr.noslowtests=1. The only known use case for doing this is for people trying to use the crypto self-tests to satisfy the FIPS 140-3 pre-operational self-testing requirements when the kernel is being validated as a FIPS 140-3 cryptographic module. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/Kconfig | 10 ------ crypto/testmgr.c | 62 +++------------------------------- include/crypto/internal/simd.h | 6 ++-- lib/crypto/Makefile | 2 +- 4 files changed, 8 insertions(+), 72 deletions(-) diff --git a/crypto/Kconfig b/crypto/Kconfig index d326d3c5544f..277fc9335767 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -186,16 +186,6 @@ config CRYPTO_SELFTESTS enabled in production kernels, unless you are trying to use these tests to fulfill a FIPS testing requirement. -config CRYPTO_MANAGER_EXTRA_TESTS - bool "Enable extra run-time crypto self tests" - depends on DEBUG_KERNEL && CRYPTO_SELFTESTS && CRYPTO_MANAGER - help - Enable extra run-time self tests of registered crypto algorithms, - including randomized fuzz tests. - - This is intended for developer use only, as these tests take much - longer to run than the normal self tests. - config CRYPTO_NULL tristate "Null algorithms" select CRYPTO_ALGAPI diff --git a/crypto/testmgr.c b/crypto/testmgr.c index 3a624c9b5525..a6707cda8328 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -45,7 +45,6 @@ static bool notests; module_param(notests, bool, 0644); MODULE_PARM_DESC(notests, "disable crypto self-tests"); -#ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS static bool noextratests; module_param(noextratests, bool, 0644); MODULE_PARM_DESC(noextratests, "disable expensive crypto self-tests"); @@ -53,7 +52,6 @@ MODULE_PARM_DESC(noextratests, "disable expensive crypto self-tests"); static unsigned int fuzz_iterations = 100; module_param(fuzz_iterations, uint, 0644); MODULE_PARM_DESC(fuzz_iterations, "number of fuzz test iterations"); -#endif #ifndef CONFIG_CRYPTO_SELFTESTS @@ -321,10 +319,9 @@ struct testvec_config { /* * The following are the lists of testvec_configs to test for each algorithm - * type when the basic crypto self-tests are enabled. They aim to provide good - * test coverage, while keeping the test time much shorter than the full fuzz - * tests so that the basic tests can be enabled in a wider range of - * circumstances. + * type when the fast crypto self-tests are enabled. They aim to provide good + * test coverage, while keeping the test time much shorter than the full tests + * so that the fast tests can be used to fulfill FIPS 140 testing requirements. */ /* Configs for skciphers and aeads */ @@ -873,8 +870,6 @@ static int prepare_keybuf(const u8 *key, unsigned int ksize, err; \ }) -#ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS - /* * The fuzz tests use prandom instead of the normal Linux RNG since they don't * need cryptographically secure random numbers. This greatly improves the @@ -1239,15 +1234,6 @@ too_long: algname); return -ENAMETOOLONG; } -#else /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */ -static void crypto_disable_simd_for_test(void) -{ -} - -static void crypto_reenable_simd_for_test(void) -{ -} -#endif /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */ static int build_hash_sglist(struct test_sglist *tsgl, const struct hash_testvec *vec, @@ -1688,7 +1674,6 @@ static int test_hash_vec(const struct hash_testvec *vec, unsigned int vec_num, return err; } -#ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS if (!noextratests) { struct rnd_state rng; struct testvec_config cfg; @@ -1706,11 +1691,9 @@ static int test_hash_vec(const struct hash_testvec *vec, unsigned int vec_num, cond_resched(); } } -#endif return 0; } -#ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS /* * Generate a hash test vector from the given implementation. * Assumes the buffers in 'vec' were already allocated. @@ -1876,17 +1859,6 @@ out: kfree_sensitive(generic_desc); return err; } -#else /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */ -static int test_hash_vs_generic_impl(const char *generic_driver, - unsigned int maxkeysize, - struct ahash_request *req, - struct shash_desc *desc, - struct test_sglist *tsgl, - u8 *hashstate) -{ - return 0; -} -#endif /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */ static int alloc_shash(const char *driver, u32 type, u32 mask, struct crypto_shash **tfm_ret, @@ -2260,7 +2232,6 @@ static int test_aead_vec(int enc, const struct aead_testvec *vec, return err; } -#ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS if (!noextratests) { struct rnd_state rng; struct testvec_config cfg; @@ -2278,12 +2249,9 @@ static int test_aead_vec(int enc, const struct aead_testvec *vec, cond_resched(); } } -#endif return 0; } -#ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS - struct aead_extra_tests_ctx { struct rnd_state rng; struct aead_request *req; @@ -2668,14 +2636,6 @@ out: kfree(ctx); return err; } -#else /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */ -static int test_aead_extra(const struct alg_test_desc *test_desc, - struct aead_request *req, - struct cipher_test_sglists *tsgls) -{ - return 0; -} -#endif /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */ static int test_aead(int enc, const struct aead_test_suite *suite, struct aead_request *req, @@ -3015,7 +2975,6 @@ static int test_skcipher_vec(int enc, const struct cipher_testvec *vec, return err; } -#ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS if (!noextratests) { struct rnd_state rng; struct testvec_config cfg; @@ -3033,11 +2992,9 @@ static int test_skcipher_vec(int enc, const struct cipher_testvec *vec, cond_resched(); } } -#endif return 0; } -#ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS /* * Generate a symmetric cipher test vector from the given implementation. * Assumes the buffers in 'vec' were already allocated. @@ -3236,14 +3193,6 @@ out: skcipher_request_free(generic_req); return err; } -#else /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */ -static int test_skcipher_vs_generic_impl(const char *generic_driver, - struct skcipher_request *req, - struct cipher_test_sglists *tsgls) -{ - return 0; -} -#endif /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */ static int test_skcipher(int enc, const struct cipher_test_suite *suite, struct skcipher_request *req, @@ -5760,9 +5709,8 @@ static void testmgr_onetime_init(void) alg_check_test_descs_order(); alg_check_testvec_configs(); -#ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS - pr_warn("alg: extra crypto tests enabled. This is intended for developer use only.\n"); -#endif + if (!noextratests) + pr_warn("alg: full crypto tests enabled. This is intended for developer use only.\n"); } static int alg_find_test(const char *alg) diff --git a/include/crypto/internal/simd.h b/include/crypto/internal/simd.h index f56049bd1660..7e7f1ac3b7fd 100644 --- a/include/crypto/internal/simd.h +++ b/include/crypto/internal/simd.h @@ -44,11 +44,9 @@ void simd_unregister_aeads(struct aead_alg *algs, int count, * * This delegates to may_use_simd(), except that this also returns false if SIMD * in crypto code has been temporarily disabled on this CPU by the crypto - * self-tests, in order to test the no-SIMD fallback code. This override is - * currently limited to configurations where the extra self-tests are enabled, - * because it might be a bit too invasive to be part of the regular self-tests. + * self-tests, in order to test the no-SIMD fallback code. */ -#ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS +#ifdef CONFIG_CRYPTO_SELFTESTS DECLARE_PER_CPU(bool, crypto_simd_disabled_for_test); #define crypto_simd_usable() \ (may_use_simd() && !this_cpu_read(crypto_simd_disabled_for_test)) diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index 599e61375938..3e79283b617d 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -62,7 +62,7 @@ libsha256-generic-y := sha256-generic.o obj-$(CONFIG_MPILIB) += mpi/ -obj-$(CONFIG_CRYPTO_MANAGER_EXTRA_TESTS) += simd.o +obj-$(CONFIG_CRYPTO_SELFTESTS) += simd.o obj-$(CONFIG_CRYPTO_LIB_SM3) += libsm3.o libsm3-y := sm3.o From 42e45119efba92461839d1473abda5e31b0e71ed Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 5 May 2025 13:33:43 -0700 Subject: [PATCH 339/382] crypto: testmgr - rename noextratests to noslowtests Rename the noextratests module parameter to noslowtests, and replace other remaining mentions of "extra" in the code with "slow". This addresses confusion regarding the word "extra" like that seen at https://lore.kernel.org/r/6cecf2de-9aa0-f6ea-0c2d-8e974a1a820b@huawei.com/. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/testmgr.c | 41 ++++++++++++++++++++--------------------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/crypto/testmgr.c b/crypto/testmgr.c index a6707cda8328..fc28000c27f5 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -43,11 +43,11 @@ MODULE_IMPORT_NS("CRYPTO_INTERNAL"); static bool notests; module_param(notests, bool, 0644); -MODULE_PARM_DESC(notests, "disable crypto self-tests"); +MODULE_PARM_DESC(notests, "disable all crypto self-tests"); -static bool noextratests; -module_param(noextratests, bool, 0644); -MODULE_PARM_DESC(noextratests, "disable expensive crypto self-tests"); +static bool noslowtests; +module_param(noslowtests, bool, 0644); +MODULE_PARM_DESC(noslowtests, "disable slow crypto self-tests"); static unsigned int fuzz_iterations = 100; module_param(fuzz_iterations, uint, 0644); @@ -1674,7 +1674,7 @@ static int test_hash_vec(const struct hash_testvec *vec, unsigned int vec_num, return err; } - if (!noextratests) { + if (!noslowtests) { struct rnd_state rng; struct testvec_config cfg; char cfgname[TESTVEC_CONFIG_NAMELEN]; @@ -1764,7 +1764,7 @@ static int test_hash_vs_generic_impl(const char *generic_driver, char cfgname[TESTVEC_CONFIG_NAMELEN]; int err; - if (noextratests) + if (noslowtests) return 0; init_rnd_state(&rng); @@ -2232,7 +2232,7 @@ static int test_aead_vec(int enc, const struct aead_testvec *vec, return err; } - if (!noextratests) { + if (!noslowtests) { struct rnd_state rng; struct testvec_config cfg; char cfgname[TESTVEC_CONFIG_NAMELEN]; @@ -2252,7 +2252,7 @@ static int test_aead_vec(int enc, const struct aead_testvec *vec, return 0; } -struct aead_extra_tests_ctx { +struct aead_slow_tests_ctx { struct rnd_state rng; struct aead_request *req; struct crypto_aead *tfm; @@ -2427,8 +2427,7 @@ static void generate_random_aead_testvec(struct rnd_state *rng, vec->alen, vec->plen, authsize, vec->klen, vec->novrfy); } -static void try_to_generate_inauthentic_testvec( - struct aead_extra_tests_ctx *ctx) +static void try_to_generate_inauthentic_testvec(struct aead_slow_tests_ctx *ctx) { int i; @@ -2447,7 +2446,7 @@ static void try_to_generate_inauthentic_testvec( * Generate inauthentic test vectors (i.e. ciphertext, AAD pairs that aren't the * result of an encryption with the key) and verify that decryption fails. */ -static int test_aead_inauthentic_inputs(struct aead_extra_tests_ctx *ctx) +static int test_aead_inauthentic_inputs(struct aead_slow_tests_ctx *ctx) { unsigned int i; int err; @@ -2482,7 +2481,7 @@ static int test_aead_inauthentic_inputs(struct aead_extra_tests_ctx *ctx) * Test the AEAD algorithm against the corresponding generic implementation, if * one is available. */ -static int test_aead_vs_generic_impl(struct aead_extra_tests_ctx *ctx) +static int test_aead_vs_generic_impl(struct aead_slow_tests_ctx *ctx) { struct crypto_aead *tfm = ctx->tfm; const char *algname = crypto_aead_alg(tfm)->base.cra_name; @@ -2586,15 +2585,15 @@ out: return err; } -static int test_aead_extra(const struct alg_test_desc *test_desc, - struct aead_request *req, - struct cipher_test_sglists *tsgls) +static int test_aead_slow(const struct alg_test_desc *test_desc, + struct aead_request *req, + struct cipher_test_sglists *tsgls) { - struct aead_extra_tests_ctx *ctx; + struct aead_slow_tests_ctx *ctx; unsigned int i; int err; - if (noextratests) + if (noslowtests) return 0; ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); @@ -2701,7 +2700,7 @@ static int alg_test_aead(const struct alg_test_desc *desc, const char *driver, if (err) goto out; - err = test_aead_extra(desc, req, tsgls); + err = test_aead_slow(desc, req, tsgls); out: free_cipher_test_sglists(tsgls); aead_request_free(req); @@ -2975,7 +2974,7 @@ static int test_skcipher_vec(int enc, const struct cipher_testvec *vec, return err; } - if (!noextratests) { + if (!noslowtests) { struct rnd_state rng; struct testvec_config cfg; char cfgname[TESTVEC_CONFIG_NAMELEN]; @@ -3077,7 +3076,7 @@ static int test_skcipher_vs_generic_impl(const char *generic_driver, char cfgname[TESTVEC_CONFIG_NAMELEN]; int err; - if (noextratests) + if (noslowtests) return 0; init_rnd_state(&rng); @@ -5709,7 +5708,7 @@ static void testmgr_onetime_init(void) alg_check_test_descs_order(); alg_check_testvec_configs(); - if (!noextratests) + if (!noslowtests) pr_warn("alg: full crypto tests enabled. This is intended for developer use only.\n"); } From 6f9d0f53f50405fcf4abf7ff45cec7baed71151c Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 5 May 2025 13:33:44 -0700 Subject: [PATCH 340/382] crypto: Kconfig - make CRYPTO_MANAGER a hidden symbol There is no reason for people configuring the kernel to be asked about CRYPTO_MANAGER, so make it a hidden symbol. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/arm/configs/milbeaut_m10v_defconfig | 1 - arch/arm/configs/pxa_defconfig | 1 - arch/mips/configs/decstation_64_defconfig | 1 - arch/mips/configs/decstation_defconfig | 1 - arch/mips/configs/decstation_r4k_defconfig | 1 - arch/mips/configs/ip28_defconfig | 1 - arch/parisc/configs/generic-64bit_defconfig | 1 - arch/sh/configs/migor_defconfig | 1 - crypto/Kconfig | 2 +- 9 files changed, 1 insertion(+), 9 deletions(-) diff --git a/arch/arm/configs/milbeaut_m10v_defconfig b/arch/arm/configs/milbeaut_m10v_defconfig index 9dd47e1d90bb..242e7d5a3f68 100644 --- a/arch/arm/configs/milbeaut_m10v_defconfig +++ b/arch/arm/configs/milbeaut_m10v_defconfig @@ -93,7 +93,6 @@ CONFIG_NLS_CODEPAGE_437=y CONFIG_NLS_ISO8859_1=y CONFIG_NLS_UTF8=y CONFIG_KEYS=y -CONFIG_CRYPTO_MANAGER=y CONFIG_CRYPTO_SELFTESTS=y # CONFIG_CRYPTO_ECHAINIV is not set CONFIG_CRYPTO_AES=y diff --git a/arch/arm/configs/pxa_defconfig b/arch/arm/configs/pxa_defconfig index edc5b4d17f77..ded4b9a5accf 100644 --- a/arch/arm/configs/pxa_defconfig +++ b/arch/arm/configs/pxa_defconfig @@ -636,7 +636,6 @@ CONFIG_NLS_ISO8859_15=m CONFIG_NLS_UTF8=m CONFIG_TIMER_STATS=y CONFIG_SECURITY=y -CONFIG_CRYPTO_MANAGER=y CONFIG_CRYPTO_CRYPTD=m CONFIG_CRYPTO_AUTHENC=m CONFIG_CRYPTO_BENCHMARK=m diff --git a/arch/mips/configs/decstation_64_defconfig b/arch/mips/configs/decstation_64_defconfig index 9655567614aa..85a4472cb058 100644 --- a/arch/mips/configs/decstation_64_defconfig +++ b/arch/mips/configs/decstation_64_defconfig @@ -168,7 +168,6 @@ CONFIG_NLS_ISO8859_14=m CONFIG_NLS_ISO8859_15=m CONFIG_NLS_UTF8=m CONFIG_CRYPTO_RSA=m -CONFIG_CRYPTO_MANAGER=y CONFIG_CRYPTO_CCM=m CONFIG_CRYPTO_GCM=m CONFIG_CRYPTO_CHACHA20POLY1305=m diff --git a/arch/mips/configs/decstation_defconfig b/arch/mips/configs/decstation_defconfig index 1539fe8eb34d..a3b2c8da2dde 100644 --- a/arch/mips/configs/decstation_defconfig +++ b/arch/mips/configs/decstation_defconfig @@ -163,7 +163,6 @@ CONFIG_NLS_ISO8859_14=m CONFIG_NLS_ISO8859_15=m CONFIG_NLS_UTF8=m CONFIG_CRYPTO_RSA=m -CONFIG_CRYPTO_MANAGER=y CONFIG_CRYPTO_CCM=m CONFIG_CRYPTO_GCM=m CONFIG_CRYPTO_CHACHA20POLY1305=m diff --git a/arch/mips/configs/decstation_r4k_defconfig b/arch/mips/configs/decstation_r4k_defconfig index 58c36720c94a..a476717b8a6a 100644 --- a/arch/mips/configs/decstation_r4k_defconfig +++ b/arch/mips/configs/decstation_r4k_defconfig @@ -163,7 +163,6 @@ CONFIG_NLS_ISO8859_14=m CONFIG_NLS_ISO8859_15=m CONFIG_NLS_UTF8=m CONFIG_CRYPTO_RSA=m -CONFIG_CRYPTO_MANAGER=y CONFIG_CRYPTO_CCM=m CONFIG_CRYPTO_GCM=m CONFIG_CRYPTO_CHACHA20POLY1305=m diff --git a/arch/mips/configs/ip28_defconfig b/arch/mips/configs/ip28_defconfig index e0040110a3ee..6db21e498faa 100644 --- a/arch/mips/configs/ip28_defconfig +++ b/arch/mips/configs/ip28_defconfig @@ -60,6 +60,5 @@ CONFIG_TMPFS_POSIX_ACL=y CONFIG_NFS_FS=y CONFIG_NFS_V3_ACL=y CONFIG_ROOT_NFS=y -CONFIG_CRYPTO_MANAGER=y # CONFIG_CRYPTO_HW is not set CONFIG_MAGIC_SYSRQ=y diff --git a/arch/parisc/configs/generic-64bit_defconfig b/arch/parisc/configs/generic-64bit_defconfig index ecc9ffcc11cd..d8cd7f858b2a 100644 --- a/arch/parisc/configs/generic-64bit_defconfig +++ b/arch/parisc/configs/generic-64bit_defconfig @@ -283,7 +283,6 @@ CONFIG_NLS_ASCII=m CONFIG_NLS_ISO8859_1=m CONFIG_NLS_ISO8859_2=m CONFIG_NLS_UTF8=m -CONFIG_CRYPTO_MANAGER=y CONFIG_CRYPTO_FCRYPT=m CONFIG_CRYPTO_ECB=m CONFIG_CRYPTO_PCBC=m diff --git a/arch/sh/configs/migor_defconfig b/arch/sh/configs/migor_defconfig index fc2010c241fb..31dbd8888aaa 100644 --- a/arch/sh/configs/migor_defconfig +++ b/arch/sh/configs/migor_defconfig @@ -87,6 +87,5 @@ CONFIG_TMPFS=y CONFIG_NFS_FS=y CONFIG_ROOT_NFS=y CONFIG_DEBUG_FS=y -CONFIG_CRYPTO_MANAGER=y # CONFIG_CRYPTO_ANSI_CPRNG is not set # CONFIG_CRYPTO_HW is not set diff --git a/crypto/Kconfig b/crypto/Kconfig index 277fc9335767..50f6e285b92a 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -148,7 +148,7 @@ config CRYPTO_HKDF select CRYPTO_HASH2 config CRYPTO_MANAGER - tristate "Cryptographic algorithm manager" + tristate select CRYPTO_MANAGER2 help Create default cryptographic template instantiations such as From 57999ed153ed7e651afecbabe0e998e75cf2d798 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 5 May 2025 13:33:45 -0700 Subject: [PATCH 341/382] crypto: testmgr - enable CRYPTO_MANAGER when CRYPTO_SELFTESTS crypto/testmgr.c is compiled only when CRYPTO_MANAGER is enabled. To make CRYPTO_SELFTESTS work as expected when CRYPTO_MANAGER doesn't get enabled for another reason, automatically set CRYPTO_MANAGER to the value of CRYPTO_ALGAPI when CRYPTO_SELFTESTS is enabled. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/Kconfig | 5 +++-- crypto/algapi.c | 3 +-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/crypto/Kconfig b/crypto/Kconfig index 50f6e285b92a..e9fee7818e27 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -149,10 +149,11 @@ config CRYPTO_HKDF config CRYPTO_MANAGER tristate + default CRYPTO_ALGAPI if CRYPTO_SELFTESTS select CRYPTO_MANAGER2 help - Create default cryptographic template instantiations such as - cbc(aes). + This provides the support for instantiating templates such as + cbc(aes), and the support for the crypto self-tests. config CRYPTO_MANAGER2 def_tristate CRYPTO_MANAGER || (CRYPTO_MANAGER!=n && CRYPTO_ALGAPI=y) diff --git a/crypto/algapi.c b/crypto/algapi.c index 9030a30c89e8..25b5519e3b71 100644 --- a/crypto/algapi.c +++ b/crypto/algapi.c @@ -274,8 +274,7 @@ static struct crypto_larval *crypto_alloc_test_larval(struct crypto_alg *alg) { struct crypto_larval *larval; - if (!IS_ENABLED(CONFIG_CRYPTO_MANAGER) || - !IS_ENABLED(CONFIG_CRYPTO_SELFTESTS) || + if (!IS_ENABLED(CONFIG_CRYPTO_SELFTESTS) || (alg->cra_flags & CRYPTO_ALG_INTERNAL)) return NULL; /* No self-test needed */ From 33cd93435cea665b24ca3f9b3d6af42afb3ba7bc Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Thu, 8 May 2025 09:26:25 +0300 Subject: [PATCH 342/382] crypto: qat/qat_6xxx - Fix NULL vs IS_ERR() check in adf_probe() The pcim_iomap_region() returns error pointers. It doesn't return NULL pointers. Update the check to match. Fixes: 17fd7514ae68 ("crypto: qat - add qat_6xxx driver") Signed-off-by: Dan Carpenter Acked-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/intel/qat/qat_6xxx/adf_drv.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/intel/qat/qat_6xxx/adf_drv.c b/drivers/crypto/intel/qat/qat_6xxx/adf_drv.c index 2531c337e0dd..132e26501621 100644 --- a/drivers/crypto/intel/qat/qat_6xxx/adf_drv.c +++ b/drivers/crypto/intel/qat/qat_6xxx/adf_drv.c @@ -156,8 +156,8 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) /* Map 64-bit PCIe BAR */ bar->virt_addr = pcim_iomap_region(pdev, bar_map[i], pci_name(pdev)); - if (!bar->virt_addr) { - ret = -ENOMEM; + if (IS_ERR(bar->virt_addr)) { + ret = PTR_ERR(bar->virt_addr); return dev_err_probe(dev, ret, "Failed to ioremap PCI region.\n"); } } From 714ca27e9bf4608fcb1f627cd5599441f448771e Mon Sep 17 00:00:00 2001 From: Yury Norov Date: Thu, 8 May 2025 15:59:50 -0400 Subject: [PATCH 343/382] crypto: iaa - Optimize rebalance_wq_table() The function opencodes for_each_cpu() by using a plain for-loop. The loop calls cpumask_weight() inside the conditional section. Because cpumask_weight() is O(1), the overall complexity of the function is O(node * node_cpus^2). Also, cpumask_nth() internally calls hweight(), which, if not hardware accelerated, is slower than cpumask_next() in for_each_cpu(). If switched to the dedicated for_each_cpu(), the rebalance_wq_table() can drop calling cpumask_weight(), together with some housekeeping code. This makes the overall complexity O(node * node_cpus), or simply speaking O(nr_cpu_ids). While there, fix opencoded for_each_possible_cpu() too. Signed-off-by: Yury Norov Signed-off-by: Herbert Xu --- drivers/crypto/intel/iaa/iaa_crypto_main.c | 35 +++++++++------------- 1 file changed, 14 insertions(+), 21 deletions(-) diff --git a/drivers/crypto/intel/iaa/iaa_crypto_main.c b/drivers/crypto/intel/iaa/iaa_crypto_main.c index e9c762124f11..23f585219fb4 100644 --- a/drivers/crypto/intel/iaa/iaa_crypto_main.c +++ b/drivers/crypto/intel/iaa/iaa_crypto_main.c @@ -894,7 +894,7 @@ out: static void rebalance_wq_table(void) { const struct cpumask *node_cpus; - int node, cpu, iaa = -1; + int node_cpu, node, cpu, iaa = 0; if (nr_iaa == 0) return; @@ -905,36 +905,29 @@ static void rebalance_wq_table(void) clear_wq_table(); if (nr_iaa == 1) { - for (cpu = 0; cpu < nr_cpus; cpu++) { - if (WARN_ON(wq_table_add_wqs(0, cpu))) { - pr_debug("could not add any wqs for iaa 0 to cpu %d!\n", cpu); - return; - } + for_each_possible_cpu(cpu) { + if (WARN_ON(wq_table_add_wqs(0, cpu))) + goto err; } return; } for_each_node_with_cpus(node) { + cpu = 0; node_cpus = cpumask_of_node(node); - for (cpu = 0; cpu < cpumask_weight(node_cpus); cpu++) { - int node_cpu = cpumask_nth(cpu, node_cpus); - - if (WARN_ON(node_cpu >= nr_cpu_ids)) { - pr_debug("node_cpu %d doesn't exist!\n", node_cpu); - return; - } - - if ((cpu % cpus_per_iaa) == 0) - iaa++; - - if (WARN_ON(wq_table_add_wqs(iaa, node_cpu))) { - pr_debug("could not add any wqs for iaa %d to cpu %d!\n", iaa, cpu); - return; - } + for_each_cpu(node_cpu, node_cpus) { + iaa = cpu / cpus_per_iaa; + if (WARN_ON(wq_table_add_wqs(iaa, node_cpu))) + goto err; + cpu++; } } + + return; +err: + pr_debug("could not add any wqs for iaa %d to cpu %d!\n", iaa, cpu); } static inline int check_completion(struct device *dev, From c66d7ebbe2fa14e41913adb421090a7426f59786 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sat, 10 May 2025 17:13:47 +0800 Subject: [PATCH 344/382] crypto: powerpc/poly1305 - Add SIMD fallback Add a SIMD fallback path for poly1305-p10 by converting the 2^64 based hash state into a 2^44 base. In order to ensure that the generic fallback is actually 2^44, add ARCH_SUPPORTS_INT128 to powerpc and make poly1305-p10 depend on it. Fixes: ba8f8624fde2 ("crypto: poly1305-p10 - Glue code for optmized Poly1305 implementation for ppc64le") Signed-off-by: Herbert Xu --- arch/powerpc/Kconfig | 1 + arch/powerpc/lib/crypto/Kconfig | 2 +- arch/powerpc/lib/crypto/poly1305-p10-glue.c | 41 +++++++++++++++++++-- 3 files changed, 40 insertions(+), 4 deletions(-) diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig index 6722625a406a..651e0c32957a 100644 --- a/arch/powerpc/Kconfig +++ b/arch/powerpc/Kconfig @@ -173,6 +173,7 @@ config PPC select ARCH_STACKWALK select ARCH_SUPPORTS_ATOMIC_RMW select ARCH_SUPPORTS_DEBUG_PAGEALLOC if PPC_BOOK3S || PPC_8xx + select ARCH_SUPPORTS_INT128 if PPC64 && CC_HAS_INT128 select ARCH_USE_BUILTIN_BSWAP select ARCH_USE_CMPXCHG_LOCKREF if PPC64 select ARCH_USE_MEMTEST diff --git a/arch/powerpc/lib/crypto/Kconfig b/arch/powerpc/lib/crypto/Kconfig index ffa541ad6d5d..6761fdb6193c 100644 --- a/arch/powerpc/lib/crypto/Kconfig +++ b/arch/powerpc/lib/crypto/Kconfig @@ -9,7 +9,7 @@ config CRYPTO_CHACHA20_P10 config CRYPTO_POLY1305_P10 tristate - depends on PPC64 && CPU_LITTLE_ENDIAN && VSX + depends on PPC64 && CPU_LITTLE_ENDIAN && VSX && ARCH_SUPPORTS_INT128 default CRYPTO_LIB_POLY1305 select CRYPTO_ARCH_HAVE_LIB_POLY1305 select CRYPTO_LIB_POLY1305_GENERIC diff --git a/arch/powerpc/lib/crypto/poly1305-p10-glue.c b/arch/powerpc/lib/crypto/poly1305-p10-glue.c index 3f1664a724b6..280c10c48c53 100644 --- a/arch/powerpc/lib/crypto/poly1305-p10-glue.c +++ b/arch/powerpc/lib/crypto/poly1305-p10-glue.c @@ -6,6 +6,7 @@ */ #include #include +#include #include #include #include @@ -18,6 +19,11 @@ asmlinkage void poly1305_emit_64(const struct poly1305_state *state, const u32 n static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_p10); +static inline bool is_state_base64(struct poly1305_block_state *state) +{ + return state->core_r.precomputed_s.r64[2]; +} + static void vsx_begin(void) { preempt_disable(); @@ -30,12 +36,35 @@ static void vsx_end(void) preempt_enable(); } +static void convert_to_base2_44(struct poly1305_block_state *state) +{ + u8 raw_key[POLY1305_BLOCK_SIZE]; + u64 h0, h1, h2; + + if (!is_state_base64(state)) + return; + + state->core_r.precomputed_s.r64[2] = 0; + put_unaligned_le64(state->core_r.key.r64[0], raw_key + 0); + put_unaligned_le64(state->core_r.key.r64[1], raw_key + 8); + poly1305_core_setkey(&state->core_r, raw_key); + + h0 = state->h.h64[0]; + h1 = state->h.h64[1]; + h2 = state->h.h64[2]; + state->h.h64[0] = h0 & 0xfffffffffffULL; + state->h.h64[1] = h0 >> 44 | (h1 & 0xffffffULL) << 20; + state->h.h64[2] = h1 >> 24 | h2 << 40; +} + void poly1305_block_init_arch(struct poly1305_block_state *dctx, const u8 raw_key[POLY1305_BLOCK_SIZE]) { - if (!static_key_enabled(&have_p10)) + dctx->core_r.precomputed_s.r64[2] = 0; + if (!static_key_enabled(&have_p10) || !crypto_simd_usable()) return poly1305_block_init_generic(dctx, raw_key); + dctx->core_r.precomputed_s.r64[2] = 1; dctx->h = (struct poly1305_state){}; dctx->core_r.key.r64[0] = get_unaligned_le64(raw_key + 0); dctx->core_r.key.r64[1] = get_unaligned_le64(raw_key + 8); @@ -45,8 +74,11 @@ EXPORT_SYMBOL_GPL(poly1305_block_init_arch); void poly1305_blocks_arch(struct poly1305_block_state *state, const u8 *src, unsigned int len, u32 padbit) { - if (!static_key_enabled(&have_p10)) + if (!static_key_enabled(&have_p10) || !is_state_base64(state) || + !crypto_simd_usable()) { + convert_to_base2_44(state); return poly1305_blocks_generic(state, src, len, padbit); + } vsx_begin(); if (len >= POLY1305_BLOCK_SIZE * 4) { poly1305_p10le_4blocks(state, src, len); @@ -66,7 +98,10 @@ void poly1305_emit_arch(const struct poly1305_state *state, u8 digest[POLY1305_DIGEST_SIZE], const u32 nonce[4]) { - if (!static_key_enabled(&have_p10)) + struct poly1305_block_state *dctx = + container_of(state, struct poly1305_block_state, h); + + if (!static_key_enabled(&have_p10) || !is_state_base64(dctx)) return poly1305_emit_generic(state, digest, nonce); poly1305_emit_64(state, nonce, digest); } From 8a4e047c6cc07676f637608a9dd675349b5de0a7 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sat, 10 May 2025 18:41:31 +0800 Subject: [PATCH 345/382] crypto: marvell/cesa - Handle zero-length skcipher requests Do not access random memory for zero-length skcipher requests. Just return 0. Fixes: f63601fd616a ("crypto: marvell/cesa - add a new driver for Marvell's CESA") Signed-off-by: Herbert Xu --- drivers/crypto/marvell/cesa/cipher.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/crypto/marvell/cesa/cipher.c b/drivers/crypto/marvell/cesa/cipher.c index cf62db50f958..48c5c8ea8c43 100644 --- a/drivers/crypto/marvell/cesa/cipher.c +++ b/drivers/crypto/marvell/cesa/cipher.c @@ -459,6 +459,9 @@ static int mv_cesa_skcipher_queue_req(struct skcipher_request *req, struct mv_cesa_skcipher_req *creq = skcipher_request_ctx(req); struct mv_cesa_engine *engine; + if (!req->cryptlen) + return 0; + ret = mv_cesa_skcipher_req_init(req, tmpl); if (ret) return ret; From 1bafd82d9a40cf09c6c40f1c09cc35b7050b1a9f Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sat, 10 May 2025 18:43:33 +0800 Subject: [PATCH 346/382] crypto: marvell/cesa - Avoid empty transfer descriptor The user may set req->src even if req->nbytes == 0. If there is no data to hash from req->src, do not generate an empty TDMA descriptor. Fixes: db509a45339f ("crypto: marvell/cesa - add TDMA support") Signed-off-by: Herbert Xu --- drivers/crypto/marvell/cesa/hash.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/marvell/cesa/hash.c b/drivers/crypto/marvell/cesa/hash.c index f150861ceaf6..6815eddc9068 100644 --- a/drivers/crypto/marvell/cesa/hash.c +++ b/drivers/crypto/marvell/cesa/hash.c @@ -663,7 +663,7 @@ static int mv_cesa_ahash_dma_req_init(struct ahash_request *req) if (ret) goto err_free_tdma; - if (iter.src.sg) { + if (iter.base.len > iter.src.op_offset) { /* * Add all the new data, inserting an operation block and * launch command between each full SRAM block-worth of From 63935e2ee1f2a371e511f853737b9efebc238bc7 Mon Sep 17 00:00:00 2001 From: John Allen Date: Mon, 12 May 2025 18:17:05 +0000 Subject: [PATCH 347/382] crypto: ccp - Add support for PCI device 0x17D8 Add a new CCP/PSP PCI device ID. Signed-off-by: John Allen Signed-off-by: Herbert Xu --- drivers/crypto/ccp/sp-pci.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/crypto/ccp/sp-pci.c b/drivers/crypto/ccp/sp-pci.c index 2ebc878da160..54ab3279f521 100644 --- a/drivers/crypto/ccp/sp-pci.c +++ b/drivers/crypto/ccp/sp-pci.c @@ -535,6 +535,7 @@ static const struct pci_device_id sp_pci_table[] = { { PCI_VDEVICE(AMD, 0x1134), (kernel_ulong_t)&dev_vdata[7] }, { PCI_VDEVICE(AMD, 0x17E0), (kernel_ulong_t)&dev_vdata[7] }, { PCI_VDEVICE(AMD, 0x156E), (kernel_ulong_t)&dev_vdata[8] }, + { PCI_VDEVICE(AMD, 0x17D8), (kernel_ulong_t)&dev_vdata[8] }, /* Last entry must be zero */ { 0, } }; From 7f3401d0a506dac688948a62ef8ef7b18cc791e1 Mon Sep 17 00:00:00 2001 From: Suman Kumar Chakraborty Date: Tue, 13 May 2025 11:25:26 +0100 Subject: [PATCH 348/382] crypto: qat - enable RAS support for GEN6 devices Enable the reporting and handling of errors for QAT GEN6 devices. Errors are categorized as correctable, non-fatal, or fatal. Error handling involves reading the error source registers (ERRSOU0 to ERRSOU3) to determine the source of the error and then decoding the actual source reading specific registers. The action taken depends on the error type: - Correctable and Non-Fatal errors. These error are logged, cleared and the corresponding counter is incremented. - Fatal errors. These errors are logged, cleared and a Function Level Reset (FLR) is scheduled. This reports and handles the following errors: - Accelerator engine (AE) correctable errors - Accelerator engine (AE) uncorrectable errors - Chassis push-pull (CPP) errors - Host interface (HI) parity errors - Internal memory parity errors - Receive interface (RI) errors - Transmit interface (TI) errors - Interface for system-on-chip (SoC) fabric (IOSF) primary command parity errors - Shared RAM and slice module (SSM) errors Signed-off-by: Suman Kumar Chakraborty Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- .../intel/qat/qat_6xxx/adf_6xxx_hw_data.c | 2 + drivers/crypto/intel/qat/qat_common/Makefile | 1 + .../intel/qat/qat_common/adf_gen6_ras.c | 818 ++++++++++++++++++ .../intel/qat/qat_common/adf_gen6_ras.h | 504 +++++++++++ 4 files changed, 1325 insertions(+) create mode 100644 drivers/crypto/intel/qat/qat_common/adf_gen6_ras.c create mode 100644 drivers/crypto/intel/qat/qat_common/adf_gen6_ras.h diff --git a/drivers/crypto/intel/qat/qat_6xxx/adf_6xxx_hw_data.c b/drivers/crypto/intel/qat/qat_6xxx/adf_6xxx_hw_data.c index 73d479383b1f..359a6447ccb8 100644 --- a/drivers/crypto/intel/qat/qat_6xxx/adf_6xxx_hw_data.c +++ b/drivers/crypto/intel/qat/qat_6xxx/adf_6xxx_hw_data.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include "adf_6xxx_hw_data.h" @@ -834,6 +835,7 @@ void adf_init_hw_data_6xxx(struct adf_hw_device_data *hw_data) adf_gen6_init_hw_csr_ops(&hw_data->csr_ops); adf_gen6_init_pf_pfvf_ops(&hw_data->pfvf_ops); adf_gen6_init_dc_ops(&hw_data->dc_ops); + adf_gen6_init_ras_ops(&hw_data->ras_ops); } void adf_clean_hw_data_6xxx(struct adf_hw_device_data *hw_data) diff --git a/drivers/crypto/intel/qat/qat_common/Makefile b/drivers/crypto/intel/qat/qat_common/Makefile index 958f9a8ac6b3..66bb295ace28 100644 --- a/drivers/crypto/intel/qat/qat_common/Makefile +++ b/drivers/crypto/intel/qat/qat_common/Makefile @@ -19,6 +19,7 @@ intel_qat-y := adf_accel_engine.o \ adf_gen4_pm.o \ adf_gen4_ras.o \ adf_gen4_vf_mig.o \ + adf_gen6_ras.o \ adf_gen6_shared.o \ adf_hw_arbiter.o \ adf_init.o \ diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen6_ras.c b/drivers/crypto/intel/qat/qat_common/adf_gen6_ras.c new file mode 100644 index 000000000000..967253082a98 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_gen6_ras.c @@ -0,0 +1,818 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2025 Intel Corporation */ +#include +#include + +#include "adf_common_drv.h" +#include "adf_gen6_ras.h" +#include "adf_sysfs_ras_counters.h" + +static void enable_errsou_reporting(void __iomem *csr) +{ + /* Enable correctable error reporting in ERRSOU0 */ + ADF_CSR_WR(csr, ADF_GEN6_ERRMSK0, 0); + + /* Enable uncorrectable error reporting in ERRSOU1 */ + ADF_CSR_WR(csr, ADF_GEN6_ERRMSK1, 0); + + /* + * Enable uncorrectable error reporting in ERRSOU2 + * but disable PM interrupt by default + */ + ADF_CSR_WR(csr, ADF_GEN6_ERRMSK2, ADF_GEN6_ERRSOU2_PM_INT_BIT); + + /* Enable uncorrectable error reporting in ERRSOU3 */ + ADF_CSR_WR(csr, ADF_GEN6_ERRMSK3, 0); +} + +static void enable_ae_error_reporting(struct adf_accel_dev *accel_dev, void __iomem *csr) +{ + u32 ae_mask = GET_HW_DATA(accel_dev)->ae_mask; + + /* Enable acceleration engine correctable error reporting */ + ADF_CSR_WR(csr, ADF_GEN6_HIAECORERRLOGENABLE_CPP0, ae_mask); + + /* Enable acceleration engine uncorrectable error reporting */ + ADF_CSR_WR(csr, ADF_GEN6_HIAEUNCERRLOGENABLE_CPP0, ae_mask); +} + +static void enable_cpp_error_reporting(struct adf_accel_dev *accel_dev, void __iomem *csr) +{ + /* Enable HI CPP agents command parity error reporting */ + ADF_CSR_WR(csr, ADF_GEN6_HICPPAGENTCMDPARERRLOGENABLE, + ADF_6XXX_HICPPAGENTCMDPARERRLOG_MASK); + + ADF_CSR_WR(csr, ADF_GEN6_CPP_CFC_ERR_CTRL, ADF_GEN6_CPP_CFC_ERR_CTRL_MASK); +} + +static void enable_ti_ri_error_reporting(void __iomem *csr) +{ + u32 reg, mask; + + /* Enable RI memory error reporting */ + mask = ADF_GEN6_RIMEM_PARERR_FATAL_MASK | ADF_GEN6_RIMEM_PARERR_CERR_MASK; + ADF_CSR_WR(csr, ADF_GEN6_RI_MEM_PAR_ERR_EN0, mask); + + /* Enable IOSF primary command parity error reporting */ + ADF_CSR_WR(csr, ADF_GEN6_RIMISCCTL, ADF_GEN6_RIMISCSTS_BIT); + + /* Enable TI internal memory parity error reporting */ + reg = ADF_CSR_RD(csr, ADF_GEN6_TI_CI_PAR_ERR_MASK); + reg &= ~ADF_GEN6_TI_CI_PAR_STS_MASK; + ADF_CSR_WR(csr, ADF_GEN6_TI_CI_PAR_ERR_MASK, reg); + + reg = ADF_CSR_RD(csr, ADF_GEN6_TI_PULL0FUB_PAR_ERR_MASK); + reg &= ~ADF_GEN6_TI_PULL0FUB_PAR_STS_MASK; + ADF_CSR_WR(csr, ADF_GEN6_TI_PULL0FUB_PAR_ERR_MASK, reg); + + reg = ADF_CSR_RD(csr, ADF_GEN6_TI_PUSHFUB_PAR_ERR_MASK); + reg &= ~ADF_GEN6_TI_PUSHFUB_PAR_STS_MASK; + ADF_CSR_WR(csr, ADF_GEN6_TI_PUSHFUB_PAR_ERR_MASK, reg); + + reg = ADF_CSR_RD(csr, ADF_GEN6_TI_CD_PAR_ERR_MASK); + reg &= ~ADF_GEN6_TI_CD_PAR_STS_MASK; + ADF_CSR_WR(csr, ADF_GEN6_TI_CD_PAR_ERR_MASK, reg); + + reg = ADF_CSR_RD(csr, ADF_GEN6_TI_TRNSB_PAR_ERR_MASK); + reg &= ~ADF_GEN6_TI_TRNSB_PAR_STS_MASK; + ADF_CSR_WR(csr, ADF_GEN6_TI_TRNSB_PAR_ERR_MASK, reg); + + /* Enable error handling in RI, TI CPP interface control registers */ + ADF_CSR_WR(csr, ADF_GEN6_RICPPINTCTL, ADF_GEN6_RICPPINTCTL_MASK); + ADF_CSR_WR(csr, ADF_GEN6_TICPPINTCTL, ADF_GEN6_TICPPINTCTL_MASK); + + /* + * Enable error detection and reporting in TIMISCSTS + * with bits 1, 2 and 30 value preserved + */ + reg = ADF_CSR_RD(csr, ADF_GEN6_TIMISCCTL); + reg &= ADF_GEN6_TIMSCCTL_RELAY_MASK; + reg |= ADF_GEN6_TIMISCCTL_BIT; + ADF_CSR_WR(csr, ADF_GEN6_TIMISCCTL, reg); +} + +static void enable_ssm_error_reporting(struct adf_accel_dev *accel_dev, + void __iomem *csr) +{ + /* Enable SSM interrupts */ + ADF_CSR_WR(csr, ADF_GEN6_INTMASKSSM, 0); +} + +static void adf_gen6_enable_ras(struct adf_accel_dev *accel_dev) +{ + void __iomem *csr = adf_get_pmisc_base(accel_dev); + + enable_errsou_reporting(csr); + enable_ae_error_reporting(accel_dev, csr); + enable_cpp_error_reporting(accel_dev, csr); + enable_ti_ri_error_reporting(csr); + enable_ssm_error_reporting(accel_dev, csr); +} + +static void disable_errsou_reporting(void __iomem *csr) +{ + u32 val; + + /* Disable correctable error reporting in ERRSOU0 */ + ADF_CSR_WR(csr, ADF_GEN6_ERRMSK0, ADF_GEN6_ERRSOU0_MASK); + + /* Disable uncorrectable error reporting in ERRSOU1 */ + ADF_CSR_WR(csr, ADF_GEN6_ERRMSK1, ADF_GEN6_ERRMSK1_MASK); + + /* Disable uncorrectable error reporting in ERRSOU2 */ + val = ADF_CSR_RD(csr, ADF_GEN6_ERRMSK2); + val |= ADF_GEN6_ERRSOU2_DIS_MASK; + ADF_CSR_WR(csr, ADF_GEN6_ERRMSK2, val); + + /* Disable uncorrectable error reporting in ERRSOU3 */ + ADF_CSR_WR(csr, ADF_GEN6_ERRMSK3, ADF_GEN6_ERRSOU3_DIS_MASK); +} + +static void disable_ae_error_reporting(void __iomem *csr) +{ + /* Disable acceleration engine correctable error reporting */ + ADF_CSR_WR(csr, ADF_GEN6_HIAECORERRLOGENABLE_CPP0, 0); + + /* Disable acceleration engine uncorrectable error reporting */ + ADF_CSR_WR(csr, ADF_GEN6_HIAEUNCERRLOGENABLE_CPP0, 0); +} + +static void disable_cpp_error_reporting(void __iomem *csr) +{ + /* Disable HI CPP agents command parity error reporting */ + ADF_CSR_WR(csr, ADF_GEN6_HICPPAGENTCMDPARERRLOGENABLE, 0); + + ADF_CSR_WR(csr, ADF_GEN6_CPP_CFC_ERR_CTRL, ADF_GEN6_CPP_CFC_ERR_CTRL_DIS_MASK); +} + +static void disable_ti_ri_error_reporting(void __iomem *csr) +{ + u32 reg; + + /* Disable RI memory error reporting */ + ADF_CSR_WR(csr, ADF_GEN6_RI_MEM_PAR_ERR_EN0, 0); + + /* Disable IOSF primary command parity error reporting */ + reg = ADF_CSR_RD(csr, ADF_GEN6_RIMISCCTL); + reg &= ~ADF_GEN6_RIMISCSTS_BIT; + ADF_CSR_WR(csr, ADF_GEN6_RIMISCCTL, reg); + + /* Disable TI internal memory parity error reporting */ + ADF_CSR_WR(csr, ADF_GEN6_TI_CI_PAR_ERR_MASK, ADF_GEN6_TI_CI_PAR_STS_MASK); + ADF_CSR_WR(csr, ADF_GEN6_TI_PULL0FUB_PAR_ERR_MASK, ADF_GEN6_TI_PULL0FUB_PAR_STS_MASK); + ADF_CSR_WR(csr, ADF_GEN6_TI_PUSHFUB_PAR_ERR_MASK, ADF_GEN6_TI_PUSHFUB_PAR_STS_MASK); + ADF_CSR_WR(csr, ADF_GEN6_TI_CD_PAR_ERR_MASK, ADF_GEN6_TI_CD_PAR_STS_MASK); + ADF_CSR_WR(csr, ADF_GEN6_TI_TRNSB_PAR_ERR_MASK, ADF_GEN6_TI_TRNSB_PAR_STS_MASK); + + /* Disable error handling in RI, TI CPP interface control registers */ + reg = ADF_CSR_RD(csr, ADF_GEN6_RICPPINTCTL); + reg &= ~ADF_GEN6_RICPPINTCTL_MASK; + ADF_CSR_WR(csr, ADF_GEN6_RICPPINTCTL, reg); + + reg = ADF_CSR_RD(csr, ADF_GEN6_TICPPINTCTL); + reg &= ~ADF_GEN6_TICPPINTCTL_MASK; + ADF_CSR_WR(csr, ADF_GEN6_TICPPINTCTL, reg); + + /* + * Disable error detection and reporting in TIMISCSTS + * with bits 1, 2 and 30 value preserved + */ + reg = ADF_CSR_RD(csr, ADF_GEN6_TIMISCCTL); + reg &= ADF_GEN6_TIMSCCTL_RELAY_MASK; + ADF_CSR_WR(csr, ADF_GEN6_TIMISCCTL, reg); +} + +static void disable_ssm_error_reporting(void __iomem *csr) +{ + /* Disable SSM interrupts */ + ADF_CSR_WR(csr, ADF_GEN6_INTMASKSSM, ADF_GEN6_INTMASKSSM_MASK); +} + +static void adf_gen6_disable_ras(struct adf_accel_dev *accel_dev) +{ + void __iomem *csr = adf_get_pmisc_base(accel_dev); + + disable_errsou_reporting(csr); + disable_ae_error_reporting(csr); + disable_cpp_error_reporting(csr); + disable_ti_ri_error_reporting(csr); + disable_ssm_error_reporting(csr); +} + +static void adf_gen6_process_errsou0(struct adf_accel_dev *accel_dev, void __iomem *csr) +{ + u32 ae, errsou; + + ae = ADF_CSR_RD(csr, ADF_GEN6_HIAECORERRLOG_CPP0); + ae &= GET_HW_DATA(accel_dev)->ae_mask; + + dev_warn(&GET_DEV(accel_dev), "Correctable error detected: %#x\n", ae); + + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_CORR); + + /* Clear interrupt from ERRSOU0 */ + ADF_CSR_WR(csr, ADF_GEN6_HIAECORERRLOG_CPP0, ae); + + errsou = ADF_CSR_RD(csr, ADF_GEN6_ERRSOU0); + if (errsou & ADF_GEN6_ERRSOU0_MASK) + dev_warn(&GET_DEV(accel_dev), "errsou0 still set: %#x\n", errsou); +} + +static void adf_handle_cpp_ae_unc(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + u32 ae; + + if (!(errsou & ADF_GEN6_ERRSOU1_CPP0_MEUNC_BIT)) + return; + + ae = ADF_CSR_RD(csr, ADF_GEN6_HIAEUNCERRLOG_CPP0); + ae &= GET_HW_DATA(accel_dev)->ae_mask; + if (ae) { + dev_err(&GET_DEV(accel_dev), "Uncorrectable error detected: %#x\n", ae); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); + ADF_CSR_WR(csr, ADF_GEN6_HIAEUNCERRLOG_CPP0, ae); + } +} + +static void adf_handle_cpp_cmd_par_err(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + u32 cmd_par_err; + + if (!(errsou & ADF_GEN6_ERRSOU1_CPP_CMDPARERR_BIT)) + return; + + cmd_par_err = ADF_CSR_RD(csr, ADF_GEN6_HICPPAGENTCMDPARERRLOG); + cmd_par_err &= ADF_6XXX_HICPPAGENTCMDPARERRLOG_MASK; + if (cmd_par_err) { + dev_err(&GET_DEV(accel_dev), "HI CPP agent command parity error: %#x\n", + cmd_par_err); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); + ADF_CSR_WR(csr, ADF_GEN6_HICPPAGENTCMDPARERRLOG, cmd_par_err); + } +} + +static void adf_handle_ri_mem_par_err(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + u32 rimem_parerr_sts; + + if (!(errsou & ADF_GEN6_ERRSOU1_RIMEM_PARERR_STS_BIT)) + return; + + rimem_parerr_sts = ADF_CSR_RD(csr, ADF_GEN6_RIMEM_PARERR_STS); + rimem_parerr_sts &= ADF_GEN6_RIMEM_PARERR_CERR_MASK | + ADF_GEN6_RIMEM_PARERR_FATAL_MASK; + if (rimem_parerr_sts & ADF_GEN6_RIMEM_PARERR_CERR_MASK) { + dev_err(&GET_DEV(accel_dev), "RI memory parity correctable error: %#x\n", + rimem_parerr_sts); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_CORR); + } + + if (rimem_parerr_sts & ADF_GEN6_RIMEM_PARERR_FATAL_MASK) { + dev_err(&GET_DEV(accel_dev), "RI memory parity fatal error: %#x\n", + rimem_parerr_sts); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); + } + + ADF_CSR_WR(csr, ADF_GEN6_RIMEM_PARERR_STS, rimem_parerr_sts); +} + +static void adf_handle_ti_ci_par_sts(struct adf_accel_dev *accel_dev, void __iomem *csr) +{ + u32 ti_ci_par_sts; + + ti_ci_par_sts = ADF_CSR_RD(csr, ADF_GEN6_TI_CI_PAR_STS); + ti_ci_par_sts &= ADF_GEN6_TI_CI_PAR_STS_MASK; + if (ti_ci_par_sts) { + dev_err(&GET_DEV(accel_dev), "TI memory parity error: %#x\n", ti_ci_par_sts); + ADF_CSR_WR(csr, ADF_GEN6_TI_CI_PAR_STS, ti_ci_par_sts); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); + } +} + +static void adf_handle_ti_pullfub_par_sts(struct adf_accel_dev *accel_dev, void __iomem *csr) +{ + u32 ti_pullfub_par_sts; + + ti_pullfub_par_sts = ADF_CSR_RD(csr, ADF_GEN6_TI_PULL0FUB_PAR_STS); + ti_pullfub_par_sts &= ADF_GEN6_TI_PULL0FUB_PAR_STS_MASK; + if (ti_pullfub_par_sts) { + dev_err(&GET_DEV(accel_dev), "TI pull parity error: %#x\n", ti_pullfub_par_sts); + ADF_CSR_WR(csr, ADF_GEN6_TI_PULL0FUB_PAR_STS, ti_pullfub_par_sts); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); + } +} + +static void adf_handle_ti_pushfub_par_sts(struct adf_accel_dev *accel_dev, void __iomem *csr) +{ + u32 ti_pushfub_par_sts; + + ti_pushfub_par_sts = ADF_CSR_RD(csr, ADF_GEN6_TI_PUSHFUB_PAR_STS); + ti_pushfub_par_sts &= ADF_GEN6_TI_PUSHFUB_PAR_STS_MASK; + if (ti_pushfub_par_sts) { + dev_err(&GET_DEV(accel_dev), "TI push parity error: %#x\n", ti_pushfub_par_sts); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); + ADF_CSR_WR(csr, ADF_GEN6_TI_PUSHFUB_PAR_STS, ti_pushfub_par_sts); + } +} + +static void adf_handle_ti_cd_par_sts(struct adf_accel_dev *accel_dev, void __iomem *csr) +{ + u32 ti_cd_par_sts; + + ti_cd_par_sts = ADF_CSR_RD(csr, ADF_GEN6_TI_CD_PAR_STS); + ti_cd_par_sts &= ADF_GEN6_TI_CD_PAR_STS_MASK; + if (ti_cd_par_sts) { + dev_err(&GET_DEV(accel_dev), "TI CD parity error: %#x\n", ti_cd_par_sts); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); + ADF_CSR_WR(csr, ADF_GEN6_TI_CD_PAR_STS, ti_cd_par_sts); + } +} + +static void adf_handle_ti_trnsb_par_sts(struct adf_accel_dev *accel_dev, void __iomem *csr) +{ + u32 ti_trnsb_par_sts; + + ti_trnsb_par_sts = ADF_CSR_RD(csr, ADF_GEN6_TI_TRNSB_PAR_STS); + ti_trnsb_par_sts &= ADF_GEN6_TI_TRNSB_PAR_STS_MASK; + if (ti_trnsb_par_sts) { + dev_err(&GET_DEV(accel_dev), "TI TRNSB parity error: %#x\n", ti_trnsb_par_sts); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); + ADF_CSR_WR(csr, ADF_GEN6_TI_TRNSB_PAR_STS, ti_trnsb_par_sts); + } +} + +static void adf_handle_iosfp_cmd_parerr(struct adf_accel_dev *accel_dev, void __iomem *csr) +{ + u32 rimiscsts; + + rimiscsts = ADF_CSR_RD(csr, ADF_GEN6_RIMISCSTS); + rimiscsts &= ADF_GEN6_RIMISCSTS_BIT; + if (rimiscsts) { + dev_err(&GET_DEV(accel_dev), "Command parity error detected on IOSFP: %#x\n", + rimiscsts); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); + ADF_CSR_WR(csr, ADF_GEN6_RIMISCSTS, rimiscsts); + } +} + +static void adf_handle_ti_err(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + if (!(errsou & ADF_GEN6_ERRSOU1_TIMEM_PARERR_STS_BIT)) + return; + + adf_handle_ti_ci_par_sts(accel_dev, csr); + adf_handle_ti_pullfub_par_sts(accel_dev, csr); + adf_handle_ti_pushfub_par_sts(accel_dev, csr); + adf_handle_ti_cd_par_sts(accel_dev, csr); + adf_handle_ti_trnsb_par_sts(accel_dev, csr); + adf_handle_iosfp_cmd_parerr(accel_dev, csr); +} + +static void adf_handle_sfi_cmd_parerr(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + if (!(errsou & ADF_GEN6_ERRSOU1_SFICMD_PARERR_BIT)) + return; + + dev_err(&GET_DEV(accel_dev), + "Command parity error detected on streaming fabric interface\n"); + + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); +} + +static void adf_gen6_process_errsou1(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + adf_handle_cpp_ae_unc(accel_dev, csr, errsou); + adf_handle_cpp_cmd_par_err(accel_dev, csr, errsou); + adf_handle_ri_mem_par_err(accel_dev, csr, errsou); + adf_handle_ti_err(accel_dev, csr, errsou); + adf_handle_sfi_cmd_parerr(accel_dev, csr, errsou); + + errsou = ADF_CSR_RD(csr, ADF_GEN6_ERRSOU1); + if (errsou & ADF_GEN6_ERRSOU1_MASK) + dev_warn(&GET_DEV(accel_dev), "errsou1 still set: %#x\n", errsou); +} + +static void adf_handle_cerrssmsh(struct adf_accel_dev *accel_dev, void __iomem *csr) +{ + u32 reg; + + reg = ADF_CSR_RD(csr, ADF_GEN6_CERRSSMSH); + reg &= ADF_GEN6_CERRSSMSH_ERROR_BIT; + if (reg) { + dev_warn(&GET_DEV(accel_dev), + "Correctable error on ssm shared memory: %#x\n", reg); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_CORR); + ADF_CSR_WR(csr, ADF_GEN6_CERRSSMSH, reg); + } +} + +static void adf_handle_uerrssmsh(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 iastatssm) +{ + u32 reg; + + if (!(iastatssm & ADF_GEN6_IAINTSTATSSM_SH_ERR_BIT)) + return; + + reg = ADF_CSR_RD(csr, ADF_GEN6_UERRSSMSH); + reg &= ADF_GEN6_UERRSSMSH_MASK; + if (reg) { + dev_err(&GET_DEV(accel_dev), + "Fatal error on ssm shared memory: %#x\n", reg); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); + ADF_CSR_WR(csr, ADF_GEN6_UERRSSMSH, reg); + } +} + +static void adf_handle_pperr_err(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 iastatssm) +{ + u32 reg; + + if (!(iastatssm & ADF_GEN6_IAINTSTATSSM_PPERR_BIT)) + return; + + reg = ADF_CSR_RD(csr, ADF_GEN6_PPERR); + reg &= ADF_GEN6_PPERR_MASK; + if (reg) { + dev_err(&GET_DEV(accel_dev), + "Fatal push or pull data error: %#x\n", reg); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); + ADF_CSR_WR(csr, ADF_GEN6_PPERR, reg); + } +} + +static void adf_handle_scmpar_err(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 iastatssm) +{ + u32 reg; + + if (!(iastatssm & ADF_GEN6_IAINTSTATSSM_SCMPAR_ERR_BIT)) + return; + + reg = ADF_CSR_RD(csr, ADF_GEN6_SSM_FERR_STATUS); + reg &= ADF_GEN6_SCM_PAR_ERR_MASK; + if (reg) { + dev_err(&GET_DEV(accel_dev), "Fatal error on SCM: %#x\n", reg); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); + ADF_CSR_WR(csr, ADF_GEN6_SSM_FERR_STATUS, reg); + } +} + +static void adf_handle_cpppar_err(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 iastatssm) +{ + u32 reg; + + if (!(iastatssm & ADF_GEN6_IAINTSTATSSM_CPPPAR_ERR_BIT)) + return; + + reg = ADF_CSR_RD(csr, ADF_GEN6_SSM_FERR_STATUS); + reg &= ADF_GEN6_CPP_PAR_ERR_MASK; + if (reg) { + dev_err(&GET_DEV(accel_dev), "Fatal error on CPP: %#x\n", reg); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); + ADF_CSR_WR(csr, ADF_GEN6_SSM_FERR_STATUS, reg); + } +} + +static void adf_handle_rfpar_err(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 iastatssm) +{ + u32 reg; + + if (!(iastatssm & ADF_GEN6_IAINTSTATSSM_RFPAR_ERR_BIT)) + return; + + reg = ADF_CSR_RD(csr, ADF_GEN6_SSM_FERR_STATUS); + reg &= ADF_GEN6_RF_PAR_ERR_MASK; + if (reg) { + dev_err(&GET_DEV(accel_dev), "Fatal error on RF Parity: %#x\n", reg); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); + ADF_CSR_WR(csr, ADF_GEN6_SSM_FERR_STATUS, reg); + } +} + +static void adf_handle_unexp_cpl_err(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 iastatssm) +{ + u32 reg; + + if (!(iastatssm & ADF_GEN6_IAINTSTATSSM_UNEXP_CPL_ERR_BIT)) + return; + + reg = ADF_CSR_RD(csr, ADF_GEN6_SSM_FERR_STATUS); + reg &= ADF_GEN6_UNEXP_CPL_ERR_MASK; + if (reg) { + dev_err(&GET_DEV(accel_dev), + "Fatal error for AXI unexpected tag/length: %#x\n", reg); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); + ADF_CSR_WR(csr, ADF_GEN6_SSM_FERR_STATUS, reg); + } +} + +static void adf_handle_iaintstatssm(struct adf_accel_dev *accel_dev, void __iomem *csr) +{ + u32 iastatssm = ADF_CSR_RD(csr, ADF_GEN6_IAINTSTATSSM); + + iastatssm &= ADF_GEN6_IAINTSTATSSM_MASK; + if (!iastatssm) + return; + + adf_handle_uerrssmsh(accel_dev, csr, iastatssm); + adf_handle_pperr_err(accel_dev, csr, iastatssm); + adf_handle_scmpar_err(accel_dev, csr, iastatssm); + adf_handle_cpppar_err(accel_dev, csr, iastatssm); + adf_handle_rfpar_err(accel_dev, csr, iastatssm); + adf_handle_unexp_cpl_err(accel_dev, csr, iastatssm); + + ADF_CSR_WR(csr, ADF_GEN6_IAINTSTATSSM, iastatssm); +} + +static void adf_handle_ssm(struct adf_accel_dev *accel_dev, void __iomem *csr, u32 errsou) +{ + if (!(errsou & ADF_GEN6_ERRSOU2_SSM_ERR_BIT)) + return; + + adf_handle_cerrssmsh(accel_dev, csr); + adf_handle_iaintstatssm(accel_dev, csr); +} + +static void adf_handle_cpp_cfc_err(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + u32 reg; + + if (!(errsou & ADF_GEN6_ERRSOU2_CPP_CFC_ERR_STATUS_BIT)) + return; + + reg = ADF_CSR_RD(csr, ADF_GEN6_CPP_CFC_ERR_STATUS); + if (reg & ADF_GEN6_CPP_CFC_ERR_STATUS_DATAPAR_BIT) { + dev_err(&GET_DEV(accel_dev), "CPP_CFC_ERR: data parity: %#x", reg); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); + } + + if (reg & ADF_GEN6_CPP_CFC_ERR_STATUS_CMDPAR_BIT) { + dev_err(&GET_DEV(accel_dev), "CPP_CFC_ERR: command parity: %#x", reg); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); + } + + if (reg & ADF_GEN6_CPP_CFC_FATAL_ERR_BIT) { + dev_err(&GET_DEV(accel_dev), "CPP_CFC_ERR: errors: %#x", reg); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); + } + + ADF_CSR_WR(csr, ADF_GEN6_CPP_CFC_ERR_STATUS_CLR, + ADF_GEN6_CPP_CFC_ERR_STATUS_CLR_MASK); +} + +static void adf_gen6_process_errsou2(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + adf_handle_ssm(accel_dev, csr, errsou); + adf_handle_cpp_cfc_err(accel_dev, csr, errsou); + + errsou = ADF_CSR_RD(csr, ADF_GEN6_ERRSOU2); + if (errsou & ADF_GEN6_ERRSOU2_MASK) + dev_warn(&GET_DEV(accel_dev), "errsou2 still set: %#x\n", errsou); +} + +static void adf_handle_timiscsts(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + u32 timiscsts; + + if (!(errsou & ADF_GEN6_ERRSOU3_TIMISCSTS_BIT)) + return; + + timiscsts = ADF_CSR_RD(csr, ADF_GEN6_TIMISCSTS); + if (timiscsts) { + dev_err(&GET_DEV(accel_dev), "Fatal error in transmit interface: %#x\n", + timiscsts); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); + } +} + +static void adf_handle_ricppintsts(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + u32 ricppintsts; + + if (!(errsou & ADF_GEN6_ERRSOU3_RICPPINTSTS_MASK)) + return; + + ricppintsts = ADF_CSR_RD(csr, ADF_GEN6_RICPPINTSTS); + ricppintsts &= ADF_GEN6_RICPPINTSTS_MASK; + if (ricppintsts) { + dev_err(&GET_DEV(accel_dev), "RI push pull error: %#x\n", ricppintsts); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); + ADF_CSR_WR(csr, ADF_GEN6_RICPPINTSTS, ricppintsts); + } +} + +static void adf_handle_ticppintsts(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + u32 ticppintsts; + + if (!(errsou & ADF_GEN6_ERRSOU3_TICPPINTSTS_MASK)) + return; + + ticppintsts = ADF_CSR_RD(csr, ADF_GEN6_TICPPINTSTS); + ticppintsts &= ADF_GEN6_TICPPINTSTS_MASK; + if (ticppintsts) { + dev_err(&GET_DEV(accel_dev), "TI push pull error: %#x\n", ticppintsts); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); + ADF_CSR_WR(csr, ADF_GEN6_TICPPINTSTS, ticppintsts); + } +} + +static void adf_handle_atufaultstatus(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + u32 max_rp_num = GET_HW_DATA(accel_dev)->num_banks; + u32 atufaultstatus; + u32 i; + + if (!(errsou & ADF_GEN6_ERRSOU3_ATUFAULTSTATUS_BIT)) + return; + + for (i = 0; i < max_rp_num; i++) { + atufaultstatus = ADF_CSR_RD(csr, ADF_GEN6_ATUFAULTSTATUS(i)); + + atufaultstatus &= ADF_GEN6_ATUFAULTSTATUS_BIT; + if (atufaultstatus) { + dev_err(&GET_DEV(accel_dev), "Ring pair (%u) ATU detected fault: %#x\n", i, + atufaultstatus); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); + ADF_CSR_WR(csr, ADF_GEN6_ATUFAULTSTATUS(i), atufaultstatus); + } + } +} + +static void adf_handle_rlterror(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + u32 rlterror; + + if (!(errsou & ADF_GEN6_ERRSOU3_RLTERROR_BIT)) + return; + + rlterror = ADF_CSR_RD(csr, ADF_GEN6_RLT_ERRLOG); + rlterror &= ADF_GEN6_RLT_ERRLOG_MASK; + if (rlterror) { + dev_err(&GET_DEV(accel_dev), "Error in rate limiting block: %#x\n", rlterror); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); + ADF_CSR_WR(csr, ADF_GEN6_RLT_ERRLOG, rlterror); + } +} + +static void adf_handle_vflr(struct adf_accel_dev *accel_dev, void __iomem *csr, u32 errsou) +{ + if (!(errsou & ADF_GEN6_ERRSOU3_VFLRNOTIFY_BIT)) + return; + + dev_err(&GET_DEV(accel_dev), "Uncorrectable error in VF\n"); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); +} + +static void adf_handle_tc_vc_map_error(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + if (!(errsou & ADF_GEN6_ERRSOU3_TC_VC_MAP_ERROR_BIT)) + return; + + dev_err(&GET_DEV(accel_dev), "Violation of PCIe TC VC mapping\n"); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); +} + +static void adf_handle_pcie_devhalt(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + if (!(errsou & ADF_GEN6_ERRSOU3_PCIE_DEVHALT_BIT)) + return; + + dev_err(&GET_DEV(accel_dev), + "DEVHALT due to an error in an incoming transaction\n"); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); +} + +static void adf_handle_pg_req_devhalt(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + if (!(errsou & ADF_GEN6_ERRSOU3_PG_REQ_DEVHALT_BIT)) + return; + + dev_err(&GET_DEV(accel_dev), + "Error due to response failure in response to a page request\n"); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); +} + +static void adf_handle_xlt_cpl_devhalt(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + if (!(errsou & ADF_GEN6_ERRSOU3_XLT_CPL_DEVHALT_BIT)) + return; + + dev_err(&GET_DEV(accel_dev), "Error status for a address translation request\n"); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); +} + +static void adf_handle_ti_int_err_devhalt(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + if (!(errsou & ADF_GEN6_ERRSOU3_TI_INT_ERR_DEVHALT_BIT)) + return; + + dev_err(&GET_DEV(accel_dev), "DEVHALT due to a TI internal memory error\n"); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); +} + +static void adf_gen6_process_errsou3(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + adf_handle_timiscsts(accel_dev, csr, errsou); + adf_handle_ricppintsts(accel_dev, csr, errsou); + adf_handle_ticppintsts(accel_dev, csr, errsou); + adf_handle_atufaultstatus(accel_dev, csr, errsou); + adf_handle_rlterror(accel_dev, csr, errsou); + adf_handle_vflr(accel_dev, csr, errsou); + adf_handle_tc_vc_map_error(accel_dev, csr, errsou); + adf_handle_pcie_devhalt(accel_dev, csr, errsou); + adf_handle_pg_req_devhalt(accel_dev, csr, errsou); + adf_handle_xlt_cpl_devhalt(accel_dev, csr, errsou); + adf_handle_ti_int_err_devhalt(accel_dev, csr, errsou); + + errsou = ADF_CSR_RD(csr, ADF_GEN6_ERRSOU3); + if (errsou & ADF_GEN6_ERRSOU3_MASK) + dev_warn(&GET_DEV(accel_dev), "errsou3 still set: %#x\n", errsou); +} + +static void adf_gen6_is_reset_required(struct adf_accel_dev *accel_dev, void __iomem *csr, + bool *reset_required) +{ + u8 reset, dev_state; + u32 gensts; + + gensts = ADF_CSR_RD(csr, ADF_GEN6_GENSTS); + dev_state = FIELD_GET(ADF_GEN6_GENSTS_DEVICE_STATE_MASK, gensts); + reset = FIELD_GET(ADF_GEN6_GENSTS_RESET_TYPE_MASK, gensts); + if (dev_state == ADF_GEN6_GENSTS_DEVHALT && reset == ADF_GEN6_GENSTS_PFLR) { + *reset_required = true; + return; + } + + if (reset == ADF_GEN6_GENSTS_COLD_RESET) + dev_err(&GET_DEV(accel_dev), "Fatal error, cold reset required\n"); + + *reset_required = false; +} + +static bool adf_gen6_handle_interrupt(struct adf_accel_dev *accel_dev, bool *reset_required) +{ + void __iomem *csr = adf_get_pmisc_base(accel_dev); + bool handled = false; + u32 errsou; + + errsou = ADF_CSR_RD(csr, ADF_GEN6_ERRSOU0); + if (errsou & ADF_GEN6_ERRSOU0_MASK) { + adf_gen6_process_errsou0(accel_dev, csr); + handled = true; + } + + errsou = ADF_CSR_RD(csr, ADF_GEN6_ERRSOU1); + if (errsou & ADF_GEN6_ERRSOU1_MASK) { + adf_gen6_process_errsou1(accel_dev, csr, errsou); + handled = true; + } + + errsou = ADF_CSR_RD(csr, ADF_GEN6_ERRSOU2); + if (errsou & ADF_GEN6_ERRSOU2_MASK) { + adf_gen6_process_errsou2(accel_dev, csr, errsou); + handled = true; + } + + errsou = ADF_CSR_RD(csr, ADF_GEN6_ERRSOU3); + if (errsou & ADF_GEN6_ERRSOU3_MASK) { + adf_gen6_process_errsou3(accel_dev, csr, errsou); + handled = true; + } + + adf_gen6_is_reset_required(accel_dev, csr, reset_required); + + return handled; +} + +void adf_gen6_init_ras_ops(struct adf_ras_ops *ras_ops) +{ + ras_ops->enable_ras_errors = adf_gen6_enable_ras; + ras_ops->disable_ras_errors = adf_gen6_disable_ras; + ras_ops->handle_interrupt = adf_gen6_handle_interrupt; +} +EXPORT_SYMBOL_GPL(adf_gen6_init_ras_ops); diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen6_ras.h b/drivers/crypto/intel/qat/qat_common/adf_gen6_ras.h new file mode 100644 index 000000000000..66ced271d173 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_gen6_ras.h @@ -0,0 +1,504 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2025 Intel Corporation */ +#ifndef ADF_GEN6_RAS_H_ +#define ADF_GEN6_RAS_H_ + +#include + +struct adf_ras_ops; + +/* Error source registers */ +#define ADF_GEN6_ERRSOU0 0x41A200 +#define ADF_GEN6_ERRSOU1 0x41A204 +#define ADF_GEN6_ERRSOU2 0x41A208 +#define ADF_GEN6_ERRSOU3 0x41A20C + +/* Error source mask registers */ +#define ADF_GEN6_ERRMSK0 0x41A210 +#define ADF_GEN6_ERRMSK1 0x41A214 +#define ADF_GEN6_ERRMSK2 0x41A218 +#define ADF_GEN6_ERRMSK3 0x41A21C + +/* ERRSOU0 Correctable error mask */ +#define ADF_GEN6_ERRSOU0_MASK BIT(0) + +#define ADF_GEN6_ERRSOU1_CPP0_MEUNC_BIT BIT(0) +#define ADF_GEN6_ERRSOU1_CPP_CMDPARERR_BIT BIT(1) +#define ADF_GEN6_ERRSOU1_RIMEM_PARERR_STS_BIT BIT(2) +#define ADF_GEN6_ERRSOU1_TIMEM_PARERR_STS_BIT BIT(3) +#define ADF_GEN6_ERRSOU1_SFICMD_PARERR_BIT BIT(4) + +#define ADF_GEN6_ERRSOU1_MASK ( \ + (ADF_GEN6_ERRSOU1_CPP0_MEUNC_BIT) | \ + (ADF_GEN6_ERRSOU1_CPP_CMDPARERR_BIT) | \ + (ADF_GEN6_ERRSOU1_RIMEM_PARERR_STS_BIT) | \ + (ADF_GEN6_ERRSOU1_TIMEM_PARERR_STS_BIT) | \ + (ADF_GEN6_ERRSOU1_SFICMD_PARERR_BIT)) + +#define ADF_GEN6_ERRMSK1_CPP0_MEUNC_BIT BIT(0) +#define ADF_GEN6_ERRMSK1_CPP_CMDPARERR_BIT BIT(1) +#define ADF_GEN6_ERRMSK1_RIMEM_PARERR_STS_BIT BIT(2) +#define ADF_GEN6_ERRMSK1_TIMEM_PARERR_STS_BIT BIT(3) +#define ADF_GEN6_ERRMSK1_IOSFCMD_PARERR_BIT BIT(4) + +#define ADF_GEN6_ERRMSK1_MASK ( \ + (ADF_GEN6_ERRMSK1_CPP0_MEUNC_BIT) | \ + (ADF_GEN6_ERRMSK1_CPP_CMDPARERR_BIT) | \ + (ADF_GEN6_ERRMSK1_RIMEM_PARERR_STS_BIT) | \ + (ADF_GEN6_ERRMSK1_TIMEM_PARERR_STS_BIT) | \ + (ADF_GEN6_ERRMSK1_IOSFCMD_PARERR_BIT)) + +/* HI AE Uncorrectable error log */ +#define ADF_GEN6_HIAEUNCERRLOG_CPP0 0x41A300 + +/* HI AE Uncorrectable error log enable */ +#define ADF_GEN6_HIAEUNCERRLOGENABLE_CPP0 0x41A320 + +/* HI AE Correctable error log */ +#define ADF_GEN6_HIAECORERRLOG_CPP0 0x41A308 + +/* HI AE Correctable error log enable */ +#define ADF_GEN6_HIAECORERRLOGENABLE_CPP0 0x41A318 + +/* HI CPP Agent Command parity error log */ +#define ADF_GEN6_HICPPAGENTCMDPARERRLOG 0x41A310 + +/* HI CPP Agent command parity error logging enable */ +#define ADF_GEN6_HICPPAGENTCMDPARERRLOGENABLE 0x41A314 + +#define ADF_6XXX_HICPPAGENTCMDPARERRLOG_MASK 0x1B + +/* RI Memory parity error status register */ +#define ADF_GEN6_RIMEM_PARERR_STS 0x41B128 + +/* RI Memory parity error reporting enable */ +#define ADF_GEN6_RI_MEM_PAR_ERR_EN0 0x41B12C + +/* + * RI Memory parity error mask + * BIT(4) - ri_tlq_phdr parity error + * BIT(5) - ri_tlq_pdata parity error + * BIT(6) - ri_tlq_nphdr parity error + * BIT(7) - ri_tlq_npdata parity error + * BIT(8) - ri_tlq_cplhdr parity error + * BIT(10) - BIT(13) - ri_tlq_cpldata[0:3] parity error + * BIT(19) - ri_cds_cmd_fifo parity error + * BIT(20) - ri_obc_ricpl_fifo parity error + * BIT(21) - ri_obc_tiricpl_fifo parity error + * BIT(22) - ri_obc_cppcpl_fifo parity error + * BIT(23) - ri_obc_pendcpl_fifo parity error + * BIT(24) - ri_cpp_cmd_fifo parity error + * BIT(25) - ri_cds_ticmd_fifo parity error + * BIT(26) - riti_cmd_fifo parity error + * BIT(27) - ri_int_msixtbl parity error + * BIT(28) - ri_int_imstbl parity error + * BIT(30) - ri_kpt_fuses parity error + */ +#define ADF_GEN6_RIMEM_PARERR_FATAL_MASK \ + (BIT(0) | BIT(1) | BIT(2) | BIT(4) | BIT(5) | BIT(6) | \ + BIT(7) | BIT(8) | BIT(18) | BIT(19) | BIT(20) | BIT(21) | \ + BIT(22) | BIT(23) | BIT(24) | BIT(25) | BIT(26) | BIT(27) | \ + BIT(28) | BIT(30)) + +#define ADF_GEN6_RIMEM_PARERR_CERR_MASK \ + (BIT(10) | BIT(11) | BIT(12) | BIT(13)) + +/* TI CI parity status */ +#define ADF_GEN6_TI_CI_PAR_STS 0x50060C + +/* TI CI parity reporting mask */ +#define ADF_GEN6_TI_CI_PAR_ERR_MASK 0x500608 + +/* + * TI CI parity status mask + * BIT(0) - CdCmdQ_sts patiry error status + * BIT(1) - CdDataQ_sts parity error status + * BIT(3) - CPP_SkidQ_sts parity error status + */ +#define ADF_GEN6_TI_CI_PAR_STS_MASK \ + (BIT(0) | BIT(1) | BIT(3)) + +/* TI PULLFUB parity status */ +#define ADF_GEN6_TI_PULL0FUB_PAR_STS 0x500618 + +/* TI PULLFUB parity error reporting mask */ +#define ADF_GEN6_TI_PULL0FUB_PAR_ERR_MASK 0x500614 + +/* + * TI PULLFUB parity status mask + * BIT(0) - TrnPullReqQ_sts parity status + * BIT(1) - TrnSharedDataQ_sts parity status + * BIT(2) - TrnPullReqDataQ_sts parity status + * BIT(4) - CPP_CiPullReqQ_sts parity status + * BIT(5) - CPP_TrnPullReqQ_sts parity status + * BIT(6) - CPP_PullidQ_sts parity status + * BIT(7) - CPP_WaitDataQ_sts parity status + * BIT(8) - CPP_CdDataQ_sts parity status + * BIT(9) - CPP_TrnDataQP0_sts parity status + * BIT(10) - BIT(11) - CPP_TrnDataQRF[00:01]_sts parity status + * BIT(12) - CPP_TrnDataQP1_sts parity status + * BIT(13) - BIT(14) - CPP_TrnDataQRF[10:11]_sts parity status + */ +#define ADF_GEN6_TI_PULL0FUB_PAR_STS_MASK \ + (BIT(0) | BIT(1) | BIT(2) | BIT(4) | BIT(5) | BIT(6) | BIT(7) | \ + BIT(8) | BIT(9) | BIT(10) | BIT(11) | BIT(12) | BIT(13) | BIT(14)) + +/* TI PUSHUB parity status */ +#define ADF_GEN6_TI_PUSHFUB_PAR_STS 0x500630 + +/* TI PUSHFUB parity error reporting mask */ +#define ADF_GEN6_TI_PUSHFUB_PAR_ERR_MASK 0x50062C + +/* + * TI PUSHUB parity status mask + * BIT(0) - SbPushReqQ_sts parity status + * BIT(1) - BIT(2) - SbPushDataQ[0:1]_sts parity status + * BIT(4) - CPP_CdPushReqQ_sts parity status + * BIT(5) - BIT(6) - CPP_CdPushDataQ[0:1]_sts parity status + * BIT(7) - CPP_SbPushReqQ_sts parity status + * BIT(8) - CPP_SbPushDataQP_sts parity status + * BIT(9) - BIT(10) - CPP_SbPushDataQRF[0:1]_sts parity status + */ +#define ADF_GEN6_TI_PUSHFUB_PAR_STS_MASK \ + (BIT(0) | BIT(1) | BIT(2) | BIT(4) | BIT(5) | \ + BIT(6) | BIT(7) | BIT(8) | BIT(9) | BIT(10)) + +/* TI CD parity status */ +#define ADF_GEN6_TI_CD_PAR_STS 0x50063C + +/* TI CD parity error mask */ +#define ADF_GEN6_TI_CD_PAR_ERR_MASK 0x500638 + +/* + * TI CD parity status mask + * BIT(0) - BIT(15) - CtxMdRam[0:15]_sts parity status + * BIT(16) - Leaf2ClusterRam_sts parity status + * BIT(17) - BIT(18) - Ring2LeafRam[0:1]_sts parity status + * BIT(19) - VirtualQ_sts parity status + * BIT(20) - DtRdQ_sts parity status + * BIT(21) - DtWrQ_sts parity status + * BIT(22) - RiCmdQ_sts parity status + * BIT(23) - BypassQ_sts parity status + * BIT(24) - DtRdQ_sc_sts parity status + * BIT(25) - DtWrQ_sc_sts parity status + */ +#define ADF_GEN6_TI_CD_PAR_STS_MASK \ + (BIT(0) | BIT(1) | BIT(2) | BIT(3) | BIT(4) | BIT(5) | BIT(6) | \ + BIT(7) | BIT(8) | BIT(9) | BIT(10) | BIT(11) | BIT(12) | BIT(13) | \ + BIT(14) | BIT(15) | BIT(16) | BIT(17) | BIT(18) | BIT(19) | BIT(20) | \ + BIT(21) | BIT(22) | BIT(23) | BIT(24) | BIT(25)) + +/* TI TRNSB parity status */ +#define ADF_GEN6_TI_TRNSB_PAR_STS 0x500648 + +/* TI TRNSB parity error reporting mask */ +#define ADF_GEN6_TI_TRNSB_PAR_ERR_MASK 0x500644 + +/* + * TI TRNSB parity status mask + * BIT(0) - TrnPHdrQP_sts parity status + * BIT(1) - TrnPHdrQRF_sts parity status + * BIT(2) - TrnPDataQP_sts parity status + * BIT(3) - BIT(6) - TrnPDataQRF[0:3]_sts parity status + * BIT(7) - TrnNpHdrQP_sts parity status + * BIT(8) - BIT(9) - TrnNpHdrQRF[0:1]_sts parity status + * BIT(10) - TrnCplHdrQ_sts parity status + * BIT(11) - TrnPutObsReqQ_sts parity status + * BIT(12) - TrnPushReqQ_sts parity status + * BIT(13) - SbSplitIdRam_sts parity status + * BIT(14) - SbReqCountQ_sts parity status + * BIT(15) - SbCplTrkRam_sts parity status + * BIT(16) - SbGetObsReqQ_sts parity status + * BIT(17) - SbEpochIdQ_sts parity status + * BIT(18) - SbAtCplHdrQ_sts parity status + * BIT(19) - SbAtCplDataQ_sts parity status + * BIT(20) - SbReqCountRam_sts parity status + * BIT(21) - SbAtCplHdrQ_sc_sts parity status + */ +#define ADF_GEN6_TI_TRNSB_PAR_STS_MASK \ + (BIT(0) | BIT(1) | BIT(2) | BIT(3) | BIT(4) | BIT(5) | BIT(6) | \ + BIT(7) | BIT(8) | BIT(9) | BIT(10) | BIT(11) | BIT(12) | \ + BIT(13) | BIT(14) | BIT(15) | BIT(16) | BIT(17) | BIT(18) | \ + BIT(19) | BIT(20) | BIT(21)) + +/* Status register to log misc error on RI */ +#define ADF_GEN6_RIMISCSTS 0x41B1B8 + +/* Status control register to log misc RI error */ +#define ADF_GEN6_RIMISCCTL 0x41B1BC + +/* + * ERRSOU2 bit mask + * BIT(0) - SSM Interrupt Mask + * BIT(1) - CFC on CPP. ORed of CFC Push error and Pull error + * BIT(2) - BIT(4) - CPP attention interrupts + * BIT(18) - PM interrupt + */ +#define ADF_GEN6_ERRSOU2_SSM_ERR_BIT BIT(0) +#define ADF_GEN6_ERRSOU2_CPP_CFC_ERR_STATUS_BIT BIT(1) +#define ADF_GEN6_ERRSOU2_CPP_CFC_ATT_INT_MASK \ + (BIT(2) | BIT(3) | BIT(4)) + +#define ADF_GEN6_ERRSOU2_PM_INT_BIT BIT(18) + +#define ADF_GEN6_ERRSOU2_MASK \ + (ADF_GEN6_ERRSOU2_SSM_ERR_BIT | \ + ADF_GEN6_ERRSOU2_CPP_CFC_ERR_STATUS_BIT) + +#define ADF_GEN6_ERRSOU2_DIS_MASK \ + (ADF_GEN6_ERRSOU2_SSM_ERR_BIT | \ + ADF_GEN6_ERRSOU2_CPP_CFC_ERR_STATUS_BIT | \ + ADF_GEN6_ERRSOU2_CPP_CFC_ATT_INT_MASK) + +#define ADF_GEN6_IAINTSTATSSM 0x28 + +/* IAINTSTATSSM error bit mask definitions */ +#define ADF_GEN6_IAINTSTATSSM_SH_ERR_BIT BIT(0) +#define ADF_GEN6_IAINTSTATSSM_PPERR_BIT BIT(2) +#define ADF_GEN6_IAINTSTATSSM_SCMPAR_ERR_BIT BIT(4) +#define ADF_GEN6_IAINTSTATSSM_CPPPAR_ERR_BIT BIT(5) +#define ADF_GEN6_IAINTSTATSSM_RFPAR_ERR_BIT BIT(6) +#define ADF_GEN6_IAINTSTATSSM_UNEXP_CPL_ERR_BIT BIT(7) + +#define ADF_GEN6_IAINTSTATSSM_MASK \ + (ADF_GEN6_IAINTSTATSSM_SH_ERR_BIT | \ + ADF_GEN6_IAINTSTATSSM_PPERR_BIT | \ + ADF_GEN6_IAINTSTATSSM_SCMPAR_ERR_BIT | \ + ADF_GEN6_IAINTSTATSSM_CPPPAR_ERR_BIT | \ + ADF_GEN6_IAINTSTATSSM_RFPAR_ERR_BIT | \ + ADF_GEN6_IAINTSTATSSM_UNEXP_CPL_ERR_BIT) + +#define ADF_GEN6_UERRSSMSH 0x18 + +/* + * UERRSSMSH error bit mask definitions + * + * BIT(0) - Indicates one uncorrectable error + * BIT(15) - Indicates multiple uncorrectable errors + * in device shared memory + */ +#define ADF_GEN6_UERRSSMSH_MASK (BIT(0) | BIT(15)) + +/* + * CERRSSMSH error bit + * BIT(0) - Indicates one correctable error + */ +#define ADF_GEN6_CERRSSMSH_ERROR_BIT (BIT(0) | BIT(15) | BIT(24)) +#define ADF_GEN6_CERRSSMSH 0x10 + +#define ADF_GEN6_INTMASKSSM 0x0 + +/* + * Error reporting mask in INTMASKSSM + * BIT(0) - Shared memory uncorrectable interrupt mask + * BIT(2) - PPERR interrupt mask + * BIT(4) - SCM parity error interrupt mask + * BIT(5) - CPP parity error interrupt mask + * BIT(6) - SHRAM RF parity error interrupt mask + * BIT(7) - AXI unexpected completion error mask + */ +#define ADF_GEN6_INTMASKSSM_MASK \ + (BIT(0) | BIT(2) | BIT(4) | BIT(5) | BIT(6) | BIT(7)) + +/* CPP push or pull error */ +#define ADF_GEN6_PPERR 0x8 + +#define ADF_GEN6_PPERR_MASK (BIT(0) | BIT(1)) + +/* + * SSM_FERR_STATUS error bit mask definitions + */ +#define ADF_GEN6_SCM_PAR_ERR_MASK BIT(5) +#define ADF_GEN6_CPP_PAR_ERR_MASK (BIT(0) | BIT(1) | BIT(2)) +#define ADF_GEN6_UNEXP_CPL_ERR_MASK (BIT(3) | BIT(4) | BIT(10) | BIT(11)) +#define ADF_GEN6_RF_PAR_ERR_MASK BIT(16) + +#define ADF_GEN6_SSM_FERR_STATUS 0x9C + +#define ADF_GEN6_CPP_CFC_ERR_STATUS 0x640C04 + +/* + * BIT(0) - Indicates one or more CPP CFC errors + * BIT(1) - Indicates multiple CPP CFC errors + * BIT(7) - Indicates CPP CFC command parity error type + * BIT(8) - Indicates CPP CFC data parity error type + */ +#define ADF_GEN6_CPP_CFC_ERR_STATUS_ERR_BIT BIT(0) +#define ADF_GEN6_CPP_CFC_ERR_STATUS_MERR_BIT BIT(1) +#define ADF_GEN6_CPP_CFC_ERR_STATUS_CMDPAR_BIT BIT(7) +#define ADF_GEN6_CPP_CFC_ERR_STATUS_DATAPAR_BIT BIT(8) +#define ADF_GEN6_CPP_CFC_FATAL_ERR_BIT \ + (ADF_GEN6_CPP_CFC_ERR_STATUS_ERR_BIT | \ + ADF_GEN6_CPP_CFC_ERR_STATUS_MERR_BIT) + +/* + * BIT(0) - Enables CFC to detect and log a push/pull data error + * BIT(1) - Enables CFC to generate interrupt to PCIEP for a CPP error + * BIT(4) - When 1 parity detection is disabled + * BIT(5) - When 1 parity detection is disabled on CPP command bus + * BIT(6) - When 1 parity detection is disabled on CPP push/pull bus + * BIT(9) - When 1 RF parity error detection is disabled + */ +#define ADF_GEN6_CPP_CFC_ERR_CTRL_MASK (BIT(0) | BIT(1)) + +#define ADF_GEN6_CPP_CFC_ERR_CTRL_DIS_MASK \ + (BIT(4) | BIT(5) | BIT(6) | BIT(9) | BIT(10)) + +#define ADF_GEN6_CPP_CFC_ERR_CTRL 0x640C00 + +/* + * BIT(0) - Clears bit(0) of ADF_GEN6_CPP_CFC_ERR_STATUS + * when an error is reported on CPP + * BIT(1) - Clears bit(1) of ADF_GEN6_CPP_CFC_ERR_STATUS + * when multiple errors are reported on CPP + * BIT(2) - Clears bit(2) of ADF_GEN6_CPP_CFC_ERR_STATUS + * when attention interrupt is reported + */ +#define ADF_GEN6_CPP_CFC_ERR_STATUS_CLR_MASK (BIT(0) | BIT(1) | BIT(2)) +#define ADF_GEN6_CPP_CFC_ERR_STATUS_CLR 0x640C08 + +/* + * ERRSOU3 bit masks + * BIT(0) - indicates error response order overflow and/or BME error + * BIT(1) - indicates RI push/pull error + * BIT(2) - indicates TI push/pull error + * BIT(5) - indicates TI pull parity error + * BIT(6) - indicates RI push parity error + * BIT(7) - indicates VFLR interrupt + * BIT(8) - indicates ring pair interrupts for ATU detected fault + * BIT(9) - indicates rate limiting error + */ +#define ADF_GEN6_ERRSOU3_TIMISCSTS_BIT BIT(0) +#define ADF_GEN6_ERRSOU3_RICPPINTSTS_MASK (BIT(1) | BIT(6)) +#define ADF_GEN6_ERRSOU3_TICPPINTSTS_MASK (BIT(2) | BIT(5)) +#define ADF_GEN6_ERRSOU3_VFLRNOTIFY_BIT BIT(7) +#define ADF_GEN6_ERRSOU3_ATUFAULTSTATUS_BIT BIT(8) +#define ADF_GEN6_ERRSOU3_RLTERROR_BIT BIT(9) +#define ADF_GEN6_ERRSOU3_TC_VC_MAP_ERROR_BIT BIT(16) +#define ADF_GEN6_ERRSOU3_PCIE_DEVHALT_BIT BIT(17) +#define ADF_GEN6_ERRSOU3_PG_REQ_DEVHALT_BIT BIT(18) +#define ADF_GEN6_ERRSOU3_XLT_CPL_DEVHALT_BIT BIT(19) +#define ADF_GEN6_ERRSOU3_TI_INT_ERR_DEVHALT_BIT BIT(20) + +#define ADF_GEN6_ERRSOU3_MASK ( \ + (ADF_GEN6_ERRSOU3_TIMISCSTS_BIT) | \ + (ADF_GEN6_ERRSOU3_RICPPINTSTS_MASK) | \ + (ADF_GEN6_ERRSOU3_TICPPINTSTS_MASK) | \ + (ADF_GEN6_ERRSOU3_VFLRNOTIFY_BIT) | \ + (ADF_GEN6_ERRSOU3_ATUFAULTSTATUS_BIT) | \ + (ADF_GEN6_ERRSOU3_RLTERROR_BIT) | \ + (ADF_GEN6_ERRSOU3_TC_VC_MAP_ERROR_BIT) | \ + (ADF_GEN6_ERRSOU3_PCIE_DEVHALT_BIT) | \ + (ADF_GEN6_ERRSOU3_PG_REQ_DEVHALT_BIT) | \ + (ADF_GEN6_ERRSOU3_XLT_CPL_DEVHALT_BIT) | \ + (ADF_GEN6_ERRSOU3_TI_INT_ERR_DEVHALT_BIT)) + +#define ADF_GEN6_ERRSOU3_DIS_MASK ( \ + (ADF_GEN6_ERRSOU3_TIMISCSTS_BIT) | \ + (ADF_GEN6_ERRSOU3_RICPPINTSTS_MASK) | \ + (ADF_GEN6_ERRSOU3_TICPPINTSTS_MASK) | \ + (ADF_GEN6_ERRSOU3_VFLRNOTIFY_BIT) | \ + (ADF_GEN6_ERRSOU3_ATUFAULTSTATUS_BIT) | \ + (ADF_GEN6_ERRSOU3_RLTERROR_BIT) | \ + (ADF_GEN6_ERRSOU3_TC_VC_MAP_ERROR_BIT)) + +/* Rate limiting error log register */ +#define ADF_GEN6_RLT_ERRLOG 0x508814 + +#define ADF_GEN6_RLT_ERRLOG_MASK (BIT(0) | BIT(1) | BIT(2) | BIT(3)) + +/* TI misc status register */ +#define ADF_GEN6_TIMISCSTS 0x50054C + +/* TI misc error reporting mask */ +#define ADF_GEN6_TIMISCCTL 0x500548 + +/* + * TI Misc error reporting control mask + * BIT(0) - Enables error detection and logging in TIMISCSTS register + * BIT(1) - It has effect only when SRIOV enabled, this bit is 0 by default + * BIT(2) - Enables the D-F-x counter within the dispatch arbiter + * to start based on the command triggered from + * BIT(30) - Disables VFLR functionality + * bits 1, 2 and 30 value should be preserved and not meant to be changed + * within RAS. + */ +#define ADF_GEN6_TIMISCCTL_BIT BIT(0) +#define ADF_GEN6_TIMSCCTL_RELAY_MASK (BIT(1) | BIT(2) | BIT(30)) + +/* RI CPP interface status register */ +#define ADF_GEN6_RICPPINTSTS 0x41A330 + +/* + * Uncorrectable error mask in RICPPINTSTS register + * BIT(0) - RI asserted the CPP error signal during a push + * BIT(1) - RI detected the CPP error signal asserted during a pull + * BIT(2) - RI detected a push data parity error + * BIT(3) - RI detected a push valid parity error + */ +#define ADF_GEN6_RICPPINTSTS_MASK (BIT(0) | BIT(1) | BIT(2) | BIT(3)) + +/* RI CPP interface register control */ +#define ADF_GEN6_RICPPINTCTL 0x41A32C + +/* + * Control bit mask for RICPPINTCTL register + * BIT(0) - value of 1 enables error detection and reporting + * on the RI CPP Push interface + * BIT(1) - value of 1 enables error detection and reporting + * on the RI CPP Pull interface + * BIT(2) - value of 1 enables error detection and reporting + * on the RI Parity + * BIT(3) - value of 1 enable checking parity on CPP + */ +#define ADF_GEN6_RICPPINTCTL_MASK \ + (BIT(0) | BIT(1) | BIT(2) | BIT(3) | BIT(4)) + +/* TI CPP interface status register */ +#define ADF_GEN6_TICPPINTSTS 0x50053C + +/* + * Uncorrectable error mask in TICPPINTSTS register + * BIT(0) - value of 1 indicates that the TI asserted + * the CPP error signal during a push + * BIT(1) - value of 1 indicates that the TI detected + * the CPP error signal asserted during a pull + * BIT(2) - value of 1 indicates that the TI detected + * a pull data parity error + */ +#define ADF_GEN6_TICPPINTSTS_MASK (BIT(0) | BIT(1) | BIT(2)) + +/* TI CPP interface status register control */ +#define ADF_GEN6_TICPPINTCTL 0x500538 + +/* + * Control bit mask for TICPPINTCTL register + * BIT(0) - value of 1 enables error detection and reporting on + * the TI CPP Push interface + * BIT(1) - value of 1 enables error detection and reporting on + * the TI CPP Push interface + * BIT(2) - value of 1 enables parity error detection and logging on + * the TI CPP Pull interface + * BIT(3) - value of 1 enables CPP CMD and Pull Data parity checking + */ +#define ADF_GEN6_TICPPINTCTL_MASK \ + (BIT(0) | BIT(1) | BIT(2) | BIT(3) | BIT(4)) + +/* ATU fault status register */ +#define ADF_GEN6_ATUFAULTSTATUS(i) (0x506000 + ((i) * 0x4)) + +#define ADF_GEN6_ATUFAULTSTATUS_BIT BIT(0) + +/* Command parity error detected on IOSFP command to QAT */ +#define ADF_GEN6_RIMISCSTS_BIT BIT(0) + +#define ADF_GEN6_GENSTS 0x41A220 +#define ADF_GEN6_GENSTS_DEVICE_STATE_MASK GENMASK(1, 0) +#define ADF_GEN6_GENSTS_RESET_TYPE_MASK GENMASK(3, 2) +#define ADF_GEN6_GENSTS_PFLR 0x1 +#define ADF_GEN6_GENSTS_COLD_RESET 0x3 +#define ADF_GEN6_GENSTS_DEVHALT 0x1 + +void adf_gen6_init_ras_ops(struct adf_ras_ops *ras_ops); + +#endif /* ADF_GEN6_RAS_H_ */ From 662c62bb83a9780c25e2d1d8bae6c2cc9a7c6663 Mon Sep 17 00:00:00 2001 From: Suman Kumar Chakraborty Date: Tue, 13 May 2025 11:25:27 +0100 Subject: [PATCH 349/382] crypto: qat - enable reporting of error counters for GEN6 devices Enable the reporting of error counters through sysfs for QAT GEN6 devices and update the ABI documentation. This enables the reporting of the following: - errors_correctable - hardware correctable errors that allow the system to recover without data loss. - errors_nonfatal: errors that can be isolated to specific in-flight requests. - errors_fatal: errors that cannot be contained to a request, requiring a Function Level Reset (FLR) upon occurrence. Signed-off-by: Suman Kumar Chakraborty Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- Documentation/ABI/testing/sysfs-driver-qat_ras | 8 ++++---- drivers/crypto/intel/qat/qat_6xxx/adf_drv.c | 2 ++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/Documentation/ABI/testing/sysfs-driver-qat_ras b/Documentation/ABI/testing/sysfs-driver-qat_ras index 176dea1e9c0a..82ceb04445ec 100644 --- a/Documentation/ABI/testing/sysfs-driver-qat_ras +++ b/Documentation/ABI/testing/sysfs-driver-qat_ras @@ -4,7 +4,7 @@ KernelVersion: 6.7 Contact: qat-linux@intel.com Description: (RO) Reports the number of correctable errors detected by the device. - This attribute is only available for qat_4xxx devices. + This attribute is only available for qat_4xxx and qat_6xxx devices. What: /sys/bus/pci/devices//qat_ras/errors_nonfatal Date: January 2024 @@ -12,7 +12,7 @@ KernelVersion: 6.7 Contact: qat-linux@intel.com Description: (RO) Reports the number of non fatal errors detected by the device. - This attribute is only available for qat_4xxx devices. + This attribute is only available for qat_4xxx and qat_6xxx devices. What: /sys/bus/pci/devices//qat_ras/errors_fatal Date: January 2024 @@ -20,7 +20,7 @@ KernelVersion: 6.7 Contact: qat-linux@intel.com Description: (RO) Reports the number of fatal errors detected by the device. - This attribute is only available for qat_4xxx devices. + This attribute is only available for qat_4xxx and qat_6xxx devices. What: /sys/bus/pci/devices//qat_ras/reset_error_counters Date: January 2024 @@ -38,4 +38,4 @@ Description: (WO) Write to resets all error counters of a device. # cat /sys/bus/pci/devices//qat_ras/errors_fatal 0 - This attribute is only available for qat_4xxx devices. + This attribute is only available for qat_4xxx and qat_6xxx devices. diff --git a/drivers/crypto/intel/qat/qat_6xxx/adf_drv.c b/drivers/crypto/intel/qat/qat_6xxx/adf_drv.c index 132e26501621..c1dc9c56fdf5 100644 --- a/drivers/crypto/intel/qat/qat_6xxx/adf_drv.c +++ b/drivers/crypto/intel/qat/qat_6xxx/adf_drv.c @@ -173,6 +173,8 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) if (ret) return dev_err_probe(dev, ret, "Failed to save pci state.\n"); + accel_dev->ras_errors.enabled = true; + adf_dbgfs_init(accel_dev); ret = devm_add_action_or_reset(dev, adf_dbgfs_cleanup, accel_dev); From 768613340842dc458c72a8ff5c6d2c8f11d42a2c Mon Sep 17 00:00:00 2001 From: "Rob Herring (Arm)" Date: Tue, 13 May 2025 13:40:28 -0500 Subject: [PATCH 350/382] dt-bindings: crypto: fsl,sec-v4.0: Add fsl,sec-v6.0 The fsl,sec-v6.0 binding is the same as the fsl,sec-v4.0 binding, so add it to the existing schema and drop the old .txt binding. The compatibles in the .txt binding don't match the 1 user. Follow the user for the schema. Signed-off-by: Rob Herring (Arm) Signed-off-by: Herbert Xu --- .../bindings/crypto/fsl,sec-v4.0.yaml | 10 +- .../devicetree/bindings/crypto/fsl-sec6.txt | 157 ------------------ 2 files changed, 9 insertions(+), 158 deletions(-) delete mode 100644 Documentation/devicetree/bindings/crypto/fsl-sec6.txt diff --git a/Documentation/devicetree/bindings/crypto/fsl,sec-v4.0.yaml b/Documentation/devicetree/bindings/crypto/fsl,sec-v4.0.yaml index f0c4a7c83568..75afa441e019 100644 --- a/Documentation/devicetree/bindings/crypto/fsl,sec-v4.0.yaml +++ b/Documentation/devicetree/bindings/crypto/fsl,sec-v4.0.yaml @@ -38,7 +38,9 @@ properties: compatible: oneOf: - items: - - const: fsl,sec-v5.4 + - enum: + - fsl,sec-v5.4 + - fsl,sec-v6.0 - const: fsl,sec-v5.0 - const: fsl,sec-v4.0 - items: @@ -93,6 +95,12 @@ patternProperties: properties: compatible: oneOf: + - items: + - const: fsl,sec-v6.0-job-ring + - const: fsl,sec-v5.2-job-ring + - const: fsl,sec-v5.0-job-ring + - const: fsl,sec-v4.4-job-ring + - const: fsl,sec-v4.0-job-ring - items: - const: fsl,sec-v5.4-job-ring - const: fsl,sec-v5.0-job-ring diff --git a/Documentation/devicetree/bindings/crypto/fsl-sec6.txt b/Documentation/devicetree/bindings/crypto/fsl-sec6.txt deleted file mode 100644 index 73b0eb950bb3..000000000000 --- a/Documentation/devicetree/bindings/crypto/fsl-sec6.txt +++ /dev/null @@ -1,157 +0,0 @@ -SEC 6 is as Freescale's Cryptographic Accelerator and Assurance Module (CAAM). -Currently Freescale powerpc chip C29X is embedded with SEC 6. -SEC 6 device tree binding include: - -SEC 6 Node - -Job Ring Node - -Full Example - -===================================================================== -SEC 6 Node - -Description - - Node defines the base address of the SEC 6 block. - This block specifies the address range of all global - configuration registers for the SEC 6 block. - For example, In C293, we could see three SEC 6 node. - -PROPERTIES - - - compatible - Usage: required - Value type: - Definition: Must include "fsl,sec-v6.0". - - - fsl,sec-era - Usage: optional - Value type: - Definition: A standard property. Define the 'ERA' of the SEC - device. - - - #address-cells - Usage: required - Value type: - Definition: A standard property. Defines the number of cells - for representing physical addresses in child nodes. - - - #size-cells - Usage: required - Value type: - Definition: A standard property. Defines the number of cells - for representing the size of physical addresses in - child nodes. - - - reg - Usage: required - Value type: - Definition: A standard property. Specifies the physical - address and length of the SEC 6 configuration registers. - - - ranges - Usage: required - Value type: - Definition: A standard property. Specifies the physical address - range of the SEC 6.0 register space (-SNVS not included). A - triplet that includes the child address, parent address, & - length. - - Note: All other standard properties (see the Devicetree Specification) - are allowed but are optional. - -EXAMPLE - crypto@a0000 { - compatible = "fsl,sec-v6.0"; - fsl,sec-era = <6>; - #address-cells = <1>; - #size-cells = <1>; - reg = <0xa0000 0x20000>; - ranges = <0 0xa0000 0x20000>; - }; - -===================================================================== -Job Ring (JR) Node - - Child of the crypto node defines data processing interface to SEC 6 - across the peripheral bus for purposes of processing - cryptographic descriptors. The specified address - range can be made visible to one (or more) cores. - The interrupt defined for this node is controlled within - the address range of this node. - - - compatible - Usage: required - Value type: - Definition: Must include "fsl,sec-v6.0-job-ring". - - - reg - Usage: required - Value type: - Definition: Specifies a two JR parameters: an offset from - the parent physical address and the length the JR registers. - - - interrupts - Usage: required - Value type: - Definition: Specifies the interrupts generated by this - device. The value of the interrupts property - consists of one interrupt specifier. The format - of the specifier is defined by the binding document - describing the node's interrupt parent. - -EXAMPLE - jr@1000 { - compatible = "fsl,sec-v6.0-job-ring"; - reg = <0x1000 0x1000>; - interrupts = <49 2 0 0>; - }; - -=================================================================== -Full Example - -Since some chips may contain more than one SEC, the dtsi contains -only the node contents, not the node itself. A chip using the SEC -should include the dtsi inside each SEC node. Example: - -In qoriq-sec6.0.dtsi: - - compatible = "fsl,sec-v6.0"; - fsl,sec-era = <6>; - #address-cells = <1>; - #size-cells = <1>; - - jr@1000 { - compatible = "fsl,sec-v6.0-job-ring", - "fsl,sec-v5.2-job-ring", - "fsl,sec-v5.0-job-ring", - "fsl,sec-v4.4-job-ring", - "fsl,sec-v4.0-job-ring"; - reg = <0x1000 0x1000>; - }; - - jr@2000 { - compatible = "fsl,sec-v6.0-job-ring", - "fsl,sec-v5.2-job-ring", - "fsl,sec-v5.0-job-ring", - "fsl,sec-v4.4-job-ring", - "fsl,sec-v4.0-job-ring"; - reg = <0x2000 0x1000>; - }; - -In the C293 device tree, we add the include of public property: - - crypto@a0000 { - /include/ "qoriq-sec6.0.dtsi" - } - - crypto@a0000 { - reg = <0xa0000 0x20000>; - ranges = <0 0xa0000 0x20000>; - - jr@1000 { - interrupts = <49 2 0 0>; - }; - - jr@2000 { - interrupts = <50 2 0 0>; - }; - }; From 7996ace661a7f910fe7c7cc5b4b1cd6d72686b83 Mon Sep 17 00:00:00 2001 From: "Rob Herring (Arm)" Date: Tue, 13 May 2025 13:40:41 -0500 Subject: [PATCH 351/382] dt-bindings: crypto: Drop obsolete mediatek,eip97-crypto The mediatek,eip97-crypto binding is half abandoned. The driver was dropped in 2020 as the Mediatek platforms use InsideSecure block and the driver for it. All the platforms except MT7623 were updated. A patch to update it was submitted, but never addressed the review comments. Link: https://lore.kernel.org/all/20210303080923.16761-1-vic.wu@mediatek.com/ Signed-off-by: Rob Herring (Arm) Signed-off-by: Herbert Xu --- .../bindings/crypto/mediatek-crypto.txt | 25 ------------------- 1 file changed, 25 deletions(-) delete mode 100644 Documentation/devicetree/bindings/crypto/mediatek-crypto.txt diff --git a/Documentation/devicetree/bindings/crypto/mediatek-crypto.txt b/Documentation/devicetree/bindings/crypto/mediatek-crypto.txt deleted file mode 100644 index 450da3661cad..000000000000 --- a/Documentation/devicetree/bindings/crypto/mediatek-crypto.txt +++ /dev/null @@ -1,25 +0,0 @@ -MediaTek cryptographic accelerators - -Required properties: -- compatible: Should be "mediatek,eip97-crypto" -- reg: Address and length of the register set for the device -- interrupts: Should contain the five crypto engines interrupts in numeric - order. These are global system and four descriptor rings. -- clocks: the clock used by the core -- clock-names: Must contain "cryp". -- power-domains: Must contain a reference to the PM domain. - - -Example: - crypto: crypto@1b240000 { - compatible = "mediatek,eip97-crypto"; - reg = <0 0x1b240000 0 0x20000>; - interrupts = , - , - , - , - ; - clocks = <ðsys CLK_ETHSYS_CRYPTO>; - clock-names = "cryp"; - power-domains = <&scpsys MT2701_POWER_DOMAIN_ETH>; - }; From aa238d34d3a98253f019e8c2e1cef0309b487fef Mon Sep 17 00:00:00 2001 From: "Rob Herring (Arm)" Date: Tue, 13 May 2025 13:40:50 -0500 Subject: [PATCH 352/382] dt-bindings: crypto: Convert amd,ccp-seattle-v1a to DT schema Convert the AMD Cryptographic Coprocessor binding to DT schema format. It's a straight forward conversion. Signed-off-by: Rob Herring (Arm) Signed-off-by: Herbert Xu --- .../bindings/crypto/amd,ccp-seattle-v1a.yaml | 38 +++++++++++++++++++ .../devicetree/bindings/crypto/amd-ccp.txt | 17 --------- 2 files changed, 38 insertions(+), 17 deletions(-) create mode 100644 Documentation/devicetree/bindings/crypto/amd,ccp-seattle-v1a.yaml delete mode 100644 Documentation/devicetree/bindings/crypto/amd-ccp.txt diff --git a/Documentation/devicetree/bindings/crypto/amd,ccp-seattle-v1a.yaml b/Documentation/devicetree/bindings/crypto/amd,ccp-seattle-v1a.yaml new file mode 100644 index 000000000000..32bf3a1c3b42 --- /dev/null +++ b/Documentation/devicetree/bindings/crypto/amd,ccp-seattle-v1a.yaml @@ -0,0 +1,38 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/amd,ccp-seattle-v1a.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: AMD Cryptographic Coprocessor (ccp) + +maintainers: + - Tom Lendacky + +properties: + compatible: + const: amd,ccp-seattle-v1a + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + dma-coherent: true + +required: + - compatible + - reg + - interrupts + +additionalProperties: false + +examples: + - | + crypto@e0100000 { + compatible = "amd,ccp-seattle-v1a"; + reg = <0xe0100000 0x10000>; + interrupts = <0 3 4>; + dma-coherent; + }; diff --git a/Documentation/devicetree/bindings/crypto/amd-ccp.txt b/Documentation/devicetree/bindings/crypto/amd-ccp.txt deleted file mode 100644 index d87579d63da6..000000000000 --- a/Documentation/devicetree/bindings/crypto/amd-ccp.txt +++ /dev/null @@ -1,17 +0,0 @@ -* AMD Cryptographic Coprocessor driver (ccp) - -Required properties: -- compatible: Should be "amd,ccp-seattle-v1a" -- reg: Address and length of the register set for the device -- interrupts: Should contain the CCP interrupt - -Optional properties: -- dma-coherent: Present if dma operations are coherent - -Example: - ccp@e0100000 { - compatible = "amd,ccp-seattle-v1a"; - reg = <0 0xe0100000 0 0x10000>; - interrupt-parent = <&gic>; - interrupts = <0 3 4>; - }; From bbd8265d457e5f4d439803acfd3d46a157a5c70b Mon Sep 17 00:00:00 2001 From: "Rob Herring (Arm)" Date: Tue, 13 May 2025 13:40:56 -0500 Subject: [PATCH 353/382] dt-bindings: crypto: Convert axis,artpec6-crypto to DT schema Convert the Axis Crypto engine binding to DT schema format. It's a straight forward conversion. Signed-off-by: Rob Herring (Arm) Reviewed-by: Krzysztof Kozlowski Signed-off-by: Herbert Xu --- .../bindings/crypto/artpec6-crypto.txt | 16 -------- .../bindings/crypto/axis,artpec6-crypto.yaml | 39 +++++++++++++++++++ 2 files changed, 39 insertions(+), 16 deletions(-) delete mode 100644 Documentation/devicetree/bindings/crypto/artpec6-crypto.txt create mode 100644 Documentation/devicetree/bindings/crypto/axis,artpec6-crypto.yaml diff --git a/Documentation/devicetree/bindings/crypto/artpec6-crypto.txt b/Documentation/devicetree/bindings/crypto/artpec6-crypto.txt deleted file mode 100644 index d9cca4875bd6..000000000000 --- a/Documentation/devicetree/bindings/crypto/artpec6-crypto.txt +++ /dev/null @@ -1,16 +0,0 @@ -Axis crypto engine with PDMA interface. - -Required properties: -- compatible : Should be one of the following strings: - "axis,artpec6-crypto" for the version in the Axis ARTPEC-6 SoC - "axis,artpec7-crypto" for the version in the Axis ARTPEC-7 SoC. -- reg: Base address and size for the PDMA register area. -- interrupts: Interrupt handle for the PDMA interrupt line. - -Example: - -crypto@f4264000 { - compatible = "axis,artpec6-crypto"; - reg = <0xf4264000 0x1000>; - interrupts = ; -}; diff --git a/Documentation/devicetree/bindings/crypto/axis,artpec6-crypto.yaml b/Documentation/devicetree/bindings/crypto/axis,artpec6-crypto.yaml new file mode 100644 index 000000000000..c91f81e3c39e --- /dev/null +++ b/Documentation/devicetree/bindings/crypto/axis,artpec6-crypto.yaml @@ -0,0 +1,39 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/crypto/axis,artpec6-crypto.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Axis ARTPEC6 crypto engine with PDMA interface + +maintainers: + - Lars Persson + +properties: + compatible: + enum: + - axis,artpec6-crypto + - axis,artpec7-crypto + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + +required: + - compatible + - reg + - interrupts + +additionalProperties: false + +examples: + - | + #include + + crypto@f4264000 { + compatible = "axis,artpec6-crypto"; + reg = <0xf4264000 0x1000>; + interrupts = ; + }; From b525ac4be3e98abe8b77311469dccaf9d15877ff Mon Sep 17 00:00:00 2001 From: "Rob Herring (Arm)" Date: Tue, 13 May 2025 13:41:02 -0500 Subject: [PATCH 354/382] dt-bindings: crypto: Convert brcm,spum-crypto to DT schema Convert the Broadcom SPUM/SPU2 binding to DT schema format. It's a straight forward conversion. Signed-off-by: Rob Herring (Arm) Signed-off-by: Herbert Xu --- .../bindings/crypto/brcm,spu-crypto.txt | 22 ---------- .../bindings/crypto/brcm,spum-crypto.yaml | 44 +++++++++++++++++++ 2 files changed, 44 insertions(+), 22 deletions(-) delete mode 100644 Documentation/devicetree/bindings/crypto/brcm,spu-crypto.txt create mode 100644 Documentation/devicetree/bindings/crypto/brcm,spum-crypto.yaml diff --git a/Documentation/devicetree/bindings/crypto/brcm,spu-crypto.txt b/Documentation/devicetree/bindings/crypto/brcm,spu-crypto.txt deleted file mode 100644 index 29b6007568eb..000000000000 --- a/Documentation/devicetree/bindings/crypto/brcm,spu-crypto.txt +++ /dev/null @@ -1,22 +0,0 @@ -The Broadcom Secure Processing Unit (SPU) hardware supports symmetric -cryptographic offload for Broadcom SoCs. A SoC may have multiple SPU hardware -blocks. - -Required properties: -- compatible: Should be one of the following: - brcm,spum-crypto - for devices with SPU-M hardware - brcm,spu2-crypto - for devices with SPU2 hardware - brcm,spu2-v2-crypto - for devices with enhanced SPU2 hardware features like SHA3 - and Rabin Fingerprint support - brcm,spum-nsp-crypto - for the Northstar Plus variant of the SPU-M hardware - -- reg: Should contain SPU registers location and length. -- mboxes: The mailbox channel to be used to communicate with the SPU. - Mailbox channels correspond to DMA rings on the device. - -Example: - crypto@612d0000 { - compatible = "brcm,spum-crypto"; - reg = <0 0x612d0000 0 0x900>; - mboxes = <&pdc0 0>; - }; diff --git a/Documentation/devicetree/bindings/crypto/brcm,spum-crypto.yaml b/Documentation/devicetree/bindings/crypto/brcm,spum-crypto.yaml new file mode 100644 index 000000000000..9a5fb61727fa --- /dev/null +++ b/Documentation/devicetree/bindings/crypto/brcm,spum-crypto.yaml @@ -0,0 +1,44 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/brcm,spum-crypto.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Broadcom SPU Crypto Offload + +maintainers: + - Rob Rice + +description: + The Broadcom Secure Processing Unit (SPU) hardware supports symmetric + cryptographic offload for Broadcom SoCs. A SoC may have multiple SPU hardware + blocks. + +properties: + compatible: + enum: + - brcm,spum-crypto + - brcm,spu2-crypto + - brcm,spu2-v2-crypto # enhanced SPU2 hardware features like SHA3 and Rabin Fingerprint support + - brcm,spum-nsp-crypto # Northstar Plus variant of the SPU-M hardware + + reg: + maxItems: 1 + + mboxes: + maxItems: 1 + +required: + - compatible + - reg + - mboxes + +additionalProperties: false + +examples: + - | + crypto@612d0000 { + compatible = "brcm,spum-crypto"; + reg = <0x612d0000 0x900>; + mboxes = <&pdc0 0>; + }; From 036454de0fec8d9789f3730f701e8df6049b5635 Mon Sep 17 00:00:00 2001 From: "Rob Herring (Arm)" Date: Tue, 13 May 2025 13:41:10 -0500 Subject: [PATCH 355/382] dt-bindings: crypto: Convert hisilicon,hip0{6,7}-sec to DT schema Convert the HiSilicon HIP06/7 Security Accelerator binding to DT schema format. It's a straight forward conversion. Signed-off-by: Rob Herring (Arm) Signed-off-by: Herbert Xu --- .../bindings/crypto/hisilicon,hip06-sec.yaml | 134 ++++++++++++++++++ .../bindings/crypto/hisilicon,hip07-sec.txt | 67 --------- 2 files changed, 134 insertions(+), 67 deletions(-) create mode 100644 Documentation/devicetree/bindings/crypto/hisilicon,hip06-sec.yaml delete mode 100644 Documentation/devicetree/bindings/crypto/hisilicon,hip07-sec.txt diff --git a/Documentation/devicetree/bindings/crypto/hisilicon,hip06-sec.yaml b/Documentation/devicetree/bindings/crypto/hisilicon,hip06-sec.yaml new file mode 100644 index 000000000000..2bfac9d1c020 --- /dev/null +++ b/Documentation/devicetree/bindings/crypto/hisilicon,hip06-sec.yaml @@ -0,0 +1,134 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/crypto/hisilicon,hip06-sec.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Hisilicon hip06/hip07 Security Accelerator + +maintainers: + - Jonathan Cameron + +properties: + compatible: + enum: + - hisilicon,hip06-sec + - hisilicon,hip07-sec + + reg: + items: + - description: Registers for backend processing engines + - description: Registers for common functionality + - description: Registers for queue 0 + - description: Registers for queue 1 + - description: Registers for queue 2 + - description: Registers for queue 3 + - description: Registers for queue 4 + - description: Registers for queue 5 + - description: Registers for queue 6 + - description: Registers for queue 7 + - description: Registers for queue 8 + - description: Registers for queue 9 + - description: Registers for queue 10 + - description: Registers for queue 11 + - description: Registers for queue 12 + - description: Registers for queue 13 + - description: Registers for queue 14 + - description: Registers for queue 15 + + interrupts: + items: + - description: SEC unit error queue interrupt + - description: Completion interrupt for queue 0 + - description: Error interrupt for queue 0 + - description: Completion interrupt for queue 1 + - description: Error interrupt for queue 1 + - description: Completion interrupt for queue 2 + - description: Error interrupt for queue 2 + - description: Completion interrupt for queue 3 + - description: Error interrupt for queue 3 + - description: Completion interrupt for queue 4 + - description: Error interrupt for queue 4 + - description: Completion interrupt for queue 5 + - description: Error interrupt for queue 5 + - description: Completion interrupt for queue 6 + - description: Error interrupt for queue 6 + - description: Completion interrupt for queue 7 + - description: Error interrupt for queue 7 + - description: Completion interrupt for queue 8 + - description: Error interrupt for queue 8 + - description: Completion interrupt for queue 9 + - description: Error interrupt for queue 9 + - description: Completion interrupt for queue 10 + - description: Error interrupt for queue 10 + - description: Completion interrupt for queue 11 + - description: Error interrupt for queue 11 + - description: Completion interrupt for queue 12 + - description: Error interrupt for queue 12 + - description: Completion interrupt for queue 13 + - description: Error interrupt for queue 13 + - description: Completion interrupt for queue 14 + - description: Error interrupt for queue 14 + - description: Completion interrupt for queue 15 + - description: Error interrupt for queue 15 + + dma-coherent: true + + iommus: + maxItems: 1 + +required: + - compatible + - reg + - interrupts + - dma-coherent + +additionalProperties: false + +examples: + - | + bus { + #address-cells = <2>; + #size-cells = <2>; + + crypto@400d2000000 { + compatible = "hisilicon,hip07-sec"; + reg = <0x400 0xd0000000 0x0 0x10000 + 0x400 0xd2000000 0x0 0x10000 + 0x400 0xd2010000 0x0 0x10000 + 0x400 0xd2020000 0x0 0x10000 + 0x400 0xd2030000 0x0 0x10000 + 0x400 0xd2040000 0x0 0x10000 + 0x400 0xd2050000 0x0 0x10000 + 0x400 0xd2060000 0x0 0x10000 + 0x400 0xd2070000 0x0 0x10000 + 0x400 0xd2080000 0x0 0x10000 + 0x400 0xd2090000 0x0 0x10000 + 0x400 0xd20a0000 0x0 0x10000 + 0x400 0xd20b0000 0x0 0x10000 + 0x400 0xd20c0000 0x0 0x10000 + 0x400 0xd20d0000 0x0 0x10000 + 0x400 0xd20e0000 0x0 0x10000 + 0x400 0xd20f0000 0x0 0x10000 + 0x400 0xd2100000 0x0 0x10000>; + interrupts = <576 4>, + <577 1>, <578 4>, + <579 1>, <580 4>, + <581 1>, <582 4>, + <583 1>, <584 4>, + <585 1>, <586 4>, + <587 1>, <588 4>, + <589 1>, <590 4>, + <591 1>, <592 4>, + <593 1>, <594 4>, + <595 1>, <596 4>, + <597 1>, <598 4>, + <599 1>, <600 4>, + <601 1>, <602 4>, + <603 1>, <604 4>, + <605 1>, <606 4>, + <607 1>, <608 4>; + dma-coherent; + iommus = <&p1_smmu_alg_a 0x600>; + }; + }; diff --git a/Documentation/devicetree/bindings/crypto/hisilicon,hip07-sec.txt b/Documentation/devicetree/bindings/crypto/hisilicon,hip07-sec.txt deleted file mode 100644 index d28fd1af01b4..000000000000 --- a/Documentation/devicetree/bindings/crypto/hisilicon,hip07-sec.txt +++ /dev/null @@ -1,67 +0,0 @@ -* Hisilicon hip07 Security Accelerator (SEC) - -Required properties: -- compatible: Must contain one of - - "hisilicon,hip06-sec" - - "hisilicon,hip07-sec" -- reg: Memory addresses and lengths of the memory regions through which - this device is controlled. - Region 0 has registers to control the backend processing engines. - Region 1 has registers for functionality common to all queues. - Regions 2-18 have registers for the 16 individual queues which are isolated - both in hardware and within the driver. -- interrupts: Interrupt specifiers. - Refer to interrupt-controller/interrupts.txt for generic interrupt client node - bindings. - Interrupt 0 is for the SEC unit error queue. - Interrupt 2N + 1 is the completion interrupt for queue N. - Interrupt 2N + 2 is the error interrupt for queue N. -- dma-coherent: The driver assumes coherent dma is possible. - -Optional properties: -- iommus: The SEC units are behind smmu-v3 iommus. - Refer to iommu/arm,smmu-v3.txt for more information. - -Example: - -p1_sec_a: crypto@400d2000000 { - compatible = "hisilicon,hip07-sec"; - reg = <0x400 0xd0000000 0x0 0x10000 - 0x400 0xd2000000 0x0 0x10000 - 0x400 0xd2010000 0x0 0x10000 - 0x400 0xd2020000 0x0 0x10000 - 0x400 0xd2030000 0x0 0x10000 - 0x400 0xd2040000 0x0 0x10000 - 0x400 0xd2050000 0x0 0x10000 - 0x400 0xd2060000 0x0 0x10000 - 0x400 0xd2070000 0x0 0x10000 - 0x400 0xd2080000 0x0 0x10000 - 0x400 0xd2090000 0x0 0x10000 - 0x400 0xd20a0000 0x0 0x10000 - 0x400 0xd20b0000 0x0 0x10000 - 0x400 0xd20c0000 0x0 0x10000 - 0x400 0xd20d0000 0x0 0x10000 - 0x400 0xd20e0000 0x0 0x10000 - 0x400 0xd20f0000 0x0 0x10000 - 0x400 0xd2100000 0x0 0x10000>; - interrupt-parent = <&p1_mbigen_sec_a>; - iommus = <&p1_smmu_alg_a 0x600>; - dma-coherent; - interrupts = <576 4>, - <577 1>, <578 4>, - <579 1>, <580 4>, - <581 1>, <582 4>, - <583 1>, <584 4>, - <585 1>, <586 4>, - <587 1>, <588 4>, - <589 1>, <590 4>, - <591 1>, <592 4>, - <593 1>, <594 4>, - <595 1>, <596 4>, - <597 1>, <598 4>, - <599 1>, <600 4>, - <601 1>, <602 4>, - <603 1>, <604 4>, - <605 1>, <606 4>, - <607 1>, <608 4>; -}; From 7eff621c460249fbbd1775faa7d4245af1b23d8d Mon Sep 17 00:00:00 2001 From: "Rob Herring (Arm)" Date: Tue, 13 May 2025 13:41:16 -0500 Subject: [PATCH 356/382] dt-bindings: crypto: Convert img,hash-accelerator to DT schema Convert the Imagination Technologies hardware hash accelerator binding to DT schema format. It's a straight forward conversion. Signed-off-by: Rob Herring (Arm) Signed-off-by: Herbert Xu --- .../bindings/crypto/img,hash-accelerator.yaml | 69 +++++++++++++++++++ .../devicetree/bindings/crypto/img-hash.txt | 27 -------- 2 files changed, 69 insertions(+), 27 deletions(-) create mode 100644 Documentation/devicetree/bindings/crypto/img,hash-accelerator.yaml delete mode 100644 Documentation/devicetree/bindings/crypto/img-hash.txt diff --git a/Documentation/devicetree/bindings/crypto/img,hash-accelerator.yaml b/Documentation/devicetree/bindings/crypto/img,hash-accelerator.yaml new file mode 100644 index 000000000000..46617561ef94 --- /dev/null +++ b/Documentation/devicetree/bindings/crypto/img,hash-accelerator.yaml @@ -0,0 +1,69 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/img,hash-accelerator.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Imagination Technologies hardware hash accelerator + +maintainers: + - James Hartley + +description: + The hash accelerator provides hardware hashing acceleration for + SHA1, SHA224, SHA256 and MD5 hashes. + +properties: + compatible: + const: img,hash-accelerator + + reg: + items: + - description: Register base address and size + - description: DMA port specifier + + interrupts: + maxItems: 1 + + dmas: + maxItems: 1 + + dma-names: + items: + - const: tx + + clocks: + items: + - description: System clock for hash block registers + - description: Hash clock for data path + + clock-names: + items: + - const: sys + - const: hash + +additionalProperties: false + +required: + - compatible + - reg + - interrupts + - dmas + - dma-names + - clocks + - clock-names + +examples: + - | + #include + #include + + hash@18149600 { + compatible = "img,hash-accelerator"; + reg = <0x18149600 0x100>, <0x18101100 0x4>; + interrupts = ; + dmas = <&dma 8 0xffffffff 0>; + dma-names = "tx"; + clocks = <&cr_periph SYS_CLK_HASH>, <&clk_periph PERIPH_CLK_ROM>; + clock-names = "sys", "hash"; + }; diff --git a/Documentation/devicetree/bindings/crypto/img-hash.txt b/Documentation/devicetree/bindings/crypto/img-hash.txt deleted file mode 100644 index 91a3d757d641..000000000000 --- a/Documentation/devicetree/bindings/crypto/img-hash.txt +++ /dev/null @@ -1,27 +0,0 @@ -Imagination Technologies hardware hash accelerator - -The hash accelerator provides hardware hashing acceleration for -SHA1, SHA224, SHA256 and MD5 hashes - -Required properties: - -- compatible : "img,hash-accelerator" -- reg : Offset and length of the register set for the module, and the DMA port -- interrupts : The designated IRQ line for the hashing module. -- dmas : DMA specifier as per Documentation/devicetree/bindings/dma/dma.txt -- dma-names : Should be "tx" -- clocks : Clock specifiers -- clock-names : "sys" Used to clock the hash block registers - "hash" Used to clock data through the accelerator - -Example: - - hash: hash@18149600 { - compatible = "img,hash-accelerator"; - reg = <0x18149600 0x100>, <0x18101100 0x4>; - interrupts = ; - dmas = <&dma 8 0xffffffff 0>; - dma-names = "tx"; - clocks = <&cr_periph SYS_CLK_HASH>, <&clk_periph PERIPH_CLK_ROM>; - clock-names = "sys", "hash"; - }; From 7d39f32c8c77ad14bfdb49f41d87c97024d0eb63 Mon Sep 17 00:00:00 2001 From: "Rob Herring (Arm)" Date: Tue, 13 May 2025 13:41:23 -0500 Subject: [PATCH 357/382] dt-bindings: crypto: Convert Marvell CESA to DT schema Convert the Marvell CESA binding to DT schema format. The marvell-cesa.txt and mv_cesa.txt are duplicate bindings. The clock properties are quite varied for each platform hence the if/then schemas. The old binding was fairly accurate with reality. The original binding didn't mention there is 1 interrupt per CESA engine. Based on users, there's a maximum of 2 engines. Signed-off-by: Rob Herring (Arm) Signed-off-by: Herbert Xu --- .../bindings/crypto/marvell,orion-crypto.yaml | 133 ++++++++++++++++++ .../bindings/crypto/marvell-cesa.txt | 44 ------ .../devicetree/bindings/crypto/mv_cesa.txt | 32 ----- 3 files changed, 133 insertions(+), 76 deletions(-) create mode 100644 Documentation/devicetree/bindings/crypto/marvell,orion-crypto.yaml delete mode 100644 Documentation/devicetree/bindings/crypto/marvell-cesa.txt delete mode 100644 Documentation/devicetree/bindings/crypto/mv_cesa.txt diff --git a/Documentation/devicetree/bindings/crypto/marvell,orion-crypto.yaml b/Documentation/devicetree/bindings/crypto/marvell,orion-crypto.yaml new file mode 100644 index 000000000000..b44d36c50ec4 --- /dev/null +++ b/Documentation/devicetree/bindings/crypto/marvell,orion-crypto.yaml @@ -0,0 +1,133 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/crypto/marvell,orion-crypto.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Marvell Cryptographic Engines And Security Accelerator + +maintainers: + - Andrew Lunn + - Boris Brezillon + +description: | + Marvell Cryptographic Engines And Security Accelerator + +properties: + compatible: + enum: + - marvell,armada-370-crypto + - marvell,armada-xp-crypto + - marvell,armada-375-crypto + - marvell,armada-38x-crypto + - marvell,dove-crypto + - marvell,kirkwood-crypto + - marvell,orion-crypto + + reg: + minItems: 1 + items: + - description: Registers region + - description: SRAM region + deprecated: true + + reg-names: + minItems: 1 + items: + - const: regs + - const: sram + deprecated: true + + interrupts: + description: One interrupt for each CESA engine + minItems: 1 + maxItems: 2 + + clocks: + description: One or two clocks for each CESA engine + minItems: 1 + maxItems: 4 + + clock-names: + minItems: 1 + items: + - const: cesa0 + - const: cesa1 + - const: cesaz0 + - const: cesaz1 + + marvell,crypto-srams: + description: Phandle(s) to crypto SRAM. + $ref: /schemas/types.yaml#/definitions/phandle-array + minItems: 1 + maxItems: 2 + items: + maxItems: 1 + + marvell,crypto-sram-size: + description: SRAM size reserved for crypto operations. + $ref: /schemas/types.yaml#/definitions/uint32 + default: 0x800 + +required: + - compatible + - reg + - reg-names + - interrupts + - marvell,crypto-srams + +allOf: + - if: + not: + properties: + compatible: + enum: + - marvell,kirkwood-crypto + - marvell,orion-crypto + then: + required: + - clocks + - if: + properties: + compatible: + contains: + enum: + - marvell,armada-370-crypto + - marvell,armada-375-crypto + - marvell,armada-38x-crypto + - marvell,armada-xp-crypto + then: + required: + - clock-names + - if: + properties: + compatible: + contains: + enum: + - marvell,armada-375-crypto + - marvell,armada-38x-crypto + then: + properties: + clocks: + minItems: 4 + clock-names: + minItems: 4 + else: + properties: + clocks: + maxItems: 2 + clock-names: + maxItems: 2 + +additionalProperties: false + +examples: + - | + crypto@30000 { + compatible = "marvell,orion-crypto"; + reg = <0x30000 0x10000>; + reg-names = "regs"; + interrupts = <22>; + marvell,crypto-srams = <&crypto_sram>; + marvell,crypto-sram-size = <0x600>; + }; diff --git a/Documentation/devicetree/bindings/crypto/marvell-cesa.txt b/Documentation/devicetree/bindings/crypto/marvell-cesa.txt deleted file mode 100644 index 28d3f2496b89..000000000000 --- a/Documentation/devicetree/bindings/crypto/marvell-cesa.txt +++ /dev/null @@ -1,44 +0,0 @@ -Marvell Cryptographic Engines And Security Accelerator - -Required properties: -- compatible: should be one of the following string - "marvell,orion-crypto" - "marvell,kirkwood-crypto" - "marvell,dove-crypto" - "marvell,armada-370-crypto" - "marvell,armada-xp-crypto" - "marvell,armada-375-crypto" - "marvell,armada-38x-crypto" -- reg: base physical address of the engine and length of memory mapped - region. Can also contain an entry for the SRAM attached to the CESA, - but this representation is deprecated and marvell,crypto-srams should - be used instead -- reg-names: "regs". Can contain an "sram" entry, but this representation - is deprecated and marvell,crypto-srams should be used instead -- interrupts: interrupt number -- clocks: reference to the crypto engines clocks. This property is not - required for orion and kirkwood platforms -- clock-names: "cesaX" and "cesazX", X should be replaced by the crypto engine - id. - This property is not required for the orion and kirkwoord - platforms. - "cesazX" clocks are not required on armada-370 platforms -- marvell,crypto-srams: phandle to crypto SRAM definitions - -Optional properties: -- marvell,crypto-sram-size: SRAM size reserved for crypto operations, if not - specified the whole SRAM is used (2KB) - - -Examples: - - crypto@90000 { - compatible = "marvell,armada-xp-crypto"; - reg = <0x90000 0x10000>; - reg-names = "regs"; - interrupts = <48>, <49>; - clocks = <&gateclk 23>, <&gateclk 23>; - clock-names = "cesa0", "cesa1"; - marvell,crypto-srams = <&crypto_sram0>, <&crypto_sram1>; - marvell,crypto-sram-size = <0x600>; - }; diff --git a/Documentation/devicetree/bindings/crypto/mv_cesa.txt b/Documentation/devicetree/bindings/crypto/mv_cesa.txt deleted file mode 100644 index d9b92e2f3138..000000000000 --- a/Documentation/devicetree/bindings/crypto/mv_cesa.txt +++ /dev/null @@ -1,32 +0,0 @@ -Marvell Cryptographic Engines And Security Accelerator - -Required properties: -- compatible: should be one of the following string - "marvell,orion-crypto" - "marvell,kirkwood-crypto" - "marvell,dove-crypto" -- reg: base physical address of the engine and length of memory mapped - region. Can also contain an entry for the SRAM attached to the CESA, - but this representation is deprecated and marvell,crypto-srams should - be used instead -- reg-names: "regs". Can contain an "sram" entry, but this representation - is deprecated and marvell,crypto-srams should be used instead -- interrupts: interrupt number -- clocks: reference to the crypto engines clocks. This property is only - required for Dove platforms -- marvell,crypto-srams: phandle to crypto SRAM definitions - -Optional properties: -- marvell,crypto-sram-size: SRAM size reserved for crypto operations, if not - specified the whole SRAM is used (2KB) - -Examples: - - crypto@30000 { - compatible = "marvell,orion-crypto"; - reg = <0x30000 0x10000>; - reg-names = "regs"; - interrupts = <22>; - marvell,crypto-srams = <&crypto_sram>; - marvell,crypto-sram-size = <0x600>; - }; From cd5a4d53069ccd7cd48d201c981d5a8a99c93fc4 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Thu, 15 May 2025 13:54:33 +0800 Subject: [PATCH 358/382] crypto: hash - Move core export and import into internel/hash.h The core export and import functions are targeted at implementors so move them into internal/hash.h. Signed-off-by: Herbert Xu --- include/crypto/hash.h | 48 ---------------------------------- include/crypto/internal/hash.h | 48 ++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 48 deletions(-) diff --git a/include/crypto/hash.h b/include/crypto/hash.h index 1760662ad70a..9fc9daaaaab4 100644 --- a/include/crypto/hash.h +++ b/include/crypto/hash.h @@ -506,18 +506,6 @@ int crypto_ahash_digest(struct ahash_request *req); */ int crypto_ahash_export(struct ahash_request *req, void *out); -/** - * crypto_ahash_export_core() - extract core state for message digest - * @req: reference to the ahash_request handle whose state is exported - * @out: output buffer of sufficient size that can hold the hash state - * - * Export the hash state without the partial block buffer. - * - * Context: Softirq or process context. - * Return: 0 if the export creation was successful; < 0 if an error occurred - */ -int crypto_ahash_export_core(struct ahash_request *req, void *out); - /** * crypto_ahash_import() - import message digest state * @req: reference to ahash_request handle the state is imported into @@ -531,18 +519,6 @@ int crypto_ahash_export_core(struct ahash_request *req, void *out); */ int crypto_ahash_import(struct ahash_request *req, const void *in); -/** - * crypto_ahash_import_core() - import core state - * @req: reference to ahash_request handle the state is imported into - * @in: buffer holding the state - * - * Import the hash state without the partial block buffer. - * - * Context: Softirq or process context. - * Return: 0 if the import was successful; < 0 if an error occurred - */ -int crypto_ahash_import_core(struct ahash_request *req, const void *in); - /** * crypto_ahash_init() - (re)initialize message digest handle * @req: ahash_request handle that already is initialized with all necessary @@ -933,18 +909,6 @@ int crypto_hash_digest(struct crypto_ahash *tfm, const u8 *data, */ int crypto_shash_export(struct shash_desc *desc, void *out); -/** - * crypto_shash_export_core() - extract core state for message digest - * @desc: reference to the operational state handle whose state is exported - * @out: output buffer of sufficient size that can hold the hash state - * - * Export the hash state without the partial block buffer. - * - * Context: Softirq or process context. - * Return: 0 if the export creation was successful; < 0 if an error occurred - */ -int crypto_shash_export_core(struct shash_desc *desc, void *out); - /** * crypto_shash_import() - import operational state * @desc: reference to the operational state handle the state imported into @@ -959,18 +923,6 @@ int crypto_shash_export_core(struct shash_desc *desc, void *out); */ int crypto_shash_import(struct shash_desc *desc, const void *in); -/** - * crypto_shash_import_core() - import core state - * @desc: reference to the operational state handle the state imported into - * @in: buffer holding the state - * - * Import the hash state without the partial block buffer. - * - * Context: Softirq or process context. - * Return: 0 if the import was successful; < 0 if an error occurred - */ -int crypto_shash_import_core(struct shash_desc *desc, const void *in); - /** * crypto_shash_init() - (re)initialize message digest * @desc: operational state handle that is already filled diff --git a/include/crypto/internal/hash.h b/include/crypto/internal/hash.h index f2bbdb74e11a..ef5ea75ac5c8 100644 --- a/include/crypto/internal/hash.h +++ b/include/crypto/internal/hash.h @@ -305,5 +305,53 @@ static inline unsigned int crypto_shash_coresize(struct crypto_shash *tfm) #define HASH_REQUEST_ZERO(name) \ memzero_explicit(__##name##_req, sizeof(__##name##_req)) +/** + * crypto_ahash_export_core() - extract core state for message digest + * @req: reference to the ahash_request handle whose state is exported + * @out: output buffer of sufficient size that can hold the hash state + * + * Export the hash state without the partial block buffer. + * + * Context: Softirq or process context. + * Return: 0 if the export creation was successful; < 0 if an error occurred + */ +int crypto_ahash_export_core(struct ahash_request *req, void *out); + +/** + * crypto_ahash_import_core() - import core state + * @req: reference to ahash_request handle the state is imported into + * @in: buffer holding the state + * + * Import the hash state without the partial block buffer. + * + * Context: Softirq or process context. + * Return: 0 if the import was successful; < 0 if an error occurred + */ +int crypto_ahash_import_core(struct ahash_request *req, const void *in); + +/** + * crypto_shash_export_core() - extract core state for message digest + * @desc: reference to the operational state handle whose state is exported + * @out: output buffer of sufficient size that can hold the hash state + * + * Export the hash state without the partial block buffer. + * + * Context: Softirq or process context. + * Return: 0 if the export creation was successful; < 0 if an error occurred + */ +int crypto_shash_export_core(struct shash_desc *desc, void *out); + +/** + * crypto_shash_import_core() - import core state + * @desc: reference to the operational state handle the state imported into + * @in: buffer holding the state + * + * Import the hash state without the partial block buffer. + * + * Context: Softirq or process context. + * Return: 0 if the import was successful; < 0 if an error occurred + */ +int crypto_shash_import_core(struct shash_desc *desc, const void *in); + #endif /* _CRYPTO_INTERNAL_HASH_H */ From c6a12f394c488cf6a7ca35c1ad51e0e88897de2e Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Thu, 15 May 2025 13:54:35 +0800 Subject: [PATCH 359/382] crypto: hash - Add export_core and import_core hooks Add export_core and import_core hooks. These are intended to be used by algorithms which are wrappers around block-only algorithms, but are not themselves block-only, e.g., hmac. Signed-off-by: Herbert Xu --- crypto/ahash.c | 22 ++++++++++++++--- crypto/shash.c | 44 +++++++++++++++++++++++++++------- include/crypto/hash.h | 10 ++++++++ include/crypto/internal/hash.h | 3 +++ 4 files changed, 68 insertions(+), 11 deletions(-) diff --git a/crypto/ahash.c b/crypto/ahash.c index 344bf1b43e71..7d96c76731ef 100644 --- a/crypto/ahash.c +++ b/crypto/ahash.c @@ -704,7 +704,7 @@ int crypto_ahash_export_core(struct ahash_request *req, void *out) if (likely(tfm->using_shash)) return crypto_shash_export_core(ahash_request_ctx(req), out); - return crypto_ahash_alg(tfm)->export(req, out); + return crypto_ahash_alg(tfm)->export_core(req, out); } EXPORT_SYMBOL_GPL(crypto_ahash_export_core); @@ -727,7 +727,7 @@ int crypto_ahash_import_core(struct ahash_request *req, const void *in) in); if (crypto_ahash_get_flags(tfm) & CRYPTO_TFM_NEED_KEY) return -ENOKEY; - return crypto_ahash_alg(tfm)->import(req, in); + return crypto_ahash_alg(tfm)->import_core(req, in); } EXPORT_SYMBOL_GPL(crypto_ahash_import_core); @@ -739,7 +739,7 @@ int crypto_ahash_import(struct ahash_request *req, const void *in) return crypto_shash_import(prepare_shash_desc(req, tfm), in); if (crypto_ahash_get_flags(tfm) & CRYPTO_TFM_NEED_KEY) return -ENOKEY; - return crypto_ahash_import_core(req, in); + return crypto_ahash_alg(tfm)->import(req, in); } EXPORT_SYMBOL_GPL(crypto_ahash_import); @@ -971,6 +971,16 @@ out_free_nhash: } EXPORT_SYMBOL_GPL(crypto_clone_ahash); +static int ahash_default_export_core(struct ahash_request *req, void *out) +{ + return -ENOSYS; +} + +static int ahash_default_import_core(struct ahash_request *req, const void *in) +{ + return -ENOSYS; +} + static int ahash_prepare_alg(struct ahash_alg *alg) { struct crypto_alg *base = &alg->halg.base; @@ -996,6 +1006,12 @@ static int ahash_prepare_alg(struct ahash_alg *alg) if (!alg->setkey) alg->setkey = ahash_nosetkey; + if (!alg->export_core || !alg->import_core) { + alg->export_core = ahash_default_export_core; + alg->import_core = ahash_default_import_core; + base->cra_flags |= CRYPTO_AHASH_ALG_NO_EXPORT_CORE; + } + return 0; } diff --git a/crypto/shash.c b/crypto/shash.c index dee391d47f51..5bc74a72d5ad 100644 --- a/crypto/shash.c +++ b/crypto/shash.c @@ -203,9 +203,10 @@ int crypto_shash_tfm_digest(struct crypto_shash *tfm, const u8 *data, } EXPORT_SYMBOL_GPL(crypto_shash_tfm_digest); -int crypto_shash_export_core(struct shash_desc *desc, void *out) +static int __crypto_shash_export(struct shash_desc *desc, void *out, + int (*export)(struct shash_desc *desc, + void *out)) { - int (*export)(struct shash_desc *desc, void *out); struct crypto_shash *tfm = desc->tfm; u8 *buf = shash_desc_ctx(desc); unsigned int plen, ss; @@ -214,7 +215,6 @@ int crypto_shash_export_core(struct shash_desc *desc, void *out) ss = crypto_shash_statesize(tfm); if (crypto_shash_block_only(tfm)) ss -= plen; - export = crypto_shash_alg(tfm)->export; if (!export) { memcpy(out, buf, ss); return 0; @@ -222,6 +222,12 @@ int crypto_shash_export_core(struct shash_desc *desc, void *out) return export(desc, out); } + +int crypto_shash_export_core(struct shash_desc *desc, void *out) +{ + return __crypto_shash_export(desc, out, + crypto_shash_alg(desc->tfm)->export_core); +} EXPORT_SYMBOL_GPL(crypto_shash_export_core); int crypto_shash_export(struct shash_desc *desc, void *out) @@ -236,13 +242,14 @@ int crypto_shash_export(struct shash_desc *desc, void *out) memcpy(out + ss - plen, buf + descsize - plen, plen); } - return crypto_shash_export_core(desc, out); + return __crypto_shash_export(desc, out, crypto_shash_alg(tfm)->export); } EXPORT_SYMBOL_GPL(crypto_shash_export); -int crypto_shash_import_core(struct shash_desc *desc, const void *in) +static int __crypto_shash_import(struct shash_desc *desc, const void *in, + int (*import)(struct shash_desc *desc, + const void *in)) { - int (*import)(struct shash_desc *desc, const void *in); struct crypto_shash *tfm = desc->tfm; unsigned int descsize, plen, ss; u8 *buf = shash_desc_ctx(desc); @@ -256,7 +263,6 @@ int crypto_shash_import_core(struct shash_desc *desc, const void *in) buf[descsize - 1] = 0; if (crypto_shash_block_only(tfm)) ss -= plen; - import = crypto_shash_alg(tfm)->import; if (!import) { memcpy(buf, in, ss); return 0; @@ -264,6 +270,12 @@ int crypto_shash_import_core(struct shash_desc *desc, const void *in) return import(desc, in); } + +int crypto_shash_import_core(struct shash_desc *desc, const void *in) +{ + return __crypto_shash_import(desc, in, + crypto_shash_alg(desc->tfm)->import_core); +} EXPORT_SYMBOL_GPL(crypto_shash_import_core); int crypto_shash_import(struct shash_desc *desc, const void *in) @@ -271,7 +283,7 @@ int crypto_shash_import(struct shash_desc *desc, const void *in) struct crypto_shash *tfm = desc->tfm; int err; - err = crypto_shash_import_core(desc, in); + err = __crypto_shash_import(desc, in, crypto_shash_alg(tfm)->import); if (crypto_shash_block_only(tfm)) { unsigned int plen = crypto_shash_blocksize(tfm) + 1; unsigned int descsize = crypto_shash_descsize(tfm); @@ -436,6 +448,16 @@ int hash_prepare_alg(struct hash_alg_common *alg) return 0; } +static int shash_default_export_core(struct shash_desc *desc, void *out) +{ + return -ENOSYS; +} + +static int shash_default_import_core(struct shash_desc *desc, const void *in) +{ + return -ENOSYS; +} + static int shash_prepare_alg(struct shash_alg *alg) { struct crypto_alg *base = &alg->halg.base; @@ -476,6 +498,12 @@ static int shash_prepare_alg(struct shash_alg *alg) BUILD_BUG_ON(MAX_ALGAPI_BLOCKSIZE >= 256); alg->descsize += base->cra_blocksize + 1; alg->statesize += base->cra_blocksize + 1; + alg->export_core = alg->export; + alg->import_core = alg->import; + } else if (!alg->export_core || !alg->import_core) { + alg->export_core = shash_default_export_core; + alg->import_core = shash_default_import_core; + base->cra_flags |= CRYPTO_AHASH_ALG_NO_EXPORT_CORE; } if (alg->descsize > HASH_MAX_DESCSIZE) diff --git a/include/crypto/hash.h b/include/crypto/hash.h index 9fc9daaaaab4..bf177cf9be10 100644 --- a/include/crypto/hash.h +++ b/include/crypto/hash.h @@ -129,6 +129,10 @@ struct ahash_request { * data so the transformation can continue from this point onward. No * data processing happens at this point. Driver must not use * req->result. + * @export_core: Export partial state without partial block. Only defined + * for algorithms that are not block-only. + * @import_core: Import partial state without partial block. Only defined + * for algorithms that are not block-only. * @init_tfm: Initialize the cryptographic transformation object. * This function is called only once at the instantiation * time, right after the transformation context was @@ -151,6 +155,8 @@ struct ahash_alg { int (*digest)(struct ahash_request *req); int (*export)(struct ahash_request *req, void *out); int (*import)(struct ahash_request *req, const void *in); + int (*export_core)(struct ahash_request *req, void *out); + int (*import_core)(struct ahash_request *req, const void *in); int (*setkey)(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen); int (*init_tfm)(struct crypto_ahash *tfm); @@ -200,6 +206,8 @@ struct shash_desc { * @digest: see struct ahash_alg * @export: see struct ahash_alg * @import: see struct ahash_alg + * @export_core: see struct ahash_alg + * @import_core: see struct ahash_alg * @setkey: see struct ahash_alg * @init_tfm: Initialize the cryptographic transformation object. * This function is called only once at the instantiation @@ -230,6 +238,8 @@ struct shash_alg { unsigned int len, u8 *out); int (*export)(struct shash_desc *desc, void *out); int (*import)(struct shash_desc *desc, const void *in); + int (*export_core)(struct shash_desc *desc, void *out); + int (*import_core)(struct shash_desc *desc, const void *in); int (*setkey)(struct crypto_shash *tfm, const u8 *key, unsigned int keylen); int (*init_tfm)(struct crypto_shash *tfm); diff --git a/include/crypto/internal/hash.h b/include/crypto/internal/hash.h index ef5ea75ac5c8..e9de2bc34a10 100644 --- a/include/crypto/internal/hash.h +++ b/include/crypto/internal/hash.h @@ -20,6 +20,9 @@ /* Set this bit if finup can deal with multiple blocks. */ #define CRYPTO_AHASH_ALG_FINUP_MAX 0x04000000 +/* This bit is set by the Crypto API if export_core is not supported. */ +#define CRYPTO_AHASH_ALG_NO_EXPORT_CORE 0x08000000 + #define HASH_FBREQ_ON_STACK(name, req) \ char __##name##_req[sizeof(struct ahash_request) + \ MAX_SYNC_HASH_REQSIZE] CRYPTO_MINALIGN_ATTR; \ From 9d7a0ab1c75365158b28a5f7fa516061ea40b1b8 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Thu, 15 May 2025 13:54:37 +0800 Subject: [PATCH 360/382] crypto: ahash - Handle partial blocks in API Provide an option to handle the partial blocks in the ahash API. Almost every hash algorithm has a block size and are only able to hash partial blocks on finalisation. As a first step disable virtual address support for algorithms with state sizes larger than HASH_MAX_STATESIZE. This is OK as virtual addresses are currently only used on synchronous fallbacks. This means ahash_do_req_chain only needs to handle synchronous fallbacks, removing the complexities of saving the request state. Also move the saved request state into the ahash_request object as nesting is no longer possible. Add a scatterlist to ahash_request to store the partial block. Signed-off-by: Herbert Xu --- crypto/ahash.c | 541 ++++++++++++++++++++---------------------- include/crypto/hash.h | 12 +- 2 files changed, 265 insertions(+), 288 deletions(-) diff --git a/crypto/ahash.c b/crypto/ahash.c index 7d96c76731ef..cf8bbe7e54c0 100644 --- a/crypto/ahash.c +++ b/crypto/ahash.c @@ -12,11 +12,13 @@ * Copyright (c) 2008 Loc Ho */ +#include #include #include #include #include #include +#include #include #include #include @@ -40,24 +42,47 @@ struct crypto_hash_walk { struct scatterlist *sg; }; -struct ahash_save_req_state { - struct ahash_request *req0; - crypto_completion_t compl; - void *data; - struct scatterlist sg; - const u8 *src; - u8 *page; - unsigned int offset; - unsigned int nbytes; - bool update; -}; - -static int ahash_save_req(struct ahash_request *req, crypto_completion_t cplt); -static void ahash_restore_req(struct ahash_request *req); -static void ahash_def_finup_done1(void *data, int err); -static int ahash_def_finup_finish1(struct ahash_request *req, int err); static int ahash_def_finup(struct ahash_request *req); +static inline bool crypto_ahash_block_only(struct crypto_ahash *tfm) +{ + return crypto_ahash_alg(tfm)->halg.base.cra_flags & + CRYPTO_AHASH_ALG_BLOCK_ONLY; +} + +static inline bool crypto_ahash_final_nonzero(struct crypto_ahash *tfm) +{ + return crypto_ahash_alg(tfm)->halg.base.cra_flags & + CRYPTO_AHASH_ALG_FINAL_NONZERO; +} + +static inline bool crypto_ahash_need_fallback(struct crypto_ahash *tfm) +{ + return crypto_ahash_alg(tfm)->halg.base.cra_flags & + CRYPTO_ALG_NEED_FALLBACK; +} + +static inline void ahash_op_done(void *data, int err, + int (*finish)(struct ahash_request *, int)) +{ + struct ahash_request *areq = data; + crypto_completion_t compl; + + compl = areq->saved_complete; + data = areq->saved_data; + if (err == -EINPROGRESS) + goto out; + + areq->base.flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; + + err = finish(areq, err); + if (err == -EINPROGRESS || err == -EBUSY) + return; + +out: + compl(data, err); +} + static int hash_walk_next(struct crypto_hash_walk *walk) { unsigned int offset = walk->offset; @@ -298,7 +323,7 @@ int crypto_ahash_setkey(struct crypto_ahash *tfm, const u8 *key, int err; err = alg->setkey(tfm, key, keylen); - if (!err && ahash_is_async(tfm)) + if (!err && crypto_ahash_need_fallback(tfm)) err = crypto_ahash_setkey(crypto_ahash_fb(tfm), key, keylen); if (unlikely(err)) { @@ -311,159 +336,47 @@ int crypto_ahash_setkey(struct crypto_ahash *tfm, const u8 *key, } EXPORT_SYMBOL_GPL(crypto_ahash_setkey); -static int ahash_reqchain_virt(struct ahash_save_req_state *state, - int err, u32 mask) -{ - struct ahash_request *req = state->req0; - struct crypto_ahash *tfm; - - tfm = crypto_ahash_reqtfm(req); - - for (;;) { - unsigned len = state->nbytes; - - if (!state->offset) - break; - - if (state->offset == len || err) { - u8 *result = req->result; - - ahash_request_set_virt(req, state->src, result, len); - state->offset = 0; - break; - } - - len -= state->offset; - - len = min(PAGE_SIZE, len); - memcpy(state->page, state->src + state->offset, len); - state->offset += len; - req->nbytes = len; - - err = crypto_ahash_alg(tfm)->update(req); - if (err == -EINPROGRESS) { - if (state->offset < state->nbytes) - err = -EBUSY; - break; - } - - if (err == -EBUSY) - break; - } - - return err; -} - -static int ahash_reqchain_finish(struct ahash_request *req0, - struct ahash_save_req_state *state, - int err, u32 mask) -{ - u8 *page; - - err = ahash_reqchain_virt(state, err, mask); - if (err == -EINPROGRESS || err == -EBUSY) - goto out; - - page = state->page; - if (page) { - memset(page, 0, PAGE_SIZE); - free_page((unsigned long)page); - } - ahash_restore_req(req0); - -out: - return err; -} - -static void ahash_reqchain_done(void *data, int err) -{ - struct ahash_save_req_state *state = data; - crypto_completion_t compl = state->compl; - - data = state->data; - - if (err == -EINPROGRESS) { - if (state->offset < state->nbytes) - return; - goto notify; - } - - err = ahash_reqchain_finish(state->req0, state, err, - CRYPTO_TFM_REQ_MAY_BACKLOG); - if (err == -EBUSY) - return; - -notify: - compl(data, err); -} - static int ahash_do_req_chain(struct ahash_request *req, - int (*op)(struct ahash_request *req)) + int (*const *op)(struct ahash_request *req)) { struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - bool update = op == crypto_ahash_alg(tfm)->update; - struct ahash_save_req_state *state; - struct ahash_save_req_state state0; - u8 *page = NULL; int err; - if (crypto_ahash_req_virt(tfm) || - !update || !ahash_request_isvirt(req)) - return op(req); + if (crypto_ahash_req_virt(tfm) || !ahash_request_isvirt(req)) + return (*op)(req); - if (update && ahash_request_isvirt(req)) { - page = (void *)__get_free_page(GFP_ATOMIC); - err = -ENOMEM; - if (!page) - goto out; - } + if (crypto_ahash_statesize(tfm) > HASH_MAX_STATESIZE) + return -ENOSYS; - state = &state0; - if (ahash_is_async(tfm)) { - err = ahash_save_req(req, ahash_reqchain_done); - if (err) - goto out_free_page; + { + u8 state[HASH_MAX_STATESIZE]; - state = req->base.data; - } + if (op == &crypto_ahash_alg(tfm)->digest) { + ahash_request_set_tfm(req, crypto_ahash_fb(tfm)); + err = crypto_ahash_digest(req); + goto out_no_state; + } - state->update = update; - state->page = page; - state->offset = 0; - state->nbytes = 0; + err = crypto_ahash_export(req, state); + ahash_request_set_tfm(req, crypto_ahash_fb(tfm)); + err = err ?: crypto_ahash_import(req, state); - if (page) - sg_init_one(&state->sg, page, PAGE_SIZE); + if (op == &crypto_ahash_alg(tfm)->finup) { + err = err ?: crypto_ahash_finup(req); + goto out_no_state; + } - if (update && ahash_request_isvirt(req) && req->nbytes) { - unsigned len = req->nbytes; - u8 *result = req->result; + err = err ?: + crypto_ahash_update(req) ?: + crypto_ahash_export(req, state); - state->src = req->svirt; - state->nbytes = len; + ahash_request_set_tfm(req, tfm); + return err ?: crypto_ahash_import(req, state); - len = min(PAGE_SIZE, len); - - memcpy(page, req->svirt, len); - state->offset = len; - - ahash_request_set_crypt(req, &state->sg, result, len); - } - - err = op(req); - if (err == -EINPROGRESS || err == -EBUSY) { - if (state->offset < state->nbytes) - err = -EBUSY; +out_no_state: + ahash_request_set_tfm(req, tfm); return err; } - - return ahash_reqchain_finish(req, state, err, ~0); - -out_free_page: - free_page((unsigned long)page); - -out: - return err; } int crypto_ahash_init(struct ahash_request *req) @@ -476,144 +389,191 @@ int crypto_ahash_init(struct ahash_request *req) return -ENOKEY; if (ahash_req_on_stack(req) && ahash_is_async(tfm)) return -EAGAIN; - return ahash_do_req_chain(req, crypto_ahash_alg(tfm)->init); + if (crypto_ahash_block_only(tfm)) { + u8 *buf = ahash_request_ctx(req); + + buf += crypto_ahash_reqsize(tfm) - 1; + *buf = 0; + } + return crypto_ahash_alg(tfm)->init(req); } EXPORT_SYMBOL_GPL(crypto_ahash_init); -static int ahash_save_req(struct ahash_request *req, crypto_completion_t cplt) +static void ahash_save_req(struct ahash_request *req, crypto_completion_t cplt) { - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - struct ahash_save_req_state *state; - - if (!ahash_is_async(tfm)) - return 0; - - state = kmalloc(sizeof(*state), GFP_ATOMIC); - if (!state) - return -ENOMEM; - - state->compl = req->base.complete; - state->data = req->base.data; + req->saved_complete = req->base.complete; + req->saved_data = req->base.data; req->base.complete = cplt; - req->base.data = state; - state->req0 = req; - - return 0; + req->base.data = req; } static void ahash_restore_req(struct ahash_request *req) { - struct ahash_save_req_state *state; - struct crypto_ahash *tfm; + req->base.complete = req->saved_complete; + req->base.data = req->saved_data; +} - tfm = crypto_ahash_reqtfm(req); - if (!ahash_is_async(tfm)) - return; +static int ahash_update_finish(struct ahash_request *req, int err) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + bool nonzero = crypto_ahash_final_nonzero(tfm); + int bs = crypto_ahash_blocksize(tfm); + u8 *blenp = ahash_request_ctx(req); + int blen; + u8 *buf; - state = req->base.data; + blenp += crypto_ahash_reqsize(tfm) - 1; + blen = *blenp; + buf = blenp - bs; - req->base.complete = state->compl; - req->base.data = state->data; - kfree(state); + if (blen) { + req->src = req->sg_head + 1; + if (sg_is_chain(req->src)) + req->src = sg_chain_ptr(req->src); + } + + req->nbytes += nonzero - blen; + + blen = err < 0 ? 0 : err + nonzero; + if (ahash_request_isvirt(req)) + memcpy(buf, req->svirt + req->nbytes - blen, blen); + else + memcpy_from_sglist(buf, req->src, req->nbytes - blen, blen); + *blenp = blen; + + ahash_restore_req(req); + + return err; +} + +static void ahash_update_done(void *data, int err) +{ + ahash_op_done(data, err, ahash_update_finish); } int crypto_ahash_update(struct ahash_request *req) { struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + bool nonzero = crypto_ahash_final_nonzero(tfm); + int bs = crypto_ahash_blocksize(tfm); + u8 *blenp = ahash_request_ctx(req); + int blen, err; + u8 *buf; if (likely(tfm->using_shash)) return shash_ahash_update(req, ahash_request_ctx(req)); if (ahash_req_on_stack(req) && ahash_is_async(tfm)) return -EAGAIN; - return ahash_do_req_chain(req, crypto_ahash_alg(tfm)->update); + if (!crypto_ahash_block_only(tfm)) + return ahash_do_req_chain(req, &crypto_ahash_alg(tfm)->update); + + blenp += crypto_ahash_reqsize(tfm) - 1; + blen = *blenp; + buf = blenp - bs; + + if (blen + req->nbytes < bs + nonzero) { + if (ahash_request_isvirt(req)) + memcpy(buf + blen, req->svirt, req->nbytes); + else + memcpy_from_sglist(buf + blen, req->src, 0, + req->nbytes); + + *blenp += req->nbytes; + return 0; + } + + if (blen) { + memset(req->sg_head, 0, sizeof(req->sg_head[0])); + sg_set_buf(req->sg_head, buf, blen); + if (req->src != req->sg_head + 1) + sg_chain(req->sg_head, 2, req->src); + req->src = req->sg_head; + req->nbytes += blen; + } + req->nbytes -= nonzero; + + ahash_save_req(req, ahash_update_done); + + err = ahash_do_req_chain(req, &crypto_ahash_alg(tfm)->update); + if (err == -EINPROGRESS || err == -EBUSY) + return err; + + return ahash_update_finish(req, err); } EXPORT_SYMBOL_GPL(crypto_ahash_update); -int crypto_ahash_final(struct ahash_request *req) +static int ahash_finup_finish(struct ahash_request *req, int err) { struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + u8 *blenp = ahash_request_ctx(req); + int blen; - if (likely(tfm->using_shash)) - return crypto_shash_final(ahash_request_ctx(req), req->result); - if (ahash_req_on_stack(req) && ahash_is_async(tfm)) - return -EAGAIN; - return ahash_do_req_chain(req, crypto_ahash_alg(tfm)->final); + blenp += crypto_ahash_reqsize(tfm) - 1; + blen = *blenp; + + if (blen) { + if (sg_is_last(req->src)) + req->src = NULL; + else { + req->src = req->sg_head + 1; + if (sg_is_chain(req->src)) + req->src = sg_chain_ptr(req->src); + } + req->nbytes -= blen; + } + + ahash_restore_req(req); + + return err; +} + +static void ahash_finup_done(void *data, int err) +{ + ahash_op_done(data, err, ahash_finup_finish); } -EXPORT_SYMBOL_GPL(crypto_ahash_final); int crypto_ahash_finup(struct ahash_request *req) { struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + int bs = crypto_ahash_blocksize(tfm); + u8 *blenp = ahash_request_ctx(req); + int blen, err; + u8 *buf; if (likely(tfm->using_shash)) return shash_ahash_finup(req, ahash_request_ctx(req)); if (ahash_req_on_stack(req) && ahash_is_async(tfm)) return -EAGAIN; - if (!crypto_ahash_alg(tfm)->finup || - (!crypto_ahash_req_virt(tfm) && ahash_request_isvirt(req))) + if (!crypto_ahash_alg(tfm)->finup) return ahash_def_finup(req); - return ahash_do_req_chain(req, crypto_ahash_alg(tfm)->finup); + if (!crypto_ahash_block_only(tfm)) + return ahash_do_req_chain(req, &crypto_ahash_alg(tfm)->finup); + + blenp += crypto_ahash_reqsize(tfm) - 1; + blen = *blenp; + buf = blenp - bs; + + if (blen) { + memset(req->sg_head, 0, sizeof(req->sg_head[0])); + sg_set_buf(req->sg_head, buf, blen); + if (!req->src) + sg_mark_end(req->sg_head); + else if (req->src != req->sg_head + 1) + sg_chain(req->sg_head, 2, req->src); + req->src = req->sg_head; + req->nbytes += blen; + } + + ahash_save_req(req, ahash_finup_done); + + err = ahash_do_req_chain(req, &crypto_ahash_alg(tfm)->finup); + if (err == -EINPROGRESS || err == -EBUSY) + return err; + + return ahash_finup_finish(req, err); } EXPORT_SYMBOL_GPL(crypto_ahash_finup); -static int ahash_def_digest_finish(struct ahash_request *req, int err) -{ - struct crypto_ahash *tfm; - - if (err) - goto out; - - tfm = crypto_ahash_reqtfm(req); - if (ahash_is_async(tfm)) - req->base.complete = ahash_def_finup_done1; - - err = crypto_ahash_update(req); - if (err == -EINPROGRESS || err == -EBUSY) - return err; - - return ahash_def_finup_finish1(req, err); - -out: - ahash_restore_req(req); - return err; -} - -static void ahash_def_digest_done(void *data, int err) -{ - struct ahash_save_req_state *state0 = data; - struct ahash_save_req_state state; - struct ahash_request *areq; - - state = *state0; - areq = state.req0; - if (err == -EINPROGRESS) - goto out; - - areq->base.flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; - - err = ahash_def_digest_finish(areq, err); - if (err == -EINPROGRESS || err == -EBUSY) - return; - -out: - state.compl(state.data, err); -} - -static int ahash_def_digest(struct ahash_request *req) -{ - int err; - - err = ahash_save_req(req, ahash_def_digest_done); - if (err) - return err; - - err = crypto_ahash_init(req); - if (err == -EINPROGRESS || err == -EBUSY) - return err; - - return ahash_def_digest_finish(req, err); -} - int crypto_ahash_digest(struct ahash_request *req) { struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); @@ -622,18 +582,15 @@ int crypto_ahash_digest(struct ahash_request *req) return shash_ahash_digest(req, prepare_shash_desc(req, tfm)); if (ahash_req_on_stack(req) && ahash_is_async(tfm)) return -EAGAIN; - if (!crypto_ahash_req_virt(tfm) && ahash_request_isvirt(req)) - return ahash_def_digest(req); if (crypto_ahash_get_flags(tfm) & CRYPTO_TFM_NEED_KEY) return -ENOKEY; - return ahash_do_req_chain(req, crypto_ahash_alg(tfm)->digest); + return ahash_do_req_chain(req, &crypto_ahash_alg(tfm)->digest); } EXPORT_SYMBOL_GPL(crypto_ahash_digest); static void ahash_def_finup_done2(void *data, int err) { - struct ahash_save_req_state *state = data; - struct ahash_request *areq = state->req0; + struct ahash_request *areq = data; if (err == -EINPROGRESS) return; @@ -644,14 +601,10 @@ static void ahash_def_finup_done2(void *data, int err) static int ahash_def_finup_finish1(struct ahash_request *req, int err) { - struct crypto_ahash *tfm; - if (err) goto out; - tfm = crypto_ahash_reqtfm(req); - if (ahash_is_async(tfm)) - req->base.complete = ahash_def_finup_done2; + req->base.complete = ahash_def_finup_done2; err = crypto_ahash_final(req); if (err == -EINPROGRESS || err == -EBUSY) @@ -664,32 +617,14 @@ out: static void ahash_def_finup_done1(void *data, int err) { - struct ahash_save_req_state *state0 = data; - struct ahash_save_req_state state; - struct ahash_request *areq; - - state = *state0; - areq = state.req0; - if (err == -EINPROGRESS) - goto out; - - areq->base.flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; - - err = ahash_def_finup_finish1(areq, err); - if (err == -EINPROGRESS || err == -EBUSY) - return; - -out: - state.compl(state.data, err); + ahash_op_done(data, err, ahash_def_finup_finish1); } static int ahash_def_finup(struct ahash_request *req) { int err; - err = ahash_save_req(req, ahash_def_finup_done1); - if (err) - return err; + ahash_save_req(req, ahash_def_finup_done1); err = crypto_ahash_update(req); if (err == -EINPROGRESS || err == -EBUSY) @@ -714,6 +649,14 @@ int crypto_ahash_export(struct ahash_request *req, void *out) if (likely(tfm->using_shash)) return crypto_shash_export(ahash_request_ctx(req), out); + if (crypto_ahash_block_only(tfm)) { + unsigned int plen = crypto_ahash_blocksize(tfm) + 1; + unsigned int reqsize = crypto_ahash_reqsize(tfm); + unsigned int ss = crypto_ahash_statesize(tfm); + u8 *buf = ahash_request_ctx(req); + + memcpy(out + ss - plen, buf + reqsize - plen, plen); + } return crypto_ahash_alg(tfm)->export(req, out); } EXPORT_SYMBOL_GPL(crypto_ahash_export); @@ -739,6 +682,12 @@ int crypto_ahash_import(struct ahash_request *req, const void *in) return crypto_shash_import(prepare_shash_desc(req, tfm), in); if (crypto_ahash_get_flags(tfm) & CRYPTO_TFM_NEED_KEY) return -ENOKEY; + if (crypto_ahash_block_only(tfm)) { + unsigned int reqsize = crypto_ahash_reqsize(tfm); + u8 *buf = ahash_request_ctx(req); + + buf[reqsize - 1] = 0; + } return crypto_ahash_alg(tfm)->import(req, in); } EXPORT_SYMBOL_GPL(crypto_ahash_import); @@ -753,7 +702,7 @@ static void crypto_ahash_exit_tfm(struct crypto_tfm *tfm) else if (tfm->__crt_alg->cra_exit) tfm->__crt_alg->cra_exit(tfm); - if (ahash_is_async(hash)) + if (crypto_ahash_need_fallback(hash)) crypto_free_ahash(crypto_ahash_fb(hash)); } @@ -770,9 +719,12 @@ static int crypto_ahash_init_tfm(struct crypto_tfm *tfm) if (tfm->__crt_alg->cra_type == &crypto_shash_type) return crypto_init_ahash_using_shash(tfm); - if (ahash_is_async(hash)) { + if (crypto_ahash_need_fallback(hash)) { fb = crypto_alloc_ahash(crypto_ahash_alg_name(hash), - 0, CRYPTO_ALG_ASYNC); + CRYPTO_ALG_REQ_VIRT, + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_REQ_VIRT | + CRYPTO_AHASH_ALG_NO_EXPORT_CORE); if (IS_ERR(fb)) return PTR_ERR(fb); @@ -797,6 +749,10 @@ static int crypto_ahash_init_tfm(struct crypto_tfm *tfm) MAX_SYNC_HASH_REQSIZE) goto out_exit_tfm; + BUILD_BUG_ON(HASH_MAX_DESCSIZE > MAX_SYNC_HASH_REQSIZE); + if (crypto_ahash_reqsize(hash) < HASH_MAX_DESCSIZE) + crypto_ahash_set_reqsize(hash, HASH_MAX_DESCSIZE); + return 0; out_exit_tfm: @@ -941,7 +897,7 @@ struct crypto_ahash *crypto_clone_ahash(struct crypto_ahash *hash) return nhash; } - if (ahash_is_async(hash)) { + if (crypto_ahash_need_fallback(hash)) { fb = crypto_clone_ahash(crypto_ahash_fb(hash)); err = PTR_ERR(fb); if (IS_ERR(fb)) @@ -1003,10 +959,23 @@ static int ahash_prepare_alg(struct ahash_alg *alg) base->cra_type = &crypto_ahash_type; base->cra_flags |= CRYPTO_ALG_TYPE_AHASH; + if ((base->cra_flags ^ CRYPTO_ALG_REQ_VIRT) & + (CRYPTO_ALG_ASYNC | CRYPTO_ALG_REQ_VIRT)) + base->cra_flags |= CRYPTO_ALG_NEED_FALLBACK; + if (!alg->setkey) alg->setkey = ahash_nosetkey; - if (!alg->export_core || !alg->import_core) { + if (base->cra_flags & CRYPTO_AHASH_ALG_BLOCK_ONLY) { + BUILD_BUG_ON(MAX_ALGAPI_BLOCKSIZE >= 256); + if (!alg->finup) + return -EINVAL; + + base->cra_reqsize += base->cra_blocksize + 1; + alg->halg.statesize += base->cra_blocksize + 1; + alg->export_core = alg->export; + alg->import_core = alg->import; + } else if (!alg->export_core || !alg->import_core) { alg->export_core = ahash_default_export_core; alg->import_core = ahash_default_import_core; base->cra_flags |= CRYPTO_AHASH_ALG_NO_EXPORT_CORE; diff --git a/include/crypto/hash.h b/include/crypto/hash.h index bf177cf9be10..05ee817a3180 100644 --- a/include/crypto/hash.h +++ b/include/crypto/hash.h @@ -8,8 +8,8 @@ #ifndef _CRYPTO_HASH_H #define _CRYPTO_HASH_H -#include #include +#include #include #include @@ -65,6 +65,10 @@ struct ahash_request { }; u8 *result; + struct scatterlist sg_head[2]; + crypto_completion_t saved_complete; + void *saved_data; + void *__ctx[] CRYPTO_MINALIGN_ATTR; }; @@ -488,7 +492,11 @@ int crypto_ahash_finup(struct ahash_request *req); * -EBUSY if queue is full and request should be resubmitted later; * other < 0 if an error occurred */ -int crypto_ahash_final(struct ahash_request *req); +static inline int crypto_ahash_final(struct ahash_request *req) +{ + req->nbytes = 0; + return crypto_ahash_finup(req); +} /** * crypto_ahash_digest() - calculate message digest for a buffer From a05a8bc5705165fe3ce9ea79048a5ff807e4651f Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Thu, 15 May 2025 13:54:40 +0800 Subject: [PATCH 361/382] crypto: hmac - Zero shash desc in setkey The shash desc needs to be zeroed after use in setkey as it is not finalised (finalisation automatically zeroes it). Also remove the final function as it's been superseded by finup. Signed-off-by: Herbert Xu --- crypto/hmac.c | 35 ++++++++++------------------------- 1 file changed, 10 insertions(+), 25 deletions(-) diff --git a/crypto/hmac.c b/crypto/hmac.c index ba36ddf50037..4517e04bfbaa 100644 --- a/crypto/hmac.c +++ b/crypto/hmac.c @@ -13,13 +13,11 @@ #include #include -#include #include #include -#include #include #include -#include +#include #include struct hmac_ctx { @@ -39,7 +37,7 @@ static int hmac_setkey(struct crypto_shash *parent, u8 *ipad = &tctx->pads[0]; u8 *opad = &tctx->pads[ss]; SHASH_DESC_ON_STACK(shash, hash); - unsigned int i; + int err, i; if (fips_enabled && (keylen < 112 / 8)) return -EINVAL; @@ -65,12 +63,14 @@ static int hmac_setkey(struct crypto_shash *parent, opad[i] ^= HMAC_OPAD_VALUE; } - return crypto_shash_init(shash) ?: - crypto_shash_update(shash, ipad, bs) ?: - crypto_shash_export(shash, ipad) ?: - crypto_shash_init(shash) ?: - crypto_shash_update(shash, opad, bs) ?: - crypto_shash_export(shash, opad); + err = crypto_shash_init(shash) ?: + crypto_shash_update(shash, ipad, bs) ?: + crypto_shash_export(shash, ipad) ?: + crypto_shash_init(shash) ?: + crypto_shash_update(shash, opad, bs) ?: + crypto_shash_export(shash, opad); + shash_desc_zero(shash); + return err; } static int hmac_export(struct shash_desc *pdesc, void *out) @@ -105,20 +105,6 @@ static int hmac_update(struct shash_desc *pdesc, return crypto_shash_update(desc, data, nbytes); } -static int hmac_final(struct shash_desc *pdesc, u8 *out) -{ - struct crypto_shash *parent = pdesc->tfm; - int ds = crypto_shash_digestsize(parent); - int ss = crypto_shash_statesize(parent); - const struct hmac_ctx *tctx = crypto_shash_ctx(parent); - const u8 *opad = &tctx->pads[ss]; - struct shash_desc *desc = shash_desc_ctx(pdesc); - - return crypto_shash_final(desc, out) ?: - crypto_shash_import(desc, opad) ?: - crypto_shash_finup(desc, out, ds, out); -} - static int hmac_finup(struct shash_desc *pdesc, const u8 *data, unsigned int nbytes, u8 *out) { @@ -222,7 +208,6 @@ static int hmac_create(struct crypto_template *tmpl, struct rtattr **tb) inst->alg.descsize = sizeof(struct shash_desc) + salg->descsize; inst->alg.init = hmac_init; inst->alg.update = hmac_update; - inst->alg.final = hmac_final; inst->alg.finup = hmac_finup; inst->alg.export = hmac_export; inst->alg.import = hmac_import; From 8cf4c341f1931c20c564ab2ee0f9eb990a606cac Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Thu, 15 May 2025 13:54:42 +0800 Subject: [PATCH 362/382] crypto: hmac - Add export_core and import_core Add export_import and import_core so that hmac can be used as a fallback by block-only drivers. Signed-off-by: Herbert Xu --- crypto/hmac.c | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/crypto/hmac.c b/crypto/hmac.c index 4517e04bfbaa..e4749a1f93dd 100644 --- a/crypto/hmac.c +++ b/crypto/hmac.c @@ -90,6 +90,22 @@ static int hmac_import(struct shash_desc *pdesc, const void *in) return crypto_shash_import(desc, in); } +static int hmac_export_core(struct shash_desc *pdesc, void *out) +{ + struct shash_desc *desc = shash_desc_ctx(pdesc); + + return crypto_shash_export_core(desc, out); +} + +static int hmac_import_core(struct shash_desc *pdesc, const void *in) +{ + const struct hmac_ctx *tctx = crypto_shash_ctx(pdesc->tfm); + struct shash_desc *desc = shash_desc_ctx(pdesc); + + desc->tfm = tctx->hash; + return crypto_shash_import_core(desc, in); +} + static int hmac_init(struct shash_desc *pdesc) { const struct hmac_ctx *tctx = crypto_shash_ctx(pdesc->tfm); @@ -177,6 +193,7 @@ static int hmac_create(struct crypto_template *tmpl, struct rtattr **tb) return -ENOMEM; spawn = shash_instance_ctx(inst); + mask |= CRYPTO_AHASH_ALG_NO_EXPORT_CORE; err = crypto_grab_shash(spawn, shash_crypto_instance(inst), crypto_attr_alg_name(tb[1]), 0, mask); if (err) @@ -211,6 +228,8 @@ static int hmac_create(struct crypto_template *tmpl, struct rtattr **tb) inst->alg.finup = hmac_finup; inst->alg.export = hmac_export; inst->alg.import = hmac_import; + inst->alg.export_core = hmac_export_core; + inst->alg.import_core = hmac_import_core; inst->alg.setkey = hmac_setkey; inst->alg.init_tfm = hmac_init_tfm; inst->alg.clone_tfm = hmac_clone_tfm; From 32a9fd8f498bffd99c5d5441015136206e351ae4 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Thu, 15 May 2025 13:54:44 +0800 Subject: [PATCH 363/382] crypto: shash - Set reqsize in shash_alg Make reqsize static for shash algorithms. Signed-off-by: Herbert Xu --- crypto/ahash.c | 1 - crypto/shash.c | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/crypto/ahash.c b/crypto/ahash.c index cf8bbe7e54c0..bf8375bb32c9 100644 --- a/crypto/ahash.c +++ b/crypto/ahash.c @@ -286,7 +286,6 @@ static int crypto_init_ahash_using_shash(struct crypto_tfm *tfm) crypto_ahash_set_flags(crt, crypto_shash_get_flags(shash) & CRYPTO_TFM_NEED_KEY); - crt->reqsize = sizeof(struct shash_desc) + crypto_shash_descsize(shash); return 0; } diff --git a/crypto/shash.c b/crypto/shash.c index 5bc74a72d5ad..37537d7995c7 100644 --- a/crypto/shash.c +++ b/crypto/shash.c @@ -511,6 +511,8 @@ static int shash_prepare_alg(struct shash_alg *alg) if (alg->statesize > HASH_MAX_STATESIZE) return -EINVAL; + base->cra_reqsize = sizeof(struct shash_desc) + alg->descsize; + return 0; } From 91b6ff579dda7660a9d11778f9dc4dd0a879de22 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Thu, 15 May 2025 13:54:47 +0800 Subject: [PATCH 364/382] crypto: algapi - Add driver template support to crypto_inst_setname Add support to crypto_inst_setname for having a driver template name that differs from the algorithm template name. Signed-off-by: Herbert Xu --- crypto/algapi.c | 8 ++++---- include/crypto/algapi.h | 12 ++++++++++-- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/crypto/algapi.c b/crypto/algapi.c index 25b5519e3b71..e604d0d8b7b4 100644 --- a/crypto/algapi.c +++ b/crypto/algapi.c @@ -923,20 +923,20 @@ const char *crypto_attr_alg_name(struct rtattr *rta) } EXPORT_SYMBOL_GPL(crypto_attr_alg_name); -int crypto_inst_setname(struct crypto_instance *inst, const char *name, - struct crypto_alg *alg) +int __crypto_inst_setname(struct crypto_instance *inst, const char *name, + const char *driver, struct crypto_alg *alg) { if (snprintf(inst->alg.cra_name, CRYPTO_MAX_ALG_NAME, "%s(%s)", name, alg->cra_name) >= CRYPTO_MAX_ALG_NAME) return -ENAMETOOLONG; if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s(%s)", - name, alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME) + driver, alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME) return -ENAMETOOLONG; return 0; } -EXPORT_SYMBOL_GPL(crypto_inst_setname); +EXPORT_SYMBOL_GPL(__crypto_inst_setname); void crypto_init_queue(struct crypto_queue *queue, unsigned int max_qlen) { diff --git a/include/crypto/algapi.h b/include/crypto/algapi.h index 423e57eca351..188eface0a11 100644 --- a/include/crypto/algapi.h +++ b/include/crypto/algapi.h @@ -146,8 +146,16 @@ void *crypto_spawn_tfm2(struct crypto_spawn *spawn); struct crypto_attr_type *crypto_get_attr_type(struct rtattr **tb); int crypto_check_attr_type(struct rtattr **tb, u32 type, u32 *mask_ret); const char *crypto_attr_alg_name(struct rtattr *rta); -int crypto_inst_setname(struct crypto_instance *inst, const char *name, - struct crypto_alg *alg); +int __crypto_inst_setname(struct crypto_instance *inst, const char *name, + const char *driver, struct crypto_alg *alg); + +#define crypto_inst_setname(inst, name, ...) \ + CONCATENATE(crypto_inst_setname_, COUNT_ARGS(__VA_ARGS__))( \ + inst, name, ##__VA_ARGS__) +#define crypto_inst_setname_1(inst, name, alg) \ + __crypto_inst_setname(inst, name, name, alg) +#define crypto_inst_setname_2(inst, name, driver, alg) \ + __crypto_inst_setname(inst, name, driver, alg) void crypto_init_queue(struct crypto_queue *queue, unsigned int max_qlen); int crypto_enqueue_request(struct crypto_queue *queue, From 8e69871836669177d2d2a5acec5c8dca3319d2f1 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Thu, 15 May 2025 13:54:49 +0800 Subject: [PATCH 365/382] crypto: testmgr - Ignore EEXIST on shash allocation Soon hmac will support ahash. For compatibility hmac still supports shash so it is possible for two hmac algorithms to be registered at the same time. The shash algorithm will have the driver name "hmac-shash(XXX-driver)". Due to a quirk in the API, there is no way to locate the shash algorithm using the name "hmac(XXX-driver)". It has to be addressed as either "hmac(XXX)" or "hmac-shash(XXX-driver)". Looking it up with "hmac(XXX-driver)" will simply trigger the creation of another instance, and on the second instantiation this will fail with EEXIST. Catch the error EEXIST along with ENOENT since it is expected. If a real shash algorithm came this way, it would be addressed using the proper name "hmac-shash(XXX-driver)". Signed-off-by: Herbert Xu --- crypto/testmgr.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crypto/testmgr.c b/crypto/testmgr.c index fc28000c27f5..ee682ad50e34 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -1869,7 +1869,7 @@ static int alloc_shash(const char *driver, u32 type, u32 mask, tfm = crypto_alloc_shash(driver, type, mask); if (IS_ERR(tfm)) { - if (PTR_ERR(tfm) == -ENOENT) { + if (PTR_ERR(tfm) == -ENOENT || PTR_ERR(tfm) == -EEXIST) { /* * This algorithm is only available through the ahash * API, not the shash API, so skip the shash tests. From c3103416d5217655d707d9417aaf66f184e3d72f Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Thu, 15 May 2025 13:54:51 +0800 Subject: [PATCH 366/382] crypto: hmac - Add ahash support Add ahash support to hmac so that drivers that can't do hmac in hardware do not have to implement duplicate copies of hmac. Signed-off-by: Herbert Xu --- crypto/ahash.c | 10 +- crypto/hmac.c | 338 +++++++++++++++++++++++++++++++-- include/crypto/hash.h | 3 +- include/crypto/internal/hash.h | 9 + 4 files changed, 345 insertions(+), 15 deletions(-) diff --git a/crypto/ahash.c b/crypto/ahash.c index bf8375bb32c9..e10bc2659ae4 100644 --- a/crypto/ahash.c +++ b/crypto/ahash.c @@ -846,7 +846,7 @@ int crypto_has_ahash(const char *alg_name, u32 type, u32 mask) } EXPORT_SYMBOL_GPL(crypto_has_ahash); -static bool crypto_hash_alg_has_setkey(struct hash_alg_common *halg) +bool crypto_hash_alg_has_setkey(struct hash_alg_common *halg) { struct crypto_alg *alg = &halg->base; @@ -855,6 +855,7 @@ static bool crypto_hash_alg_has_setkey(struct hash_alg_common *halg) return __crypto_ahash_alg(alg)->setkey != ahash_nosetkey; } +EXPORT_SYMBOL_GPL(crypto_hash_alg_has_setkey); struct crypto_ahash *crypto_clone_ahash(struct crypto_ahash *hash) { @@ -1077,5 +1078,12 @@ int crypto_hash_digest(struct crypto_ahash *tfm, const u8 *data, } EXPORT_SYMBOL_GPL(crypto_hash_digest); +void ahash_free_singlespawn_instance(struct ahash_instance *inst) +{ + crypto_drop_spawn(ahash_instance_ctx(inst)); + kfree(inst); +} +EXPORT_SYMBOL_GPL(ahash_free_singlespawn_instance); + MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Asynchronous cryptographic hash type"); diff --git a/crypto/hmac.c b/crypto/hmac.c index e4749a1f93dd..148af460ae97 100644 --- a/crypto/hmac.c +++ b/crypto/hmac.c @@ -26,6 +26,12 @@ struct hmac_ctx { u8 pads[]; }; +struct ahash_hmac_ctx { + struct crypto_ahash *hash; + /* Contains 'u8 ipad[statesize];', then 'u8 opad[statesize];' */ + u8 pads[]; +}; + static int hmac_setkey(struct crypto_shash *parent, const u8 *inkey, unsigned int keylen) { @@ -173,21 +179,17 @@ static void hmac_exit_tfm(struct crypto_shash *parent) crypto_free_shash(tctx->hash); } -static int hmac_create(struct crypto_template *tmpl, struct rtattr **tb) +static int __hmac_create_shash(struct crypto_template *tmpl, + struct rtattr **tb, u32 mask) { struct shash_instance *inst; struct crypto_shash_spawn *spawn; struct crypto_alg *alg; struct shash_alg *salg; - u32 mask; int err; int ds; int ss; - err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_SHASH, &mask); - if (err) - return err; - inst = kzalloc(sizeof(*inst) + sizeof(*spawn), GFP_KERNEL); if (!inst) return -ENOMEM; @@ -212,7 +214,8 @@ static int hmac_create(struct crypto_template *tmpl, struct rtattr **tb) ss < alg->cra_blocksize) goto err_free_inst; - err = crypto_inst_setname(shash_crypto_instance(inst), tmpl->name, alg); + err = crypto_inst_setname(shash_crypto_instance(inst), "hmac", + "hmac-shash", alg); if (err) goto err_free_inst; @@ -245,20 +248,329 @@ err_free_inst: return err; } -static struct crypto_template hmac_tmpl = { - .name = "hmac", - .create = hmac_create, - .module = THIS_MODULE, +static int hmac_setkey_ahash(struct crypto_ahash *parent, + const u8 *inkey, unsigned int keylen) +{ + struct ahash_hmac_ctx *tctx = crypto_ahash_ctx(parent); + struct crypto_ahash *fb = crypto_ahash_fb(tctx->hash); + int ds = crypto_ahash_digestsize(parent); + int bs = crypto_ahash_blocksize(parent); + int ss = crypto_ahash_statesize(parent); + HASH_REQUEST_ON_STACK(req, fb); + u8 *opad = &tctx->pads[ss]; + u8 *ipad = &tctx->pads[0]; + int err, i; + + if (fips_enabled && (keylen < 112 / 8)) + return -EINVAL; + + ahash_request_set_callback(req, 0, NULL, NULL); + + if (keylen > bs) { + ahash_request_set_virt(req, inkey, ipad, keylen); + err = crypto_ahash_digest(req); + if (err) + goto out_zero_req; + + keylen = ds; + } else + memcpy(ipad, inkey, keylen); + + memset(ipad + keylen, 0, bs - keylen); + memcpy(opad, ipad, bs); + + for (i = 0; i < bs; i++) { + ipad[i] ^= HMAC_IPAD_VALUE; + opad[i] ^= HMAC_OPAD_VALUE; + } + + ahash_request_set_virt(req, ipad, NULL, bs); + err = crypto_ahash_init(req) ?: + crypto_ahash_update(req) ?: + crypto_ahash_export(req, ipad); + + ahash_request_set_virt(req, opad, NULL, bs); + err = err ?: + crypto_ahash_init(req) ?: + crypto_ahash_update(req) ?: + crypto_ahash_export(req, opad); + +out_zero_req: + HASH_REQUEST_ZERO(req); + return err; +} + +static int hmac_export_ahash(struct ahash_request *preq, void *out) +{ + return crypto_ahash_export(ahash_request_ctx(preq), out); +} + +static int hmac_import_ahash(struct ahash_request *preq, const void *in) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(preq); + struct ahash_hmac_ctx *tctx = crypto_ahash_ctx(tfm); + struct ahash_request *req = ahash_request_ctx(preq); + + ahash_request_set_tfm(req, tctx->hash); + return crypto_ahash_import(req, in); +} + +static int hmac_export_core_ahash(struct ahash_request *preq, void *out) +{ + return crypto_ahash_export_core(ahash_request_ctx(preq), out); +} + +static int hmac_import_core_ahash(struct ahash_request *preq, const void *in) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(preq); + struct ahash_hmac_ctx *tctx = crypto_ahash_ctx(tfm); + struct ahash_request *req = ahash_request_ctx(preq); + + ahash_request_set_tfm(req, tctx->hash); + return crypto_ahash_import_core(req, in); +} + +static int hmac_init_ahash(struct ahash_request *preq) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(preq); + struct ahash_hmac_ctx *tctx = crypto_ahash_ctx(tfm); + + return hmac_import_ahash(preq, &tctx->pads[0]); +} + +static int hmac_update_ahash(struct ahash_request *preq) +{ + struct ahash_request *req = ahash_request_ctx(preq); + + ahash_request_set_callback(req, ahash_request_flags(preq), + preq->base.complete, preq->base.data); + if (ahash_request_isvirt(preq)) + ahash_request_set_virt(req, preq->svirt, NULL, preq->nbytes); + else + ahash_request_set_crypt(req, preq->src, NULL, preq->nbytes); + return crypto_ahash_update(req); +} + +static int hmac_finup_finish(struct ahash_request *preq, unsigned int mask) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(preq); + struct ahash_request *req = ahash_request_ctx(preq); + struct ahash_hmac_ctx *tctx = crypto_ahash_ctx(tfm); + int ds = crypto_ahash_digestsize(tfm); + int ss = crypto_ahash_statesize(tfm); + const u8 *opad = &tctx->pads[ss]; + + ahash_request_set_callback(req, ahash_request_flags(preq) & ~mask, + preq->base.complete, preq->base.data); + ahash_request_set_virt(req, preq->result, preq->result, ds); + return crypto_ahash_import(req, opad) ?: + crypto_ahash_finup(req); + +} + +static void hmac_finup_done(void *data, int err) +{ + struct ahash_request *preq = data; + + if (err) + goto out; + + err = hmac_finup_finish(preq, CRYPTO_TFM_REQ_MAY_SLEEP); + if (err == -EINPROGRESS || err == -EBUSY) + return; + +out: + ahash_request_complete(preq, err); +} + +static int hmac_finup_ahash(struct ahash_request *preq) +{ + struct ahash_request *req = ahash_request_ctx(preq); + + ahash_request_set_callback(req, ahash_request_flags(preq), + hmac_finup_done, preq); + if (ahash_request_isvirt(preq)) + ahash_request_set_virt(req, preq->svirt, preq->result, + preq->nbytes); + else + ahash_request_set_crypt(req, preq->src, preq->result, + preq->nbytes); + return crypto_ahash_finup(req) ?: + hmac_finup_finish(preq, 0); +} + +static int hmac_digest_ahash(struct ahash_request *preq) +{ + return hmac_init_ahash(preq) ?: + hmac_finup_ahash(preq); +} + +static int hmac_init_ahash_tfm(struct crypto_ahash *parent) +{ + struct ahash_instance *inst = ahash_alg_instance(parent); + struct ahash_hmac_ctx *tctx = crypto_ahash_ctx(parent); + struct crypto_ahash *hash; + + hash = crypto_spawn_ahash(ahash_instance_ctx(inst)); + if (IS_ERR(hash)) + return PTR_ERR(hash); + + if (crypto_ahash_reqsize(parent) < sizeof(struct ahash_request) + + crypto_ahash_reqsize(hash)) + return -EINVAL; + + tctx->hash = hash; + return 0; +} + +static int hmac_clone_ahash_tfm(struct crypto_ahash *dst, + struct crypto_ahash *src) +{ + struct ahash_hmac_ctx *sctx = crypto_ahash_ctx(src); + struct ahash_hmac_ctx *dctx = crypto_ahash_ctx(dst); + struct crypto_ahash *hash; + + hash = crypto_clone_ahash(sctx->hash); + if (IS_ERR(hash)) + return PTR_ERR(hash); + + dctx->hash = hash; + return 0; +} + +static void hmac_exit_ahash_tfm(struct crypto_ahash *parent) +{ + struct ahash_hmac_ctx *tctx = crypto_ahash_ctx(parent); + + crypto_free_ahash(tctx->hash); +} + +static int hmac_create_ahash(struct crypto_template *tmpl, struct rtattr **tb, + u32 mask) +{ + struct crypto_ahash_spawn *spawn; + struct ahash_instance *inst; + struct crypto_alg *alg; + struct hash_alg_common *halg; + int ds, ss, err; + + inst = kzalloc(sizeof(*inst) + sizeof(*spawn), GFP_KERNEL); + if (!inst) + return -ENOMEM; + spawn = ahash_instance_ctx(inst); + + mask |= CRYPTO_AHASH_ALG_NO_EXPORT_CORE; + err = crypto_grab_ahash(spawn, ahash_crypto_instance(inst), + crypto_attr_alg_name(tb[1]), 0, mask); + if (err) + goto err_free_inst; + halg = crypto_spawn_ahash_alg(spawn); + alg = &halg->base; + + /* The underlying hash algorithm must not require a key */ + err = -EINVAL; + if (crypto_hash_alg_needs_key(halg)) + goto err_free_inst; + + ds = halg->digestsize; + ss = halg->statesize; + if (ds > alg->cra_blocksize || ss < alg->cra_blocksize) + goto err_free_inst; + + err = crypto_inst_setname(ahash_crypto_instance(inst), tmpl->name, alg); + if (err) + goto err_free_inst; + + inst->alg.halg.base.cra_flags = alg->cra_flags & + CRYPTO_ALG_INHERITED_FLAGS; + inst->alg.halg.base.cra_flags |= CRYPTO_ALG_REQ_VIRT; + inst->alg.halg.base.cra_priority = alg->cra_priority + 100; + inst->alg.halg.base.cra_blocksize = alg->cra_blocksize; + inst->alg.halg.base.cra_ctxsize = sizeof(struct ahash_hmac_ctx) + + (ss * 2); + inst->alg.halg.base.cra_reqsize = sizeof(struct ahash_request) + + alg->cra_reqsize; + + inst->alg.halg.digestsize = ds; + inst->alg.halg.statesize = ss; + inst->alg.init = hmac_init_ahash; + inst->alg.update = hmac_update_ahash; + inst->alg.finup = hmac_finup_ahash; + inst->alg.digest = hmac_digest_ahash; + inst->alg.export = hmac_export_ahash; + inst->alg.import = hmac_import_ahash; + inst->alg.export_core = hmac_export_core_ahash; + inst->alg.import_core = hmac_import_core_ahash; + inst->alg.setkey = hmac_setkey_ahash; + inst->alg.init_tfm = hmac_init_ahash_tfm; + inst->alg.clone_tfm = hmac_clone_ahash_tfm; + inst->alg.exit_tfm = hmac_exit_ahash_tfm; + + inst->free = ahash_free_singlespawn_instance; + + err = ahash_register_instance(tmpl, inst); + if (err) { +err_free_inst: + ahash_free_singlespawn_instance(inst); + } + return err; +} + +static int hmac_create(struct crypto_template *tmpl, struct rtattr **tb) +{ + struct crypto_attr_type *algt; + u32 mask; + + algt = crypto_get_attr_type(tb); + if (IS_ERR(algt)) + return PTR_ERR(algt); + + mask = crypto_algt_inherited_mask(algt); + + if (!((algt->type ^ CRYPTO_ALG_TYPE_AHASH) & + algt->mask & CRYPTO_ALG_TYPE_MASK)) + return hmac_create_ahash(tmpl, tb, mask); + + if ((algt->type ^ CRYPTO_ALG_TYPE_SHASH) & + algt->mask & CRYPTO_ALG_TYPE_MASK) + return -EINVAL; + + return __hmac_create_shash(tmpl, tb, mask); +} + +static int hmac_create_shash(struct crypto_template *tmpl, struct rtattr **tb) +{ + u32 mask; + int err; + + err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_SHASH, &mask); + if (err) + return err == -EINVAL ? -ENOENT : err; + + return __hmac_create_shash(tmpl, tb, mask); +} + +static struct crypto_template hmac_tmpls[] = { + { + .name = "hmac", + .create = hmac_create, + .module = THIS_MODULE, + }, + { + .name = "hmac-shash", + .create = hmac_create_shash, + .module = THIS_MODULE, + }, }; static int __init hmac_module_init(void) { - return crypto_register_template(&hmac_tmpl); + return crypto_register_templates(hmac_tmpls, ARRAY_SIZE(hmac_tmpls)); } static void __exit hmac_module_exit(void) { - crypto_unregister_template(&hmac_tmpl); + crypto_unregister_templates(hmac_tmpls, ARRAY_SIZE(hmac_tmpls)); } module_init(hmac_module_init); diff --git a/include/crypto/hash.h b/include/crypto/hash.h index 05ee817a3180..6f6b9de12cd3 100644 --- a/include/crypto/hash.h +++ b/include/crypto/hash.h @@ -185,7 +185,8 @@ struct shash_desc { * containing a 'struct s390_sha_ctx'. */ #define HASH_MAX_DESCSIZE (sizeof(struct shash_desc) + 360) -#define MAX_SYNC_HASH_REQSIZE HASH_MAX_DESCSIZE +#define MAX_SYNC_HASH_REQSIZE (sizeof(struct ahash_request) + \ + HASH_MAX_DESCSIZE) #define SHASH_DESC_ON_STACK(shash, ctx) \ char __##shash##_desc[sizeof(struct shash_desc) + HASH_MAX_DESCSIZE] \ diff --git a/include/crypto/internal/hash.h b/include/crypto/internal/hash.h index e9de2bc34a10..0f85c543f80b 100644 --- a/include/crypto/internal/hash.h +++ b/include/crypto/internal/hash.h @@ -67,6 +67,7 @@ int crypto_register_ahashes(struct ahash_alg *algs, int count); void crypto_unregister_ahashes(struct ahash_alg *algs, int count); int ahash_register_instance(struct crypto_template *tmpl, struct ahash_instance *inst); +void ahash_free_singlespawn_instance(struct ahash_instance *inst); int shash_no_setkey(struct crypto_shash *tfm, const u8 *key, unsigned int keylen); @@ -76,12 +77,20 @@ static inline bool crypto_shash_alg_has_setkey(struct shash_alg *alg) return alg->setkey != shash_no_setkey; } +bool crypto_hash_alg_has_setkey(struct hash_alg_common *halg); + static inline bool crypto_shash_alg_needs_key(struct shash_alg *alg) { return crypto_shash_alg_has_setkey(alg) && !(alg->base.cra_flags & CRYPTO_ALG_OPTIONAL_KEY); } +static inline bool crypto_hash_alg_needs_key(struct hash_alg_common *alg) +{ + return crypto_hash_alg_has_setkey(alg) && + !(alg->base.cra_flags & CRYPTO_ALG_OPTIONAL_KEY); +} + int crypto_grab_ahash(struct crypto_ahash_spawn *spawn, struct crypto_instance *inst, const char *name, u32 type, u32 mask); From c2a813545ed4810540dcc8ebdcb21092fda80be5 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Thu, 15 May 2025 13:54:54 +0800 Subject: [PATCH 367/382] crypto: testmgr - Use ahash for generic tfm As shash is being phased out, use ahash for the generic tfm. Signed-off-by: Herbert Xu --- crypto/testmgr.c | 37 ++++++++++++++++++------------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/crypto/testmgr.c b/crypto/testmgr.c index ee682ad50e34..72005074a5c2 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -1699,7 +1699,7 @@ static int test_hash_vec(const struct hash_testvec *vec, unsigned int vec_num, * Assumes the buffers in 'vec' were already allocated. */ static void generate_random_hash_testvec(struct rnd_state *rng, - struct shash_desc *desc, + struct ahash_request *req, struct hash_testvec *vec, unsigned int maxkeysize, unsigned int maxdatasize, @@ -1721,16 +1721,17 @@ static void generate_random_hash_testvec(struct rnd_state *rng, vec->ksize = prandom_u32_inclusive(rng, 1, maxkeysize); generate_random_bytes(rng, (u8 *)vec->key, vec->ksize); - vec->setkey_error = crypto_shash_setkey(desc->tfm, vec->key, - vec->ksize); + vec->setkey_error = crypto_ahash_setkey( + crypto_ahash_reqtfm(req), vec->key, vec->ksize); /* If the key couldn't be set, no need to continue to digest. */ if (vec->setkey_error) goto done; } /* Digest */ - vec->digest_error = crypto_shash_digest(desc, vec->plaintext, - vec->psize, (u8 *)vec->digest); + vec->digest_error = crypto_hash_digest( + crypto_ahash_reqtfm(req), vec->plaintext, + vec->psize, (u8 *)vec->digest); done: snprintf(name, max_namelen, "\"random: psize=%u ksize=%u\"", vec->psize, vec->ksize); @@ -1755,8 +1756,8 @@ static int test_hash_vs_generic_impl(const char *generic_driver, const char *driver = crypto_ahash_driver_name(tfm); struct rnd_state rng; char _generic_driver[CRYPTO_MAX_ALG_NAME]; - struct crypto_shash *generic_tfm = NULL; - struct shash_desc *generic_desc = NULL; + struct ahash_request *generic_req = NULL; + struct crypto_ahash *generic_tfm = NULL; unsigned int i; struct hash_testvec vec = { 0 }; char vec_name[64]; @@ -1779,7 +1780,7 @@ static int test_hash_vs_generic_impl(const char *generic_driver, if (strcmp(generic_driver, driver) == 0) /* Already the generic impl? */ return 0; - generic_tfm = crypto_alloc_shash(generic_driver, 0, 0); + generic_tfm = crypto_alloc_ahash(generic_driver, 0, 0); if (IS_ERR(generic_tfm)) { err = PTR_ERR(generic_tfm); if (err == -ENOENT) { @@ -1798,27 +1799,25 @@ static int test_hash_vs_generic_impl(const char *generic_driver, goto out; } - generic_desc = kzalloc(sizeof(*desc) + - crypto_shash_descsize(generic_tfm), GFP_KERNEL); - if (!generic_desc) { + generic_req = ahash_request_alloc(generic_tfm, GFP_KERNEL); + if (!generic_req) { err = -ENOMEM; goto out; } - generic_desc->tfm = generic_tfm; /* Check the algorithm properties for consistency. */ - if (digestsize != crypto_shash_digestsize(generic_tfm)) { + if (digestsize != crypto_ahash_digestsize(generic_tfm)) { pr_err("alg: hash: digestsize for %s (%u) doesn't match generic impl (%u)\n", driver, digestsize, - crypto_shash_digestsize(generic_tfm)); + crypto_ahash_digestsize(generic_tfm)); err = -EINVAL; goto out; } - if (blocksize != crypto_shash_blocksize(generic_tfm)) { + if (blocksize != crypto_ahash_blocksize(generic_tfm)) { pr_err("alg: hash: blocksize for %s (%u) doesn't match generic impl (%u)\n", - driver, blocksize, crypto_shash_blocksize(generic_tfm)); + driver, blocksize, crypto_ahash_blocksize(generic_tfm)); err = -EINVAL; goto out; } @@ -1837,7 +1836,7 @@ static int test_hash_vs_generic_impl(const char *generic_driver, } for (i = 0; i < fuzz_iterations * 8; i++) { - generate_random_hash_testvec(&rng, generic_desc, &vec, + generate_random_hash_testvec(&rng, generic_req, &vec, maxkeysize, maxdatasize, vec_name, sizeof(vec_name)); generate_random_testvec_config(&rng, cfg, cfgname, @@ -1855,8 +1854,8 @@ out: kfree(vec.key); kfree(vec.plaintext); kfree(vec.digest); - crypto_free_shash(generic_tfm); - kfree_sensitive(generic_desc); + ahash_request_free(generic_req); + crypto_free_ahash(generic_tfm); return err; } From 18c438b228558e05ede7dccf947a6547516fc0c7 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Thu, 15 May 2025 13:54:56 +0800 Subject: [PATCH 368/382] crypto: testmgr - Add hash export format testing Ensure that the hash state can be exported to and imported from the generic algorithm. Signed-off-by: Herbert Xu --- crypto/testmgr.c | 95 ++++++++++++++++++++++++++++++---- crypto/testmgr.h | 2 + include/crypto/internal/hash.h | 6 +++ 3 files changed, 94 insertions(+), 9 deletions(-) diff --git a/crypto/testmgr.c b/crypto/testmgr.c index 72005074a5c2..737064b31480 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -17,10 +17,19 @@ */ #include -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include #include #include +#include #include #include #include @@ -28,14 +37,6 @@ #include #include #include -#include -#include -#include -#include -#include -#include -#include -#include #include "internal.h" @@ -1464,6 +1465,49 @@ static int check_nonfinal_ahash_op(const char *op, int err, return 0; } +static int check_ahash_export(struct ahash_request *req, + const struct hash_testvec *vec, + const char *vec_name, + const struct testvec_config *cfg, + const char *driver, u8 *hashstate) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + const unsigned int digestsize = crypto_ahash_digestsize(tfm); + HASH_FBREQ_ON_STACK(fbreq, req); + int err; + + if (!vec->state) + return 0; + + err = crypto_ahash_export(req, hashstate); + if (err) { + pr_err("alg: ahash: %s mixed export() failed with err %d on test vector %s, cfg=\"%s\"\n", + driver, err, vec_name, cfg->name); + return err; + } + err = crypto_ahash_import(req, vec->state); + if (err) { + pr_err("alg: ahash: %s mixed import() failed with err %d on test vector %s, cfg=\"%s\"\n", + driver, err, vec_name, cfg->name); + return err; + } + err = crypto_ahash_import(fbreq, hashstate); + if (err) { + pr_err("alg: ahash: %s fallback import() failed with err %d on test vector %s, cfg=\"%s\"\n", + crypto_ahash_driver_name(crypto_ahash_reqtfm(fbreq)), err, vec_name, cfg->name); + return err; + } + ahash_request_set_crypt(fbreq, NULL, hashstate, 0); + testmgr_poison(hashstate, digestsize + TESTMGR_POISON_LEN); + err = crypto_ahash_final(fbreq); + if (err) { + pr_err("alg: ahash: %s fallback final() failed with err %d on test vector %s, cfg=\"%s\"\n", + crypto_ahash_driver_name(crypto_ahash_reqtfm(fbreq)), err, vec_name, cfg->name); + return err; + } + return check_hash_result("ahash export", hashstate, digestsize, vec, vec_name, driver, cfg); +} + /* Test one hash test vector in one configuration, using the ahash API */ static int test_ahash_vec_cfg(const struct hash_testvec *vec, const char *vec_name, @@ -1609,6 +1653,10 @@ static int test_ahash_vec_cfg(const struct hash_testvec *vec, driver, vec_name, cfg); if (err) return err; + err = check_ahash_export(req, vec, vec_name, cfg, + driver, hashstate); + if (err) + return err; err = do_ahash_op(crypto_ahash_final, req, &wait, cfg->nosimd); if (err) { pr_err("alg: ahash: %s final() failed with err %d on test vector %s, cfg=\"%s\"\n", @@ -1732,6 +1780,17 @@ static void generate_random_hash_testvec(struct rnd_state *rng, vec->digest_error = crypto_hash_digest( crypto_ahash_reqtfm(req), vec->plaintext, vec->psize, (u8 *)vec->digest); + + if (vec->digest_error || !vec->state) + goto done; + + ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP, NULL, NULL); + ahash_request_set_virt(req, vec->plaintext, (u8 *)vec->digest, + vec->psize); + crypto_ahash_init(req); + crypto_ahash_update(req); + crypto_ahash_export(req, (u8 *)vec->state); + done: snprintf(name, max_namelen, "\"random: psize=%u ksize=%u\"", vec->psize, vec->ksize); @@ -1750,6 +1809,7 @@ static int test_hash_vs_generic_impl(const char *generic_driver, { struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); const unsigned int digestsize = crypto_ahash_digestsize(tfm); + const unsigned int statesize = crypto_ahash_statesize(tfm); const unsigned int blocksize = crypto_ahash_blocksize(tfm); const unsigned int maxdatasize = (2 * PAGE_SIZE) - TESTMGR_POISON_LEN; const char *algname = crypto_hash_alg_common(tfm)->base.cra_name; @@ -1822,6 +1882,22 @@ static int test_hash_vs_generic_impl(const char *generic_driver, goto out; } + if (crypto_hash_no_export_core(tfm) || + crypto_hash_no_export_core(generic_tfm)) + ; + else if (statesize != crypto_ahash_statesize(generic_tfm)) { + pr_err("alg: hash: statesize for %s (%u) doesn't match generic impl (%u)\n", + driver, statesize, + crypto_ahash_statesize(generic_tfm)); + err = -EINVAL; + goto out; + } else { + vec.state = kmalloc(statesize, GFP_KERNEL); + err = -ENOMEM; + if (!vec.state) + goto out; + } + /* * Now generate test vectors using the generic implementation, and test * the other implementation against them. @@ -1854,6 +1930,7 @@ out: kfree(vec.key); kfree(vec.plaintext); kfree(vec.digest); + kfree(vec.state); ahash_request_free(generic_req); crypto_free_ahash(generic_tfm); return err; diff --git a/crypto/testmgr.h b/crypto/testmgr.h index 32d099ac9e73..5cf455a708b8 100644 --- a/crypto/testmgr.h +++ b/crypto/testmgr.h @@ -29,6 +29,7 @@ * hash_testvec: structure to describe a hash (message digest) test * @key: Pointer to key (NULL if none) * @plaintext: Pointer to source data + * @state: Pointer to expected state * @digest: Pointer to expected digest * @psize: Length of source data in bytes * @ksize: Length of @key in bytes (0 if no key) @@ -39,6 +40,7 @@ struct hash_testvec { const char *key; const char *plaintext; + const char *state; const char *digest; unsigned int psize; unsigned short ksize; diff --git a/include/crypto/internal/hash.h b/include/crypto/internal/hash.h index 0f85c543f80b..f052afa6e7b0 100644 --- a/include/crypto/internal/hash.h +++ b/include/crypto/internal/hash.h @@ -91,6 +91,12 @@ static inline bool crypto_hash_alg_needs_key(struct hash_alg_common *alg) !(alg->base.cra_flags & CRYPTO_ALG_OPTIONAL_KEY); } +static inline bool crypto_hash_no_export_core(struct crypto_ahash *tfm) +{ + return crypto_hash_alg_common(tfm)->base.cra_flags & + CRYPTO_AHASH_ALG_NO_EXPORT_CORE; +} + int crypto_grab_ahash(struct crypto_ahash_spawn *spawn, struct crypto_instance *inst, const char *name, u32 type, u32 mask); From 3d73909bddc2ebb3224a8bc2e5ce00e9df70c15d Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Thu, 15 May 2025 16:28:08 +0800 Subject: [PATCH 369/382] crypto: lrw - Only add ecb if it is not already there Only add ecb to the cipher name if it isn't already ecb. Also use memcmp instead of strncmp since these strings are all stored in an array of length CRYPTO_MAX_ALG_NAME. Fixes: 700cb3f5fe75 ("crypto: lrw - Convert to skcipher") Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-lkp/202505151503.d8a6cf10-lkp@intel.com Signed-off-by: Herbert Xu --- crypto/lrw.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crypto/lrw.c b/crypto/lrw.c index e7f0368f8c97..dd403b800513 100644 --- a/crypto/lrw.c +++ b/crypto/lrw.c @@ -322,7 +322,7 @@ static int lrw_create(struct crypto_template *tmpl, struct rtattr **tb) err = crypto_grab_skcipher(spawn, skcipher_crypto_instance(inst), cipher_name, 0, mask); - if (err == -ENOENT) { + if (err == -ENOENT && memcmp(cipher_name, "ecb(", 4)) { err = -ENAMETOOLONG; if (snprintf(ecb_name, CRYPTO_MAX_ALG_NAME, "ecb(%s)", cipher_name) >= CRYPTO_MAX_ALG_NAME) @@ -356,7 +356,7 @@ static int lrw_create(struct crypto_template *tmpl, struct rtattr **tb) /* Alas we screwed up the naming so we have to mangle the * cipher name. */ - if (!strncmp(cipher_name, "ecb(", 4)) { + if (!memcmp(cipher_name, "ecb(", 4)) { int len; len = strscpy(ecb_name, cipher_name + 4, sizeof(ecb_name)); From 270b6f13454cb7f2f7058c50df64df409c5dcf55 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Thu, 15 May 2025 16:34:04 +0800 Subject: [PATCH 370/382] crypto: xts - Only add ecb if it is not already there Only add ecb to the cipher name if it isn't already ecb. Also use memcmp instead of strncmp since these strings are all stored in an array of length CRYPTO_MAX_ALG_NAME. Fixes: f1c131b45410 ("crypto: xts - Convert to skcipher") Signed-off-by: Herbert Xu --- crypto/xts.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crypto/xts.c b/crypto/xts.c index 1a9edd55a3a2..3da8f5e053d6 100644 --- a/crypto/xts.c +++ b/crypto/xts.c @@ -363,7 +363,7 @@ static int xts_create(struct crypto_template *tmpl, struct rtattr **tb) err = crypto_grab_skcipher(&ctx->spawn, skcipher_crypto_instance(inst), cipher_name, 0, mask); - if (err == -ENOENT) { + if (err == -ENOENT && memcmp(cipher_name, "ecb(", 4)) { err = -ENAMETOOLONG; if (snprintf(name, CRYPTO_MAX_ALG_NAME, "ecb(%s)", cipher_name) >= CRYPTO_MAX_ALG_NAME) @@ -397,7 +397,7 @@ static int xts_create(struct crypto_template *tmpl, struct rtattr **tb) /* Alas we screwed up the naming so we have to mangle the * cipher name. */ - if (!strncmp(cipher_name, "ecb(", 4)) { + if (!memcmp(cipher_name, "ecb(", 4)) { int len; len = strscpy(name, cipher_name + 4, sizeof(name)); From 57b1e1c0901c9a035cd19b3b08bc8b87edbf69c1 Mon Sep 17 00:00:00 2001 From: Bharat Bhushan Date: Fri, 16 May 2025 14:14:40 +0530 Subject: [PATCH 371/382] crypto: octeontx2 - Initialize cptlfs device info once Function otx2_cptlf_set_dev_info() initializes common fields of cptlfs data-struct. This function is called every time a cptlf is initialized but this needs be done once for a cptlf block. So this initialization is moved to early device probe code to avoid redundant initialization. Signed-off-by: Bharat Bhushan Signed-off-by: Herbert Xu --- drivers/crypto/marvell/octeontx2/otx2_cptpf_main.c | 6 ++++++ drivers/crypto/marvell/octeontx2/otx2_cptpf_mbox.c | 5 ----- drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c | 2 -- drivers/crypto/marvell/octeontx2/otx2_cptvf_main.c | 5 +++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptpf_main.c b/drivers/crypto/marvell/octeontx2/otx2_cptpf_main.c index 12971300296d..687b6c7d7674 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cptpf_main.c +++ b/drivers/crypto/marvell/octeontx2/otx2_cptpf_main.c @@ -639,6 +639,12 @@ static int cptpf_device_init(struct otx2_cptpf_dev *cptpf) /* Disable all cores */ ret = otx2_cpt_disable_all_cores(cptpf); + otx2_cptlf_set_dev_info(&cptpf->lfs, cptpf->pdev, cptpf->reg_base, + &cptpf->afpf_mbox, BLKADDR_CPT0); + if (cptpf->has_cpt1) + otx2_cptlf_set_dev_info(&cptpf->cpt1_lfs, cptpf->pdev, + cptpf->reg_base, &cptpf->afpf_mbox, + BLKADDR_CPT1); return ret; } diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptpf_mbox.c b/drivers/crypto/marvell/octeontx2/otx2_cptpf_mbox.c index ec1ac7e836a3..3eb45bb91296 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cptpf_mbox.c +++ b/drivers/crypto/marvell/octeontx2/otx2_cptpf_mbox.c @@ -264,8 +264,6 @@ static int handle_msg_rx_inline_ipsec_lf_cfg(struct otx2_cptpf_dev *cptpf, return -ENOENT; } - otx2_cptlf_set_dev_info(&cptpf->lfs, cptpf->pdev, cptpf->reg_base, - &cptpf->afpf_mbox, BLKADDR_CPT0); cptpf->lfs.global_slot = 0; cptpf->lfs.ctx_ilen_ovrd = cfg_req->ctx_ilen_valid; cptpf->lfs.ctx_ilen = cfg_req->ctx_ilen; @@ -278,9 +276,6 @@ static int handle_msg_rx_inline_ipsec_lf_cfg(struct otx2_cptpf_dev *cptpf, if (cptpf->has_cpt1) { cptpf->rsrc_req_blkaddr = BLKADDR_CPT1; - otx2_cptlf_set_dev_info(&cptpf->cpt1_lfs, cptpf->pdev, - cptpf->reg_base, &cptpf->afpf_mbox, - BLKADDR_CPT1); cptpf->cpt1_lfs.global_slot = num_lfs; cptpf->cpt1_lfs.ctx_ilen_ovrd = cfg_req->ctx_ilen_valid; cptpf->cpt1_lfs.ctx_ilen = cfg_req->ctx_ilen; diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c b/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c index 42c5484ce66a..78367849c3d5 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c +++ b/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c @@ -1513,8 +1513,6 @@ int otx2_cpt_discover_eng_capabilities(struct otx2_cptpf_dev *cptpf) if (ret) goto delete_grps; - otx2_cptlf_set_dev_info(lfs, cptpf->pdev, cptpf->reg_base, - &cptpf->afpf_mbox, BLKADDR_CPT0); ret = otx2_cptlf_init(lfs, OTX2_CPT_ALL_ENG_GRPS_MASK, OTX2_CPT_QUEUE_HI_PRIO, 1); if (ret) diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptvf_main.c b/drivers/crypto/marvell/octeontx2/otx2_cptvf_main.c index d84eebdf2fa8..11e351a48efe 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cptvf_main.c +++ b/drivers/crypto/marvell/octeontx2/otx2_cptvf_main.c @@ -283,8 +283,6 @@ static int cptvf_lf_init(struct otx2_cptvf_dev *cptvf) lfs_num = cptvf->lfs.kvf_limits; - otx2_cptlf_set_dev_info(lfs, cptvf->pdev, cptvf->reg_base, - &cptvf->pfvf_mbox, cptvf->blkaddr); ret = otx2_cptlf_init(lfs, eng_grp_msk, OTX2_CPT_QUEUE_HI_PRIO, lfs_num); if (ret) @@ -396,6 +394,9 @@ static int otx2_cptvf_probe(struct pci_dev *pdev, cptvf_hw_ops_get(cptvf); + otx2_cptlf_set_dev_info(&cptvf->lfs, cptvf->pdev, cptvf->reg_base, + &cptvf->pfvf_mbox, cptvf->blkaddr); + ret = otx2_cptvf_send_caps_msg(cptvf); if (ret) { dev_err(&pdev->dev, "Couldn't get CPT engine capabilities.\n"); From d15cbe7137caae0de4fcd368e6f4d89b42b6538b Mon Sep 17 00:00:00 2001 From: Bharat Bhushan Date: Fri, 16 May 2025 14:14:41 +0530 Subject: [PATCH 372/382] crypto: octeontx2 - Use dynamic allocated memory region for lmtst Current driver uses static LMTST region allocated by firmware. Firmware allocated memory for LMTST is available in PF/VF BAR2. Using this memory have performance impact as this is mapped as device memory. There is another option to allocate contiguous memory at run time and map this in LMT MAP table with the help of AF driver. With this patch dynamic allocated memory is used for LMTST. Also add myself as maintainer for crypto octeontx2 driver Signed-off-by: Bharat Bhushan Signed-off-by: Herbert Xu --- MAINTAINERS | 1 + drivers/crypto/marvell/octeontx2/cn10k_cpt.c | 89 ++++++++++++++----- drivers/crypto/marvell/octeontx2/cn10k_cpt.h | 1 + .../marvell/octeontx2/otx2_cpt_common.h | 1 + .../marvell/octeontx2/otx2_cpt_mbox_common.c | 25 ++++++ drivers/crypto/marvell/octeontx2/otx2_cptlf.c | 5 +- drivers/crypto/marvell/octeontx2/otx2_cptlf.h | 12 ++- .../marvell/octeontx2/otx2_cptpf_main.c | 12 ++- .../marvell/octeontx2/otx2_cptpf_mbox.c | 1 + .../marvell/octeontx2/otx2_cptvf_main.c | 16 ++-- .../marvell/octeontx2/otx2_cptvf_mbox.c | 1 + 11 files changed, 125 insertions(+), 39 deletions(-) diff --git a/MAINTAINERS b/MAINTAINERS index b951660f0990..85f7619e06b6 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -14290,6 +14290,7 @@ F: include/uapi/drm/armada_drm.h MARVELL CRYPTO DRIVER M: Srujana Challa +M: Bharat Bhushan L: linux-crypto@vger.kernel.org S: Maintained F: drivers/crypto/marvell/ diff --git a/drivers/crypto/marvell/octeontx2/cn10k_cpt.c b/drivers/crypto/marvell/octeontx2/cn10k_cpt.c index 5cae8fafa151..d4aab9e20f2a 100644 --- a/drivers/crypto/marvell/octeontx2/cn10k_cpt.c +++ b/drivers/crypto/marvell/octeontx2/cn10k_cpt.c @@ -6,6 +6,7 @@ #include "otx2_cptvf.h" #include "otx2_cptlf.h" #include "cn10k_cpt.h" +#include "otx2_cpt_common.h" static void cn10k_cpt_send_cmd(union otx2_cpt_inst_s *cptinst, u32 insts_num, struct otx2_cptlf_info *lf); @@ -27,7 +28,7 @@ static struct cpt_hw_ops cn10k_hw_ops = { static void cn10k_cpt_send_cmd(union otx2_cpt_inst_s *cptinst, u32 insts_num, struct otx2_cptlf_info *lf) { - void __iomem *lmtline = lf->lmtline; + void *lmtline = lf->lfs->lmt_info.base + (lf->slot * LMTLINE_SIZE); u64 val = (lf->slot & 0x7FF); u64 tar_addr = 0; @@ -41,15 +42,49 @@ static void cn10k_cpt_send_cmd(union otx2_cpt_inst_s *cptinst, u32 insts_num, dma_wmb(); /* Copy CPT command to LMTLINE */ - memcpy_toio(lmtline, cptinst, insts_num * OTX2_CPT_INST_SIZE); + memcpy(lmtline, cptinst, insts_num * OTX2_CPT_INST_SIZE); cn10k_lmt_flush(val, tar_addr); } +void cn10k_cpt_lmtst_free(struct pci_dev *pdev, struct otx2_cptlfs_info *lfs) +{ + struct otx2_lmt_info *lmt_info = &lfs->lmt_info; + + if (!lmt_info->base) + return; + + dma_free_attrs(&pdev->dev, lmt_info->size, + lmt_info->base - lmt_info->align, + lmt_info->iova - lmt_info->align, + DMA_ATTR_FORCE_CONTIGUOUS); +} +EXPORT_SYMBOL_NS_GPL(cn10k_cpt_lmtst_free, "CRYPTO_DEV_OCTEONTX2_CPT"); + +static int cn10k_cpt_lmtst_alloc(struct pci_dev *pdev, + struct otx2_cptlfs_info *lfs, u32 size) +{ + struct otx2_lmt_info *lmt_info = &lfs->lmt_info; + dma_addr_t align_iova; + dma_addr_t iova; + + lmt_info->base = dma_alloc_attrs(&pdev->dev, size, &iova, GFP_KERNEL, + DMA_ATTR_FORCE_CONTIGUOUS); + if (!lmt_info->base) + return -ENOMEM; + + align_iova = ALIGN((u64)iova, LMTLINE_ALIGN); + lmt_info->iova = align_iova; + lmt_info->align = align_iova - iova; + lmt_info->size = size; + lmt_info->base += lmt_info->align; + return 0; +} + int cn10k_cptpf_lmtst_init(struct otx2_cptpf_dev *cptpf) { struct pci_dev *pdev = cptpf->pdev; - resource_size_t size; - u64 lmt_base; + u32 size; + int ret; if (!test_bit(CN10K_LMTST, &cptpf->cap_flag)) { cptpf->lfs.ops = &otx2_hw_ops; @@ -57,18 +92,19 @@ int cn10k_cptpf_lmtst_init(struct otx2_cptpf_dev *cptpf) } cptpf->lfs.ops = &cn10k_hw_ops; - lmt_base = readq(cptpf->reg_base + RVU_PF_LMTLINE_ADDR); - if (!lmt_base) { - dev_err(&pdev->dev, "PF LMTLINE address not configured\n"); - return -ENOMEM; + size = OTX2_CPT_MAX_VFS_NUM * LMTLINE_SIZE + LMTLINE_ALIGN; + ret = cn10k_cpt_lmtst_alloc(pdev, &cptpf->lfs, size); + if (ret) { + dev_err(&pdev->dev, "PF-%d LMTLINE memory allocation failed\n", + cptpf->pf_id); + return ret; } - size = pci_resource_len(pdev, PCI_MBOX_BAR_NUM); - size -= ((1 + cptpf->max_vfs) * MBOX_SIZE); - cptpf->lfs.lmt_base = devm_ioremap_wc(&pdev->dev, lmt_base, size); - if (!cptpf->lfs.lmt_base) { - dev_err(&pdev->dev, - "Mapping of PF LMTLINE address failed\n"); - return -ENOMEM; + + ret = otx2_cpt_lmtst_tbl_setup_msg(&cptpf->lfs); + if (ret) { + dev_err(&pdev->dev, "PF-%d: LMTST Table setup failed\n", + cptpf->pf_id); + cn10k_cpt_lmtst_free(pdev, &cptpf->lfs); } return 0; @@ -78,18 +114,25 @@ EXPORT_SYMBOL_NS_GPL(cn10k_cptpf_lmtst_init, "CRYPTO_DEV_OCTEONTX2_CPT"); int cn10k_cptvf_lmtst_init(struct otx2_cptvf_dev *cptvf) { struct pci_dev *pdev = cptvf->pdev; - resource_size_t offset, size; + u32 size; + int ret; if (!test_bit(CN10K_LMTST, &cptvf->cap_flag)) return 0; - offset = pci_resource_start(pdev, PCI_MBOX_BAR_NUM); - size = pci_resource_len(pdev, PCI_MBOX_BAR_NUM); - /* Map VF LMILINE region */ - cptvf->lfs.lmt_base = devm_ioremap_wc(&pdev->dev, offset, size); - if (!cptvf->lfs.lmt_base) { - dev_err(&pdev->dev, "Unable to map BAR4\n"); - return -ENOMEM; + size = cptvf->lfs.lfs_num * LMTLINE_SIZE + LMTLINE_ALIGN; + ret = cn10k_cpt_lmtst_alloc(pdev, &cptvf->lfs, size); + if (ret) { + dev_err(&pdev->dev, "VF-%d LMTLINE memory allocation failed\n", + cptvf->vf_id); + return ret; + } + + ret = otx2_cpt_lmtst_tbl_setup_msg(&cptvf->lfs); + if (ret) { + dev_err(&pdev->dev, "VF-%d: LMTST Table setup failed\n", + cptvf->vf_id); + cn10k_cpt_lmtst_free(pdev, &cptvf->lfs); } return 0; diff --git a/drivers/crypto/marvell/octeontx2/cn10k_cpt.h b/drivers/crypto/marvell/octeontx2/cn10k_cpt.h index 92be3ecf570f..ea5990048c21 100644 --- a/drivers/crypto/marvell/octeontx2/cn10k_cpt.h +++ b/drivers/crypto/marvell/octeontx2/cn10k_cpt.h @@ -50,6 +50,7 @@ static inline u8 otx2_cpt_get_uc_compcode(union otx2_cpt_res_s *result) int cn10k_cptpf_lmtst_init(struct otx2_cptpf_dev *cptpf); int cn10k_cptvf_lmtst_init(struct otx2_cptvf_dev *cptvf); +void cn10k_cpt_lmtst_free(struct pci_dev *pdev, struct otx2_cptlfs_info *lfs); void cn10k_cpt_ctx_flush(struct pci_dev *pdev, u64 cptr, bool inval); int cn10k_cpt_hw_ctx_init(struct pci_dev *pdev, struct cn10k_cpt_errata_ctx *er_ctx); diff --git a/drivers/crypto/marvell/octeontx2/otx2_cpt_common.h b/drivers/crypto/marvell/octeontx2/otx2_cpt_common.h index 84f2bb0ca11c..d529bcb03775 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cpt_common.h +++ b/drivers/crypto/marvell/octeontx2/otx2_cpt_common.h @@ -209,5 +209,6 @@ int otx2_cpt_detach_rsrcs_msg(struct otx2_cptlfs_info *lfs); int otx2_cpt_msix_offset_msg(struct otx2_cptlfs_info *lfs); int otx2_cpt_sync_mbox_msg(struct otx2_mbox *mbox); int otx2_cpt_lf_reset_msg(struct otx2_cptlfs_info *lfs, int slot); +int otx2_cpt_lmtst_tbl_setup_msg(struct otx2_cptlfs_info *lfs); #endif /* __OTX2_CPT_COMMON_H */ diff --git a/drivers/crypto/marvell/octeontx2/otx2_cpt_mbox_common.c b/drivers/crypto/marvell/octeontx2/otx2_cpt_mbox_common.c index b8b7c8a3c0ca..95f3de3a34eb 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cpt_mbox_common.c +++ b/drivers/crypto/marvell/octeontx2/otx2_cpt_mbox_common.c @@ -255,3 +255,28 @@ int otx2_cpt_lf_reset_msg(struct otx2_cptlfs_info *lfs, int slot) return ret; } EXPORT_SYMBOL_NS_GPL(otx2_cpt_lf_reset_msg, "CRYPTO_DEV_OCTEONTX2_CPT"); + +int otx2_cpt_lmtst_tbl_setup_msg(struct otx2_cptlfs_info *lfs) +{ + struct otx2_mbox *mbox = lfs->mbox; + struct pci_dev *pdev = lfs->pdev; + struct lmtst_tbl_setup_req *req; + + req = (struct lmtst_tbl_setup_req *) + otx2_mbox_alloc_msg_rsp(mbox, 0, sizeof(*req), + sizeof(struct msg_rsp)); + if (!req) { + dev_err(&pdev->dev, "RVU MBOX failed to alloc message.\n"); + return -EFAULT; + } + + req->hdr.id = MBOX_MSG_LMTST_TBL_SETUP; + req->hdr.sig = OTX2_MBOX_REQ_SIG; + req->hdr.pcifunc = 0; + + req->use_local_lmt_region = true; + req->lmt_iova = lfs->lmt_info.iova; + + return otx2_cpt_send_mbox_msg(mbox, pdev); +} +EXPORT_SYMBOL_NS_GPL(otx2_cpt_lmtst_tbl_setup_msg, "CRYPTO_DEV_OCTEONTX2_CPT"); diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptlf.c b/drivers/crypto/marvell/octeontx2/otx2_cptlf.c index b5d66afcc030..dc7c7a2650a5 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cptlf.c +++ b/drivers/crypto/marvell/octeontx2/otx2_cptlf.c @@ -433,10 +433,7 @@ int otx2_cptlf_init(struct otx2_cptlfs_info *lfs, u8 eng_grp_mask, int pri, for (slot = 0; slot < lfs->lfs_num; slot++) { lfs->lf[slot].lfs = lfs; lfs->lf[slot].slot = slot; - if (lfs->lmt_base) - lfs->lf[slot].lmtline = lfs->lmt_base + - (slot * LMTLINE_SIZE); - else + if (!lfs->lmt_info.base) lfs->lf[slot].lmtline = lfs->reg_base + OTX2_CPT_RVU_FUNC_ADDR_S(BLKADDR_LMT, slot, OTX2_CPT_LMT_LF_LMTLINEX(0)); diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptlf.h b/drivers/crypto/marvell/octeontx2/otx2_cptlf.h index bd8604be2952..6e004a5568d8 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cptlf.h +++ b/drivers/crypto/marvell/octeontx2/otx2_cptlf.h @@ -105,11 +105,19 @@ struct cpt_hw_ops { gfp_t gfp); }; +#define LMTLINE_SIZE 128 +#define LMTLINE_ALIGN 128 +struct otx2_lmt_info { + void *base; + dma_addr_t iova; + u32 size; + u8 align; +}; + struct otx2_cptlfs_info { /* Registers start address of VF/PF LFs are attached to */ void __iomem *reg_base; -#define LMTLINE_SIZE 128 - void __iomem *lmt_base; + struct otx2_lmt_info lmt_info; struct pci_dev *pdev; /* Device LFs are attached to */ struct otx2_cptlf_info lf[OTX2_CPT_MAX_LFS_NUM]; struct otx2_mbox *mbox; diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptpf_main.c b/drivers/crypto/marvell/octeontx2/otx2_cptpf_main.c index 687b6c7d7674..1c5c262af48d 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cptpf_main.c +++ b/drivers/crypto/marvell/octeontx2/otx2_cptpf_main.c @@ -792,19 +792,19 @@ static int otx2_cptpf_probe(struct pci_dev *pdev, cptpf->max_vfs = pci_sriov_get_totalvfs(pdev); cptpf->kvf_limits = 1; - err = cn10k_cptpf_lmtst_init(cptpf); + /* Initialize CPT PF device */ + err = cptpf_device_init(cptpf); if (err) goto unregister_intr; - /* Initialize CPT PF device */ - err = cptpf_device_init(cptpf); + err = cn10k_cptpf_lmtst_init(cptpf); if (err) goto unregister_intr; /* Initialize engine groups */ err = otx2_cpt_init_eng_grps(pdev, &cptpf->eng_grps); if (err) - goto unregister_intr; + goto free_lmtst; err = sysfs_create_group(&dev->kobj, &cptpf_sysfs_group); if (err) @@ -820,6 +820,8 @@ sysfs_grp_del: sysfs_remove_group(&dev->kobj, &cptpf_sysfs_group); cleanup_eng_grps: otx2_cpt_cleanup_eng_grps(pdev, &cptpf->eng_grps); +free_lmtst: + cn10k_cpt_lmtst_free(pdev, &cptpf->lfs); unregister_intr: cptpf_disable_afpf_mbox_intr(cptpf); destroy_afpf_mbox: @@ -854,6 +856,8 @@ static void otx2_cptpf_remove(struct pci_dev *pdev) cptpf_disable_afpf_mbox_intr(cptpf); /* Destroy AF-PF mbox */ cptpf_afpf_mbox_destroy(cptpf); + /* Free LMTST memory */ + cn10k_cpt_lmtst_free(pdev, &cptpf->lfs); pci_set_drvdata(pdev, NULL); } diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptpf_mbox.c b/drivers/crypto/marvell/octeontx2/otx2_cptpf_mbox.c index 3eb45bb91296..12c0e966fa65 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cptpf_mbox.c +++ b/drivers/crypto/marvell/octeontx2/otx2_cptpf_mbox.c @@ -502,6 +502,7 @@ static void process_afpf_mbox_msg(struct otx2_cptpf_dev *cptpf, case MBOX_MSG_CPT_INLINE_IPSEC_CFG: case MBOX_MSG_NIX_INLINE_IPSEC_CFG: case MBOX_MSG_CPT_LF_RESET: + case MBOX_MSG_LMTST_TBL_SETUP: break; default: diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptvf_main.c b/drivers/crypto/marvell/octeontx2/otx2_cptvf_main.c index 11e351a48efe..56904bdfd6e8 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cptvf_main.c +++ b/drivers/crypto/marvell/octeontx2/otx2_cptvf_main.c @@ -376,10 +376,6 @@ static int otx2_cptvf_probe(struct pci_dev *pdev, otx2_cpt_set_hw_caps(pdev, &cptvf->cap_flag); - ret = cn10k_cptvf_lmtst_init(cptvf); - if (ret) - goto clear_drvdata; - /* Initialize PF<=>VF mailbox */ ret = cptvf_pfvf_mbox_init(cptvf); if (ret) @@ -405,13 +401,19 @@ static int otx2_cptvf_probe(struct pci_dev *pdev, if (cptvf->eng_caps[OTX2_CPT_SE_TYPES] & BIT_ULL(35)) cptvf->lfs.ops->cpt_sg_info_create = cn10k_sgv2_info_create; - /* Initialize CPT LFs */ - ret = cptvf_lf_init(cptvf); + ret = cn10k_cptvf_lmtst_init(cptvf); if (ret) goto unregister_interrupts; + /* Initialize CPT LFs */ + ret = cptvf_lf_init(cptvf); + if (ret) + goto free_lmtst; + return 0; +free_lmtst: + cn10k_cpt_lmtst_free(pdev, &cptvf->lfs); unregister_interrupts: cptvf_disable_pfvf_mbox_intrs(cptvf); destroy_pfvf_mbox: @@ -435,6 +437,8 @@ static void otx2_cptvf_remove(struct pci_dev *pdev) cptvf_disable_pfvf_mbox_intrs(cptvf); /* Destroy PF-VF mbox */ cptvf_pfvf_mbox_destroy(cptvf); + /* Free LMTST memory */ + cn10k_cpt_lmtst_free(pdev, &cptvf->lfs); pci_set_drvdata(pdev, NULL); } diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptvf_mbox.c b/drivers/crypto/marvell/octeontx2/otx2_cptvf_mbox.c index d9fa5f6e204d..931b72580fd9 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cptvf_mbox.c +++ b/drivers/crypto/marvell/octeontx2/otx2_cptvf_mbox.c @@ -134,6 +134,7 @@ static void process_pfvf_mbox_mbox_msg(struct otx2_cptvf_dev *cptvf, sizeof(cptvf->eng_caps)); break; case MBOX_MSG_CPT_LF_RESET: + case MBOX_MSG_LMTST_TBL_SETUP: break; default: dev_err(&cptvf->pdev->dev, "Unsupported msg %d received.\n", From c822831b426307a6ca426621504d3c7f99765a39 Mon Sep 17 00:00:00 2001 From: Ovidiu Panait Date: Fri, 16 May 2025 15:06:56 +0300 Subject: [PATCH 373/382] crypto: sun8i-ce - move fallback ahash_request to the end of the struct 'struct ahash_request' has a flexible array at the end, so it must be the last member in a struct, to avoid overwriting other struct members. Therefore, move 'fallback_req' to the end of the 'sun8i_ce_hash_reqctx' struct. Fixes: 56f6d5aee88d ("crypto: sun8i-ce - support hash algorithms") Signed-off-by: Ovidiu Panait Signed-off-by: Herbert Xu --- drivers/crypto/allwinner/sun8i-ce/sun8i-ce.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce.h b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce.h index 3b5c2af013d0..83df4d719053 100644 --- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce.h +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce.h @@ -308,8 +308,8 @@ struct sun8i_ce_hash_tfm_ctx { * @flow: the flow to use for this request */ struct sun8i_ce_hash_reqctx { - struct ahash_request fallback_req; int flow; + struct ahash_request fallback_req; // keep at the end }; /* From 52e8ae868a824045f204a848b445e4747acad2a7 Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Sat, 17 May 2025 15:26:29 -0500 Subject: [PATCH 374/382] crypto: ccp - Add missing bootloader info reg for pspv5 The bootloader info reg for pspv5 is the same as pspv4. Signed-off-by: Mario Limonciello Signed-off-by: Herbert Xu --- drivers/crypto/ccp/sp-pci.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/crypto/ccp/sp-pci.c b/drivers/crypto/ccp/sp-pci.c index 54ab3279f521..ee996d0ae443 100644 --- a/drivers/crypto/ccp/sp-pci.c +++ b/drivers/crypto/ccp/sp-pci.c @@ -440,6 +440,7 @@ static const struct psp_vdata pspv5 = { .cmdresp_reg = 0x10944, /* C2PMSG_17 */ .cmdbuff_addr_lo_reg = 0x10948, /* C2PMSG_18 */ .cmdbuff_addr_hi_reg = 0x1094c, /* C2PMSG_19 */ + .bootloader_info_reg = 0x109ec, /* C2PMSG_59 */ .feature_reg = 0x109fc, /* C2PMSG_63 */ .inten_reg = 0x10510, /* P2CMSG_INTEN */ .intsts_reg = 0x10514, /* P2CMSG_INTSTS */ From 72942d65385641fb91b467f2ea210b4ca6bda58e Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Sat, 17 May 2025 15:26:30 -0500 Subject: [PATCH 375/382] crypto: ccp - Add missing tee info reg for teev2 The tee info reg for teev2 is the same as teev1. Signed-off-by: Mario Limonciello Signed-off-by: Herbert Xu --- drivers/crypto/ccp/sp-pci.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/crypto/ccp/sp-pci.c b/drivers/crypto/ccp/sp-pci.c index ee996d0ae443..e1be2072d680 100644 --- a/drivers/crypto/ccp/sp-pci.c +++ b/drivers/crypto/ccp/sp-pci.c @@ -375,6 +375,7 @@ static const struct tee_vdata teev1 = { static const struct tee_vdata teev2 = { .ring_wptr_reg = 0x10950, /* C2PMSG_20 */ .ring_rptr_reg = 0x10954, /* C2PMSG_21 */ + .info_reg = 0x109e8, /* C2PMSG_58 */ }; static const struct platform_access_vdata pa_v1 = { From 16766292c697f31ccfc338d4040f2d5f03df777c Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Tue, 20 May 2025 10:35:29 +0800 Subject: [PATCH 376/382] Revert "crypto: powerpc/poly1305 - Add SIMD fallback" This reverts commit c66d7ebbe2fa14e41913adb421090a7426f59786. Link: https://lore.kernel.org/all/02c22854-eebf-4ad1-b89e-8c2b65ab8236@csgroup.eu/ Signed-off-by: Herbert Xu --- arch/powerpc/Kconfig | 1 - arch/powerpc/lib/crypto/Kconfig | 2 +- arch/powerpc/lib/crypto/poly1305-p10-glue.c | 41 ++------------------- 3 files changed, 4 insertions(+), 40 deletions(-) diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig index 651e0c32957a..6722625a406a 100644 --- a/arch/powerpc/Kconfig +++ b/arch/powerpc/Kconfig @@ -173,7 +173,6 @@ config PPC select ARCH_STACKWALK select ARCH_SUPPORTS_ATOMIC_RMW select ARCH_SUPPORTS_DEBUG_PAGEALLOC if PPC_BOOK3S || PPC_8xx - select ARCH_SUPPORTS_INT128 if PPC64 && CC_HAS_INT128 select ARCH_USE_BUILTIN_BSWAP select ARCH_USE_CMPXCHG_LOCKREF if PPC64 select ARCH_USE_MEMTEST diff --git a/arch/powerpc/lib/crypto/Kconfig b/arch/powerpc/lib/crypto/Kconfig index 6761fdb6193c..ffa541ad6d5d 100644 --- a/arch/powerpc/lib/crypto/Kconfig +++ b/arch/powerpc/lib/crypto/Kconfig @@ -9,7 +9,7 @@ config CRYPTO_CHACHA20_P10 config CRYPTO_POLY1305_P10 tristate - depends on PPC64 && CPU_LITTLE_ENDIAN && VSX && ARCH_SUPPORTS_INT128 + depends on PPC64 && CPU_LITTLE_ENDIAN && VSX default CRYPTO_LIB_POLY1305 select CRYPTO_ARCH_HAVE_LIB_POLY1305 select CRYPTO_LIB_POLY1305_GENERIC diff --git a/arch/powerpc/lib/crypto/poly1305-p10-glue.c b/arch/powerpc/lib/crypto/poly1305-p10-glue.c index 280c10c48c53..3f1664a724b6 100644 --- a/arch/powerpc/lib/crypto/poly1305-p10-glue.c +++ b/arch/powerpc/lib/crypto/poly1305-p10-glue.c @@ -6,7 +6,6 @@ */ #include #include -#include #include #include #include @@ -19,11 +18,6 @@ asmlinkage void poly1305_emit_64(const struct poly1305_state *state, const u32 n static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_p10); -static inline bool is_state_base64(struct poly1305_block_state *state) -{ - return state->core_r.precomputed_s.r64[2]; -} - static void vsx_begin(void) { preempt_disable(); @@ -36,35 +30,12 @@ static void vsx_end(void) preempt_enable(); } -static void convert_to_base2_44(struct poly1305_block_state *state) -{ - u8 raw_key[POLY1305_BLOCK_SIZE]; - u64 h0, h1, h2; - - if (!is_state_base64(state)) - return; - - state->core_r.precomputed_s.r64[2] = 0; - put_unaligned_le64(state->core_r.key.r64[0], raw_key + 0); - put_unaligned_le64(state->core_r.key.r64[1], raw_key + 8); - poly1305_core_setkey(&state->core_r, raw_key); - - h0 = state->h.h64[0]; - h1 = state->h.h64[1]; - h2 = state->h.h64[2]; - state->h.h64[0] = h0 & 0xfffffffffffULL; - state->h.h64[1] = h0 >> 44 | (h1 & 0xffffffULL) << 20; - state->h.h64[2] = h1 >> 24 | h2 << 40; -} - void poly1305_block_init_arch(struct poly1305_block_state *dctx, const u8 raw_key[POLY1305_BLOCK_SIZE]) { - dctx->core_r.precomputed_s.r64[2] = 0; - if (!static_key_enabled(&have_p10) || !crypto_simd_usable()) + if (!static_key_enabled(&have_p10)) return poly1305_block_init_generic(dctx, raw_key); - dctx->core_r.precomputed_s.r64[2] = 1; dctx->h = (struct poly1305_state){}; dctx->core_r.key.r64[0] = get_unaligned_le64(raw_key + 0); dctx->core_r.key.r64[1] = get_unaligned_le64(raw_key + 8); @@ -74,11 +45,8 @@ EXPORT_SYMBOL_GPL(poly1305_block_init_arch); void poly1305_blocks_arch(struct poly1305_block_state *state, const u8 *src, unsigned int len, u32 padbit) { - if (!static_key_enabled(&have_p10) || !is_state_base64(state) || - !crypto_simd_usable()) { - convert_to_base2_44(state); + if (!static_key_enabled(&have_p10)) return poly1305_blocks_generic(state, src, len, padbit); - } vsx_begin(); if (len >= POLY1305_BLOCK_SIZE * 4) { poly1305_p10le_4blocks(state, src, len); @@ -98,10 +66,7 @@ void poly1305_emit_arch(const struct poly1305_state *state, u8 digest[POLY1305_DIGEST_SIZE], const u32 nonce[4]) { - struct poly1305_block_state *dctx = - container_of(state, struct poly1305_block_state, h); - - if (!static_key_enabled(&have_p10) || !is_state_base64(dctx)) + if (!static_key_enabled(&have_p10)) return poly1305_emit_generic(state, digest, nonce); poly1305_emit_64(state, nonce, digest); } From bc8169003b41e89fe7052e408cf9fdbecb4017fe Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Tue, 20 May 2025 10:39:29 +0800 Subject: [PATCH 377/382] crypto: powerpc/poly1305 - add depends on BROKEN for now As discussed in the thread containing https://lore.kernel.org/linux-crypto/20250510053308.GB505731@sol/, the Power10-optimized Poly1305 code is currently not safe to call in softirq context. Disable it for now. It can be re-enabled once it is fixed. Fixes: ba8f8624fde2 ("crypto: poly1305-p10 - Glue code for optmized Poly1305 implementation for ppc64le") Cc: stable@vger.kernel.org Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/powerpc/lib/crypto/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/powerpc/lib/crypto/Kconfig b/arch/powerpc/lib/crypto/Kconfig index ffa541ad6d5d..3f9e1bbd9905 100644 --- a/arch/powerpc/lib/crypto/Kconfig +++ b/arch/powerpc/lib/crypto/Kconfig @@ -10,6 +10,7 @@ config CRYPTO_CHACHA20_P10 config CRYPTO_POLY1305_P10 tristate depends on PPC64 && CPU_LITTLE_ENDIAN && VSX + depends on BROKEN # Needs to be fixed to work in softirq context default CRYPTO_LIB_POLY1305 select CRYPTO_ARCH_HAVE_LIB_POLY1305 select CRYPTO_LIB_POLY1305_GENERIC From 0413bcf0fc460a68a2a7a8354aee833293d7d693 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Thu, 8 May 2025 13:22:16 +0800 Subject: [PATCH 378/382] crypto: marvell/cesa - Do not chain submitted requests This driver tries to chain requests together before submitting them to hardware in order to reduce completion interrupts. However, it even extends chains that have already been submitted to hardware. This is dangerous because there is no way of knowing whether the hardware has already read the DMA memory in question or not. Fix this by splitting the chain list into two. One for submitted requests and one for requests that have not yet been submitted. Only extend the latter. Reported-by: Klaus Kudielka Fixes: 85030c5168f1 ("crypto: marvell - Add support for chaining crypto requests in TDMA mode") Cc: Signed-off-by: Herbert Xu --- drivers/crypto/marvell/cesa/cesa.c | 2 +- drivers/crypto/marvell/cesa/cesa.h | 9 +++-- drivers/crypto/marvell/cesa/tdma.c | 53 ++++++++++++++++++------------ 3 files changed, 39 insertions(+), 25 deletions(-) diff --git a/drivers/crypto/marvell/cesa/cesa.c b/drivers/crypto/marvell/cesa/cesa.c index fa08f10e6f3f..9c21f5d835d2 100644 --- a/drivers/crypto/marvell/cesa/cesa.c +++ b/drivers/crypto/marvell/cesa/cesa.c @@ -94,7 +94,7 @@ static int mv_cesa_std_process(struct mv_cesa_engine *engine, u32 status) static int mv_cesa_int_process(struct mv_cesa_engine *engine, u32 status) { - if (engine->chain.first && engine->chain.last) + if (engine->chain_hw.first && engine->chain_hw.last) return mv_cesa_tdma_process(engine, status); return mv_cesa_std_process(engine, status); diff --git a/drivers/crypto/marvell/cesa/cesa.h b/drivers/crypto/marvell/cesa/cesa.h index d215a6bed6bc..50ca1039fdaa 100644 --- a/drivers/crypto/marvell/cesa/cesa.h +++ b/drivers/crypto/marvell/cesa/cesa.h @@ -440,8 +440,10 @@ struct mv_cesa_dev { * SRAM * @queue: fifo of the pending crypto requests * @load: engine load counter, useful for load balancing - * @chain: list of the current tdma descriptors being processed - * by this engine. + * @chain_hw: list of the current tdma descriptors being processed + * by the hardware. + * @chain_sw: list of the current tdma descriptors that will be + * submitted to the hardware. * @complete_queue: fifo of the processed requests by the engine * * Structure storing CESA engine information. @@ -463,7 +465,8 @@ struct mv_cesa_engine { struct gen_pool *pool; struct crypto_queue queue; atomic_t load; - struct mv_cesa_tdma_chain chain; + struct mv_cesa_tdma_chain chain_hw; + struct mv_cesa_tdma_chain chain_sw; struct list_head complete_queue; int irq; }; diff --git a/drivers/crypto/marvell/cesa/tdma.c b/drivers/crypto/marvell/cesa/tdma.c index 388a06e180d6..243305354420 100644 --- a/drivers/crypto/marvell/cesa/tdma.c +++ b/drivers/crypto/marvell/cesa/tdma.c @@ -38,6 +38,15 @@ void mv_cesa_dma_step(struct mv_cesa_req *dreq) { struct mv_cesa_engine *engine = dreq->engine; + spin_lock_bh(&engine->lock); + if (engine->chain_sw.first == dreq->chain.first) { + engine->chain_sw.first = NULL; + engine->chain_sw.last = NULL; + } + engine->chain_hw.first = dreq->chain.first; + engine->chain_hw.last = dreq->chain.last; + spin_unlock_bh(&engine->lock); + writel_relaxed(0, engine->regs + CESA_SA_CFG); mv_cesa_set_int_mask(engine, CESA_SA_INT_ACC0_IDMA_DONE); @@ -96,25 +105,27 @@ void mv_cesa_dma_prepare(struct mv_cesa_req *dreq, void mv_cesa_tdma_chain(struct mv_cesa_engine *engine, struct mv_cesa_req *dreq) { - if (engine->chain.first == NULL && engine->chain.last == NULL) { - engine->chain.first = dreq->chain.first; - engine->chain.last = dreq->chain.last; - } else { - struct mv_cesa_tdma_desc *last; + struct mv_cesa_tdma_desc *last = engine->chain_sw.last; - last = engine->chain.last; + /* + * Break the DMA chain if the request being queued needs the IV + * regs to be set before lauching the request. + */ + if (!last || dreq->chain.first->flags & CESA_TDMA_SET_STATE) + engine->chain_sw.first = dreq->chain.first; + else { last->next = dreq->chain.first; - engine->chain.last = dreq->chain.last; - - /* - * Break the DMA chain if the CESA_TDMA_BREAK_CHAIN is set on - * the last element of the current chain, or if the request - * being queued needs the IV regs to be set before lauching - * the request. - */ - if (!(last->flags & CESA_TDMA_BREAK_CHAIN) && - !(dreq->chain.first->flags & CESA_TDMA_SET_STATE)) - last->next_dma = cpu_to_le32(dreq->chain.first->cur_dma); + last->next_dma = cpu_to_le32(dreq->chain.first->cur_dma); + } + last = dreq->chain.last; + engine->chain_sw.last = last; + /* + * Break the DMA chain if the CESA_TDMA_BREAK_CHAIN is set on + * the last element of the current chain. + */ + if (last->flags & CESA_TDMA_BREAK_CHAIN) { + engine->chain_sw.first = NULL; + engine->chain_sw.last = NULL; } } @@ -127,7 +138,7 @@ int mv_cesa_tdma_process(struct mv_cesa_engine *engine, u32 status) tdma_cur = readl(engine->regs + CESA_TDMA_CUR); - for (tdma = engine->chain.first; tdma; tdma = next) { + for (tdma = engine->chain_hw.first; tdma; tdma = next) { spin_lock_bh(&engine->lock); next = tdma->next; spin_unlock_bh(&engine->lock); @@ -149,12 +160,12 @@ int mv_cesa_tdma_process(struct mv_cesa_engine *engine, u32 status) &backlog); /* Re-chaining to the next request */ - engine->chain.first = tdma->next; + engine->chain_hw.first = tdma->next; tdma->next = NULL; /* If this is the last request, clear the chain */ - if (engine->chain.first == NULL) - engine->chain.last = NULL; + if (engine->chain_hw.first == NULL) + engine->chain_hw.last = NULL; spin_unlock_bh(&engine->lock); ctx = crypto_tfm_ctx(req->tfm); From 35de409aa30269a3b106fe957a95f7a2b7e21a60 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 23 May 2025 17:20:59 +0800 Subject: [PATCH 379/382] Revert "crypto: testmgr - Add hash export format testing" This reverts commit 18c438b228558e05ede7dccf947a6547516fc0c7. The s390 hmac and sha3 algorithms are failing the test. Revert the change until they have been fixed. Reported-by: Ingo Franzki Link: https://lore.kernel.org/all/623a7fcb-b4cb-48e6-9833-57ad2b32a252@linux.ibm.com/ Signed-off-by: Herbert Xu --- crypto/testmgr.c | 95 ++++------------------------------ crypto/testmgr.h | 2 - include/crypto/internal/hash.h | 6 --- 3 files changed, 9 insertions(+), 94 deletions(-) diff --git a/crypto/testmgr.c b/crypto/testmgr.c index 737064b31480..72005074a5c2 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -17,19 +17,10 @@ */ #include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include #include #include #include -#include #include #include #include @@ -37,6 +28,14 @@ #include #include #include +#include +#include +#include +#include +#include +#include +#include +#include #include "internal.h" @@ -1465,49 +1464,6 @@ static int check_nonfinal_ahash_op(const char *op, int err, return 0; } -static int check_ahash_export(struct ahash_request *req, - const struct hash_testvec *vec, - const char *vec_name, - const struct testvec_config *cfg, - const char *driver, u8 *hashstate) -{ - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - const unsigned int digestsize = crypto_ahash_digestsize(tfm); - HASH_FBREQ_ON_STACK(fbreq, req); - int err; - - if (!vec->state) - return 0; - - err = crypto_ahash_export(req, hashstate); - if (err) { - pr_err("alg: ahash: %s mixed export() failed with err %d on test vector %s, cfg=\"%s\"\n", - driver, err, vec_name, cfg->name); - return err; - } - err = crypto_ahash_import(req, vec->state); - if (err) { - pr_err("alg: ahash: %s mixed import() failed with err %d on test vector %s, cfg=\"%s\"\n", - driver, err, vec_name, cfg->name); - return err; - } - err = crypto_ahash_import(fbreq, hashstate); - if (err) { - pr_err("alg: ahash: %s fallback import() failed with err %d on test vector %s, cfg=\"%s\"\n", - crypto_ahash_driver_name(crypto_ahash_reqtfm(fbreq)), err, vec_name, cfg->name); - return err; - } - ahash_request_set_crypt(fbreq, NULL, hashstate, 0); - testmgr_poison(hashstate, digestsize + TESTMGR_POISON_LEN); - err = crypto_ahash_final(fbreq); - if (err) { - pr_err("alg: ahash: %s fallback final() failed with err %d on test vector %s, cfg=\"%s\"\n", - crypto_ahash_driver_name(crypto_ahash_reqtfm(fbreq)), err, vec_name, cfg->name); - return err; - } - return check_hash_result("ahash export", hashstate, digestsize, vec, vec_name, driver, cfg); -} - /* Test one hash test vector in one configuration, using the ahash API */ static int test_ahash_vec_cfg(const struct hash_testvec *vec, const char *vec_name, @@ -1653,10 +1609,6 @@ static int test_ahash_vec_cfg(const struct hash_testvec *vec, driver, vec_name, cfg); if (err) return err; - err = check_ahash_export(req, vec, vec_name, cfg, - driver, hashstate); - if (err) - return err; err = do_ahash_op(crypto_ahash_final, req, &wait, cfg->nosimd); if (err) { pr_err("alg: ahash: %s final() failed with err %d on test vector %s, cfg=\"%s\"\n", @@ -1780,17 +1732,6 @@ static void generate_random_hash_testvec(struct rnd_state *rng, vec->digest_error = crypto_hash_digest( crypto_ahash_reqtfm(req), vec->plaintext, vec->psize, (u8 *)vec->digest); - - if (vec->digest_error || !vec->state) - goto done; - - ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP, NULL, NULL); - ahash_request_set_virt(req, vec->plaintext, (u8 *)vec->digest, - vec->psize); - crypto_ahash_init(req); - crypto_ahash_update(req); - crypto_ahash_export(req, (u8 *)vec->state); - done: snprintf(name, max_namelen, "\"random: psize=%u ksize=%u\"", vec->psize, vec->ksize); @@ -1809,7 +1750,6 @@ static int test_hash_vs_generic_impl(const char *generic_driver, { struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); const unsigned int digestsize = crypto_ahash_digestsize(tfm); - const unsigned int statesize = crypto_ahash_statesize(tfm); const unsigned int blocksize = crypto_ahash_blocksize(tfm); const unsigned int maxdatasize = (2 * PAGE_SIZE) - TESTMGR_POISON_LEN; const char *algname = crypto_hash_alg_common(tfm)->base.cra_name; @@ -1882,22 +1822,6 @@ static int test_hash_vs_generic_impl(const char *generic_driver, goto out; } - if (crypto_hash_no_export_core(tfm) || - crypto_hash_no_export_core(generic_tfm)) - ; - else if (statesize != crypto_ahash_statesize(generic_tfm)) { - pr_err("alg: hash: statesize for %s (%u) doesn't match generic impl (%u)\n", - driver, statesize, - crypto_ahash_statesize(generic_tfm)); - err = -EINVAL; - goto out; - } else { - vec.state = kmalloc(statesize, GFP_KERNEL); - err = -ENOMEM; - if (!vec.state) - goto out; - } - /* * Now generate test vectors using the generic implementation, and test * the other implementation against them. @@ -1930,7 +1854,6 @@ out: kfree(vec.key); kfree(vec.plaintext); kfree(vec.digest); - kfree(vec.state); ahash_request_free(generic_req); crypto_free_ahash(generic_tfm); return err; diff --git a/crypto/testmgr.h b/crypto/testmgr.h index 5cf455a708b8..32d099ac9e73 100644 --- a/crypto/testmgr.h +++ b/crypto/testmgr.h @@ -29,7 +29,6 @@ * hash_testvec: structure to describe a hash (message digest) test * @key: Pointer to key (NULL if none) * @plaintext: Pointer to source data - * @state: Pointer to expected state * @digest: Pointer to expected digest * @psize: Length of source data in bytes * @ksize: Length of @key in bytes (0 if no key) @@ -40,7 +39,6 @@ struct hash_testvec { const char *key; const char *plaintext; - const char *state; const char *digest; unsigned int psize; unsigned short ksize; diff --git a/include/crypto/internal/hash.h b/include/crypto/internal/hash.h index f052afa6e7b0..0f85c543f80b 100644 --- a/include/crypto/internal/hash.h +++ b/include/crypto/internal/hash.h @@ -91,12 +91,6 @@ static inline bool crypto_hash_alg_needs_key(struct hash_alg_common *alg) !(alg->base.cra_flags & CRYPTO_ALG_OPTIONAL_KEY); } -static inline bool crypto_hash_no_export_core(struct crypto_ahash *tfm) -{ - return crypto_hash_alg_common(tfm)->base.cra_flags & - CRYPTO_AHASH_ALG_NO_EXPORT_CORE; -} - int crypto_grab_ahash(struct crypto_ahash_spawn *spawn, struct crypto_instance *inst, const char *name, u32 type, u32 mask); From 0a3cf32da469ff1df6e016f5f82b439a63d14461 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Mon, 19 May 2025 18:29:38 +0800 Subject: [PATCH 380/382] crypto: api - Redo lookup on EEXIST When two crypto algorithm lookups occur at the same time with different names for the same algorithm, e.g., ctr(aes-generic) and ctr(aes), they will both be instantiated. However, only one of them can be registered. The second instantiation will fail with EEXIST. Avoid failing the second lookup by making it retry, but only once because there are tricky names such as gcm_base(ctr(aes),ghash) that will always fail, despite triggering instantiation and EEXIST. Reported-by: Ingo Franzki Fixes: 2825982d9d66 ("[CRYPTO] api: Added event notification") Signed-off-by: Herbert Xu --- crypto/api.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/crypto/api.c b/crypto/api.c index 133d9b626922..5724d62e9d07 100644 --- a/crypto/api.c +++ b/crypto/api.c @@ -219,10 +219,19 @@ again: if (crypto_is_test_larval(larval)) crypto_larval_kill(larval); alg = ERR_PTR(-ETIMEDOUT); - } else if (!alg) { + } else if (!alg || PTR_ERR(alg) == -EEXIST) { + int err = alg ? -EEXIST : -EAGAIN; + + /* + * EEXIST is expected because two probes can be scheduled + * at the same time with one using alg_name and the other + * using driver_name. Do a re-lookup but do not retry in + * case we hit a quirk like gcm_base(ctr(aes),...) which + * will never match. + */ alg = &larval->alg; alg = crypto_alg_lookup(alg->cra_name, type, mask) ?: - ERR_PTR(-EAGAIN); + ERR_PTR(err); } else if (IS_ERR(alg)) ; else if (crypto_is_test_larval(larval) && From 61fc01f8f7f8e299b9fed5a5e5e068ef0e40593d Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Thu, 22 May 2025 11:21:41 +0100 Subject: [PATCH 381/382] crypto: qat - add missing header inclusion Without this header, the build of the new qat_6xxx driver fails when CONFIG_PCI_IOV is not set: In file included from drivers/crypto/intel/qat/qat_common/adf_gen6_shared.c:7: drivers/crypto/intel/qat/qat_common/adf_gen4_pfvf.h: In function 'adf_gen4_init_pf_pfvf_ops': drivers/crypto/intel/qat/qat_common/adf_gen4_pfvf.h:13:34: error: 'adf_pfvf_comms_disabled' undeclared (first use in this function) 13 | pfvf_ops->enable_comms = adf_pfvf_comms_disabled; | ^~~~~~~~~~~~~~~~~~~~~~~ Fixes: 17fd7514ae68 ("crypto: qat - add qat_6xxx driver") Signed-off-by: Arnd Bergmann Acked-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/intel/qat/qat_common/adf_gen2_pfvf.h | 1 + drivers/crypto/intel/qat/qat_common/adf_gen4_pfvf.h | 1 + 2 files changed, 2 insertions(+) diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen2_pfvf.h b/drivers/crypto/intel/qat/qat_common/adf_gen2_pfvf.h index a716545a764c..34a63cf40db2 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_gen2_pfvf.h +++ b/drivers/crypto/intel/qat/qat_common/adf_gen2_pfvf.h @@ -5,6 +5,7 @@ #include #include "adf_accel_devices.h" +#include "adf_common_drv.h" #define ADF_GEN2_ERRSOU3 (0x3A000 + 0x0C) #define ADF_GEN2_ERRSOU5 (0x3A000 + 0xD8) diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen4_pfvf.h b/drivers/crypto/intel/qat/qat_common/adf_gen4_pfvf.h index 17d1b774d4a8..2c8708117f70 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_gen4_pfvf.h +++ b/drivers/crypto/intel/qat/qat_common/adf_gen4_pfvf.h @@ -4,6 +4,7 @@ #define ADF_GEN4_PFVF_H #include "adf_accel_devices.h" +#include "adf_common_drv.h" #ifdef CONFIG_PCI_IOV void adf_gen4_init_pf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops); From 2297554f01df6d3d4e98a3915c183ce3e491740a Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Sun, 18 May 2025 12:32:12 -0700 Subject: [PATCH 382/382] x86/fpu: Fix irq_fpu_usable() to return false during CPU onlining irq_fpu_usable() incorrectly returned true before the FPU is initialized. The x86 CPU onlining code can call sha256() to checksum AMD microcode images, before the FPU is initialized. Since sha256() recently gained a kernel-mode FPU optimized code path, a crash occurred in kernel_fpu_begin_mask() during hotplug CPU onlining. (The crash did not occur during boot-time CPU onlining, since the optimized sha256() code is not enabled until subsys_initcalls run.) Fix this by making irq_fpu_usable() return false before fpu__init_cpu() has run. To do this without adding any additional overhead to irq_fpu_usable(), replace the existing per-CPU bool in_kernel_fpu with kernel_fpu_allowed which tracks both initialization and usage rather than just usage. The initial state is false; FPU initialization sets it to true; kernel-mode FPU sections toggle it to false and then back to true; and CPU offlining restores it to the initial state of false. Fixes: 11d7956d526f ("crypto: x86/sha256 - implement library instead of shash") Reported-by: Ayush Jain Closes: https://lore.kernel.org/r/20250516112217.GBaCcf6Yoc6LkIIryP@fat_crate.local Signed-off-by: Eric Biggers Tested-by: Ayush Jain Reviewed-by: Thomas Gleixner Signed-off-by: Herbert Xu --- arch/x86/include/asm/fpu/api.h | 1 + arch/x86/kernel/fpu/core.c | 34 +++++++++++++++++++++------------- arch/x86/kernel/fpu/init.c | 3 +++ arch/x86/kernel/smpboot.c | 6 ++++++ 4 files changed, 31 insertions(+), 13 deletions(-) diff --git a/arch/x86/include/asm/fpu/api.h b/arch/x86/include/asm/fpu/api.h index f42de5f05e7e..3ebeee2644de 100644 --- a/arch/x86/include/asm/fpu/api.h +++ b/arch/x86/include/asm/fpu/api.h @@ -126,6 +126,7 @@ static inline void fpstate_init_soft(struct swregs_state *soft) {} #endif /* State tracking */ +DECLARE_PER_CPU(bool, kernel_fpu_allowed); DECLARE_PER_CPU(struct fpu *, fpu_fpregs_owner_ctx); /* Process cleanup */ diff --git a/arch/x86/kernel/fpu/core.c b/arch/x86/kernel/fpu/core.c index 91d6341f281f..399f43aa78d5 100644 --- a/arch/x86/kernel/fpu/core.c +++ b/arch/x86/kernel/fpu/core.c @@ -43,8 +43,11 @@ struct fpu_state_config fpu_user_cfg __ro_after_init; */ struct fpstate init_fpstate __ro_after_init; -/* Track in-kernel FPU usage */ -static DEFINE_PER_CPU(bool, in_kernel_fpu); +/* + * Track FPU initialization and kernel-mode usage. 'true' means the FPU is + * initialized and is not currently being used by the kernel: + */ +DEFINE_PER_CPU(bool, kernel_fpu_allowed); /* * Track which context is using the FPU on the CPU: @@ -61,15 +64,18 @@ bool irq_fpu_usable(void) return false; /* - * In kernel FPU usage already active? This detects any explicitly - * nested usage in task or softirq context, which is unsupported. It - * also detects attempted usage in a hardirq that has interrupted a - * kernel-mode FPU section. + * Return false in the following cases: + * + * - FPU is not yet initialized. This can happen only when the call is + * coming from CPU onlining, for example for microcode checksumming. + * - The kernel is already using the FPU, either because of explicit + * nesting (which should never be done), or because of implicit + * nesting when a hardirq interrupted a kernel-mode FPU section. + * + * The single boolean check below handles both cases: */ - if (this_cpu_read(in_kernel_fpu)) { - WARN_ON_FPU(!in_hardirq()); + if (!this_cpu_read(kernel_fpu_allowed)) return false; - } /* * When not in NMI or hard interrupt context, FPU can be used in: @@ -431,9 +437,10 @@ void kernel_fpu_begin_mask(unsigned int kfpu_mask) fpregs_lock(); WARN_ON_FPU(!irq_fpu_usable()); - WARN_ON_FPU(this_cpu_read(in_kernel_fpu)); - this_cpu_write(in_kernel_fpu, true); + /* Toggle kernel_fpu_allowed to false: */ + WARN_ON_FPU(!this_cpu_read(kernel_fpu_allowed)); + this_cpu_write(kernel_fpu_allowed, false); if (!(current->flags & (PF_KTHREAD | PF_USER_WORKER)) && !test_thread_flag(TIF_NEED_FPU_LOAD)) { @@ -453,9 +460,10 @@ EXPORT_SYMBOL_GPL(kernel_fpu_begin_mask); void kernel_fpu_end(void) { - WARN_ON_FPU(!this_cpu_read(in_kernel_fpu)); + /* Toggle kernel_fpu_allowed back to true: */ + WARN_ON_FPU(this_cpu_read(kernel_fpu_allowed)); + this_cpu_write(kernel_fpu_allowed, true); - this_cpu_write(in_kernel_fpu, false); if (!irqs_disabled()) fpregs_unlock(); } diff --git a/arch/x86/kernel/fpu/init.c b/arch/x86/kernel/fpu/init.c index 998a08f17e33..1975c37c3668 100644 --- a/arch/x86/kernel/fpu/init.c +++ b/arch/x86/kernel/fpu/init.c @@ -51,6 +51,9 @@ void fpu__init_cpu(void) { fpu__init_cpu_generic(); fpu__init_cpu_xstate(); + + /* Start allowing kernel-mode FPU: */ + this_cpu_write(kernel_fpu_allowed, true); } static bool __init fpu__probe_without_cpuid(void) diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c index d6cf1e23c2a3..2901f5cfd825 100644 --- a/arch/x86/kernel/smpboot.c +++ b/arch/x86/kernel/smpboot.c @@ -1188,6 +1188,12 @@ void cpu_disable_common(void) remove_siblinginfo(cpu); + /* + * Stop allowing kernel-mode FPU. This is needed so that if the CPU is + * brought online again, the initial state is not allowed: + */ + this_cpu_write(kernel_fpu_allowed, false); + /* It's now safe to remove this processor from the online map */ lock_vector_lock(); remove_cpu_from_maps(cpu);