diff --git a/.wolfssl_known_macro_extras b/.wolfssl_known_macro_extras index 4f88a806c0..282a474c01 100644 --- a/.wolfssl_known_macro_extras +++ b/.wolfssl_known_macro_extras @@ -231,6 +231,7 @@ CONFIG_WOLFSSL_XMSS CONFIG_WOLFTPM CONFIG_WOLFTPM_EXAMPLE_NAME_ESPRESSIF CONFIG_X86 +CONFIG_X86_64 CONV_WITH_DIV CPA_CY_API_VERSION_NUM_MAJOR CPA_CY_API_VERSION_NUM_MINOR @@ -799,6 +800,7 @@ WC_STM32_RNG_DIAG WC_STM32_RNG_NO_NIST_INIT WC_STM32_SAES_DIAG WC_STRICT_SIG +WC_SVR_DONT_USE_NATIVE_REG_BUFS WC_USE_PIE_FENCEPOSTS_FOR_FIPS WC_WANT_FLAG_DONT_USE_VECTOR_OPS WIFIESPAT diff --git a/linuxkm/linuxkm_wc_port.h b/linuxkm/linuxkm_wc_port.h index 51db536567..7a53f7a28d 100644 --- a/linuxkm/linuxkm_wc_port.h +++ b/linuxkm/linuxkm_wc_port.h @@ -259,6 +259,12 @@ #endif #endif + #if defined(WC_SVR_DONT_USE_NATIVE_REG_BUFS) + #undef WC_SVR_USE_NATIVE_REG_BUFS + #elif defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS) && !defined(WC_SVR_USE_NATIVE_REG_BUFS) + #define WC_SVR_USE_NATIVE_REG_BUFS + #endif + #if defined(HAVE_FIPS) && FIPS_VERSION3_LT(7, 0, 0) #if defined(HAVE_HASHDRBG) && \ defined(HAVE_ENTROPY_MEMUSE) && \ @@ -740,8 +746,13 @@ #if defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS) && \ defined(CONFIG_X86) - extern __must_check int allocate_wolfcrypt_linuxkm_fpu_states(void); - extern void free_wolfcrypt_linuxkm_fpu_states(void); + extern __must_check int wc_linuxkm_allocate_svr_states(void); + extern void wc_linuxkm_free_svr_states(void); + WOLFSSL_API void wc_svr_disallowed_count_reset(void); + #ifdef WC_SVR_USE_NATIVE_REG_BUFS + WOLFSSL_LOCAL __must_check int wc_linuxkm_svr_native_is_ready(void); + #endif + WOLFSSL_API __must_check unsigned long long int wc_svr_disallowed_count_current(void); WOLFSSL_API __must_check int wc_can_save_vector_registers_x86(void); WOLFSSL_API __must_check int wc_save_vector_registers_x86(enum wc_svr_flags flags); WOLFSSL_API void wc_restore_vector_registers_x86(enum wc_svr_flags flags); @@ -1247,9 +1258,9 @@ #ifdef WOLFSSL_USE_SAVE_VECTOR_REGISTERS #ifdef CONFIG_X86 - typeof(allocate_wolfcrypt_linuxkm_fpu_states) *allocate_wolfcrypt_linuxkm_fpu_states; + typeof(wc_linuxkm_allocate_svr_states) *wc_linuxkm_allocate_svr_states; typeof(wc_can_save_vector_registers_x86) *wc_can_save_vector_registers_x86; - typeof(free_wolfcrypt_linuxkm_fpu_states) *free_wolfcrypt_linuxkm_fpu_states; + typeof(wc_linuxkm_free_svr_states) *wc_linuxkm_free_svr_states; typeof(wc_restore_vector_registers_x86) *wc_restore_vector_registers_x86; typeof(wc_save_vector_registers_x86) *wc_save_vector_registers_x86; #else /* !CONFIG_X86 */ @@ -1606,9 +1617,9 @@ #define get_current WC_PIE_INDIRECT_SYM(get_current) #if defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS) && defined(CONFIG_X86) - #define allocate_wolfcrypt_linuxkm_fpu_states WC_PIE_INDIRECT_SYM(allocate_wolfcrypt_linuxkm_fpu_states) + #define wc_linuxkm_allocate_svr_states WC_PIE_INDIRECT_SYM(wc_linuxkm_allocate_svr_states) #define wc_can_save_vector_registers_x86 WC_PIE_INDIRECT_SYM(wc_can_save_vector_registers_x86) - #define free_wolfcrypt_linuxkm_fpu_states WC_PIE_INDIRECT_SYM(free_wolfcrypt_linuxkm_fpu_states) + #define wc_linuxkm_free_svr_states WC_PIE_INDIRECT_SYM(wc_linuxkm_free_svr_states) #define wc_restore_vector_registers_x86 WC_PIE_INDIRECT_SYM(wc_restore_vector_registers_x86) #define wc_save_vector_registers_x86 WC_PIE_INDIRECT_SYM(wc_save_vector_registers_x86) #elif defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS) diff --git a/linuxkm/lkcapi_sha_glue.c b/linuxkm/lkcapi_sha_glue.c index 414f9d718a..1ad9379690 100644 --- a/linuxkm/lkcapi_sha_glue.c +++ b/linuxkm/lkcapi_sha_glue.c @@ -2123,7 +2123,9 @@ static int wc_linuxkm_rng_bank_init(struct wc_rng_bank *ctx) int ret; word32 flags = WC_RNG_BANK_FLAG_CAN_WAIT; -#if defined(HAVE_FIPS) && FIPS_VERSION3_LT(7,0,0) +#if defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS) && \ + !defined(WC_SVR_USE_NATIVE_REG_BUFS) && \ + defined(HAVE_FIPS) && FIPS_VERSION3_LT(7,0,0) /* before v7, the SHA-2 implementations couldn't dynamically switch between * C and asm in a given wc_Sha256 instance. */ @@ -2205,6 +2207,11 @@ static struct wc_rng_bank_inst *linuxkm_get_drbg(struct wc_rng_bank *ctx) { WC_RNG_BANK_FLAG_CAN_WAIT | WC_RNG_BANK_FLAG_PREFER_AFFINITY_INST; +#ifdef WC_SVR_USE_NATIVE_REG_BUFS + if (wc_linuxkm_svr_native_is_ready()) + flags |= WC_RNG_BANK_FLAG_AFFINITY_LOCK; + else +#endif if (wc_linuxkm_can_block()) flags |= WC_RNG_BANK_FLAG_AFFINITY_LOCK; else diff --git a/linuxkm/module_hooks.c b/linuxkm/module_hooks.c index 888343b9da..c661cbe65d 100644 --- a/linuxkm/module_hooks.c +++ b/linuxkm/module_hooks.c @@ -192,10 +192,14 @@ int wc_lkm_LockMutex(wolfSSL_Mutex* m) m->irq_flags = irq_flags; return 0; } - if (irq_count() != 0) { + if (! wc_linuxkm_can_block()) { /* Note, this catches calls while SAVE_VECTOR_REGISTERS()ed as * required, because in_softirq() is always true while saved, * even for WC_FPU_INHIBITED_FLAG contexts. + * + * It also catches non-interrupt atomic callers -- tasks holding a + * spinlock or running with IRQs off -- which must not reach the + * cond_resched() retry loop below. */ spin_lock_irqsave(&m->lock, irq_flags); m->irq_flags = irq_flags; @@ -1522,9 +1526,9 @@ static int set_up_wolfssl_linuxkm_pie_redirect_table(void) { wolfssl_linuxkm_pie_redirect_table.get_current = my_get_current_thread; #if defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS) && defined(CONFIG_X86) - wolfssl_linuxkm_pie_redirect_table.allocate_wolfcrypt_linuxkm_fpu_states = allocate_wolfcrypt_linuxkm_fpu_states; + wolfssl_linuxkm_pie_redirect_table.wc_linuxkm_allocate_svr_states = wc_linuxkm_allocate_svr_states; wolfssl_linuxkm_pie_redirect_table.wc_can_save_vector_registers_x86 = wc_can_save_vector_registers_x86; - wolfssl_linuxkm_pie_redirect_table.free_wolfcrypt_linuxkm_fpu_states = free_wolfcrypt_linuxkm_fpu_states; + wolfssl_linuxkm_pie_redirect_table.wc_linuxkm_free_svr_states = wc_linuxkm_free_svr_states; wolfssl_linuxkm_pie_redirect_table.wc_restore_vector_registers_x86 = wc_restore_vector_registers_x86; wolfssl_linuxkm_pie_redirect_table.wc_save_vector_registers_x86 = wc_save_vector_registers_x86; #elif defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS) diff --git a/linuxkm/simd_hammer/Makefile b/linuxkm/simd_hammer/Makefile new file mode 100644 index 0000000000..b0401bb4d0 --- /dev/null +++ b/linuxkm/simd_hammer/Makefile @@ -0,0 +1,29 @@ +# Makefile for simd_hammer.ko -- stress testing for kernel FPU/SIMD context handling +# +# Copyright (C) 2006-2026 wolfSSL Inc. +# +# This file is part of wolfSSL. +# +# wolfSSL is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# wolfSSL is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA + +obj-m := simd_hammer.o + +KDIR ?= /lib/modules/$(shell uname -r)/build + +default: + $(MAKE) -C $(KDIR) M=$(CURDIR) modules + +clean: + $(MAKE) -C $(KDIR) M=$(CURDIR) clean diff --git a/linuxkm/simd_hammer/simd_hammer.c b/linuxkm/simd_hammer/simd_hammer.c new file mode 100644 index 0000000000..40e818f197 --- /dev/null +++ b/linuxkm/simd_hammer/simd_hammer.c @@ -0,0 +1,714 @@ +/* simd_hammer.c -- stress testing for kernel FPU/SIMD context handling + * + * Copyright (C) 2006-2026 wolfSSL Inc. + * + * This file is part of wolfSSL. + * + * wolfSSL is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * wolfSSL is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA + */ + +/* + * simd_hammer -- simulate a heavy multithreaded kernel workload to test SIMD + * handling in libwolfssl.ko. + * + * Hammer modes (per-CPU pinned kthreads): + * hammer_mode=wolfcrypt sync-skcipher traffic through the LKCAPI. + * NOTE: wolfSSL's SVR glue brackets its own FPU + * sections with local_bh_disable(), so this mode + * doesn't create softirq-visible collisions -- it + * exists to demonstrate that fact, and to measure + * probe lateness (softirq blackout) caused by + * bh-disabled crypto sections. + * hammer_mode=rawfpu bare kernel_fpu_begin()/udelay/kernel_fpu_end() + * sections, emulating foreign FPU users (raid6, + * ZFS, other crypto modules). On pre-6.15 kernels + * these sections are softirq-interruptible, so the + * probe collides with them at rate ~= HZ x duty. + * On 6.15+ kernel_fpu_begin() itself disables bh + * (commit d02198550423) and the collision class is + * structurally extinct -- expect zero, on any build. + * + * Probe contexts (per-CPU, pinned): + * probe_ctx=softirq TIMER_SOFTIRQ (timer_list) -- ESP-receive-class. + * probe_ctx=hardirq hrtimer in hard-interrupt context -- emulates + * get_random_bytes()-from-irq-handler-class callers. + * + * Each probe samples may_use_simd() at entry (the interrupted context is + * frozen underneath, so the sample holds for the whole call), then performs + * a sync shash digest through the LKCAPI and records the result. Against a + * with-fallback module: !simd probes succeed via the C path (errs stays 0). + * Against a no-fallback module: !simd probes fail (errs tracks no_simd, and + * last_err records the mapped errno). + * + * Lateness stats: scheduled-vs-actual delta per probe, reported avg/max. + * Under bh-disabled crypto sections, softirq probes are deferred until + * local_bh_enable(); lateness quantifies that blackout. + * + * Note that the test is x86-only, non-PREEMPT_RT, with no CPU-hotplug handling. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if LINUX_VERSION_CODE < KERNEL_VERSION(6, 2, 0) + /* self-rearm is gated on c->stop, so plain del_timer_sync is safe here */ + #define timer_shutdown_sync(t) del_timer_sync(t) +#endif + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 16, 0) + #define wc_simd_timer_container_of(var, t, field) timer_container_of(var, t, field) +#else + #define wc_simd_timer_container_of(var, t, field) from_timer(var, t, field) +#endif + +/* sidestep "flush-left function calls" warnings from check-source-text: */ +#define MODULE_PARAM module_param + +static char *hammer_mode = "rawfpu"; +MODULE_PARAM(hammer_mode, charp, 0444); +MODULE_PARM_DESC(hammer_mode, "\"rawfpu\" (foreign-FPU-user emulation) or \"wolfcrypt\""); + +static char *hammer_alg = "xts-aes-wolfcrypt"; +MODULE_PARAM(hammer_alg, charp, 0444); +MODULE_PARM_DESC(hammer_alg, "sync skcipher driver name (hammer_mode=wolfcrypt)"); + +static unsigned int hammer_keylen = 64; +MODULE_PARAM(hammer_keylen, uint, 0444); + +static unsigned int hammer_bytes = 64 * 1024; +MODULE_PARAM(hammer_bytes, uint, 0444); +MODULE_PARM_DESC(hammer_bytes, "bytes per wolfcrypt hammer op"); + +static unsigned int raw_section_us = 200; +MODULE_PARAM(raw_section_us, uint, 0444); +MODULE_PARM_DESC(raw_section_us, "kernel_fpu section length, us (hammer_mode=rawfpu)"); + +static unsigned int hammer_pause_us; +MODULE_PARAM(hammer_pause_us, uint, 0444); +MODULE_PARM_DESC(hammer_pause_us, "sleep between hammer ops (duty throttle; 0 = flat out)"); + +static char *probe_ctx = "softirq"; +MODULE_PARAM(probe_ctx, charp, 0444); +MODULE_PARM_DESC(probe_ctx, "\"softirq\" (timer_list) or \"hardirq\" (hrtimer)"); + +static char *probe_alg = "sha256"; +MODULE_PARAM(probe_alg, charp, 0444); +MODULE_PARM_DESC(probe_alg, "sync shash driver name for the probe"); + +static char *probe_kind = "shash"; +MODULE_PARAM(probe_kind, charp, 0444); +MODULE_PARM_DESC(probe_kind, + "\"shash\" (probe_alg digest), \"rng\" (crypto_rng_get_bytes " + "on probe_alg, e.g. probe_alg=stdrng), or \"grb\" " + "(get_random_bytes, the kernel randomness path)"); + +static unsigned int probe_bytes = 4096; +MODULE_PARAM(probe_bytes, uint, 0444); + +static unsigned int probe_interval = 1; +MODULE_PARAM(probe_interval, uint, 0444); +MODULE_PARM_DESC(probe_interval, "probe period in jiffies (both contexts)"); + +static unsigned int report_secs = 10; +MODULE_PARAM(report_secs, uint, 0444); + +#define WC_SIMD_MAX_DIGEST 64 + +struct wc_simd_pcpu { + int cpu; + struct task_struct *hammer; + bool hammer_failed; + struct timer_list probe_timer; + struct hrtimer probe_hrtimer; + ktime_t hr_period; + bool timer_live; + bool hrtimer_live; + struct shash_desc *desc; + u8 *pbuf; + u8 digest[WC_SIMD_MAX_DIGEST]; + bool stop; + /* counters: written only by the respective CPUs' probe / hammer contexts */ + unsigned long probes; + unsigned long simd_ok; + unsigned long no_simd; + unsigned long errs; + int last_err; + u64 late_sum_us; + u64 late_max_us; + /* Duration of the probe call itself. Lateness says the probe was + * delayed getting in; this says how long it took once inside, which is + * what an unbounded wait or a long IRQs-off section actually looks + * like. A call that stalls and then succeeds is invisible to a + * success count and obvious here. */ + u64 dur_sum_ns; + u64 dur_max_ns; +}; + +static struct wc_simd_pcpu *wc_simd_pc; +static struct crypto_shash *wc_simd_probe_tfm; +static struct crypto_rng *wc_simd_probe_rng; +static struct task_struct *wc_simd_report_task; +static bool wc_simd_raw_mode; +static bool wc_simd_hardirq_probe; + +enum wc_simd_probe_kind { + WC_SIMD_PROBE_SHASH = 0, + WC_SIMD_PROBE_RNG, + WC_SIMD_PROBE_GRB +}; +static enum wc_simd_probe_kind wc_simd_probe_kind; + +/* ---------------- probe core (any context) ---------------- */ + +static void wc_simd_probe_once(struct wc_simd_pcpu *c, u64 late_us) +{ + bool simd = may_use_simd(); + int ret; + + c->probes++; + if (simd) + c->simd_ok++; + else + c->no_simd++; + + c->late_sum_us += late_us; + if (late_us > c->late_max_us) + c->late_max_us = late_us; + + { + u64 t0 = ktime_get_ns(), dur; + + switch (wc_simd_probe_kind) { + case WC_SIMD_PROBE_RNG: + /* Exercises the registered stdrng, i.e. wolfCrypt's DRBG when + * the module is loaded at higher cra_priority. */ + ret = crypto_rng_get_bytes(wc_simd_probe_rng, c->pbuf, probe_bytes); + break; + case WC_SIMD_PROBE_GRB: + /* Exercises the kernel randomness path itself, including any + * get_random_bytes hook the module under test installs. */ + get_random_bytes(c->pbuf, probe_bytes); + ret = 0; + break; + case WC_SIMD_PROBE_SHASH: + default: + ret = crypto_shash_digest(c->desc, c->pbuf, probe_bytes, c->digest); + break; + } + + dur = ktime_get_ns() - t0; + c->dur_sum_ns += dur; + if (dur > c->dur_max_ns) + c->dur_max_ns = dur; + } + + if (unlikely(ret)) { + c->errs++; + c->last_err = ret; + } +} + +/* ---------------- softirq probe (timer_list) ---------------- */ + +static void wc_simd_probe_timer_fn(struct timer_list *t) +{ + struct wc_simd_pcpu *c = wc_simd_timer_container_of(c, t, probe_timer); + unsigned long sched_for = t->expires; + u64 late_us = (u64)jiffies_to_usecs(jiffies - sched_for); + + wc_simd_probe_once(c, late_us); + + if (!READ_ONCE(c->stop)) + mod_timer(&c->probe_timer, jiffies + probe_interval); +} + +/* ---------------- hardirq probe (hrtimer) ---------------- */ + +static enum hrtimer_restart wc_simd_probe_hrtimer_fn(struct hrtimer *t) +{ + struct wc_simd_pcpu *c = container_of(t, struct wc_simd_pcpu, probe_hrtimer); + s64 late_ns = ktime_to_ns(ktime_sub(ktime_get(), hrtimer_get_expires(t))); + + wc_simd_probe_once(c, late_ns > 0 ? (u64)late_ns / 1000 : 0); + + if (READ_ONCE(c->stop)) + return HRTIMER_NORESTART; + hrtimer_forward_now(t, c->hr_period); + return HRTIMER_RESTART; +} + +static void wc_simd_start_hrtimer_on_cpu(void *arg) +{ + struct wc_simd_pcpu *c = arg; + + hrtimer_start(&c->probe_hrtimer, c->hr_period, HRTIMER_MODE_REL_PINNED); +} + +/* ---------------- hammers ---------------- */ + +/* Foreign-FPU-SIMD-user emulation: simulate workload from other modules that + * use vector and FP registers wrapped in kernel_fpu_{begin,end}(): + * + * raid6_pq -- vectorized RAID-6 P/Q parity calculation + * + * crc32c-intel / crc32-pclmul / crct10dif-pclmul -- ext4/btrfs metadata + * checksums, iSCSI and NVMe-TCP data digests, T10-DIF generation + * + * xor_blocks -- vectorized MD RAID5 parity and btrfs raid56 + * + * EFI runtime services -- efi_fpu_begin() wraps every efi_call + * + * chacha20, poly1305, curve25519, etc. -- non-FIPS vectorized crypto + * implementations used by WireGuard etc. + * + * cached tfm handles bound to built-in vectorized implementations at kernel + * init -- IMA/EVM measurement hashes (in-tree vectorized SHA-256) + * + * Native accelerated FIPS-algorithm crypto implementations -- aesni-intel, + * sha*-ssse3/SHA-NI, ghash-clmulni, etc. -- these can be requested + * explicitly from both kernel and user (AF_ALG) space, bypassing algorithm + * priority, and inducing contention. Raw GHASH in particular has no public + * support in wolfCrypt, hence no driver, so its vectorized in-tree + * implementations remain the sole accelerated providers for any + * ghash/gcm_base composition. + * + ******************************************************************************* + * + * On 6.15+ kernels, kernel_fpu_begin() itself disables bh, precluding this + * contention in practice, if not in principle. Hard IRQs remain subject to + * SIMD disablement, but no in-tree hard IRQ handlers call into the LKCAPI. + * + * On pre-6.15, crypto users are preempt-disabled but softirq-interruptible; + * impacted softirq handlers encounter SIMD-forbidden contexts. This can be + * mitigated by backporting kernel commit d02198550423. + * + * On all kernels, LINUXKM_DRBG_GET_RANDOM_BYTES is subject to SIMD disablement, + * as it is called from hard IRQ contexts, wherein SAVE_VECTOR_REGISTERS*() + * always returns WC_ACCEL_INHIBIT_E. + */ + +static int wc_simd_hammer_raw_fn(void *arg) +{ + (void)arg; + + while (!kthread_should_stop()) { + kernel_fpu_begin(); + udelay(raw_section_us); + kernel_fpu_end(); + + if (hammer_pause_us) + usleep_range(hammer_pause_us, hammer_pause_us + hammer_pause_us / 8 + 1); + else + cond_resched(); + } + return 0; +} + +static int wc_simd_hammer_wc_fn(void *arg) +{ + struct wc_simd_pcpu *c = arg; + struct crypto_sync_skcipher *tfm = NULL; + u8 *buf = NULL; + u8 key[64]; + u8 iv[16]; + struct scatterlist sg; + int ret; + + if (hammer_keylen > sizeof(key)) { + pr_err("ERROR: %s: hammer_keylen %u > %zu\n", __func__, hammer_keylen, sizeof(key)); + goto fail_idle; + } + + tfm = crypto_alloc_sync_skcipher(hammer_alg, 0, 0); + if (IS_ERR(tfm)) { + pr_err("ERROR: %s: cpu%d: crypto_alloc_sync_skcipher(\"%s\") = %ld " + "(is the module under test loaded? check /proc/crypto)\n", + __func__, c->cpu, hammer_alg, PTR_ERR(tfm)); + tfm = NULL; + goto fail_idle; + } + + get_random_bytes(key, hammer_keylen); + ret = crypto_sync_skcipher_setkey(tfm, key, hammer_keylen); + memzero_explicit(key, sizeof(key)); + if (ret) { + pr_err("ERROR: %s: cpu%d: setkey(%u) = %d\n", __func__, c->cpu, hammer_keylen, ret); + goto fail_idle; + } + + buf = kmalloc(hammer_bytes, GFP_KERNEL); + if (!buf) { + pr_err("ERROR: %s: cpu%d: kmalloc(%u) failed\n", __func__, c->cpu, hammer_bytes); + goto fail_idle; + } + get_random_bytes(buf, hammer_bytes); + get_random_bytes(iv, sizeof(iv)); + + while (!kthread_should_stop()) { + SYNC_SKCIPHER_REQUEST_ON_STACK(req, tfm); + + sg_init_one(&sg, buf, hammer_bytes); + skcipher_request_set_sync_tfm(req, tfm); + skcipher_request_set_callback(req, 0, NULL, NULL); + skcipher_request_set_crypt(req, &sg, &sg, hammer_bytes, iv); + + ret = crypto_skcipher_encrypt(req); + skcipher_request_zero(req); + if (unlikely(ret)) { + c->errs++; + c->last_err = ret; + } + + if (hammer_pause_us) + usleep_range(hammer_pause_us, hammer_pause_us + hammer_pause_us / 8 + 1); + else + cond_resched(); + } + goto out; + +fail_idle: + c->hammer_failed = true; + while (!kthread_should_stop()) + schedule_timeout_interruptible(HZ); +out: + kfree(buf); + if (tfm) + crypto_free_sync_skcipher(tfm); + return 0; +} + +/* ---------------- reporting ---------------- */ + +struct wc_simd_totals { + unsigned long p, s, n, e; + u64 late_sum, late_max; + u64 dur_sum, dur_max; + int last_err; + bool failed; +}; + +static void wc_simd_sum(struct wc_simd_totals *t) +{ + int cpu; + + memset(t, 0, sizeof(*t)); + for_each_online_cpu(cpu) { + struct wc_simd_pcpu *c = &wc_simd_pc[cpu]; + u64 lm = READ_ONCE(c->late_max_us); + int le = READ_ONCE(c->last_err); + + t->p += READ_ONCE(c->probes); + t->s += READ_ONCE(c->simd_ok); + t->n += READ_ONCE(c->no_simd); + t->e += READ_ONCE(c->errs); + t->late_sum += READ_ONCE(c->late_sum_us); + if (lm > t->late_max) + t->late_max = lm; + t->dur_sum += READ_ONCE(c->dur_sum_ns); + { + u64 dm = READ_ONCE(c->dur_max_ns); + + if (dm > t->dur_max) + t->dur_max = dm; + } + if (le) + t->last_err = le; + t->failed |= READ_ONCE(c->hammer_failed); + } +} + +static int wc_simd_report_fn(void *unused) +{ + unsigned long last_p = 0, last_n = 0, last_e = 0; + + while (!kthread_should_stop()) { + struct wc_simd_totals t; + unsigned long dp, dn, de, n_per_min_x100, e_per_min_x100; + u64 late_avg, dur_avg; + + schedule_timeout_interruptible(report_secs * HZ); + if (kthread_should_stop()) + break; + + wc_simd_sum(&t); + dp = t.p - last_p; + dn = t.n - last_n; + de = t.e - last_e; + last_p = t.p; + last_n = t.n; + last_e = t.e; + n_per_min_x100 = report_secs ? (dn * 6000UL) / report_secs : 0; + e_per_min_x100 = report_secs ? (de * 6000UL) / report_secs : 0; + late_avg = t.p ? div64_u64(t.late_sum, t.p) : 0; + dur_avg = t.p ? div64_u64(t.dur_sum, t.p) : 0; + + pr_info("simd_hammer summary: probes=%lu simd=%lu NO_SIMD=%lu ERRS=%lu(last=%d) " + "late avg=%lluus max=%lluus dur avg=%lluns max=%lluns | " + "last %us: +%lu NO_SIMD (%lu.%02lu/min), " + "+%lu ERRS (%lu.%02lu/min)%s\n", + t.p, t.s, t.n, t.e, t.last_err, + late_avg, t.late_max, dur_avg, t.dur_max, report_secs, + dn, n_per_min_x100 / 100, n_per_min_x100 % 100, + de, e_per_min_x100 / 100, e_per_min_x100 % 100, + t.failed ? " [HAMMER FAILED on >=1 cpu]" : ""); + } + return 0; +} + +/* ---------------- init / exit ---------------- */ + +static void wc_simd_teardown(void) +{ + int cpu; + + if (wc_simd_report_task) { + kthread_stop(wc_simd_report_task); + wc_simd_report_task = NULL; + } + + if (wc_simd_pc) { + for_each_online_cpu(cpu) + WRITE_ONCE(wc_simd_pc[cpu].stop, true); + + for_each_online_cpu(cpu) { + struct wc_simd_pcpu *c = &wc_simd_pc[cpu]; + + if (c->timer_live) { + timer_shutdown_sync(&c->probe_timer); + c->timer_live = false; + } + if (c->hrtimer_live) { + hrtimer_cancel(&c->probe_hrtimer); + c->hrtimer_live = false; + } + } + for_each_online_cpu(cpu) { + struct wc_simd_pcpu *c = &wc_simd_pc[cpu]; + + if (c->hammer) { + kthread_stop(c->hammer); + c->hammer = NULL; + } + kfree(c->desc); + kfree(c->pbuf); + } + kfree(wc_simd_pc); + wc_simd_pc = NULL; + } + + if (wc_simd_probe_tfm) { + crypto_free_shash(wc_simd_probe_tfm); + wc_simd_probe_tfm = NULL; + } + + if (wc_simd_probe_rng) { + crypto_free_rng(wc_simd_probe_rng); + wc_simd_probe_rng = NULL; + } +} + +static int __init wc_simd_init(void) +{ + int cpu, ret; + int (*hammer_fn)(void *); + + if (!probe_bytes || !probe_interval) + return -EINVAL; + + if (!strcmp(hammer_mode, "rawfpu")) { + wc_simd_raw_mode = true; + hammer_fn = wc_simd_hammer_raw_fn; + if (!raw_section_us || raw_section_us > 1000) { + pr_err("ERROR: %s: raw_section_us must be 1..1000\n", __func__); + return -EINVAL; + } + } else if (!strcmp(hammer_mode, "wolfcrypt")) { + wc_simd_raw_mode = false; + hammer_fn = wc_simd_hammer_wc_fn; + if (!hammer_bytes) + return -EINVAL; + } else { + pr_err("ERROR: %s: hammer_mode must be \"rawfpu\" or \"wolfcrypt\"\n", __func__); + return -EINVAL; + } + + if (!strcmp(probe_ctx, "hardirq")) + wc_simd_hardirq_probe = true; + else if (!strcmp(probe_ctx, "softirq")) + wc_simd_hardirq_probe = false; + else { + pr_err("ERROR: %s: probe_ctx must be \"softirq\" or \"hardirq\"\n", __func__); + return -EINVAL; + } + + if (!strcmp(probe_kind, "shash")) + wc_simd_probe_kind = WC_SIMD_PROBE_SHASH; + else if (!strcmp(probe_kind, "rng")) + wc_simd_probe_kind = WC_SIMD_PROBE_RNG; + else if (!strcmp(probe_kind, "grb")) + wc_simd_probe_kind = WC_SIMD_PROBE_GRB; + else { + pr_err("ERROR: %s: probe_kind must be \"shash\", \"rng\" or \"grb\"\n", + __func__); + return -EINVAL; + } + + if (wc_simd_probe_kind == WC_SIMD_PROBE_SHASH) { + wc_simd_probe_tfm = crypto_alloc_shash(probe_alg, 0, 0); + if (IS_ERR(wc_simd_probe_tfm)) { + ret = PTR_ERR(wc_simd_probe_tfm); + wc_simd_probe_tfm = NULL; + pr_err("ERROR: %s: crypto_alloc_shash(\"%s\") = %d " + "(is the module under test loaded? check /proc/crypto)\n", + __func__, probe_alg, ret); + return ret; + } + if (crypto_shash_digestsize(wc_simd_probe_tfm) > WC_SIMD_MAX_DIGEST) { + ret = -EINVAL; + goto err; + } + } else if (wc_simd_probe_kind == WC_SIMD_PROBE_RNG) { + /* ONE shared tfm for every CPU, deliberately. A tfm per CPU would + * hand each probe its own private DRBG and measure nothing about + * the shared path under test. */ + wc_simd_probe_rng = crypto_alloc_rng(probe_alg, 0, 0); + if (IS_ERR(wc_simd_probe_rng)) { + ret = PTR_ERR(wc_simd_probe_rng); + wc_simd_probe_rng = NULL; + pr_err("ERROR: %s: crypto_alloc_rng(\"%s\") = %d " + "(is the module under test loaded? check /proc/crypto)\n", + __func__, probe_alg, ret); + return ret; + } + } + + wc_simd_pc = kcalloc(nr_cpu_ids, sizeof(*wc_simd_pc), GFP_KERNEL); + if (!wc_simd_pc) { + ret = -ENOMEM; + goto err; + } + + for_each_online_cpu(cpu) { + struct wc_simd_pcpu *c = &wc_simd_pc[cpu]; + + c->cpu = cpu; + + c->pbuf = kmalloc(probe_bytes, GFP_KERNEL); + if (!c->pbuf) { + ret = -ENOMEM; + goto err; + } + get_random_bytes(c->pbuf, probe_bytes); + + if (wc_simd_probe_kind == WC_SIMD_PROBE_SHASH) { + c->desc = kmalloc(sizeof(*c->desc) + + crypto_shash_descsize(wc_simd_probe_tfm), + GFP_KERNEL); + if (!c->desc) { + ret = -ENOMEM; + goto err; + } + c->desc->tfm = wc_simd_probe_tfm; + } + + c->hammer = kthread_create(hammer_fn, c, "wc_simd_hammer/%d", cpu); + if (IS_ERR(c->hammer)) { + ret = PTR_ERR(c->hammer); + c->hammer = NULL; + goto err; + } + kthread_bind(c->hammer, cpu); + wake_up_process(c->hammer); + + if (wc_simd_hardirq_probe) { + c->hr_period = ns_to_ktime((u64)jiffies_to_usecs(probe_interval) + * NSEC_PER_USEC); +#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 15, 0) + hrtimer_setup(&c->probe_hrtimer, wc_simd_probe_hrtimer_fn, + CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED); +#else + hrtimer_init(&c->probe_hrtimer, CLOCK_MONOTONIC, + HRTIMER_MODE_REL_PINNED); + c->probe_hrtimer.function = wc_simd_probe_hrtimer_fn; +#endif + ret = smp_call_function_single(cpu, wc_simd_start_hrtimer_on_cpu, c, 1); + if (ret) + goto err; + c->hrtimer_live = true; + } else { + timer_setup(&c->probe_timer, wc_simd_probe_timer_fn, TIMER_PINNED); + c->probe_timer.expires = jiffies + probe_interval; + add_timer_on(&c->probe_timer, cpu); + c->timer_live = true; + } + } + + wc_simd_report_task = kthread_run(wc_simd_report_fn, NULL, "wc_simd_report"); + if (IS_ERR(wc_simd_report_task)) { + ret = PTR_ERR(wc_simd_report_task); + wc_simd_report_task = NULL; + goto err; + } + + pr_info("simd_hammer loaded: hammer=%s%s%s probe=%s:%s(%s) %u B every %u jiffies, " + "%u cpus, HZ=%d\n", + hammer_mode, + wc_simd_raw_mode ? "" : " alg=", wc_simd_raw_mode ? "" : hammer_alg, + probe_kind, probe_alg, probe_ctx, probe_bytes, probe_interval, + num_online_cpus(), HZ); + return 0; + +err: + wc_simd_teardown(); + return ret; +} + +static void __exit wc_simd_exit(void) +{ + struct wc_simd_totals t; + + wc_simd_sum(&t); + wc_simd_teardown(); + pr_info("simd_hammer unloaded: final probes=%lu simd=%lu NO_SIMD=%lu ERRS=%lu " + "(last=%d) late max=%lluus dur max=%lluns\n", + t.p, t.s, t.n, t.e, t.last_err, t.late_max, t.dur_max); +} + +module_init(wc_simd_init); +module_exit(wc_simd_exit); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("!may_use_simd() collision generator + fallback-vs-error discriminator for wolfCrypt LKCAPI"); diff --git a/linuxkm/x86_vector_register_glue.c b/linuxkm/x86_vector_register_glue.c index 2313cd80d4..9fb0b2ad97 100644 --- a/linuxkm/x86_vector_register_glue.c +++ b/linuxkm/x86_vector_register_glue.c @@ -23,6 +23,9 @@ /* included by linuxkm/module_hooks.c */ #ifndef WC_SKIP_INCLUDED_C_FILES +#include +#include + #if !defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS) || !defined(CONFIG_X86) #error x86_vector_register_glue.c included in non-vectorized/non-x86 project. #endif @@ -35,77 +38,193 @@ #define VRG_PR_WARN_X pr_warn_once #endif -static unsigned int wc_linuxkm_fpu_states_n_tracked = 0; +static unsigned int wc_linuxkm_svr_states_n_tracked = 0; -struct wc_thread_fpu_count_ent { - volatile pid_t pid; - unsigned int fpu_state; +struct wc_thread_svr_count_ent { + volatile pid_t pid; /* sync pivot, __atomic access only. */ + unsigned int fpu_state; /* plain, owner-private under the nonpreemptibility invariant. */ + volatile unsigned long reserved_at; /* volatile, read live by foreign diagnostics. */ }; -struct wc_thread_fpu_count_ent *wc_linuxkm_fpu_states = NULL; +struct wc_thread_svr_count_ent *wc_linuxkm_svr_states = NULL; + +#define WC_SVR_COUNT_MASK 0x1fffffffU +#define WC_SVR_INHIBITED_FLAG 0x40000000U +#define WC_SVR_BH_HELD_FLAG 0x20000000U + +#define WC_SVR_FREE_SLOT_PID 0 +#define WC_SVR_IDLE_PID ((pid_t)(-1)) +#define WC_SVR_PID_SLOT_ID() (__extension__ (task_pid_nr(current) ? : WC_SVR_IDLE_PID)) -#define WC_FPU_COUNT_MASK 0x3fffffffU -#define WC_FPU_INHIBITED_FLAG 0x40000000U +/* On targets with 64 bit longs, jiffies starts at INITIAL_JIFFIES and climbs + * monotonically -- a zero ->reserved_at means that slot has not been reserved + * since wc_linuxkm_allocate_svr_states(). On 32 bit long targets it + * wraps periodically, and WC_SVR_SLOT_AGE_MS() is best effort (used only in log + * messages). + */ +#if HZ <= MSEC_PER_SEC && !(MSEC_PER_SEC % HZ) + #define WC_SVR_SLOT_AGE_MS(slot) ((slot)->reserved_at ? \ + ((long)jiffies - (long)(slot)->reserved_at) * (MSEC_PER_SEC / HZ) : \ + -1L) +#else + #define WC_SVR_SLOT_AGE_MS(slot) ((slot)->reserved_at ? \ + (long)jiffies_to_msecs(jiffies - (slot)->reserved_at) : \ + -1L) +#endif -WARN_UNUSED_RESULT int allocate_wolfcrypt_linuxkm_fpu_states(void) +#ifdef WOLFSSL_LINUXKM_VERBOSE_DEBUG +/* separately tracked count of softirq-contention events (contrast with + * wc_svr_disallowed_count, below). */ +static atomic_long_t softirq_SVR_err_count; +static atomic_long_t hardirq_SVR_err_count; +static atomic_long_t NMI_SVR_err_count; +static atomic_long_t other_SVR_err_count; +#endif + +#ifdef WC_SVR_USE_NATIVE_REG_BUFS + +struct wc_svr_native_ctx_state { + unsigned int depth; /* same-context nesting count. Save only on 0->1, + * restore only on 1->0. Owner-private: only code + * executing in this context class on this CPU + * reads or writes it, and context classes on one + * CPU interleave strictly (run-to-completion), so + * no atomics are needed. + */ + unsigned int pin_preempt; /* set at init for the softirq entry: on + * PREEMPT_RT, serving-softirq is preemptible, + * which would break both the frozen-cause + * precondition and register ownership, so the + * section holds preempt_disable(). On + * mainline this is a free nop-nesting. + * Hardirq needs no pin. + */ + u8 *save_area; /* this CPU+context's slice of + * wc_svr_native_save_mem, 64-byte aligned for + * XSAVE. */ +}; + +struct wc_svr_native_cpu_state { + struct wc_svr_native_ctx_state softirq; + struct wc_svr_native_ctx_state hardirq; +}; + +static void wc_svr_native_init(void); +static int wc_svr_native_check_busy(void); +static void wc_svr_native_free(void); +static inline struct wc_svr_native_cpu_state *wc_svr_native_here(void); +static WARN_UNUSED_RESULT int wc_svr_native_save( + struct wc_svr_native_ctx_state *ctx); +static void wc_svr_native_restore(struct wc_svr_native_ctx_state *ctx, + enum wc_svr_flags flags); + +static int wc_svr_native_ready = 0; + +#endif /* WC_SVR_USE_NATIVE_REG_BUFS */ + +WARN_UNUSED_RESULT int wc_linuxkm_allocate_svr_states(void) { - if (wc_linuxkm_fpu_states != NULL) { + if (wc_linuxkm_svr_states != NULL) { #ifdef HAVE_FIPS - /* see note below in wc_linuxkm_fpu_state_assoc_unlikely(). */ + /* see note below in wc_linuxkm_svr_state_assoc_unlikely(). */ +#ifdef WC_SVR_USE_NATIVE_REG_BUFS + /* If the first allocation was the FIPS pre-Init lazy path, native + * init deferred itself out of the atomic context; this task-context + * repeat call is the retry (no-op once ready). + */ + /* If the first allocation was the FIPS pre-Init lazy path and + * native init could not complete there (NMI defer, allocation + * failure), this repeat call is the retry (no-op once ready). + */ + wc_svr_native_init(); +#endif return 0; #else static int warned_for_repeat_alloc = 0; if (! warned_for_repeat_alloc) { pr_err("BUG: attempt at repeat allocation" - " in allocate_wolfcrypt_linuxkm_fpu_states.\n"); + " in wc_linuxkm_allocate_svr_states.\n"); warned_for_repeat_alloc = 1; } return BAD_STATE_E; #endif } - wc_linuxkm_fpu_states_n_tracked = nr_cpu_ids; + wc_linuxkm_svr_states_n_tracked = nr_cpu_ids; - wc_linuxkm_fpu_states = - (struct wc_thread_fpu_count_ent *)malloc( - wc_linuxkm_fpu_states_n_tracked * sizeof(wc_linuxkm_fpu_states[0])); + wc_linuxkm_svr_states = + (struct wc_thread_svr_count_ent *)malloc( + wc_linuxkm_svr_states_n_tracked * sizeof(wc_linuxkm_svr_states[0])); - if (! wc_linuxkm_fpu_states) { + if (! wc_linuxkm_svr_states) { pr_err("ERROR: allocation of %lu bytes for " - "wc_linuxkm_fpu_states failed.\n", - nr_cpu_ids * sizeof(wc_linuxkm_fpu_states[0])); + "wc_linuxkm_svr_states failed.\n", + nr_cpu_ids * sizeof(wc_linuxkm_svr_states[0])); return MEMORY_E; } - XMEMSET(wc_linuxkm_fpu_states, 0, wc_linuxkm_fpu_states_n_tracked - * sizeof(wc_linuxkm_fpu_states[0])); + XMEMSET(wc_linuxkm_svr_states, 0, wc_linuxkm_svr_states_n_tracked + * sizeof(wc_linuxkm_svr_states[0])); + +#ifdef WC_SVR_USE_NATIVE_REG_BUFS + /* Best-effort: any failure leaves wc_svr_native_ready clear and the + * module on its long-standing refusal semantics. + */ + wc_svr_native_init(); +#endif return 0; } -void free_wolfcrypt_linuxkm_fpu_states(void) { - struct wc_thread_fpu_count_ent *i, *i_endptr; +void wc_linuxkm_free_svr_states(void) { + struct wc_thread_svr_count_ent *i, *i_endptr; pid_t i_pid; + int seen_errors = 0; - if (wc_linuxkm_fpu_states == NULL) + if (wc_linuxkm_svr_states == NULL) return; - for (i = wc_linuxkm_fpu_states, - i_endptr = &wc_linuxkm_fpu_states[wc_linuxkm_fpu_states_n_tracked]; +#ifdef WOLFSSL_LINUXKM_VERBOSE_DEBUG + pr_info("IRQ INFO: WC_ACCEL_INHIBIT_E totals at module shutdown: %ld softirq, %ld hardirq, %ld NMI, %ld other\n", + atomic_long_read(&softirq_SVR_err_count), + atomic_long_read(&hardirq_SVR_err_count), + atomic_long_read(&NMI_SVR_err_count), + atomic_long_read(&other_SVR_err_count)); +#endif + +#ifdef WC_SVR_USE_NATIVE_REG_BUFS + seen_errors += wc_svr_native_check_busy(); +#endif + + for (i = wc_linuxkm_svr_states, + i_endptr = &wc_linuxkm_svr_states[wc_linuxkm_svr_states_n_tracked]; i < i_endptr; ++i) { i_pid = __atomic_load_n(&i->pid, __ATOMIC_CONSUME); - if (i_pid == 0) + if (i_pid == WC_SVR_FREE_SLOT_PID) continue; - if (i->fpu_state != 0) { - pr_err("ERROR: free_wolfcrypt_linuxkm_fpu_states called" - " with nonzero state 0x%x for PID %d.\n", i->fpu_state, i_pid); - i->fpu_state = 0; - } + /* Any occupied slot blocks deallocation. A slot with fpu_state + * still zero is a claim in flight (pid published, state not yet + * stored) -- freeing under it is a use-after-free for the + * claimant, so it is no safer than a nonzero state. */ + pr_err("ERROR: wc_linuxkm_free_svr_states called" + " with occupied slot: state 0x%x for PID %d, age %ld ms.\n", i->fpu_state, i_pid, + WC_SVR_SLOT_AGE_MS(i)); + ++seen_errors; + } + + if (seen_errors > 0) { + pr_crit("ERROR: wc_linuxkm_free_svr_states encountered" + " %d errors -- can't free current wc_linuxkm_svr_states.\n", seen_errors); + return; } - free(wc_linuxkm_fpu_states); - wc_linuxkm_fpu_states = NULL; + free(wc_linuxkm_svr_states); + wc_linuxkm_svr_states = NULL; + +#ifdef WC_SVR_USE_NATIVE_REG_BUFS + wc_svr_native_free(); +#endif } /* lock-free O(1)-lookup CPU-local storage facility for tracking recursive fpu @@ -114,14 +233,14 @@ void free_wolfcrypt_linuxkm_fpu_states(void) { * caller must have already locked itself on its CPU before entering this, or * entering the streamlined inline version of it below. */ -static struct wc_thread_fpu_count_ent *wc_linuxkm_fpu_state_assoc_unlikely(int create_p) { +static struct wc_thread_svr_count_ent *wc_linuxkm_svr_state_assoc_unlikely(int create_p) { int my_cpu = raw_smp_processor_id(); - pid_t my_pid = task_pid_nr(current), slot_pid; - struct wc_thread_fpu_count_ent *slot; + pid_t my_pid = WC_SVR_PID_SLOT_ID(), slot_pid; + struct wc_thread_svr_count_ent *slot; { static int _warned_on_null = 0; - if (wc_linuxkm_fpu_states == NULL) + if (wc_linuxkm_svr_states == NULL) { #ifdef HAVE_FIPS /* FIPS needs to use SHA256 for the core verify HMAC, before @@ -129,13 +248,13 @@ static struct wc_thread_fpu_count_ent *wc_linuxkm_fpu_state_assoc_unlikely(int c * dependency loop on intelasm builds, we allocate here. * this is not thread-safe and doesn't need to be. */ - int ret = allocate_wolfcrypt_linuxkm_fpu_states(); + int ret = wc_linuxkm_allocate_svr_states(); if (ret != 0) #endif { if (_warned_on_null == 0) { - pr_err("BUG: wc_linuxkm_fpu_state_assoc called by PID %d" - " before allocate_wolfcrypt_linuxkm_fpu_states.\n", my_pid); + pr_err("BUG: wc_linuxkm_svr_state_assoc called by PID %d" + " before wc_linuxkm_allocate_svr_states.\n", my_pid); #ifdef WOLFSSL_LINUXKM_VERBOSE_DEBUG dump_stack(); #endif @@ -146,15 +265,15 @@ static struct wc_thread_fpu_count_ent *wc_linuxkm_fpu_state_assoc_unlikely(int c } } - slot = &wc_linuxkm_fpu_states[my_cpu]; + slot = &wc_linuxkm_svr_states[my_cpu]; slot_pid = __atomic_load_n(&slot->pid, __ATOMIC_CONSUME); if (slot_pid == my_pid) { if (create_p) { static int _warned_on_redundant_create_p = 0; if (_warned_on_redundant_create_p < 10) { - pr_err("BUG: wc_linuxkm_fpu_state_assoc called with create_p=1 by" + pr_err("BUG: wc_linuxkm_svr_state_assoc called with create_p=1 by" " PID %d on CPU %d with CPU slot already reserved by" - " said PID.\n", my_pid, my_cpu); + " said PID (age %ld ms).\n", my_pid, my_cpu, WC_SVR_SLOT_AGE_MS(slot)); #ifdef WOLFSSL_LINUXKM_VERBOSE_DEBUG dump_stack(); #endif @@ -164,21 +283,49 @@ static struct wc_thread_fpu_count_ent *wc_linuxkm_fpu_state_assoc_unlikely(int c return slot; } if (create_p) { - if (slot_pid == 0) { + if (slot_pid == WC_SVR_FREE_SLOT_PID) { __atomic_store_n(&slot->pid, my_pid, __ATOMIC_RELEASE); + slot->reserved_at = jiffies; return slot; + } else if (slot_pid == WC_SVR_IDLE_PID) { + /* A WC_SVR_IDLE_PID reservation on a live system is always a live + * reservation or a bug -- these reservations originate with softirq + * handlers on the idle threads, and crashes in those handlers + * summarily panic the kernel. + * + * If an idle-context cycle nonetheless goes unbalanced, there is + * no way to recover. The exposure is confined to this library -- + * consumers never invoke save/restore directly -- and balance is + * an enforced invariant (single-exit discipline, fuzzer + * coverage); this branch is that invariant's audit, not its + * substitute. + * + * Ultimately, recovery is impossible because WC_SVR_IDLE_PID names a + * context class, not a task, so nothing analogous to find_get_pid() + * below could decide whether this record is a stale leftover or a + * live idle-context bracket. Recovery couldn't close the hazard + * anyway -- the next idle-context save on this CPU fast-path would + * match a stale record as its own live nesting and proceed without + * kernel_fpu_begin(). A record in this state is proof of an + * unbalanced idle-context save/restore; report it and fail. + * Save/restore balance discipline, not runtime recovery, is what + * keeps this branch unreachable. + */ + pr_err_ratelimited("BUG: wc_linuxkm_svr_state_assoc_unlikely found WC_SVR_IDLE_PID in the slot for calling CPU %d PID %d (age %ld ms) requesting outermost vector register save -- CPU is acceleration-degraded.\n", my_cpu, my_pid, WC_SVR_SLOT_AGE_MS(slot)); + return NULL; } else { struct pid *slot_pid_struct; - /* if the slot is already occupied, that can be benign-ish due to a + /* if the slot is already occupied, that can be benign-ish due to an * unwanted migration, or due to a process crashing in kernel mode. * it will require fixup either here, or by the thread that owns the * slot, which will happen when it releases its lock. */ slot_pid_struct = find_get_pid(slot_pid); if (slot_pid_struct == NULL) { - if (__atomic_compare_exchange_n(&slot->pid, &slot_pid, my_pid, 0, __ATOMIC_SEQ_CST, __ATOMIC_ACQUIRE)) { - pr_warn("WARNING: wc_linuxkm_fpu_state_assoc_unlikely fixed up orphaned slot on CPU %d owned by dead PID %d.\n", my_cpu, slot_pid); + if (__atomic_compare_exchange_n(&slot->pid, &slot_pid, my_pid, 0 /* weak */, __ATOMIC_SEQ_CST, __ATOMIC_ACQUIRE)) { + pr_warn("WARNING: wc_linuxkm_svr_state_assoc_unlikely fixed up orphaned slot on CPU %d owned by dead PID %d (age %ld ms).\n", my_cpu, slot_pid, WC_SVR_SLOT_AGE_MS(slot)); + slot->reserved_at = jiffies; return slot; } } else { @@ -189,9 +336,9 @@ static struct wc_thread_fpu_count_ent *wc_linuxkm_fpu_state_assoc_unlikely(int c { static int _warned_on_mismatched_pid = 0; if (_warned_on_mismatched_pid < 10) { - pr_warn("WARNING: wc_linuxkm_fpu_state_assoc called by pid %d on CPU %d" - " but CPU slot already reserved by pid %d.\n", - my_pid, my_cpu, slot_pid); + pr_warn("WARNING: wc_linuxkm_svr_state_assoc called by pid %d on CPU %d" + " but CPU slot already reserved by pid %d (age %ld ms).\n", + my_pid, my_cpu, slot_pid, WC_SVR_SLOT_AGE_MS(slot)); #ifdef WOLFSSL_LINUXKM_VERBOSE_DEBUG dump_stack(); #endif @@ -201,33 +348,41 @@ static struct wc_thread_fpu_count_ent *wc_linuxkm_fpu_state_assoc_unlikely(int c return NULL; } + } else if (my_pid == WC_SVR_IDLE_PID) { + /* unique per value, not per task - all swappers share WC_SVR_IDLE_PID, so a + * scan match may be another CPU's live hold. + */ + return NULL; } else { /* check for migration. this can happen despite our best efforts if any * I/O occurred while locked, e.g. kernel messages like "uninitialized * urandom read". since we're locked now, we can safely migrate the - * entry in wc_linuxkm_fpu_states[], freeing up the slot on the previous + * entry in wc_linuxkm_svr_states[], freeing up the slot on the previous * cpu. */ unsigned int cpu_i; - for (cpu_i = 0; cpu_i < wc_linuxkm_fpu_states_n_tracked; ++cpu_i) { + for (cpu_i = 0; cpu_i < wc_linuxkm_svr_states_n_tracked; ++cpu_i) { if (__atomic_load_n( - &wc_linuxkm_fpu_states[cpu_i].pid, + &wc_linuxkm_svr_states[cpu_i].pid, __ATOMIC_CONSUME) == my_pid) { - wc_linuxkm_fpu_states[my_cpu] = wc_linuxkm_fpu_states[cpu_i]; - __atomic_store_n(&wc_linuxkm_fpu_states[cpu_i].fpu_state, 0, + wc_linuxkm_svr_states[my_cpu] = wc_linuxkm_svr_states[cpu_i]; + __atomic_store_n(&wc_linuxkm_svr_states[cpu_i].fpu_state, 0, __ATOMIC_RELEASE); - __atomic_store_n(&wc_linuxkm_fpu_states[cpu_i].pid, 0, + __atomic_store_n(&wc_linuxkm_svr_states[cpu_i].pid, WC_SVR_FREE_SLOT_PID, __ATOMIC_RELEASE); - return &wc_linuxkm_fpu_states[my_cpu]; + /* don't clear the .reserved_at member -- it's invalidated by + * the .pid assignment, and it might prove useful + * forensically. */ + return &wc_linuxkm_svr_states[my_cpu]; } } return NULL; } } -static inline struct wc_thread_fpu_count_ent *wc_linuxkm_fpu_state_assoc( +static inline struct wc_thread_svr_count_ent *wc_linuxkm_svr_state_assoc( int create_p, int assume_fpu_began) { int my_cpu = raw_smp_processor_id(); /* my_cpu is only trustworthy if we're @@ -237,9 +392,9 @@ static inline struct wc_thread_fpu_count_ent *wc_linuxkm_fpu_state_assoc( * failing that, if create_p. */ pid_t my_pid, slot_pid; - struct wc_thread_fpu_count_ent *slot; + struct wc_thread_svr_count_ent *slot; - if (unlikely(wc_linuxkm_fpu_states == NULL)) { + if (unlikely(wc_linuxkm_svr_states == NULL)) { if (! assume_fpu_began) { /* this was just a quick check for whether we're in a recursive * wc_save_vector_registers_x86(). we're not. @@ -247,16 +402,16 @@ static inline struct wc_thread_fpu_count_ent *wc_linuxkm_fpu_state_assoc( return NULL; } else - return wc_linuxkm_fpu_state_assoc_unlikely(create_p); + return wc_linuxkm_svr_state_assoc_unlikely(create_p); } - my_pid = task_pid_nr(current); + my_pid = WC_SVR_PID_SLOT_ID(); - slot = &wc_linuxkm_fpu_states[my_cpu]; + slot = &wc_linuxkm_svr_states[my_cpu]; slot_pid = __atomic_load_n(&slot->pid, __ATOMIC_CONSUME); if (slot_pid == my_pid) { if (unlikely(create_p)) - return wc_linuxkm_fpu_state_assoc_unlikely(create_p); + return wc_linuxkm_svr_state_assoc_unlikely(create_p); else return slot; } @@ -265,67 +420,116 @@ static inline struct wc_thread_fpu_count_ent *wc_linuxkm_fpu_state_assoc( * wc_save_vector_registers_x86(). we're not. * * if we're in a softirq context, we'll always wind up here, because - * processes with entries in wc_linuxkm_fpu_states[] always have + * processes with entries in wc_linuxkm_svr_states[] always have * softirqs inhibited. */ return NULL; } if (likely(create_p)) { - if (likely(slot_pid == 0)) { + if (likely(slot_pid == WC_SVR_FREE_SLOT_PID)) { __atomic_store_n(&slot->pid, my_pid, __ATOMIC_RELEASE); + slot->reserved_at = jiffies; return slot; } else { - return wc_linuxkm_fpu_state_assoc_unlikely(create_p); + return wc_linuxkm_svr_state_assoc_unlikely(create_p); } } else { - return wc_linuxkm_fpu_state_assoc_unlikely(create_p); + return wc_linuxkm_svr_state_assoc_unlikely(create_p); } } -static void wc_linuxkm_fpu_state_release_unlikely( - struct wc_thread_fpu_count_ent *ent) +static void wc_linuxkm_svr_state_release_unlikely( + struct wc_thread_svr_count_ent *ent) { if (ent->fpu_state != 0) { static int warned_nonzero_fpu_state = 0; if (! warned_nonzero_fpu_state) { - VRG_PR_ERR_X("ERROR: wc_linuxkm_fpu_state_free for pid %d on CPU %d" - " with nonzero fpu_state 0x%x.\n", ent->pid, raw_smp_processor_id(), ent->fpu_state); + VRG_PR_ERR_X("ERROR: wc_linuxkm_svr_state_release for pid %d on CPU %d" + " with nonzero fpu_state 0x%x (age %ld ms).\n", ent->pid, + raw_smp_processor_id(), ent->fpu_state, WC_SVR_SLOT_AGE_MS(ent)); warned_nonzero_fpu_state = 1; } ent->fpu_state = 0; } - __atomic_store_n(&ent->pid, 0, __ATOMIC_RELEASE); + __atomic_store_n(&ent->pid, WC_SVR_FREE_SLOT_PID, __ATOMIC_RELEASE); } -static inline void wc_linuxkm_fpu_state_release( - struct wc_thread_fpu_count_ent *ent) +static inline void wc_linuxkm_svr_state_release( + struct wc_thread_svr_count_ent *ent) { if (unlikely(ent->fpu_state != 0)) - return wc_linuxkm_fpu_state_release_unlikely(ent); - __atomic_store_n(&ent->pid, 0, __ATOMIC_RELEASE); + return wc_linuxkm_svr_state_release_unlikely(ent); + __atomic_store_n(&ent->pid, WC_SVR_FREE_SLOT_PID, __ATOMIC_RELEASE); +} + +/* Note that a volatile is used here deliberately, rather than an atomic, to + * avoid frivolous overhead. wc_svr_disallowed_count is intrinsically only + * precise and reliable when the module is single-threaded (i.e. during module + * wolfssl_init()). Incrementing it atomically would gain nothing meaningful + * but incur the atomic tax. + */ +static volatile unsigned long long int wc_svr_disallowed_count = 0; + +void wc_svr_disallowed_count_reset(void) { + wc_svr_disallowed_count = 0; +} + +unsigned long long int wc_svr_disallowed_count_current(void) { + return wc_svr_disallowed_count; +} + +static inline void wc_svr_disallowed_count_increment(void) { + wc_svr_disallowed_count = wc_svr_disallowed_count + 1; } WARN_UNUSED_RESULT int wc_can_save_vector_registers_x86(void) { - struct wc_thread_fpu_count_ent *pstate; + struct wc_thread_svr_count_ent *pstate; /* check for hard interrupt context (unusable current->pid) preemptively. * if we're in a softirq context we'll catch that below with * a second preempt_count() check. */ - if (((preempt_count() & (NMI_MASK | HARDIRQ_MASK)) != 0) || (task_pid_nr(current) == 0)) + if ((preempt_count() & (NMI_MASK | HARDIRQ_MASK)) != 0) { +#ifdef WC_SVR_USE_NATIVE_REG_BUFS + /* Plain-flavor saves in hardirq are natively serviceable; only NMI + * and recursion-count saturation still answer no. + */ + if (likely(wc_svr_native_ready) && (! in_nmi())) { + struct wc_svr_native_ctx_state *nctx = + &wc_svr_native_here()->hardirq; + if (likely(nctx->depth != ~0U)) + return 1; + } +#endif /* WC_SVR_USE_NATIVE_REG_BUFS */ +#ifdef DEBUG_VECTOR_REGISTER_ACCESS_HARDIRQ_INFO + pr_info("HARDIRQ_INFO: wc_can_save_vector_registers_x86() with preempt_count 0x%x, PID %d, CPU %d\n", + preempt_count(), task_pid_nr(current), raw_smp_processor_id()); + dump_stack(); +#endif + wc_svr_disallowed_count_increment(); + #ifdef WOLFSSL_LINUXKM_VERBOSE_DEBUG + if (in_nmi()) + atomic_long_inc(&NMI_SVR_err_count); + else + atomic_long_inc(&hardirq_SVR_err_count); + #endif return 0; + } - /* Check if we're already saved, per wc_linuxkm_fpu_states. */ - pstate = wc_linuxkm_fpu_state_assoc(0, 0); + /* Check if we're already saved, per wc_linuxkm_svr_states. */ + pstate = wc_linuxkm_svr_state_assoc(0, 0); if ((pstate != NULL) && (pstate->fpu_state != 0U)) { - if (unlikely(pstate->fpu_state & WC_FPU_INHIBITED_FLAG)) + if (unlikely(pstate->fpu_state & WC_SVR_INHIBITED_FLAG)) { + wc_svr_disallowed_count_increment(); return 0; - if (unlikely((pstate->fpu_state & WC_FPU_COUNT_MASK) - == WC_FPU_COUNT_MASK)) + } + if (unlikely((pstate->fpu_state & WC_SVR_COUNT_MASK) + == WC_SVR_COUNT_MASK)) { /* would overflow */ + wc_svr_disallowed_count_increment(); return 0; } else { return 1; @@ -333,65 +537,125 @@ WARN_UNUSED_RESULT int wc_can_save_vector_registers_x86(void) } #ifdef DEBUG_VECTOR_REGISTER_ACCESS_FUZZING - if (SAVE_VECTOR_REGISTERS2_fuzzer() != 0) + if (SAVE_VECTOR_REGISTERS2_fuzzer() != 0) { + wc_svr_disallowed_count_increment(); return 0; + } #endif if ((preempt_count() == 0) || may_use_simd()) return 1; - else + else { +#ifdef WC_SVR_USE_NATIVE_REG_BUFS + if (likely(wc_svr_native_ready) && in_serving_softirq()) { + struct wc_svr_native_ctx_state *nctx = + &wc_svr_native_here()->softirq; + if (likely(nctx->depth != ~0U)) + return 1; + } +#endif + wc_svr_disallowed_count_increment(); + #ifdef WOLFSSL_LINUXKM_VERBOSE_DEBUG + atomic_long_inc(&softirq_SVR_err_count); + #endif return 0; + } } WARN_UNUSED_RESULT int wc_save_vector_registers_x86(enum wc_svr_flags flags) { - struct wc_thread_fpu_count_ent *pstate; + struct wc_thread_svr_count_ent *pstate; + unsigned int new_state_flags = 0; - /* check for hard interrupt context (unusable current->pid) preemptively. - * if we're in a softirq context we'll catch that below with + /* Check for hard interrupt context (unusable current->pid) preemptively. + * If we're in a softirq context we'll catch that below with * a second look at preempt_count(). + * + * Note that this is not actually an abnormal condition in any way -- + * e.g. with LINUXKM_DRBG_GET_RANDOM_BYTES, get_random_u32() and the + * like called from hard IRQ handlers land here, and we return + * WC_ACCEL_INHIBIT_E for graceful fallback to C. */ - if (((preempt_count() & (NMI_MASK | HARDIRQ_MASK)) != 0) || (task_pid_nr(current) == 0)) { - if (! (flags & (WC_SVR_FLAG_INHIBIT | WC_SVR_FLAG_MAYBE_INHIBIT))) { - VRG_PR_WARN_X("WARNING: wc_save_vector_registers_x86(0x%x) called with preempt_count 0x%x and pid %d on CPU %d.\n", (unsigned)flags, preempt_count(), task_pid_nr(current), raw_smp_processor_id()); - #ifdef WOLFSSL_LINUXKM_VERBOSE_DEBUG - dump_stack(); - #endif + if ((preempt_count() & (NMI_MASK | HARDIRQ_MASK)) != 0) { +#ifdef WC_SVR_USE_NATIVE_REG_BUFS + /* Native service for plain-flavor saves in hardirq. The in_nmi() + * disposition must stay ahead of any may_use_simd() call -- + * irq_fpu_usable() carries WARN_ON_ONCE(in_nmi()). NMI, the + * inhibit flavors, and !wc_svr_native_ready keep the long-standing + * graceful refusal below. + */ + if (likely(wc_svr_native_ready) && (! in_nmi()) && + ((flags & (WC_SVR_FLAG_INHIBIT | WC_SVR_FLAG_MAYBE_INHIBIT)) + == 0)) + { + struct wc_svr_native_ctx_state *nctx = + &wc_svr_native_here()->hardirq; +#ifdef DEBUG_VECTOR_REGISTER_ACCESS_FUZZING + /* mirror the existing paths: injection at outermost depth only. + */ + if ((flags & WC_SVR_FLAG_FUZZ) && (nctx->depth == 0U)) { + int ret = SAVE_VECTOR_REGISTERS2_fuzzer(); + if (ret != 0) { + wc_svr_disallowed_count_increment(); + return ret; + } + } +#endif + return wc_svr_native_save(nctx); } +#endif /* WC_SVR_USE_NATIVE_REG_BUFS */ +#ifdef DEBUG_VECTOR_REGISTER_ACCESS_HARDIRQ_INFO + pr_info("HARDIRQ_INFO: wc_save_vector_registers_x86() with preempt_count 0x%x, PID %d, CPU %d\n", + preempt_count(), task_pid_nr(current), raw_smp_processor_id()); + dump_stack(); +#endif + wc_svr_disallowed_count_increment(); return WC_ACCEL_INHIBIT_E; } - pstate = wc_linuxkm_fpu_state_assoc(0, 0); + pstate = wc_linuxkm_svr_state_assoc(0, 0); /* allow for nested calls */ if (pstate && (pstate->fpu_state != 0U)) { - if (flags & WC_SVR_FLAG_MAYBE_INHIBIT) { + if (unlikely((pstate->fpu_state & WC_SVR_BH_HELD_FLAG) && (softirq_count() == 0))) { + VRG_PR_ERR_X("BUG: wc_save_vector_registers_x86(): zero softirq_count in nested call (depth %u, age %ld ms) after local_bh_disable() on CPU %d.\n", + (pstate->fpu_state & WC_SVR_COUNT_MASK), + WC_SVR_SLOT_AGE_MS(pstate), + raw_smp_processor_id()); + } + if (unlikely(flags & WC_SVR_FLAG_MAYBE_INHIBIT)) { VRG_PR_WARN_X("BUG: wc_save_vector_registers_x86() called by pid %d on CPU %d " - "with _MAYBE_INHIBIT flag at non-outermost depth %u.\n", task_pid_nr(current), + "with _MAYBE_INHIBIT flag in nested call (depth %u, age %ld ms).\n", task_pid_nr(current), raw_smp_processor_id(), - (pstate->fpu_state & WC_FPU_COUNT_MASK)); + (pstate->fpu_state & WC_SVR_COUNT_MASK), + WC_SVR_SLOT_AGE_MS(pstate)); #ifdef WOLFSSL_LINUXKM_VERBOSE_DEBUG dump_stack(); #endif + wc_svr_disallowed_count_increment(); return BAD_STATE_E; } - if (pstate->fpu_state & WC_FPU_INHIBITED_FLAG) { + if (unlikely(pstate->fpu_state & WC_SVR_INHIBITED_FLAG)) { /* don't allow recursive inhibit calls when already inhibited -- * it would add no functionality and require keeping a separate * count of inhibit recursions. */ + wc_svr_disallowed_count_increment(); return WC_ACCEL_INHIBIT_E; } - if (unlikely((pstate->fpu_state & WC_FPU_COUNT_MASK) - == WC_FPU_COUNT_MASK)) + if (unlikely((pstate->fpu_state & WC_SVR_COUNT_MASK) + == WC_SVR_COUNT_MASK)) { pr_err("ERROR: wc_save_vector_registers_x86 recursion register overflow for " - "pid %d on CPU %d.\n", pstate->pid, raw_smp_processor_id()); + "pid %d on CPU %d (age %ld ms).\n", pstate->pid, raw_smp_processor_id(), + WC_SVR_SLOT_AGE_MS(pstate)); + wc_svr_disallowed_count_increment(); return BAD_STATE_E; } - if (flags & WC_SVR_FLAG_INHIBIT) { + if (unlikely(flags & WC_SVR_FLAG_INHIBIT)) { ++pstate->fpu_state; - pstate->fpu_state |= WC_FPU_INHIBITED_FLAG; + pstate->fpu_state |= WC_SVR_INHIBITED_FLAG; + wc_svr_disallowed_count_increment(); return 0; } else { @@ -408,8 +672,10 @@ WARN_UNUSED_RESULT int wc_save_vector_registers_x86(enum wc_svr_flags flags) */ { int ret = WC_CHECK_FOR_INTR_SIGNALS(); - if (ret) + if (ret) { + wc_svr_disallowed_count_increment(); return ret; + } } #endif @@ -421,109 +687,197 @@ WARN_UNUSED_RESULT int wc_save_vector_registers_x86(enum wc_svr_flags flags) if (ret != 0) { if (flags & WC_SVR_FLAG_MAYBE_INHIBIT) flags |= WC_SVR_FLAG_INHIBIT; - else + else { + wc_svr_disallowed_count_increment(); return ret; + } } } #endif if ((flags & WC_SVR_FLAG_MAYBE_INHIBIT) && - ((preempt_count() != 0) && !may_use_simd())) + ((preempt_count() != 0) && !may_use_simd()) +#ifdef WC_SVR_USE_NATIVE_REG_BUFS + && unlikely(! wc_svr_native_ready) +#endif + ) { + wc_svr_disallowed_count_increment(); return WC_ACCEL_INHIBIT_E; /* not an error here, just a * short-circuit result. */ } if (flags & WC_SVR_FLAG_INHIBIT) { - if ((preempt_count() != 0) && !may_use_simd()) + if ((preempt_count() != 0) && !may_use_simd()) { + wc_svr_disallowed_count_increment(); return WC_ACCEL_INHIBIT_E; /* not an error here, just a * short-circuit result. */ + } /* we need to inhibit migration and softirqs here to assure that we can * support recursive calls safely, i.e. without mistaking a softirq * context for a recursion. + * + * pre-5.11, migrate_disable() either doesn't exist or is a no-op -- + * there, we lean on pinning from the bh offset or irq disablement. */ #if defined(CONFIG_SMP) && !defined(CONFIG_PREEMPT_COUNT) && \ - (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0)) + (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 11, 0)) migrate_disable(); #endif - local_bh_disable(); - if (preempt_count() == 0) { - VRG_PR_ERR_X("BUG: wc_save_vector_registers_x86(): zero preempt_count after local_bh_disable() on CPU %d.\n", - raw_smp_processor_id()); - #if defined(CONFIG_SMP) && !defined(CONFIG_PREEMPT_COUNT) && \ - (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0)) - migrate_enable(); - #endif - local_bh_enable(); - return WC_ACCEL_INHIBIT_E; + if (! irqs_disabled()) { + local_bh_disable(); + new_state_flags |= WC_SVR_BH_HELD_FLAG; + + if (softirq_count() == 0) { + VRG_PR_ERR_X("BUG: wc_save_vector_registers_x86(): zero softirq_count in outermost call after local_bh_disable() on CPU %d.\n", + raw_smp_processor_id()); + local_bh_enable(); + #if defined(CONFIG_SMP) && !defined(CONFIG_PREEMPT_COUNT) && \ + (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 11, 0)) + migrate_enable(); + #endif + wc_svr_disallowed_count_increment(); + return WC_ACCEL_INHIBIT_E; + } } - pstate = wc_linuxkm_fpu_state_assoc(1, 1); + pstate = wc_linuxkm_svr_state_assoc(1, 1); if (pstate == NULL) { + if (new_state_flags & WC_SVR_BH_HELD_FLAG) + local_bh_enable(); #if defined(CONFIG_SMP) && !defined(CONFIG_PREEMPT_COUNT) && \ - (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0)) + (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 11, 0)) migrate_enable(); #endif - local_bh_enable(); + wc_svr_disallowed_count_increment(); return BAD_STATE_E; } pstate->fpu_state = - WC_FPU_INHIBITED_FLAG + 1U; + (WC_SVR_INHIBITED_FLAG | new_state_flags) + 1U; + wc_svr_disallowed_count_increment(); return 0; } if ((preempt_count() == 0) || may_use_simd()) { /* fpregs_lock() calls either local_bh_disable() or preempt_disable() - * depending on CONFIG_PREEMPT_RT -- we call both, explicitly. + * depending on CONFIG_PREEMPT_RT -- we call both, explicitly, with the + * sole exception that local_bh_disable() is necessarily omitted if the + * caller has irqs_disabled(). This exception is critical: without it, + * a caller in a critical section get local_bh_enable()d when it calls + * wc_restore_vector_registers_x86(), reenabling interrupts during its + * critical section. * - * empirically, on some kernels, kernel_fpu_begin() doesn't reliably - * disable softirqs, indeed doesn't make preempt_count() nonzero, which - * breaks our locking algorithm. we sidestep this completely by - * explicitly disabling softirq's, preemption, and migration. - * helpfully, the calls to do that are all guaranteed recursion-safe. + * Note: pre-6.15, kernel_fpu_begin() is preempt_disable()-only -- it + * never defers softirqs, and on !CONFIG_PREEMPT_COUNT configs it leaves + * preempt_count() zero, which would break our locking algorithm. 6.15+ + * (mainline commit d02198550423) disables bh in kernel_fpu_begin(), + * except for irqs-off callers. We sidestep the variance completely by + * making the disables ourselves; helpfully, all these calls are + * recursion-safe. */ #if defined(CONFIG_SMP) && !defined(CONFIG_PREEMPT_COUNT) && \ - (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0)) + (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 11, 0)) migrate_disable(); #endif - local_bh_disable(); + + if (! irqs_disabled()) { + local_bh_disable(); + new_state_flags |= WC_SVR_BH_HELD_FLAG; + } + #if IS_ENABLED(CONFIG_PREEMPT_RT) preempt_disable(); #endif kernel_fpu_begin(); - pstate = wc_linuxkm_fpu_state_assoc(1, 1); + pstate = wc_linuxkm_svr_state_assoc(1, 1); if (pstate == NULL) { kernel_fpu_end(); #if IS_ENABLED(CONFIG_PREEMPT_RT) preempt_enable(); #endif - local_bh_enable(); + if (new_state_flags & WC_SVR_BH_HELD_FLAG) + local_bh_enable(); #if defined(CONFIG_SMP) && !defined(CONFIG_PREEMPT_COUNT) && \ - (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0)) + (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 11, 0)) migrate_enable(); #endif + wc_svr_disallowed_count_increment(); return BAD_STATE_E; } - /* set msb to 0 to trigger kernel_fpu_end() at cleanup. */ - pstate->fpu_state = 1U; + pstate->fpu_state = new_state_flags | 1U; - if (preempt_count() == 0) { - VRG_PR_ERR_X("BUG: wc_save_vector_registers_x86(): zero preempt_count after kernel_fpu_begin() on CPU %d.\n", + if ((new_state_flags & WC_SVR_BH_HELD_FLAG) && (softirq_count() == 0)) { + VRG_PR_ERR_X("BUG: wc_save_vector_registers_x86(): zero softirq_count after local_bh_disable() on CPU %d.\n", raw_smp_processor_id()); } return 0; - } else { - VRG_PR_WARN_X("WARNING: wc_save_vector_registers_x86 called with no saved state and nonzero preempt_count 0x%x on CPU %d.\n", preempt_count(), raw_smp_processor_id()); + } else { + static DEFINE_RATELIMIT_STATE(vrg_contend_rs, HZ, 1); + +#ifdef WC_SVR_USE_NATIVE_REG_BUFS + /* Native service for softirq contention (canonically: this softirq + * interrupted a foreign kernel_fpu section, on a pre-6.15 kernel + * where kernel_fpu_begin() doesn't defer softirqs). Whatever holds + * may_use_simd() false is frozen beneath us until we return, so the + * section is exact and invisible. The plain flavor and (when + * native is ready) the _MAYBE_INHIBIT flavor reach this branch -- + * _MAYBE's legacy short-circuit above is bypassed so that native + * can serve its vector-preferred arm; _INHIBIT still short-circuits + * above. The fuzzer has already had its chance, and nested saves + * inside an open native softirq section land back here + * (may_use_simd() is still false) and count up. A nested call inside a *legacy* + * (kernel_fpu-backed) softirq section never gets here -- it is + * intercepted by the pid-match quick-check above. + */ + if (likely(wc_svr_native_ready) && in_serving_softirq()) + return wc_svr_native_save(&wc_svr_native_here()->softirq); +#endif /* WC_SVR_USE_NATIVE_REG_BUFS */ + #ifdef WOLFSSL_LINUXKM_VERBOSE_DEBUG - dump_stack(); + atomic_long_inc(&softirq_SVR_err_count); #endif + + if (preempt_count() != 0) { + /* this path is normal on pre-6.15 kernels, where kernel_fpu_begin() + * doesn't local_bh_disable(), but on 6.15+ it's a warnable + * anomaly. */ + #if LINUX_VERSION_CODE < KERNEL_VERSION(6, 15, 0) + + #ifdef WOLFSSL_LINUXKM_VERBOSE_DEBUG + if (__ratelimit(&vrg_contend_rs)) { + pr_info("INFO: !may_use_simd() in wc_save_vector_registers_x86() on CPU %d PID %d (%s) with preempt_count 0x%x.\n", raw_smp_processor_id(), task_pid_nr(current), current->comm, preempt_count()); + dump_stack(); + } + #endif /* WOLFSSL_LINUXKM_VERBOSE_DEBUG */ + + #else /* >=6.15.0 */ + + if (__ratelimit(&vrg_contend_rs)) { + pr_warn("WARNING: !may_use_simd() in wc_save_vector_registers_x86 called with no saved state on CPU %d PID %d (%s) with preempt_count 0x%x.\n", raw_smp_processor_id(), task_pid_nr(current), current->comm, preempt_count()); + #ifdef WOLFSSL_LINUXKM_VERBOSE_DEBUG + dump_stack(); + #endif + } + + #endif /* >=6.15.0 */ + } + else { + if (__ratelimit(&vrg_contend_rs)) { + pr_warn("WARNING: !may_use_simd() in wc_save_vector_registers_x86 called with no saved state on CPU %d PID %d (%s) with preempt_count 0x%x.\n", raw_smp_processor_id(), task_pid_nr(current), current->comm, preempt_count()); + #ifdef WOLFSSL_LINUXKM_VERBOSE_DEBUG + dump_stack(); + #endif + } + } + + wc_svr_disallowed_count_increment(); return WC_ACCEL_INHIBIT_E; } @@ -532,15 +886,44 @@ WARN_UNUSED_RESULT int wc_save_vector_registers_x86(enum wc_svr_flags flags) void wc_restore_vector_registers_x86(enum wc_svr_flags flags) { - struct wc_thread_fpu_count_ent *pstate; + struct wc_thread_svr_count_ent *pstate; + unsigned int cur_fpu_state; - if (((preempt_count() & (NMI_MASK | HARDIRQ_MASK)) != 0) || (task_pid_nr(current) == 0)) { + if ((preempt_count() & (NMI_MASK | HARDIRQ_MASK)) != 0) { +#ifdef WC_SVR_USE_NATIVE_REG_BUFS + if (likely(wc_svr_native_ready) && (! in_nmi())) { + struct wc_svr_native_ctx_state *nctx = + &wc_svr_native_here()->hardirq; + if (likely(nctx->depth > 0U)) { + wc_svr_native_restore(nctx, flags); + return; + } + /* depth == 0 in hardirq is a pairing bug; fall through to the + * long-standing diagnostic. + */ + } +#endif /* WC_SVR_USE_NATIVE_REG_BUFS */ VRG_PR_WARN_X("BUG: wc_restore_vector_registers_x86() called from interrupt handler on CPU %d.\n", raw_smp_processor_id()); return; } - pstate = wc_linuxkm_fpu_state_assoc(0, 1); +#ifdef WC_SVR_USE_NATIVE_REG_BUFS + if (likely(wc_svr_native_ready) && in_serving_softirq()) { + struct wc_svr_native_ctx_state *nctx = + &wc_svr_native_here()->softirq; + if (nctx->depth > 0U) { + wc_svr_native_restore(nctx, flags); + return; + } + /* depth == 0: a legacy softirq section (kernel_fpu-backed, opened + * while may_use_simd() was true) closes through the existing path + * below. + */ + } +#endif /* WC_SVR_USE_NATIVE_REG_BUFS */ + + pstate = wc_linuxkm_svr_state_assoc(0, 1); if (unlikely(pstate == NULL)) { VRG_PR_WARN_X("BUG: wc_restore_vector_registers_x86() called by pid %d on CPU %d " "with no saved state.\n", task_pid_nr(current), @@ -551,23 +934,24 @@ void wc_restore_vector_registers_x86(enum wc_svr_flags flags) return; } - if ((--pstate->fpu_state & WC_FPU_COUNT_MASK) > 0U) { + if ((--pstate->fpu_state & WC_SVR_COUNT_MASK) > 0U) { if (flags & WC_SVR_FLAG_MAYBE_INHIBIT) { VRG_PR_WARN_X("BUG: wc_restore_vector_registers_x86() called by pid %d on CPU %d " - "with _MAYBE_INHIBIT flag at non-outermost depth %u.\n", task_pid_nr(current), + "with _MAYBE_INHIBIT flag at non-outermost depth %u (age %ld ms).\n", task_pid_nr(current), raw_smp_processor_id(), - (pstate->fpu_state & WC_FPU_COUNT_MASK) + 1U); + (pstate->fpu_state & WC_SVR_COUNT_MASK) + 1U, + WC_SVR_SLOT_AGE_MS(pstate)); #ifdef WOLFSSL_LINUXKM_VERBOSE_DEBUG dump_stack(); #endif } if (flags & WC_SVR_FLAG_INHIBIT) { - if (pstate->fpu_state & WC_FPU_INHIBITED_FLAG) - pstate->fpu_state &= ~WC_FPU_INHIBITED_FLAG; + if (pstate->fpu_state & WC_SVR_INHIBITED_FLAG) + pstate->fpu_state &= ~WC_SVR_INHIBITED_FLAG; else { VRG_PR_WARN_X("BUG: wc_restore_vector_registers_x86() called by pid %d on CPU %d " - "with _INHIBIT flag but saved state isn't _INHIBITED_.\n", task_pid_nr(current), - raw_smp_processor_id()); + "with _INHIBIT flag but saved state isn't _INHIBITED_ (age %ld ms).\n", + task_pid_nr(current), raw_smp_processor_id(), WC_SVR_SLOT_AGE_MS(pstate)); #ifdef WOLFSSL_LINUXKM_VERBOSE_DEBUG dump_stack(); #endif @@ -576,29 +960,44 @@ void wc_restore_vector_registers_x86(enum wc_svr_flags flags) return; } - if (pstate->fpu_state == 0U) { - wc_linuxkm_fpu_state_release(pstate); + cur_fpu_state = pstate->fpu_state; + + if ((pstate->fpu_state & ~WC_SVR_BH_HELD_FLAG) == 0U) { + pstate->fpu_state = 0; + wc_linuxkm_svr_state_release(pstate); kernel_fpu_end(); #if IS_ENABLED(CONFIG_PREEMPT_RT) preempt_enable(); #endif - local_bh_enable(); - } else if (unlikely(pstate->fpu_state & WC_FPU_INHIBITED_FLAG)) { + if (cur_fpu_state & WC_SVR_BH_HELD_FLAG) { + if (softirq_count() == 0) { + VRG_PR_ERR_X("BUG: wc_restore_vector_registers_x86(): zero softirq_count after local_bh_disable() on CPU %d.\n", + raw_smp_processor_id()); + } + local_bh_enable(); + } + } else if (unlikely(pstate->fpu_state & WC_SVR_INHIBITED_FLAG)) { if (unlikely(! (flags & (WC_SVR_FLAG_INHIBIT | WC_SVR_FLAG_MAYBE_INHIBIT)))) { VRG_PR_WARN_X("BUG: wc_restore_vector_registers_x86() called by pid %d on CPU %d " - "without _INHIBIT flag but saved state is _INHIBITED_.\n", task_pid_nr(current), - raw_smp_processor_id()); + "without _INHIBIT flag but saved state is _INHIBITED_ (age %ld ms).\n", + task_pid_nr(current), raw_smp_processor_id(), WC_SVR_SLOT_AGE_MS(pstate)); #ifdef WOLFSSL_LINUXKM_VERBOSE_DEBUG dump_stack(); #endif } pstate->fpu_state = 0U; - wc_linuxkm_fpu_state_release(pstate); - local_bh_enable(); + wc_linuxkm_svr_state_release(pstate); + if (cur_fpu_state & WC_SVR_BH_HELD_FLAG) { + if (softirq_count() == 0) { + VRG_PR_ERR_X("BUG: wc_restore_vector_registers_x86(): zero softirq_count after local_bh_disable() on CPU %d.\n", + raw_smp_processor_id()); + } + local_bh_enable(); + } } #if defined(CONFIG_SMP) && !defined(CONFIG_PREEMPT_COUNT) && \ - (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0)) + (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 11, 0)) migrate_enable(); #endif @@ -607,4 +1006,717 @@ void wc_restore_vector_registers_x86(enum wc_svr_flags flags) return; } +#ifdef WC_SVR_USE_NATIVE_REG_BUFS + +/* Native per-CPU register save buffers for softirq and hardirq contexts. + * + * When may_use_simd() is false, kernel_fpu_begin() is unusable, but the + * vector register file itself is not: an exact save of every register + * component our kernels can write, followed by our own vector computation, + * followed by an exact restore, is invisible to the interrupted context and + * to every kernel bookkeeping mechanism (in_kernel_fpu, TIF_NEED_FPU_LOAD, + * fpu_fpregs_owner_ctx), all of which track register *contents*, which we + * restore bit-exactly. The one obligation is exactness, and the one + * precondition is that whatever made may_use_simd() false stays true for the + * duration of our section. That holds by run-to-completion: the condition's + * owner is beneath us on this CPU and cannot resume until we return. + * + * Two context classes are served, each with a dedicated per-CPU save buffer: + * + * - hardirq: previously refused outright (WC_ACCEL_INHIBIT_E). Handlers + * run with IRQs disabled, so a hardirq section can be interrupted only + * by NMI, which is not served (below) and whose kernel handlers do not + * touch the FPU. + * + * - softirq when !may_use_simd(): previously refused with a ratelimited + * warning. Reachable when a softirq interrupts a foreign + * kernel_fpu section (pre-6.15 kernels, where kernel_fpu_begin() is + * preempt_disable()-only and softirqs can run over an open section). + * A hardirq arriving over our softirq section lands in the hardirq + * buffer; the two never share. + * + * NMI context remains refused in this revision: the register mechanism below + * would serve it (a fourth buffer), but everything reachable under an NMI + * bracket must additionally be lock-free against the interrupted context, + * and that audit is module-wide, not glue-local. Note also that + * irq_fpu_usable() carries WARN_ON_ONCE(in_nmi()) on current kernels, so the + * in_nmi() disposition below must stay ahead of any may_use_simd() call. + * + * Task context needs no buffer here: its save buffer is the task struct, + * maintained by kernel_fpu_begin()/end() on the existing paths, which remain + * in sole charge of task-context sections. + * + * Same-context recursion (an outer bracket in a PK algorithm, an inner + * bracket in a hash it calls) is a pure depth count: the registers are + * already owned by this context, so only the 0->1 transition saves and only + * the 1->0 transition restores. Recursion never consumes a second buffer. + * As on the existing paths, an inner section does not preserve the outer's + * register contents -- the standing wolfCrypt discipline that live vector + * state never spans a C-level call is what makes the count sufficient. + * + * The native state is deliberately NOT placed in struct + * wc_thread_svr_count_ent: these buffers belong to the CPU, while ents + * belong to pids and are copied between CPU slots by the migration scan in + * wc_linuxkm_svr_state_assoc_unlikely(). A CPU-owned pointer inside a + * pid-owned struct would follow the pid to the wrong CPU. + * + * Interweave invariants with the existing machinery: + * + * - A pid-keyed (legacy) section always holds softirqs off (or IRQs off) + * for its duration, so a softirq can never run over an open legacy + * section of the task it interrupted. Hence the recursion quick-check + * in wc_save_vector_registers_x86() and the native depth check below + * partition cleanly: a nested call inside a legacy softirq section is + * caught by the pid match (in_kernel_fpu makes may_use_simd() false, but + * the quick-check runs first); a nested call inside a native softirq + * section reaches the native depth check because may_use_simd() is still + * false (its cause is frozen beneath us). + * + * - The plain save flavor is served natively in both classes, and the + * _MAYBE_INHIBIT flavor ("vector if possible, else say so") is served + * natively in softirq -- native IS the vector-possible arm, so its + * legacy short-circuit is bypassed when wc_svr_native_ready. The + * _INHIBIT flavor (a no-vector section request) keeps its existing + * short-circuit semantics in all contexts (WC_ACCEL_INHIBIT_E, + * tolerated by all callers), as does _MAYBE_INHIBIT in hardirq; + * native service for those, and for NMI, is left for a later revision. + */ + +/* XSAVE requested-feature bitmap: x87 (0), SSE (1), AVX/YMM (2), and the + * AVX-512 components opmask (5), ZMM_Hi256 (6), Hi16_ZMM (7) -- every + * component a wolfCrypt vector routine can write, and nothing else. The + * effective mask is this ANDed with XCR0, so components the OS hasn't + * enabled (which the wolfCrypt dispatchers therefore won't use) drop out. + * Components outside the mask (AMX, PKRU, MPX) are neither saved nor + * restored nor touched by wolfCrypt code, so they ride through unchanged. + */ +#define WC_SVR_NATIVE_XFEATURE_MASK 0x00e7U + +static struct wc_svr_native_cpu_state *wc_svr_native_states = NULL; +static u8 *wc_svr_native_save_mem = NULL; /* raw allocation */ +static unsigned int wc_svr_native_save_size = 0; /* per-buffer, 64-multiple */ +static u32 wc_svr_native_mask_lo = 0; +static u32 wc_svr_native_mask_hi = 0; /* always 0; kept for the asm + * constraints and for clarity + * that EDX:EAX is a 64 bit + * RFBM. */ +static int wc_svr_native_use_xsave = 0; +static int wc_svr_native_have_sse = 0; /* CPUID.1:EDX[25], cached at init; + * gates the ldmxcsr in + * wc_svr_native_regs_save(). */ + +/* wc_svr_native_cpuid_count() is open-coded, like xgetbv below: the kernel's + * cpuid helpers have been migrating between asm/processor.h, asm/cpuid.h, and + * asm/cpuid/api.h across our supported kernel span, and this file targets + * exactly one ISA. Constraints mirror the kernel's native_cpuid(). + */ +static inline void wc_svr_native_cpuid_count(u32 leaf, u32 subleaf, + u32 *a, u32 *b, u32 *c, u32 *d) +{ + __asm__ __volatile__("cpuid" + : "=a" (*a), "=b" (*b), "=c" (*c), "=d" (*d) + : "0" (leaf), "2" (subleaf) + : "memory"); +} + +static inline struct wc_svr_native_cpu_state *wc_svr_native_here(void) +{ + /* Callers are in hardirq or serving-softirq (or, for the selftest, + * preempt-disabled task context), so raw_smp_processor_id() is stable. + */ + return &wc_svr_native_states[raw_smp_processor_id()]; +} + +/* Exact save of the masked register components into a 64-byte-aligned area, + * then normalization of MXCSR to its default, mirroring + * kernel_fpu_begin_mask()'s KFPU_MXCSR: the interrupted context may have + * unmasked exception bits set, under which a wolfCrypt SSE/AVX instruction + * could fault. The interrupted MXCSR is in the save image and comes back at + * restore. + * + * Plain XSAVE/XRSTOR (never XSAVEOPT/XSAVES/XSAVEC): the modified + * optimization keys on the last-XRSTOR address, and reusing per-context + * areas under XSAVEOPT can legally skip stores. Our XRSTOR to a foreign + * area merely invalidates the kernel's own tracking address, making the + * task's next XSAVES a full save -- conservative and correct. The 64-bit + * forms are used on x86_64 so the x87 FIP/FDP come back at full width -- + * with the 32-bit forms an exact restore would truncate them, a visible + * state change for the interrupted context. + * + * No vector-register clobber lists: like the rest of this file, this + * compiles with -mno-sse in force, so the compiler never holds values in + * vector registers. "memory" clobbers order the asm against the depth + * bookkeeping. + */ +static void wc_svr_native_regs_save(u8 *area) +{ + if (wc_svr_native_use_xsave) { +#ifdef CONFIG_X86_64 + __asm__ __volatile__("xsave64 (%0)" + : : "r" (area), + "a" (wc_svr_native_mask_lo), + "d" (wc_svr_native_mask_hi) + : "memory"); +#else + __asm__ __volatile__("xsave (%0)" + : : "r" (area), + "a" (wc_svr_native_mask_lo), + "d" (wc_svr_native_mask_hi) + : "memory"); +#endif + } + else { +#ifdef CONFIG_X86_64 + __asm__ __volatile__("fxsave64 (%0)" : : "r" (area) : "memory"); +#else + __asm__ __volatile__("fxsave (%0)" : : "r" (area) : "memory"); +#endif + } + + if (wc_svr_native_have_sse) { + static const u32 wc_svr_native_mxcsr_default = 0x1f80; + __asm__ __volatile__("ldmxcsr %0" + : : "m" (wc_svr_native_mxcsr_default)); + } +} + +static void wc_svr_native_regs_restore(u8 *area) +{ + if (wc_svr_native_use_xsave) { +#ifdef CONFIG_X86_64 + __asm__ __volatile__("xrstor64 (%0)" + : : "r" (area), + "a" (wc_svr_native_mask_lo), + "d" (wc_svr_native_mask_hi) + : "memory"); +#else + __asm__ __volatile__("xrstor (%0)" + : : "r" (area), + "a" (wc_svr_native_mask_lo), + "d" (wc_svr_native_mask_hi) + : "memory"); +#endif + } + else { +#ifdef CONFIG_X86_64 + __asm__ __volatile__("fxrstor64 (%0)" : : "r" (area) : "memory"); +#else + __asm__ __volatile__("fxrstor (%0)" : : "r" (area) : "memory"); +#endif + } +} + +/* Register-file round-trip selftest, run once at init from task context. + * + * Loads a known pattern into the registers, saves, destroys the registers, + * restores, reads the registers back, and compares; then corrupts one byte + * of the save image (offset 160 = XMM0 byte 0 in the legacy region -- + * deliberately not the XSAVE header, which XRSTOR validates and would #GP + * on) and proves the corruption is detected. A negative control converts + * "the comparison is miswired" from silent to loud. + * + * The test brackets itself with its own mechanism -- IRQs off, exact save + * of the live register file into this CPU's hardirq buffer, pattern + * round-trip in the softirq buffer, exact restore -- rather than with + * kernel_fpu_begin()/end(). The kernel bracket was an incidental + * dependency, and its gating predicate (may_use_simd() -> + * irq_fpu_usable()) has drifted across kernel versions, spuriously + * refusing task-context init on some (observed on linux-next, 2026-08). + * The self-bracket depends only on the ISA, and doubles as a live + * demonstration of the invisibility property being certified: the + * interrupted state here is the insmod task's own user registers, restored + * bit-exactly. (If XSAVE/XRSTOR themselves misbehaved, the bracket could + * not contain the damage -- but that contingency is a CPU that doesn't + * implement its own spec, and the kernel_fpu bracket had the same exposure + * through its XRSTOR.) + * + * Scope: xmm0-15 (and ymm0-15 when AVX is enabled) are exercised. The + * AVX-512 components in the mask (opmask, ZMM_Hi256, Hi16_ZMM) ride through + * XSAVE/XRSTOR on the same terms but are not pattern-tested here. + */ +#ifdef CONFIG_X86_64 + #define WC_SVR_NATIVE_ST_NREG 16 +#else + #define WC_SVR_NATIVE_ST_NREG 8 +#endif +#define WC_SVR_NATIVE_ST_MAX_STRIDE 32 + +static void wc_svr_native_st_load(const u8 *p, int use_avx) +{ + if (use_avx) { + __asm__ __volatile__( + "vmovdqu 0x000(%0), %%ymm0\n\t" + "vmovdqu 0x020(%0), %%ymm1\n\t" + "vmovdqu 0x040(%0), %%ymm2\n\t" + "vmovdqu 0x060(%0), %%ymm3\n\t" + "vmovdqu 0x080(%0), %%ymm4\n\t" + "vmovdqu 0x0a0(%0), %%ymm5\n\t" + "vmovdqu 0x0c0(%0), %%ymm6\n\t" + "vmovdqu 0x0e0(%0), %%ymm7\n\t" +#ifdef CONFIG_X86_64 + "vmovdqu 0x100(%0), %%ymm8\n\t" + "vmovdqu 0x120(%0), %%ymm9\n\t" + "vmovdqu 0x140(%0), %%ymm10\n\t" + "vmovdqu 0x160(%0), %%ymm11\n\t" + "vmovdqu 0x180(%0), %%ymm12\n\t" + "vmovdqu 0x1a0(%0), %%ymm13\n\t" + "vmovdqu 0x1c0(%0), %%ymm14\n\t" + "vmovdqu 0x1e0(%0), %%ymm15\n\t" +#endif + : : "r" (p) : "memory"); + } + else { + __asm__ __volatile__( + "movdqu 0x00(%0), %%xmm0\n\t" + "movdqu 0x10(%0), %%xmm1\n\t" + "movdqu 0x20(%0), %%xmm2\n\t" + "movdqu 0x30(%0), %%xmm3\n\t" + "movdqu 0x40(%0), %%xmm4\n\t" + "movdqu 0x50(%0), %%xmm5\n\t" + "movdqu 0x60(%0), %%xmm6\n\t" + "movdqu 0x70(%0), %%xmm7\n\t" +#ifdef CONFIG_X86_64 + "movdqu 0x80(%0), %%xmm8\n\t" + "movdqu 0x90(%0), %%xmm9\n\t" + "movdqu 0xa0(%0), %%xmm10\n\t" + "movdqu 0xb0(%0), %%xmm11\n\t" + "movdqu 0xc0(%0), %%xmm12\n\t" + "movdqu 0xd0(%0), %%xmm13\n\t" + "movdqu 0xe0(%0), %%xmm14\n\t" + "movdqu 0xf0(%0), %%xmm15\n\t" +#endif + : : "r" (p) : "memory"); + } +} + +static void wc_svr_native_st_store(u8 *p, int use_avx) +{ + if (use_avx) { + __asm__ __volatile__( + "vmovdqu %%ymm0, 0x000(%0)\n\t" + "vmovdqu %%ymm1, 0x020(%0)\n\t" + "vmovdqu %%ymm2, 0x040(%0)\n\t" + "vmovdqu %%ymm3, 0x060(%0)\n\t" + "vmovdqu %%ymm4, 0x080(%0)\n\t" + "vmovdqu %%ymm5, 0x0a0(%0)\n\t" + "vmovdqu %%ymm6, 0x0c0(%0)\n\t" + "vmovdqu %%ymm7, 0x0e0(%0)\n\t" +#ifdef CONFIG_X86_64 + "vmovdqu %%ymm8, 0x100(%0)\n\t" + "vmovdqu %%ymm9, 0x120(%0)\n\t" + "vmovdqu %%ymm10, 0x140(%0)\n\t" + "vmovdqu %%ymm11, 0x160(%0)\n\t" + "vmovdqu %%ymm12, 0x180(%0)\n\t" + "vmovdqu %%ymm13, 0x1a0(%0)\n\t" + "vmovdqu %%ymm14, 0x1c0(%0)\n\t" + "vmovdqu %%ymm15, 0x1e0(%0)\n\t" +#endif + : : "r" (p) : "memory"); + } + else { + __asm__ __volatile__( + "movdqu %%xmm0, 0x00(%0)\n\t" + "movdqu %%xmm1, 0x10(%0)\n\t" + "movdqu %%xmm2, 0x20(%0)\n\t" + "movdqu %%xmm3, 0x30(%0)\n\t" + "movdqu %%xmm4, 0x40(%0)\n\t" + "movdqu %%xmm5, 0x50(%0)\n\t" + "movdqu %%xmm6, 0x60(%0)\n\t" + "movdqu %%xmm7, 0x70(%0)\n\t" +#ifdef CONFIG_X86_64 + "movdqu %%xmm8, 0x80(%0)\n\t" + "movdqu %%xmm9, 0x90(%0)\n\t" + "movdqu %%xmm10, 0xa0(%0)\n\t" + "movdqu %%xmm11, 0xb0(%0)\n\t" + "movdqu %%xmm12, 0xc0(%0)\n\t" + "movdqu %%xmm13, 0xd0(%0)\n\t" + "movdqu %%xmm14, 0xe0(%0)\n\t" + "movdqu %%xmm15, 0xf0(%0)\n\t" +#endif + : : "r" (p) : "memory"); + } +} + +static WARN_UNUSED_RESULT int wc_svr_native_selftest(void) +{ + int use_avx = wc_svr_native_use_xsave && + ((wc_svr_native_mask_lo & 0x4U) != 0U); + unsigned int stride = use_avx ? 32U : 16U; + unsigned int st_len = WC_SVR_NATIVE_ST_NREG * stride; + u8 *want = NULL, *got = NULL; + u8 *area, *bracket; + unsigned long irqflags; + unsigned int i; + int ret; + + want = (u8 *)malloc(WC_SVR_NATIVE_ST_NREG * WC_SVR_NATIVE_ST_MAX_STRIDE); + got = (u8 *)malloc(WC_SVR_NATIVE_ST_NREG * WC_SVR_NATIVE_ST_MAX_STRIDE); + if ((want == NULL) || (got == NULL)) { + ret = MEMORY_E; + goto out; + } + + for (i = 0; i < st_len; ++i) + want[i] = (u8)((i * 251U) + 3U); + + /* No context guard here: the self-bracket below needs only an + * irqs-savable, non-NMI context (wc_svr_native_init() disposes of NMI + * before calling), and init contexts are legitimately atomic-ish -- + * the FIPS pre-Init lazy path arrives inside the outermost save's own + * bh-off/kernel_fpu bracket, and wolfCrypt_Init() itself has been + * observed arriving with irqs disabled and preempt_count 0 (non-FIPS + * 5.10/5.15, 2026-08). An + * earlier in_interrupt() guard here silently disabled the mechanism on + * exactly those paths. + */ + + local_irq_save(irqflags); /* quiesce the CPU: no migration, no + * preemption, no interrupts -- sole register + * file ownership for the bracket's duration. + * Deliberately unconditional under + * already_locked: the enclosing bracket holds + * softirqs off, but on pre-6.15 kernels a + * foreign hardirq kernel_fpu user is excluded + * only when in_kernel_fpu is set, which the + * _INHIBIT-flavored lazy path doesn't do -- + * and mid-selftest our pattern registers + * belong to no fpstate, so a foreign save + * would leak them into the interrupted task's + * user state. IRQs-off closes that window on + * all vintages, and nests harmlessly. + */ + + bracket = wc_svr_native_here()->hardirq.save_area; + wc_svr_native_regs_save(bracket); + + /* The softirq buffer of this CPU serves as scratch: no sections are open + * at init, and XSAVE never writes XCOMP_BV, so the zeroed-header + * invariant established below survives this use. Same for the hardirq + * buffer's service as the bracket area above. + */ + area = wc_svr_native_here()->softirq.save_area; + + wc_svr_native_st_load(want, use_avx); + wc_svr_native_regs_save(area); + XMEMSET(got, 0, st_len); + wc_svr_native_st_load(got, use_avx); /* destroy the registers, as + * wolfCrypt would. */ + wc_svr_native_regs_restore(area); + wc_svr_native_st_store(got, use_avx); /* what the interrupted context + * would see. */ + + ret = (XMEMCMP(want, got, st_len) == 0) ? 0 : -1; + + if (ret == 0) { + /* Negative control: a corrupted image must be detected. */ + area[160] ^= 0xffU; + wc_svr_native_st_load(want, use_avx); + wc_svr_native_regs_save(area); + area[160] ^= 0xffU; + XMEMSET(got, 0, st_len); + wc_svr_native_st_load(got, use_avx); + wc_svr_native_regs_restore(area); + wc_svr_native_st_store(got, use_avx); + ret = (XMEMCMP(want, got, st_len) != 0) ? 0 : -1; + if (ret != 0) + pr_err("ERROR: wc_svr_native_selftest: negative control not" + " detected.\n"); + } + else { + pr_err("ERROR: wc_svr_native_selftest: register file round-trip" + " mismatch.\n"); + } + + wc_svr_native_regs_restore(bracket); + local_irq_restore(irqflags); + + if (ret != 0) + ret = BAD_STATE_E; + +out: + if (want != NULL) + free(want); + if (got != NULL) + free(got); + return ret; +} + +/* Size and allocate the save buffers, prove the mechanism with the selftest, + * and only then mark it ready. Any failure leaves wc_svr_native_ready + * clear, and every native branch below then falls through to the + * long-standing refusal semantics -- the capability is additive, never + * load-bearing for correctness. + * + * Called from wc_linuxkm_allocate_svr_states() (task context, including the + * FIPS pre-Init lazy-allocation path). Idempotent for the FIPS repeat-call + * pattern. + */ +static void wc_svr_native_init(void) +{ + size_t total; + u8 *aligned; + unsigned int cpu_i; + int wc_svr_native_have_osxsave, wc_svr_native_have_fxsr; + + if (wc_svr_native_states != NULL) + return; + + if (in_nmi()) { + /* Nothing below is possible in NMI; a later allocate call retries. + * Every other context is serviceable: the selftest brackets itself + * with irqs off, and the allocations ride the same malloc mapping + * as the pre-existing lazy-path table allocation. + */ + return; + } + +#ifdef WOLFSSL_LINUXKM_VERBOSE_DEBUG + if (in_interrupt() || irqs_disabled()) { + /* Not a problem -- just record the context so atypical init + * environments are visible in the log (FIPS pre-Init lazy path, + * irqs-off wolfCrypt_Init(), etc). + */ + pr_info("wc_svr_native_init in atomic context: preempt_count 0x%x," + " irqs_disabled %d.\n", + preempt_count(), irqs_disabled() ? 1 : 0); + } +#endif + + { + u32 eax, ebx, ecx, edx; + wc_svr_native_cpuid_count(1, 0, &eax, &ebx, &ecx, &edx); + wc_svr_native_have_sse = (int)((edx >> 25) & 1U); + wc_svr_native_have_osxsave = (int)((ecx >> 27) & 1U); + wc_svr_native_have_fxsr = (int)((edx >> 24) & 1U); + } + + if (wc_svr_native_have_osxsave) { + /* CPUID.1:ECX[27] mirrors live CR4.OSXSAVE, so noxsave/clearcpuid + * boots read 0 here and degrade to the FXSAVE arm, consistently + * with the wolfCrypt dispatchers (which likewise see no OSXSAVE and + * select no AVX+ paths). + */ + u32 eax, ebx, ecx, edx; + u32 xcr0_lo, xcr0_hi; + + __asm__ __volatile__("xgetbv" + : "=a" (xcr0_lo), "=d" (xcr0_hi) + : "c" (0)); + wc_svr_native_mask_lo = xcr0_lo & WC_SVR_NATIVE_XFEATURE_MASK; + wc_svr_native_mask_hi = 0; + + /* CPUID.(EAX=0DH,ECX=0):ECX = maximum standard-format XSAVE area + * size over all XCR0-enablable components -- an upper bound for our + * subset mask on any core in the package. + */ + wc_svr_native_cpuid_count(0x0d, 0, &eax, &ebx, &ecx, &edx); + wc_svr_native_save_size = (ecx + 63U) & ~63U; + if (wc_svr_native_save_size < 576U) { + /* legacy region (512) + XSAVE header (64) is the floor; + * anything smaller is CPUID nonsense. + */ + pr_err("ERROR: wc_svr_native_init: implausible XSAVE area size" + " %u.\n", wc_svr_native_save_size); + return; + } + wc_svr_native_use_xsave = 1; + } + else if (wc_svr_native_have_fxsr) { + /* No OSXSAVE means no YMM+ state is architecturally enabled, so the + * wolfCrypt dispatchers won't select AVX+ paths, and FXSAVE's + * x87+SSE coverage is the complete writable set. + */ + wc_svr_native_save_size = 512U; + wc_svr_native_use_xsave = 0; + } + else { + pr_warn("WARNING: wc_svr_native_init: no XSAVE or FXSR -- native" + " register save buffers disabled.\n"); + return; + } + + wc_svr_native_states = (struct wc_svr_native_cpu_state *)malloc( + (size_t)nr_cpu_ids * sizeof(struct wc_svr_native_cpu_state)); + if (wc_svr_native_states == NULL) { + pr_err("ERROR: allocation of %zu bytes for wc_svr_native_states" + " failed.\n", + (size_t)nr_cpu_ids * sizeof(struct wc_svr_native_cpu_state)); + return; + } + XMEMSET(wc_svr_native_states, 0, + (size_t)nr_cpu_ids * sizeof(struct wc_svr_native_cpu_state)); + + /* One allocation, two 64-aligned buffers per CPU. Zero-initialized and + * never re-zeroed: XSAVE writes XSTATE_BV but never XCOMP_BV, and + * standard-format XRSTOR requires XCOMP_BV == 0, so the zeroing at birth + * is the invariant that keeps every later XRSTOR well-formed (SDM vol. 1 + * ch. 13). + */ + total = (size_t)nr_cpu_ids * 2U * (size_t)wc_svr_native_save_size + 63U; + wc_svr_native_save_mem = (u8 *)malloc(total); + if (wc_svr_native_save_mem == NULL) { + pr_err("ERROR: allocation of %zu bytes for native register save" + " buffers failed.\n", total); + free(wc_svr_native_states); + wc_svr_native_states = NULL; + return; + } + XMEMSET(wc_svr_native_save_mem, 0, total); + + aligned = PTR_ALIGN(wc_svr_native_save_mem, 64); + /* cast: nr_cpu_ids is plain int on pre-4.15 kernels (treewide + * unsigned conversion), tripping -Wsign-compare there. */ + for (cpu_i = 0; cpu_i < (unsigned int)nr_cpu_ids; ++cpu_i) { + struct wc_svr_native_cpu_state *cst = &wc_svr_native_states[cpu_i]; + cst->softirq.save_area = + aligned + ((size_t)cpu_i * 2U * wc_svr_native_save_size); + cst->softirq.pin_preempt = 1; + cst->hardirq.save_area = + cst->softirq.save_area + wc_svr_native_save_size; + cst->hardirq.pin_preempt = 0; + } + + if (wc_svr_native_selftest() != 0) { + free(wc_svr_native_save_mem); + wc_svr_native_save_mem = NULL; + free(wc_svr_native_states); + wc_svr_native_states = NULL; + return; + } + + wc_svr_native_ready = 1; + +#ifdef WOLFSSL_LINUXKM_VERBOSE_DEBUG + pr_info("wolfCrypt: native register save buffers enabled (%s, %u bytes" + " x 2 x %u CPUs).\n", + wc_svr_native_use_xsave ? "xsave" : "fxsave", + wc_svr_native_save_size, nr_cpu_ids); +#endif +} + +/* Returns the number of still-open native sections (0 when teardown is + * safe), mirroring the occupied-slot scan in wc_linuxkm_free_svr_states(). + */ +static int wc_svr_native_check_busy(void) +{ + unsigned int cpu_i; + int busy = 0; + + if (wc_svr_native_states == NULL) + return 0; + + for (cpu_i = 0; cpu_i < (unsigned int)nr_cpu_ids; ++cpu_i) { + if (wc_svr_native_states[cpu_i].softirq.depth != 0) { + pr_err("ERROR: wc_linuxkm_free_svr_states called with open native" + " softirq section on CPU %u (depth %u).\n", + cpu_i, wc_svr_native_states[cpu_i].softirq.depth); + ++busy; + } + if (wc_svr_native_states[cpu_i].hardirq.depth != 0) { + pr_err("ERROR: wc_linuxkm_free_svr_states called with open native" + " hardirq section on CPU %u (depth %u).\n", + cpu_i, wc_svr_native_states[cpu_i].hardirq.depth); + ++busy; + } + } + return busy; +} + +static void wc_svr_native_free(void) +{ + wc_svr_native_ready = 0; + if (wc_svr_native_save_mem != NULL) { + free(wc_svr_native_save_mem); + wc_svr_native_save_mem = NULL; + } + if (wc_svr_native_states != NULL) { + free(wc_svr_native_states); + wc_svr_native_states = NULL; + } + wc_svr_native_save_size = 0; + wc_svr_native_use_xsave = 0; +} + +/* Runtime readiness accessor for out-of-file policy decisions -- notably + * the FIPSdepth > 0U) { + if (unlikely(ctx->depth == ~0U)) { + pr_err_ratelimited("ERROR: wc_svr_native_save recursion count overflow on" + " CPU %d.\n", raw_smp_processor_id()); + wc_svr_disallowed_count_increment(); + #ifdef WOLFSSL_LINUXKM_VERBOSE_DEBUG + if (in_nmi()) + atomic_long_inc(&NMI_SVR_err_count); + else if (preempt_count() & HARDIRQ_MASK) + atomic_long_inc(&hardirq_SVR_err_count); + else if (preempt_count()) + atomic_long_inc(&softirq_SVR_err_count); + else + atomic_long_inc(&other_SVR_err_count); + #endif + return BAD_STATE_E; + } + ++ctx->depth; + return 0; + } + + if (ctx->pin_preempt) + preempt_disable(); + + wc_svr_native_regs_save(ctx->save_area); + ctx->depth = 1; + + return 0; +} + +static void wc_svr_native_restore(struct wc_svr_native_ctx_state *ctx, + enum wc_svr_flags flags) +{ + if (unlikely(flags & WC_SVR_FLAG_INHIBIT)) { + /* Native sections are opened by the plain and _MAYBE_INHIBIT + * flavors (the latter when native service preempts the legacy + * short-circuit), so _MAYBE-paired restores are legitimate here; + * only an _INHIBIT-flavored restore against an open native section + * is a caller pairing bug -- _INHIBIT never opens one. + */ + VRG_PR_WARN_X("BUG: wc_restore_vector_registers_x86() with inhibit" + " flags 0x%x on open native section (CPU %d," + " depth %u).\n", + (unsigned int)flags, raw_smp_processor_id(), + ctx->depth); + } + + if (ctx->depth > 1U) { + --ctx->depth; + return; + } + + wc_svr_native_regs_restore(ctx->save_area); + ctx->depth = 0; + + if (ctx->pin_preempt) + preempt_enable(); +} + +#endif /* WC_SVR_USE_NATIVE_REG_BUFS */ + #endif /* !WC_SKIP_INCLUDED_C_FILES */ diff --git a/wolfcrypt/src/wc_port.c b/wolfcrypt/src/wc_port.c index ef5403e7eb..d624840fc5 100644 --- a/wolfcrypt/src/wc_port.c +++ b/wolfcrypt/src/wc_port.c @@ -530,9 +530,9 @@ int wolfCrypt_Init(void) #endif #if defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS) && defined(WOLFSSL_LINUXKM) - ret = allocate_wolfcrypt_linuxkm_fpu_states(); + ret = wc_linuxkm_allocate_svr_states(); if (ret != 0) { - WOLFSSL_MSG("allocate_wolfcrypt_linuxkm_fpu_states failed"); + WOLFSSL_MSG("wc_linuxkm_allocate_svr_states failed"); WOLFCRYPT_INIT_RAISE_BAD_STATE(); } #endif @@ -893,7 +893,7 @@ int wolfCrypt_Cleanup(void) wolfSSL_CleanupHandle(); #endif #if defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS) && defined(WOLFSSL_LINUXKM) - free_wolfcrypt_linuxkm_fpu_states(); + wc_linuxkm_free_svr_states(); #endif #ifdef HAVE_ENTROPY_MEMUSE