# HG changeset patch # User Taylor R Campbell # Date 1783005493 0 # Thu Jul 02 15:18:13 2026 +0000 # Branch trunk # Node ID c239a5a9d48af75b96b9ed2b746363dbfbc8be08 # Parent 8ec04006c4d9e99089e7219275f8355c2fc2a2f2 # EXP-Topic riastradh-pr60393-hashlockedatomicoops atomic_store(9): Fix this properly on hashlocked-atomic ports. Turns out the only such port (at the moment) is sparc -- hppa doesn't define this, even in GENERIC.MP, which we'll have to fix if we ever want to take advantage of the advanced technology of multiprocessor HPPA hardware. To make this work, just add atomic_swap_8 and atomic_swap_16 to the kernel on such ports. (We should maybe just always provide these in the kernel, on all architectures, but this is simpler for now.) PR kern/60393: hash-locked atomic_store_* is all kinds of busted diff -r 8ec04006c4d9 -r c239a5a9d48a common/lib/libc/arch/sparc/atomic/Makefile.inc --- a/common/lib/libc/arch/sparc/atomic/Makefile.inc Thu Jul 02 14:05:42 2026 +0000 +++ b/common/lib/libc/arch/sparc/atomic/Makefile.inc Thu Jul 02 15:18:13 2026 +0000 @@ -52,6 +52,13 @@ SRCS+= atomic_nand_16_cas.c atomic_nand_ . if (${LIB} == "kern" || ${LIB} == "rump") SRCS+= atomic_cas.S + +# atomic_swap_8 and atomic_swap_16 are needed for atomic_store_* in +# sys/atomic.h to cooperate with concurrent atomic_r/m/w operations. +# (Could define atomic_store_* explicitly here but reusing +# atomic_swap_* is simpler, even if not perfectly optimal.) +SRCS+= atomic_swap_16_cas.c +SRCS+= atomic_swap_8_cas.c . endif . if (${LIB} == "c" || ${LIB} == "pthread") diff -r 8ec04006c4d9 -r c239a5a9d48a common/lib/libc/arch/sparc/atomic/atomic_cas.S --- a/common/lib/libc/arch/sparc/atomic/atomic_cas.S Thu Jul 02 14:05:42 2026 +0000 +++ b/common/lib/libc/arch/sparc/atomic/atomic_cas.S Thu Jul 02 15:18:13 2026 +0000 @@ -125,20 +125,26 @@ 2: * XXX NOTE! The interlock trick only works if EVERYTHING writes to * XXX the memory cell through this code path! */ -ENTRY(_atomic_cas_32) - ACQUIRE_INTERLOCK - ! %o4 has saved PSR value - ! %o5 has interlock address +#define ATOMIC_CAS(nbits, ld, st) \ +ENTRY(_atomic_cas_##nbits) \ + ACQUIRE_INTERLOCK ;\ + /* %o4 has saved PSR value */ ;\ + /* %o5 has interlock address */ ;\ + ;\ + ld [%o0], %o3 /* get old value */ ;\ + cmp %o1, %o3 /* old == new? */ ;\ + beq,a 3f /* yes, do the store */ ;\ + st %o2, [%o0] /* (in the delay slot) */ ;\ + ;\ +3: RELEASE_INTERLOCK ;\ + ;\ + retl ;\ + mov %o3, %o0 /* return old value */ ;\ +END(_atomic_cas_##nbits) - ld [%o0], %o3 ! get old value - cmp %o1, %o3 ! old == new? - beq,a 3f ! yes, do the store - st %o2, [%o0] ! (in the delay slot) - -3: RELEASE_INTERLOCK - - retl - mov %o3, %o0 ! return old value +ATOMIC_CAS(32, ld, st) +ATOMIC_CAS(16, lduh, stuh) +ATOMIC_CAS(8, ldub, stub) ATOMIC_OP_ALIAS(atomic_cas_32,_atomic_cas_32) ATOMIC_OP_ALIAS(atomic_cas_uint,_atomic_cas_32) diff -r 8ec04006c4d9 -r c239a5a9d48a sys/sys/atomic.h --- a/sys/sys/atomic.h Thu Jul 02 14:05:42 2026 +0000 +++ b/sys/sys/atomic.h Thu Jul 02 15:18:13 2026 +0000 @@ -497,46 +497,35 @@ void kcsan_atomic_store(volatile void *, }) #ifdef __HAVE_HASHLOCKED_ATOMICS +ATOMIC_PROTO_SWAP(8, uint8_t, uint8_t, uint8_t); +ATOMIC_PROTO_SWAP(16, uint16_t, uint16_t, uint16_t); +#if defined(KASAN) +#define atomic_swap_8 kasan_atomic_swap_8 +#define atomic_swap_16 kasan_atomic_swap_16 +#elif defined(KCSAN) +#define atomic_swap_8 kcsan_atomic_swap_8 +#define atomic_swap_16 kcsan_atomic_swap_16 +#elif defined(KMSAN) +#define atomic_cas_8 kmsan_atomic_cas_8 +#define atomic_cas_16 kmsan_atomic_cas_16 +#endif static __inline __always_inline void __do_atomic_store(volatile void *p, const void *q, size_t size) { - volatile uint32_t *p32 = (volatile uint32_t *)((uintptr_t)p & ~3); switch (size) { case 1: { uint8_t v; -#if _BYTE_ORDER == _LITTLE_ENDIAN - unsigned s = 8 * ((uintptr_t)p & 3); -#elif _BYTE_ORDER == _BIG_ENDIAN - unsigned s = 8 * (3 - ((uintptr_t)p & 3)); -#else -# error atomic endianness kablooie -#endif - uint32_t o, n, m = ~(0xffU << s); memcpy(&v, q, 1); - do { - o = atomic_load_relaxed(p32); - n = (o & m) | ((uint32_t)v << s); - } while (atomic_cas_32(p32, o, n) != o); + (void)atomic_swap_8(p, v); break; } case 2: { uint16_t v; -#if _BYTE_ORDER == _LITTLE_ENDIAN - unsigned s = 8 * ((uintptr_t)p & 2); -#elif _BYTE_ORDER == _BIG_ENDIAN - unsigned s = 8 * (2 - ((uintptr_t)p & 2)); -#else -# error atomic endianness kablooie -#endif - uint32_t o, n, m = ~(0xffffU << s); memcpy(&v, q, 2); - do { - o = atomic_load_relaxed(p32); - n = (o & m) | ((uint32_t)v << s); - } while (atomic_cas_32(p32, o, n) != o); + (void)atomic_swap_16(p, v); break; } case 4: { # HG changeset patch # User Taylor R Campbell # Date 1783005720 0 # Thu Jul 02 15:22:00 2026 +0000 # Branch trunk # Node ID 15309b831a7d55eb29fe13c57f11f03a34c24b39 # Parent c239a5a9d48af75b96b9ed2b746363dbfbc8be08 # EXP-Topic riastradh-pr60393-hashlockedatomicoops atomic_store(9): Hash-locked atomics matter only if MULTIPROCESSOR. No need on uniprocessor kernels: a machine load or store instruction can't interfere with a hash-locked atomic r/m/w on another CPU because there's no other CPU. PR kern/60393: hash-locked atomic_store_* is all kinds of busted diff -r c239a5a9d48a -r 15309b831a7d sys/kern/subr_csan.c --- a/sys/kern/subr_csan.c Thu Jul 02 15:18:13 2026 +0000 +++ b/sys/kern/subr_csan.c Thu Jul 02 15:22:00 2026 +0000 @@ -31,6 +31,10 @@ #include __KERNEL_RCSID(0, "$NetBSD: subr_csan.c,v 1.14 2022/07/30 14:13:27 riastradh Exp $"); +#ifdef _KERNEL_OPT +#include "opt_multiprocessor.h" +#endif + #include #include #include @@ -615,7 +619,7 @@ void kcsan_atomic_store(volatile void *p, const void *v, int size) { kcsan_access((uintptr_t)p, size, true, true, __RET_ADDR); -#ifdef __HAVE_HASHLOCKED_ATOMICS +#if defined __HAVE_HASHLOCKED_ATOMICS && defined MULTIPROCESSOR __do_atomic_store(p, v, size); #else switch (size) { diff -r c239a5a9d48a -r 15309b831a7d sys/sys/atomic.h --- a/sys/sys/atomic.h Thu Jul 02 15:18:13 2026 +0000 +++ b/sys/sys/atomic.h Thu Jul 02 15:22:00 2026 +0000 @@ -42,6 +42,7 @@ #include "opt_kasan.h" #include "opt_kcsan.h" #include "opt_kmsan.h" +#include "opt_multiprocessor.h" #endif #if defined(KASAN) @@ -496,7 +497,7 @@ void kcsan_atomic_store(volatile void *, __DO_ATOMIC_STORE(__as_ptr, __as_val); \ }) -#ifdef __HAVE_HASHLOCKED_ATOMICS +#if defined __HAVE_HASHLOCKED_ATOMICS && defined MULTIPROCESSOR ATOMIC_PROTO_SWAP(8, uint8_t, uint8_t, uint8_t); ATOMIC_PROTO_SWAP(16, uint16_t, uint16_t, uint16_t); #if defined(KASAN) # HG changeset patch # User Taylor R Campbell # Date 1783005781 0 # Thu Jul 02 15:23:01 2026 +0000 # Branch trunk # Node ID 1229e8637142be5817859a24402761024030a699 # Parent 15309b831a7d55eb29fe13c57f11f03a34c24b39 # EXP-Topic riastradh-pr60393-hashlockedatomicoops hppa: Define __HAVE_HASHLOCKED_ATOMICS. This way, in MULTIPROCESSOR kernels, atomic_store_* will correctly cooperate with hash-locked atomic_cas_* on other CPUs. (On !MULTIPROCESSOR kernels, this should make no difference.) Or at least, it would if we ever got around to implementing atomic_cas_* for multiprocessor hppa! PR kern/60393: hash-locked atomic_store_* is all kinds of busted diff -r 15309b831a7d -r 1229e8637142 sys/arch/hppa/include/types.h --- a/sys/arch/hppa/include/types.h Thu Jul 02 15:22:00 2026 +0000 +++ b/sys/arch/hppa/include/types.h Thu Jul 02 15:23:01 2026 +0000 @@ -89,6 +89,7 @@ typedef int __register_t; #define __HAVE_COMMON___TLS_GET_ADDR #define __HAVE_CPU_LWP_SETPRIVATE #define __HAVE_FUNCTION_DESCRIPTORS /* function ptrs may be descriptors */ +#define __HAVE_HASHLOCKED_ATOMICS #define __HAVE_MM_MD_DIRECT_MAPPED_PHYS #define __HAVE_MM_MD_KERNACC #define __HAVE_NEW_STYLE_BUS_H