diff -r 8ec04006c4d9 -r 781927823d71 common/lib/libc/arch/sparc/atomic/Makefile.inc --- a/common/lib/libc/arch/sparc/atomic/Makefile.inc Thu Jul 02 14:05:42 2026 +0000 +++ b/common/lib/libc/arch/sparc/atomic/Makefile.inc Thu Jul 02 15:23:01 2026 +0000 @@ -52,6 +52,13 @@ SRCS+= atomic_nand_16_cas.c atomic_nand_ . if (${LIB} == "kern" || ${LIB} == "rump") SRCS+= atomic_cas.S + +# atomic_swap_8 and atomic_swap_16 are needed for atomic_store_* in +# sys/atomic.h to cooperate with concurrent atomic_r/m/w operations. +# (Could define atomic_store_* explicitly here but reusing +# atomic_swap_* is simpler, even if not perfectly optimal.) +SRCS+= atomic_swap_16_cas.c +SRCS+= atomic_swap_8_cas.c . endif . if (${LIB} == "c" || ${LIB} == "pthread") diff -r 8ec04006c4d9 -r 781927823d71 common/lib/libc/arch/sparc/atomic/atomic_cas.S --- a/common/lib/libc/arch/sparc/atomic/atomic_cas.S Thu Jul 02 14:05:42 2026 +0000 +++ b/common/lib/libc/arch/sparc/atomic/atomic_cas.S Thu Jul 02 15:23:01 2026 +0000 @@ -125,20 +125,26 @@ 2: * XXX NOTE! The interlock trick only works if EVERYTHING writes to * XXX the memory cell through this code path! */ -ENTRY(_atomic_cas_32) - ACQUIRE_INTERLOCK - ! %o4 has saved PSR value - ! %o5 has interlock address +#define ATOMIC_CAS(nbits, ld, st) \ +ENTRY(_atomic_cas_##nbits) \ + ACQUIRE_INTERLOCK ;\ + /* %o4 has saved PSR value */ ;\ + /* %o5 has interlock address */ ;\ + ;\ + ld [%o0], %o3 /* get old value */ ;\ + cmp %o1, %o3 /* old == new? */ ;\ + beq,a 3f /* yes, do the store */ ;\ + st %o2, [%o0] /* (in the delay slot) */ ;\ + ;\ +3: RELEASE_INTERLOCK ;\ + ;\ + retl ;\ + mov %o3, %o0 /* return old value */ ;\ +END(_atomic_cas_##nbits) - ld [%o0], %o3 ! get old value - cmp %o1, %o3 ! old == new? - beq,a 3f ! yes, do the store - st %o2, [%o0] ! (in the delay slot) - -3: RELEASE_INTERLOCK - - retl - mov %o3, %o0 ! return old value +ATOMIC_CAS(32, ld, st) +ATOMIC_CAS(16, lduh, stuh) +ATOMIC_CAS(8, ldub, stub) ATOMIC_OP_ALIAS(atomic_cas_32,_atomic_cas_32) ATOMIC_OP_ALIAS(atomic_cas_uint,_atomic_cas_32) diff -r 8ec04006c4d9 -r 781927823d71 common/lib/libc/atomic/atomic_swap_16_cas.c --- a/common/lib/libc/atomic/atomic_swap_16_cas.c Thu Jul 02 14:05:42 2026 +0000 +++ b/common/lib/libc/atomic/atomic_swap_16_cas.c Thu Jul 02 15:23:01 2026 +0000 @@ -31,11 +31,21 @@ #include "atomic_op_namespace.h" +#ifdef _KERNEL_OPT +#include "opt_multiprocessor.h" +#endif + +#include + #include +#if !defined(_KERNEL) || defined(MULTIPROCESSOR) + +#ifndef _KERNEL uint16_t atomic_swap_16(volatile uint16_t *addr, uint16_t new) asm("__sync_lock_test_and_set_2"); +#endif uint16_t atomic_swap_16(volatile uint16_t *addr, uint16_t new) @@ -50,3 +60,5 @@ atomic_swap_16(volatile uint16_t *addr, } crt_alias(__atomic_exchange_2,__sync_lock_test_and_set_2) + +#endif diff -r 8ec04006c4d9 -r 781927823d71 common/lib/libc/atomic/atomic_swap_8_cas.c --- a/common/lib/libc/atomic/atomic_swap_8_cas.c Thu Jul 02 14:05:42 2026 +0000 +++ b/common/lib/libc/atomic/atomic_swap_8_cas.c Thu Jul 02 15:23:01 2026 +0000 @@ -31,11 +31,21 @@ #include "atomic_op_namespace.h" +#ifdef _KERNEL_OPT +#include "opt_multiprocessor.h" +#endif + +#include + #include +#if !defined(_KERNEL) || defined(MULTIPROCESSOR) + +#ifndef _KERNEL uint8_t atomic_swap_8(volatile uint8_t *addr, uint8_t new) asm("__sync_lock_test_and_set_1"); +#endif uint8_t atomic_swap_8(volatile uint8_t *addr, uint8_t new) @@ -50,3 +60,5 @@ atomic_swap_8(volatile uint8_t *addr, ui } crt_alias(__atomic_exchange_1,__sync_lock_test_and_set_1) + +#endif diff -r 8ec04006c4d9 -r 781927823d71 sys/arch/hppa/include/types.h --- a/sys/arch/hppa/include/types.h Thu Jul 02 14:05:42 2026 +0000 +++ b/sys/arch/hppa/include/types.h Thu Jul 02 15:23:01 2026 +0000 @@ -36,6 +36,10 @@ #ifndef _HPPA_TYPES_H_ #define _HPPA_TYPES_H_ +#ifdef _KERNEL_OPT +#include "opt_multiprocessor.h" +#endif + #include #include @@ -89,6 +93,9 @@ typedef int __register_t; #define __HAVE_COMMON___TLS_GET_ADDR #define __HAVE_CPU_LWP_SETPRIVATE #define __HAVE_FUNCTION_DESCRIPTORS /* function ptrs may be descriptors */ +#if !defined(_KERNEL) || defined(MULTIPROCESSOR) +#define __HAVE_HASHLOCKED_ATOMICS +#endif #define __HAVE_MM_MD_DIRECT_MAPPED_PHYS #define __HAVE_MM_MD_KERNACC #define __HAVE_NEW_STYLE_BUS_H diff -r 8ec04006c4d9 -r 781927823d71 sys/sys/atomic.h --- a/sys/sys/atomic.h Thu Jul 02 14:05:42 2026 +0000 +++ b/sys/sys/atomic.h Thu Jul 02 15:23:01 2026 +0000 @@ -33,7 +33,6 @@ #define _SYS_ATOMIC_H_ #include -#include #if !defined(_KERNEL) && !defined(_STANDALONE) #include #endif @@ -497,46 +496,35 @@ void kcsan_atomic_store(volatile void *, }) #ifdef __HAVE_HASHLOCKED_ATOMICS +ATOMIC_PROTO_SWAP(8, uint8_t, uint8_t, uint8_t); +ATOMIC_PROTO_SWAP(16, uint16_t, uint16_t, uint16_t); +#if defined(KASAN) +#define atomic_swap_8 kasan_atomic_swap_8 +#define atomic_swap_16 kasan_atomic_swap_16 +#elif defined(KCSAN) +#define atomic_swap_8 kcsan_atomic_swap_8 +#define atomic_swap_16 kcsan_atomic_swap_16 +#elif defined(KMSAN) +#define atomic_cas_8 kmsan_atomic_cas_8 +#define atomic_cas_16 kmsan_atomic_cas_16 +#endif static __inline __always_inline void __do_atomic_store(volatile void *p, const void *q, size_t size) { - volatile uint32_t *p32 = (volatile uint32_t *)((uintptr_t)p & ~3); switch (size) { case 1: { uint8_t v; -#if _BYTE_ORDER == _LITTLE_ENDIAN - unsigned s = 8 * ((uintptr_t)p & 3); -#elif _BYTE_ORDER == _BIG_ENDIAN - unsigned s = 8 * (3 - ((uintptr_t)p & 3)); -#else -# error atomic endianness kablooie -#endif - uint32_t o, n, m = ~(0xffU << s); memcpy(&v, q, 1); - do { - o = atomic_load_relaxed(p32); - n = (o & m) | ((uint32_t)v << s); - } while (atomic_cas_32(p32, o, n) != o); + (void)atomic_swap_8(p, v); break; } case 2: { uint16_t v; -#if _BYTE_ORDER == _LITTLE_ENDIAN - unsigned s = 8 * ((uintptr_t)p & 2); -#elif _BYTE_ORDER == _BIG_ENDIAN - unsigned s = 8 * (2 - ((uintptr_t)p & 2)); -#else -# error atomic endianness kablooie -#endif - uint32_t o, n, m = ~(0xffffU << s); memcpy(&v, q, 2); - do { - o = atomic_load_relaxed(p32); - n = (o & m) | ((uint32_t)v << s); - } while (atomic_cas_32(p32, o, n) != o); + (void)atomic_swap_16(p, v); break; } case 4: {