# HG changeset patch # User Taylor R Campbell # Date 1783005493 0 # Thu Jul 02 15:18:13 2026 +0000 # Branch trunk # Node ID 96d89e979aed96151281e91b9a1b96c5fbe3c716 # Parent 8ec04006c4d9e99089e7219275f8355c2fc2a2f2 # EXP-Topic riastradh-pr60393-hashlockedatomicoops atomic_store(9): Fix this properly on hashlocked-atomic ports. Turns out the only such port (at the moment) is sparc -- hppa doesn't define this, even in GENERIC.MP, which we'll have to fix if we ever want to take advantage of the advanced technology of multiprocessor HPPA hardware. To make this work, just add atomic_swap_8 and atomic_swap_16 to the kernel on such ports. (We should maybe just always provide these in the kernel, on all architectures, but this is simpler for now.) PR kern/60393: hash-locked atomic_store_* is all kinds of busted diff -r 8ec04006c4d9 -r 96d89e979aed common/lib/libc/arch/sparc/atomic/Makefile.inc --- a/common/lib/libc/arch/sparc/atomic/Makefile.inc Thu Jul 02 14:05:42 2026 +0000 +++ b/common/lib/libc/arch/sparc/atomic/Makefile.inc Thu Jul 02 15:18:13 2026 +0000 @@ -52,6 +52,13 @@ SRCS+= atomic_nand_16_cas.c atomic_nand_ . if (${LIB} == "kern" || ${LIB} == "rump") SRCS+= atomic_cas.S + +# atomic_swap_8 and atomic_swap_16 are needed for atomic_store_* in +# sys/atomic.h to cooperate with concurrent atomic_r/m/w operations. +# (Could define atomic_store_* explicitly here but reusing +# atomic_swap_* is simpler, even if not perfectly optimal.) +SRCS+= atomic_swap_16_cas.c +SRCS+= atomic_swap_8_cas.c . endif . if (${LIB} == "c" || ${LIB} == "pthread") diff -r 8ec04006c4d9 -r 96d89e979aed common/lib/libc/arch/sparc/atomic/atomic_cas.S --- a/common/lib/libc/arch/sparc/atomic/atomic_cas.S Thu Jul 02 14:05:42 2026 +0000 +++ b/common/lib/libc/arch/sparc/atomic/atomic_cas.S Thu Jul 02 15:18:13 2026 +0000 @@ -125,20 +125,26 @@ 2: * XXX NOTE! The interlock trick only works if EVERYTHING writes to * XXX the memory cell through this code path! */ -ENTRY(_atomic_cas_32) - ACQUIRE_INTERLOCK - ! %o4 has saved PSR value - ! %o5 has interlock address +#define ATOMIC_CAS(nbits, ld, st) \ +ENTRY(_atomic_cas_##nbits) \ + ACQUIRE_INTERLOCK ;\ + /* %o4 has saved PSR value */ ;\ + /* %o5 has interlock address */ ;\ + ;\ + ld [%o0], %o3 /* get old value */ ;\ + cmp %o1, %o3 /* old == new? */ ;\ + beq,a 3f /* yes, do the store */ ;\ + st %o2, [%o0] /* (in the delay slot) */ ;\ + ;\ +3: RELEASE_INTERLOCK ;\ + ;\ + retl ;\ + mov %o3, %o0 /* return old value */ ;\ +END(_atomic_cas_##nbits) - ld [%o0], %o3 ! get old value - cmp %o1, %o3 ! old == new? - beq,a 3f ! yes, do the store - st %o2, [%o0] ! (in the delay slot) - -3: RELEASE_INTERLOCK - - retl - mov %o3, %o0 ! return old value +ATOMIC_CAS(32, ld, st) +ATOMIC_CAS(16, lduh, stuh) +ATOMIC_CAS(8, ldub, stub) ATOMIC_OP_ALIAS(atomic_cas_32,_atomic_cas_32) ATOMIC_OP_ALIAS(atomic_cas_uint,_atomic_cas_32) diff -r 8ec04006c4d9 -r 96d89e979aed common/lib/libc/atomic/atomic_swap_16_cas.c --- a/common/lib/libc/atomic/atomic_swap_16_cas.c Thu Jul 02 14:05:42 2026 +0000 +++ b/common/lib/libc/atomic/atomic_swap_16_cas.c Thu Jul 02 15:18:13 2026 +0000 @@ -31,11 +31,21 @@ #include "atomic_op_namespace.h" +#ifdef _KERNEL_OPT +#include "opt_multiprocessor.h" +#endif + +#include + #include +#if !defined(_KERNEL) || defined(MULTIPROCESSOR) + +#ifndef _KERNEL uint16_t atomic_swap_16(volatile uint16_t *addr, uint16_t new) asm("__sync_lock_test_and_set_2"); +#endif uint16_t atomic_swap_16(volatile uint16_t *addr, uint16_t new) @@ -50,3 +60,5 @@ atomic_swap_16(volatile uint16_t *addr, } crt_alias(__atomic_exchange_2,__sync_lock_test_and_set_2) + +#endif diff -r 8ec04006c4d9 -r 96d89e979aed common/lib/libc/atomic/atomic_swap_8_cas.c --- a/common/lib/libc/atomic/atomic_swap_8_cas.c Thu Jul 02 14:05:42 2026 +0000 +++ b/common/lib/libc/atomic/atomic_swap_8_cas.c Thu Jul 02 15:18:13 2026 +0000 @@ -31,11 +31,21 @@ #include "atomic_op_namespace.h" +#ifdef _KERNEL_OPT +#include "opt_multiprocessor.h" +#endif + +#include + #include +#if !defined(_KERNEL) || defined(MULTIPROCESSOR) + +#ifndef _KERNEL uint8_t atomic_swap_8(volatile uint8_t *addr, uint8_t new) asm("__sync_lock_test_and_set_1"); +#endif uint8_t atomic_swap_8(volatile uint8_t *addr, uint8_t new) @@ -50,3 +60,5 @@ atomic_swap_8(volatile uint8_t *addr, ui } crt_alias(__atomic_exchange_1,__sync_lock_test_and_set_1) + +#endif diff -r 8ec04006c4d9 -r 96d89e979aed sys/sys/atomic.h --- a/sys/sys/atomic.h Thu Jul 02 14:05:42 2026 +0000 +++ b/sys/sys/atomic.h Thu Jul 02 15:18:13 2026 +0000 @@ -33,7 +33,6 @@ #define _SYS_ATOMIC_H_ #include -#include #if !defined(_KERNEL) && !defined(_STANDALONE) #include #endif @@ -497,46 +496,35 @@ void kcsan_atomic_store(volatile void *, }) #ifdef __HAVE_HASHLOCKED_ATOMICS +ATOMIC_PROTO_SWAP(8, uint8_t, uint8_t, uint8_t); +ATOMIC_PROTO_SWAP(16, uint16_t, uint16_t, uint16_t); +#if defined(KASAN) +#define atomic_swap_8 kasan_atomic_swap_8 +#define atomic_swap_16 kasan_atomic_swap_16 +#elif defined(KCSAN) +#define atomic_swap_8 kcsan_atomic_swap_8 +#define atomic_swap_16 kcsan_atomic_swap_16 +#elif defined(KMSAN) +#define atomic_cas_8 kmsan_atomic_cas_8 +#define atomic_cas_16 kmsan_atomic_cas_16 +#endif static __inline __always_inline void __do_atomic_store(volatile void *p, const void *q, size_t size) { - volatile uint32_t *p32 = (volatile uint32_t *)((uintptr_t)p & ~3); switch (size) { case 1: { uint8_t v; -#if _BYTE_ORDER == _LITTLE_ENDIAN - unsigned s = 8 * ((uintptr_t)p & 3); -#elif _BYTE_ORDER == _BIG_ENDIAN - unsigned s = 8 * (3 - ((uintptr_t)p & 3)); -#else -# error atomic endianness kablooie -#endif - uint32_t o, n, m = ~(0xffU << s); memcpy(&v, q, 1); - do { - o = atomic_load_relaxed(p32); - n = (o & m) | ((uint32_t)v << s); - } while (atomic_cas_32(p32, o, n) != o); + (void)atomic_swap_8(p, v); break; } case 2: { uint16_t v; -#if _BYTE_ORDER == _LITTLE_ENDIAN - unsigned s = 8 * ((uintptr_t)p & 2); -#elif _BYTE_ORDER == _BIG_ENDIAN - unsigned s = 8 * (2 - ((uintptr_t)p & 2)); -#else -# error atomic endianness kablooie -#endif - uint32_t o, n, m = ~(0xffffU << s); memcpy(&v, q, 2); - do { - o = atomic_load_relaxed(p32); - n = (o & m) | ((uint32_t)v << s); - } while (atomic_cas_32(p32, o, n) != o); + (void)atomic_swap_16(p, v); break; } case 4: { # HG changeset patch # User Taylor R Campbell # Date 1783005781 0 # Thu Jul 02 15:23:01 2026 +0000 # Branch trunk # Node ID 781927823d71bd1eb78b675b64e7609861ffe36c # Parent 96d89e979aed96151281e91b9a1b96c5fbe3c716 # EXP-Topic riastradh-pr60393-hashlockedatomicoops WIP: hppa: Define __HAVE_HASHLOCKED_ATOMICS if MULTIPROCESSOR. XXX Missing atomic_cas/swap_8/16 in lock_stubs.S. This way, in MULTIPROCESSOR kernels, atomic_store_* will correctly cooperate with hash-locked atomic_cas_* on other CPUs. (On !MULTIPROCESSOR kernels, this should make no difference.) PR kern/60393: hash-locked atomic_store_* is all kinds of busted diff -r 96d89e979aed -r 781927823d71 sys/arch/hppa/include/types.h --- a/sys/arch/hppa/include/types.h Thu Jul 02 15:18:13 2026 +0000 +++ b/sys/arch/hppa/include/types.h Thu Jul 02 15:23:01 2026 +0000 @@ -36,6 +36,10 @@ #ifndef _HPPA_TYPES_H_ #define _HPPA_TYPES_H_ +#ifdef _KERNEL_OPT +#include "opt_multiprocessor.h" +#endif + #include #include @@ -89,6 +93,9 @@ typedef int __register_t; #define __HAVE_COMMON___TLS_GET_ADDR #define __HAVE_CPU_LWP_SETPRIVATE #define __HAVE_FUNCTION_DESCRIPTORS /* function ptrs may be descriptors */ +#if !defined(_KERNEL) || defined(MULTIPROCESSOR) +#define __HAVE_HASHLOCKED_ATOMICS +#endif #define __HAVE_MM_MD_DIRECT_MAPPED_PHYS #define __HAVE_MM_MD_KERNACC #define __HAVE_NEW_STYLE_BUS_H