Index: arch/amd64/amd64/genassym.cf =================================================================== RCS file: /cvsroot/src/sys/arch/amd64/amd64/genassym.cf,v retrieving revision 1.82 diff -u -p -r1.82 genassym.cf --- arch/amd64/amd64/genassym.cf 17 Feb 2020 09:09:48 -0000 1.82 +++ arch/amd64/amd64/genassym.cf 27 Feb 2020 22:36:04 -0000 @@ -70,7 +70,6 @@ include "opt_xen.h" endif quote #define __MUTEX_PRIVATE -quote #define __RWLOCK_PRIVATE include include @@ -78,7 +77,6 @@ include include include include -include include include include @@ -343,15 +341,6 @@ define MTX_IPL offsetof(struct kmutex, define MTX_LOCK offsetof(struct kmutex, u.s.mtxs_lock) define MTX_OWNER offsetof(struct kmutex, u.mtxa_owner) -define RW_OWNER offsetof(struct krwlock, rw_owner) -define RW_WRITE_LOCKED RW_WRITE_LOCKED -define RW_WRITE_WANTED RW_WRITE_WANTED -define RW_READ_INCR RW_READ_INCR -define RW_HAS_WAITERS RW_HAS_WAITERS -define RW_THREAD RW_THREAD -define RW_READER RW_READER -define RW_WRITER RW_WRITER - define EV_COUNT offsetof(struct evcnt, ev_count) define OPTERON_MSR_PASSCODE OPTERON_MSR_PASSCODE Index: arch/amd64/amd64/lock_stubs.S =================================================================== RCS file: /cvsroot/src/sys/arch/amd64/amd64/lock_stubs.S,v retrieving revision 1.35 diff -u -p -r1.35 lock_stubs.S --- arch/amd64/amd64/lock_stubs.S 8 Dec 2019 20:00:56 -0000 1.35 +++ arch/amd64/amd64/lock_stubs.S 27 Feb 2020 22:36:04 -0000 @@ -185,126 +185,6 @@ ENTRY(mutex_spin_exit) END(mutex_spin_exit) -/* - * void rw_enter(krwlock_t *rwl, krw_t op); - * - * Acquire one hold on a RW lock. - */ -ENTRY(rw_enter) - cmpl $RW_READER, %esi - jne 2f - - /* - * Reader: this is the most common case. - */ - movq (%rdi), %rax -0: - testb $(RW_WRITE_LOCKED|RW_WRITE_WANTED), %al - jnz 3f - leaq RW_READ_INCR(%rax), %rdx - LOCK - cmpxchgq %rdx, (%rdi) - jnz 1f - RET -1: - jmp 0b - - /* - * Writer: if the compare-and-set fails, don't bother retrying. - */ -2: movq CPUVAR(CURLWP), %rcx - xorq %rax, %rax - orq $RW_WRITE_LOCKED, %rcx - LOCK - cmpxchgq %rcx, (%rdi) - jnz 3f - RET -3: - jmp _C_LABEL(rw_vector_enter) -END(rw_enter) - -/* - * void rw_exit(krwlock_t *rwl); - * - * Release one hold on a RW lock. - */ -ENTRY(rw_exit) - movq (%rdi), %rax - testb $RW_WRITE_LOCKED, %al - jnz 2f - - /* - * Reader - */ -0: testb $RW_HAS_WAITERS, %al - jnz 3f - cmpq $RW_READ_INCR, %rax - jb 3f - leaq -RW_READ_INCR(%rax), %rdx - LOCK - cmpxchgq %rdx, (%rdi) - jnz 1f - ret -1: - jmp 0b - - /* - * Writer - */ -2: leaq -RW_WRITE_LOCKED(%rax), %rdx - subq CPUVAR(CURLWP), %rdx - jnz 3f - LOCK - cmpxchgq %rdx, (%rdi) - jnz 3f - ret - -3: jmp _C_LABEL(rw_vector_exit) -END(rw_exit) - -/* - * int rw_tryenter(krwlock_t *rwl, krw_t op); - * - * Try to acquire one hold on a RW lock. - */ -ENTRY(rw_tryenter) - cmpl $RW_READER, %esi - jne 2f - - /* - * Reader: this is the most common case. - */ - movq (%rdi), %rax -0: - testb $(RW_WRITE_LOCKED|RW_WRITE_WANTED), %al - jnz 4f - leaq RW_READ_INCR(%rax), %rdx - LOCK - cmpxchgq %rdx, (%rdi) - jnz 1f - movl %edx, %eax /* nonzero */ - RET -1: - jmp 0b - - /* - * Writer: if the compare-and-set fails, don't bother retrying. - */ -2: movq CPUVAR(CURLWP), %rcx - xorq %rax, %rax - orq $RW_WRITE_LOCKED, %rcx - LOCK - cmpxchgq %rcx, (%rdi) - movl $0, %eax - setz %al -3: - RET - ret -4: - xorl %eax, %eax - jmp 3b -END(rw_tryenter) - #endif /* LOCKDEBUG */ /* Index: arch/hppa/hppa/genassym.cf =================================================================== RCS file: /cvsroot/src/sys/arch/hppa/hppa/genassym.cf,v retrieving revision 1.3 diff -u -p -r1.3 genassym.cf --- arch/hppa/hppa/genassym.cf 20 Feb 2020 08:27:38 -0000 1.3 +++ arch/hppa/hppa/genassym.cf 27 Feb 2020 22:36:06 -0000 @@ -38,7 +38,6 @@ include "opt_multiprocessor.h" endif quote #define __MUTEX_PRIVATE -quote #define __RWLOCK_PRIVATE include include @@ -48,7 +47,6 @@ include include include include -include include include @@ -108,15 +106,6 @@ define MTX_LOCK offsetof(struct kmutex, define MTX_OWNER offsetof(struct kmutex, mtx_owner) define MTX_WAITERS offsetof(struct kmutex, mtx_waiters) -define RW_OWNER offsetof(struct krwlock, rw_owner) -define RW_WRITE_LOCKED RW_WRITE_LOCKED -define RW_WRITE_WANTED RW_WRITE_WANTED -define RW_READ_INCR RW_READ_INCR -define RW_HAS_WAITERS RW_HAS_WAITERS -define RW_THREAD RW_THREAD -define RW_READER RW_READER -define RW_WRITER RW_WRITER - # saved state fields struct trapframe member TF_FLAGS tf_flags Index: arch/i386/i386/genassym.cf =================================================================== RCS file: /cvsroot/src/sys/arch/i386/i386/genassym.cf,v retrieving revision 1.119 diff -u -p -r1.119 genassym.cf --- arch/i386/i386/genassym.cf 17 Feb 2020 09:09:48 -0000 1.119 +++ arch/i386/i386/genassym.cf 27 Feb 2020 22:36:06 -0000 @@ -71,7 +71,6 @@ include "opt_xen.h" endif quote #define __MUTEX_PRIVATE -quote #define __RWLOCK_PRIVATE include include @@ -79,7 +78,6 @@ include include include include -include include include @@ -351,15 +349,6 @@ define MTX_IPL offsetof(struct kmutex, define MTX_LOCK offsetof(struct kmutex, mtx_lock) define MTX_OWNER offsetof(struct kmutex, u.mtxa_owner) -define RW_OWNER offsetof(struct krwlock, rw_owner) -define RW_WRITE_LOCKED RW_WRITE_LOCKED -define RW_WRITE_WANTED RW_WRITE_WANTED -define RW_READ_INCR RW_READ_INCR -define RW_HAS_WAITERS RW_HAS_WAITERS -define RW_THREAD RW_THREAD -define RW_READER RW_READER -define RW_WRITER RW_WRITER - define EV_COUNT offsetof(struct evcnt, ev_count) define OPTERON_MSR_PASSCODE OPTERON_MSR_PASSCODE Index: arch/i386/i386/lock_stubs.S =================================================================== RCS file: /cvsroot/src/sys/arch/i386/i386/lock_stubs.S,v retrieving revision 1.32 diff -u -p -r1.32 lock_stubs.S --- arch/i386/i386/lock_stubs.S 8 Dec 2019 20:00:56 -0000 1.32 +++ arch/i386/i386/lock_stubs.S 27 Feb 2020 22:36:07 -0000 @@ -97,132 +97,6 @@ ENTRY(mutex_exit) jmp _C_LABEL(mutex_vector_exit) END(mutex_exit) -/* - * void rw_enter(krwlock_t *rwl, krw_t op); - * - * Acquire one hold on a RW lock. - */ -ENTRY(rw_enter) - movl 4(%esp), %edx - cmpl $RW_READER, 8(%esp) - jne 2f - - /* - * Reader - */ - movl (%edx), %eax -0: - testb $(RW_WRITE_LOCKED|RW_WRITE_WANTED), %al - jnz 3f - leal RW_READ_INCR(%eax), %ecx - LOCK(2) - cmpxchgl %ecx, (%edx) - jnz 1f - RET(2) -1: - jmp 0b - - /* - * Writer - */ -2: xorl %eax, %eax - movl %fs:CPU_INFO_CURLWP(%eax), %ecx - orl $RW_WRITE_LOCKED, %ecx - LOCK(3) - cmpxchgl %ecx, (%edx) - jnz 3f - RET(3) -3: - jmp _C_LABEL(rw_vector_enter) -END(rw_enter) - -/* - * void rw_exit(krwlock_t *rwl); - * - * Release one hold on a RW lock. - */ -ENTRY(rw_exit) - movl 4(%esp), %edx - movl (%edx), %eax - testb $RW_WRITE_LOCKED, %al - jnz 2f - - /* - * Reader - */ -0: testb $RW_HAS_WAITERS, %al - jnz 3f - cmpl $RW_READ_INCR, %eax - jb 3f - leal -RW_READ_INCR(%eax), %ecx - LOCK(4) - cmpxchgl %ecx, (%edx) - jnz 1f - ret -1: - jmp 0b - - /* - * Writer - */ -2: leal -RW_WRITE_LOCKED(%eax), %ecx - subl CPUVAR(CURLWP), %ecx - jnz 3f - LOCK(5) - cmpxchgl %ecx, (%edx) - jnz 3f - ret - - /* - * Slow path. - */ -3: jmp _C_LABEL(rw_vector_exit) -END(rw_exit) - -/* - * int rw_tryenter(krwlock_t *rwl, krw_t op); - * - * Try to acquire one hold on a RW lock. - */ -ENTRY(rw_tryenter) - movl 4(%esp), %edx - cmpl $RW_READER, 8(%esp) - jne 2f - - /* - * Reader - */ - movl (%edx), %eax -0: - testb $(RW_WRITE_LOCKED|RW_WRITE_WANTED), %al - jnz 4f - leal RW_READ_INCR(%eax), %ecx - LOCK(12) - cmpxchgl %ecx, (%edx) - jnz 1f - movl %edx, %eax /* nonzero */ - RET(4) -1: - jmp 0b - - /* - * Writer - */ -2: - xorl %eax, %eax - movl %fs:CPU_INFO_CURLWP(%eax), %ecx - orl $RW_WRITE_LOCKED, %ecx - LOCK(13) - cmpxchgl %ecx, (%edx) - movl $0, %eax - setz %al -3: - RET(5) -4: - xorl %eax, %eax - jmp 3b -END(rw_tryenter) - #ifndef XENPV /* Index: arch/powerpc/oea/genassym.cf =================================================================== RCS file: /cvsroot/src/sys/arch/powerpc/oea/genassym.cf,v retrieving revision 1.27 diff -u -p -r1.27 genassym.cf --- arch/powerpc/oea/genassym.cf 15 Jul 2018 05:16:44 -0000 1.27 +++ arch/powerpc/oea/genassym.cf 27 Feb 2020 22:36:07 -0000 @@ -36,7 +36,6 @@ include "opt_ppcarch.h" include include include -include include include Index: arch/powerpc/powerpc/genassym.cf =================================================================== RCS file: /cvsroot/src/sys/arch/powerpc/powerpc/genassym.cf,v retrieving revision 1.12 diff -u -p -r1.12 genassym.cf --- arch/powerpc/powerpc/genassym.cf 8 Jan 2020 17:38:42 -0000 1.12 +++ arch/powerpc/powerpc/genassym.cf 27 Feb 2020 22:36:07 -0000 @@ -44,7 +44,6 @@ include "opt_ppcarch.h" quote #define __MUTEX_PRIVATE -quote #define __RWLOCK_PRIVATE quote #define __INTR_PRIVATE include @@ -54,7 +53,6 @@ include include include include -include include @@ -215,17 +213,8 @@ ifdef __HAVE_MUTEX_SPIN_STUBS define __HAVE_MUTEX_SPIN_STUBS __HAVE_MUTEX_SPIN_STUBS endif -ifdef __HAVE_RW_STUBS -define __HAVE_RW_STUBS __HAVE_RW_STUBS -endif - define MTX_OWNER offsetof(struct kmutex, mtx_owner) define MTX_LOCK offsetof(struct kmutex, mtx_lock) define MTX_IPL offsetof(struct kmutex, mtx_ipl) -define RW_OWNER offsetof(struct krwlock, rw_owner) -define RW_WRITE_LOCKED RW_WRITE_LOCKED -define RW_READ_INCR RW_READ_INCR -define RW_READER RW_READER - define MSR_PR 31 - ilog2(PSL_PR) Index: arch/powerpc/powerpc/lock_stubs.S =================================================================== RCS file: /cvsroot/src/sys/arch/powerpc/powerpc/lock_stubs.S,v retrieving revision 1.11 diff -u -p -r1.11 lock_stubs.S --- arch/powerpc/powerpc/lock_stubs.S 21 Jan 2020 04:30:14 -0000 1.11 +++ arch/powerpc/powerpc/lock_stubs.S 27 Feb 2020 22:36:07 -0000 @@ -98,96 +98,4 @@ ENTRY(mutex_exit) #endif /* !LOCKDEBUG */ -/* - * void rw_enter(krwlock_t *krw, krw_t op); - */ -#if RW_READ_INCR != 32 -#error RW_READ_INCR != 32, clrrXi need fixing -#endif -#if RW_OWNER != 0 -#error RW_OWNER != 0, ldptr should be ldptru -#endif - -#if __HAVE_RW_STUBS - -ENTRY(rw_enter) - cmpwi %r3,RW_READER - bne- 1f - - ldptr %r9,RW_OWNER(%r3) - clrrptri %r9,%r9,5 /* clear low 5 bits */ - addi %r7,%r9,RW_READ_INCR - b 2f -1: - li %r9,0 - ori %r7,%r13,RW_WRITE_LOCKED - -2: lptrarx %r10,0,%r3 - cmpptr %r10,%r9 - bne- 3f - stptrcx. %r7,0,%r3 - bne- 2b - ISYNC - blr - -3: b _C_LABEL(rw_vector_enter) - -/* - * bool rw_tryenter(krwlock_t *krw, krw_t op); - */ -ENTRY(rw_tryenter) - cmpwi %r3,RW_READER - bne- 1f - - ldptr %r9,RW_OWNER(%r3) - clrrptri %r9,%r9,5 /* clear low 5 bits */ - addi %r7,%r9,RW_READ_INCR - b 2f - -1: li %r9,0 - ori %r7,%r13,RW_WRITE_LOCKED - -2: lptrarx %r10,0,%r3 - cmpptr %r10,%r9 - bne- 3f - stptrcx. %r7,0,%r3 - bne- 2b - - ISYNC - li %r3,1 - blr - -3: li %r3,0 - blr - -/* - * void rw_exit(krwlock_t *krw, krw_t op); - */ -ENTRY(rw_exit) - ldptr %r9,RW_OWNER(%r3) - SYNC - andi. %r0,%r9,RW_WRITE_LOCKED - bne- 1f - - clrrptri. %r9,%r9,5 /* clear low 5 bits */ - beq- 3f /* if 0, no reader, go slow */ - - addi %r7,%r9,-RW_READ_INCR - b 2f -1: - li %r7,0 - ori %r9,%r13,RW_WRITE_LOCKED - -2: lptrarx %r10,0,%r3 - cmpptr %r10,%r9 - bne- 3f - stptrcx. %r7,0,%r3 - bne- 2b - - blr - -3: b _C_LABEL(rw_vector_exit) - -#endif /* __HAVE_RW_STUBS */ - #endif /* __HAVE_MUTEX_STUBS */ Index: arch/riscv/riscv/genassym.cf =================================================================== RCS file: /cvsroot/src/sys/arch/riscv/riscv/genassym.cf,v retrieving revision 1.7 diff -u -p -r1.7 genassym.cf --- arch/riscv/riscv/genassym.cf 8 Jan 2020 17:38:42 -0000 1.7 +++ arch/riscv/riscv/genassym.cf 27 Feb 2020 22:36:07 -0000 @@ -29,7 +29,6 @@ #+ quote #define __MUTEX_PRIVATE -quote #define __RWLOCK_PRIVATE quote #define __INTR_PRIVATE quote #define __PMAP_PRIVATE @@ -40,7 +39,6 @@ include include include include -include include @@ -164,15 +162,6 @@ ifdef __HAVE_MUTEX_SPIN_STUBS define __HAVE_MUTEX_SPIN_STUBS 1 endif -ifdef __HAVE_RW_STUBS -define __HAVE_RW_STUBS 1 -endif - -define RW_OWNER offsetof(struct krwlock, rw_owner) -define RW_WRITE_LOCKED RW_WRITE_LOCKED -define RW_READ_INCR RW_READ_INCR -define RW_READER RW_READER - define VM_MIN_KERNEL_ADDRESS VM_MIN_KERNEL_ADDRESS define VM_MAX_KERNEL_ADDRESS VM_MAX_KERNEL_ADDRESS Index: arch/usermode/usermode/genassym.cf =================================================================== RCS file: /cvsroot/src/sys/arch/usermode/usermode/genassym.cf,v retrieving revision 1.2 diff -u -p -r1.2 genassym.cf --- arch/usermode/usermode/genassym.cf 21 Oct 2009 16:07:00 -0000 1.2 +++ arch/usermode/usermode/genassym.cf 27 Feb 2020 22:36:08 -0000 @@ -27,7 +27,5 @@ # quote #define __MUTEX_PRIVATE -quote #define __RWLOCK_PRIVATE include -include Index: arch/vax/vax/genassym.cf =================================================================== RCS file: /cvsroot/src/sys/arch/vax/vax/genassym.cf,v retrieving revision 1.55 diff -u -p -r1.55 genassym.cf --- arch/vax/vax/genassym.cf 20 Feb 2020 08:27:40 -0000 1.55 +++ arch/vax/vax/genassym.cf 27 Feb 2020 22:36:08 -0000 @@ -25,7 +25,6 @@ # quote #define __MUTEX_PRIVATE -quote #define __RWLOCK_PRIVATE include include @@ -169,12 +168,3 @@ define PSL_HIGHIPL PSL_HIGHIPL define MTX_OWNER offsetof(struct kmutex, mtx_owner) define MTX_LOCK offsetof(struct kmutex, mtx_lock) define MTX_IPL offsetof(struct kmutex, mtx_ipl) - -define RW_OWNER offsetof(struct krwlock, rw_owner) -define RW_HAS_WAITERS RW_HAS_WAITERS -define RW_WRITE_LOCKED RW_WRITE_LOCKED -define RW_WRITE_WANTED RW_WRITE_WANTED -define RW_READ_INCR RW_READ_INCR -define RW_THREAD RW_THREAD -define RW_READER RW_READER -define RW_WRITER RW_WRITER Index: arch/vax/vax/lock_stubs.S =================================================================== RCS file: /cvsroot/src/sys/arch/vax/vax/lock_stubs.S,v retrieving revision 1.18 diff -u -p -r1.18 lock_stubs.S --- arch/vax/vax/lock_stubs.S 9 Dec 2013 09:35:16 -0000 1.18 +++ arch/vax/vax/lock_stubs.S 27 Feb 2020 22:36:08 -0000 @@ -144,114 +144,6 @@ ENTRY(mutex_spin_exit, 0) ret #endif -#if RW_READER != 0 -#error RW_READER != 0, change tstl to cmpl $RW_READER -#endif -#if RW_HAS_WAITERS != 1 -#error RW_HAS_WAITERS != 1, don't use blbs -#endif -#if RW_OWNER != 0 -#error RW_OWNER != 0, need to add to loads -#endif -/* - * void rw_enter(krwlock_t *rwl, krw_t op); - */ -ENTRY(rw_enter, 0) - movl 4(%ap), %r1 /* grab rwlock ptr */ - tstl 8(%ap) /* is this a reader op? */ - bneq 2f /* nope, branch to writer */ - - movl (%r1), %r2 /* get owner field */ - bitl $(RW_WRITE_LOCKED|RW_WRITE_WANTED), %r2 - /* write active or pending? */ - bneq 3f /* yep, go slow */ - addl3 $RW_READ_INCR, %r2, %r3 /* incr. reader count (new) */ -#ifndef MULTIPROCESSOR - mfpr $PR_SSP, %r4 - addl3 $CI_CAS_ADDR, L_CPU(%r4), %r4 /* r4 == &curcpu()->ci_cas_addr */ -#endif -1: bsbw _do_cas+2 /* do the compare-and-swap */ - cmpl %r0, %r2 /* did it succeed? */ - bneq 3f /* nope, go slow */ - ret /* yep, return */ - -2: clrl %r2 /* get old value (zero) */ - mfpr $PR_SSP, %r3 /* get new value (curlwp) */ -#ifndef MULTIPROCESSOR - addl3 $CI_CAS_ADDR, L_CPU(%r3), %r4 /* r4 == &curcpu()->ci_cas_addr */ -#endif - bisl2 $RW_WRITE_LOCKED, %r3 /* show that it's a write */ - brb 1b /* do the compare-and-swap */ - -3: callg (%ap), _C_LABEL(rw_vector_enter) - ret - -/* - * void rw_exit(krwlock_t *rwl, krw_t op); - */ -ENTRY(rw_exit, 0) - movl 4(%ap), %r1 /* grab rwlock ptr */ - movl (%r1), %r2 /* grab owner (old) */ - bitl $RW_WRITE_LOCKED, %r2 /* is it write locked? */ - bneq 2f /* yep, do the write case */ - - blbs %r2, 3f /* RW_HAS_WAITERS mbz */ - subl3 $RW_READ_INCR, %r2, %r3 /* decr. reader count (new) */ - blss 3f /* if less than 0, go slow */ -#ifndef MULTIPROCESSOR - mfpr $PR_SSP, %r4 /* get curlwp */ - addl3 $CI_CAS_ADDR, L_CPU(%r4), %r4 /* r4 == &curcpu()->ci_cas_addr */ -#endif -1: bsbw _do_cas+2 /* do the compare-and-swap */ - cmpl %r0, %r2 /* did it succeed? */ - bneq 3f /* nope, go slow */ - ret /* yes, return */ - -2: mfpr $PR_SSP, %r2 /* get old (curlwp) */ -#ifndef MULTIPROCESSOR - addl3 $CI_CAS_ADDR, L_CPU(%r2), %r4 /* r4 == &curcpu()->ci_cas_addr */ -#endif - bisl2 $RW_WRITE_LOCKED, %r2 /* show that it's a write */ - clrl %r3 /* get new (zero) */ - brb 1b /* do the compare-and-swap */ - -3: callg (%ap), _C_LABEL(rw_vector_exit) - ret - -/* - * bool rw_tryenter(krwlock_t *krw, krw_t op); - */ -ENTRY(rw_tryenter, 0) - movl 4(%ap), %r1 /* get rwlock ptr */ - tstl 8(%ap) /* is this a reader op? */ - bneq 3f /* nope, branch to writer */ - - movl (%r1), %r2 /* get owner field (old) */ - bitl $(RW_WRITE_LOCKED|RW_WRITE_WANTED), %r2 - /* write active or pending? */ - bneq 2f /* yes, return failure */ - addl3 $RW_READ_INCR, %r2, %r3 /* incr reader count (new) */ -#ifndef MULTIPROCESSOR - mfpr $PR_SSP, %r4 - addl3 $CI_CAS_ADDR, L_CPU(%r4), %r4 /* r4 == &curcpu()->ci_cas_addr */ -#endif -1: bsbw _do_cas+2 /* do the compare-and-swap */ - cmpl %r0, %r2 /* did it succeed? */ - bneq 2f /* no, we failed. */ - movl $1,%r0 /* yes, indicate success */ - ret /* return */ -2: clrl %r0 /* indicate failure */ - ret /* return */ - -3: clrl %r2 /* set old value (0) */ - mfpr $PR_SSP, %r3 /* set new value (curlwp) */ -#ifndef MULTIPROCESSOR - addl3 $CI_CAS_ADDR, L_CPU(%r3), %r4 /* r4 == &curcpu()->ci_cas_addr */ -#endif - bisl2 $RW_WRITE_LOCKED, %r3 /* show that it's a write */ - brb 1b /* do the compare-and-swap */ -#endif /* LOCKDEBUG */ - /* * _atomic_cas_32(volatile uint32_t *p, uint32_t old, uint32_t new); */ Index: kern/kern_rwlock.c =================================================================== RCS file: /cvsroot/src/sys/kern/kern_rwlock.c,v retrieving revision 1.65 diff -u -p -r1.65 kern_rwlock.c --- kern/kern_rwlock.c 22 Feb 2020 21:24:45 -0000 1.65 +++ kern/kern_rwlock.c 27 Feb 2020 22:36:11 -0000 @@ -38,10 +38,10 @@ * Richard McDougall. * * The NetBSD implementation differs from that described in the book, in - * that the locks are partially adaptive. Lock waiters spin wait while a - * lock is write held and the holder is still running on a CPU. The method - * of choosing which threads to awaken when a lock is released also differs, - * mainly to take account of the partially adaptive behaviour. + * that the locks are adaptive. Lock waiters spin wait while a lock is held + * and the holder is still running on a CPU. The method of choosing which + * threads to awaken when a lock is released also differs, to take account + * of the partially adaptive behaviour. */ #include @@ -65,8 +65,6 @@ __KERNEL_RCSID(0, "$NetBSD: kern_rwlock. #include -#include - /* * LOCKDEBUG */ @@ -110,19 +108,6 @@ do { \ #define RW_MEMBAR_PRODUCER() membar_producer() #endif -/* - * For platforms that do not provide stubs, or for the LOCKDEBUG case. - */ -#ifdef LOCKDEBUG -#undef __HAVE_RW_STUBS -#endif - -#ifndef __HAVE_RW_STUBS -__strong_alias(rw_enter,rw_vector_enter); -__strong_alias(rw_exit,rw_vector_exit); -__strong_alias(rw_tryenter,rw_vector_tryenter); -#endif - static void rw_abort(const char *, size_t, krwlock_t *, const char *); static void rw_dump(const volatile void *, lockop_printer_t); static lwp_t *rw_owner(wchan_t); @@ -155,6 +140,18 @@ rw_cas(krwlock_t *rw, uintptr_t o, uintp } /* + * rw_and: + * + * Do an atomic AND on the lock word. + */ +static inline void +rw_and(krwlock_t *rw, uintptr_t m) +{ + + atomic_and_ulong(&rw->rw_owner, m); +} + +/* * rw_swap: * * Do an atomic swap of the lock word. This is used only when it's @@ -173,6 +170,75 @@ rw_swap(krwlock_t *rw, uintptr_t o, uint } /* + * rw_enter_lwp: + * + * Helper - when acquring a lock, record the new hold. + */ +static inline uintptr_t +rw_enter_lwp(krwlock_t *rw, lwp_t *l) +{ + int i; + + KASSERT(kpreempt_disabled()); + + for (i = 0; i < __arraycount(l->l_rwlocks); i++) { + if (__predict_true(l->l_rwlocks[i] == NULL)) { + l->l_rwlocks[i] = rw; + /* + * Clear the write wanted flag on every acquire to + * give readers a chance once again. + */ + return ~RW_WRITE_WANTED; + } + } + + /* + * Nowhere to track the hold so we lose: temporarily disable + * spinning on the lock. + */ + return ~(RW_WRITE_WANTED | RW_SPIN); +} + +/* + * rw_exit_lwp: + * + * Helper - when releasing a lock, stop tracking the hold. + */ +static inline void +rw_exit_lwp(krwlock_t *rw, lwp_t *l) +{ + int i; + + KASSERT(kpreempt_disabled()); + + for (i = 0; i < __arraycount(l->l_rwlocks); i++) { + if (__predict_true(l->l_rwlocks[i] == rw)) { + l->l_rwlocks[i] = NULL; + return; + } + } +} + +/* + * rw_switch: + * + * Called by mi_switch() to indicate that an LWP is going off the CPU. + */ +void +rw_switch(void) +{ + lwp_t *l = curlwp; + int i; + + for (i = 0; i < __arraycount(l->l_rwlocks); i++) { + if (l->l_rwlocks[i] != NULL) { + rw_and(l->l_rwlocks[i], ~RW_SPIN); + /* Leave in place for exit to clear. */ + } + } +} + +/* * rw_dump: * * Dump the contents of a rwlock structure. @@ -212,15 +278,10 @@ void _rw_init(krwlock_t *rw, uintptr_t return_address) { -#ifdef LOCKDEBUG - /* XXX only because the assembly stubs can't handle RW_NODEBUG */ if (LOCKDEBUG_ALLOC(rw, &rwlock_lockops, return_address)) - rw->rw_owner = 0; + rw->rw_owner = RW_SPIN; else - rw->rw_owner = RW_NODEBUG; -#else - rw->rw_owner = 0; -#endif + rw->rw_owner = RW_SPIN | RW_NODEBUG; } void @@ -239,55 +300,65 @@ void rw_destroy(krwlock_t *rw) { - RW_ASSERT(rw, (rw->rw_owner & ~RW_NODEBUG) == 0); + RW_ASSERT(rw, (rw->rw_owner & ~(RW_NODEBUG | RW_SPIN)) == 0); LOCKDEBUG_FREE((rw->rw_owner & RW_NODEBUG) == 0, rw); } /* - * rw_oncpu: + * rw_must_sleep: * - * Return true if an rwlock owner is running on a CPU in the system. - * If the target is waiting on the kernel big lock, then we must - * release it. This is necessary to avoid deadlock. + * Return true spinning is not possible and the lock requestor must + * sleep. This can happen in a couple of cases: + * + * 1) If an LWP on this CPU (possibly curlwp, or an LWP that curlwp has + * interupted) holds kernel_lock, we can't spin without a deadlock. + * The CPU that holds the rwlock may be blocked trying to acquire + * kernel_lock, or there may be an unseen chain of dependant locks. + * This LWP needs to sleep (and thereby directly drop kernel_lock, or + * permit the interrupted LWP that holds kernel_lock to complete its + * work). + * + * 2) If the lock holder is pinned beneath a soft interrupt LWP on this + * CPU, the curlwp must sleep for the lock holder to make progress and + * release the lock. */ static bool -rw_oncpu(uintptr_t owner) +rw_must_sleep(krwlock_t *rw, lwp_t *l) { -#ifdef MULTIPROCESSOR - struct cpu_info *ci; - lwp_t *l; + lwp_t *t; + int i; KASSERT(kpreempt_disabled()); - if ((owner & (RW_WRITE_LOCKED|RW_HAS_WAITERS)) != RW_WRITE_LOCKED) { - return false; - } + /* Kernel lock held ? */ + if (curcpu()->ci_biglock_count != 0) + return true; /* - * See lwp_dtor() why dereference of the LWP pointer is safe. - * We must have kernel preemption disabled for that. + * Walk down the chain of LWPs pinned beneath this one and look for + * a tracked hold of the lock we're trying to acquire. */ - l = (lwp_t *)(owner & RW_THREAD); - ci = l->l_cpu; - - if (ci && ci->ci_curlwp == l) { - /* Target is running; do we need to block? */ - return (ci->ci_biglock_wanted != l); + for (t = l->l_switchto; t != NULL; t = t->l_switchto) { + KASSERT(cpu_softintr_p()); + KASSERT(t->l_cpu == curcpu()); + for (i = 0; i < __arraycount(t->l_rwlocks); i++) { + if (t->l_rwlocks[i] == rw) + return true; + } } -#endif - /* Not running. It may be safe to block now. */ return false; } /* * rw_vector_enter: * - * Acquire a rwlock. + * The slow path for acquiring a rwlock, that considers all conditions. + * Marked __noinline to prevent the compiler pulling it into rw_enter(). */ -void -rw_vector_enter(krwlock_t *rw, const krw_t op) +static void __noinline +rw_vector_enter(krwlock_t *rw, const krw_t op, uintptr_t mask, uintptr_t ra) { - uintptr_t owner, incr, need_wait, set_wait, curthread, next; + uintptr_t owner, incr, need_wait, set_wait, next; turnstile_t *ts; int queue; lwp_t *l; @@ -297,12 +368,8 @@ rw_vector_enter(krwlock_t *rw, const krw LOCKSTAT_COUNTER(spincnt); LOCKSTAT_FLAG(lsflag); - l = curlwp; - curthread = (uintptr_t)l; - RW_ASSERT(rw, !cpu_intr_p()); - RW_ASSERT(rw, curthread != 0); - RW_WANTLOCK(rw, op); + RW_ASSERT(rw, kpreempt_disabled()); if (panicstr == NULL) { KDASSERT(pserialize_not_in_read_section()); @@ -318,14 +385,15 @@ rw_vector_enter(krwlock_t *rw, const krw * therefore we can use an add operation to set them, which * means an add operation for both cases. */ - if (__predict_true(op == RW_READER)) { + l = curlwp; + if (op == RW_READER) { incr = RW_READ_INCR; set_wait = RW_HAS_WAITERS; need_wait = RW_WRITE_LOCKED | RW_WRITE_WANTED; queue = TS_READER_Q; } else { RW_ASSERT(rw, op == RW_WRITER); - incr = curthread | RW_WRITE_LOCKED; + incr = (uintptr_t)l | RW_WRITE_LOCKED; set_wait = RW_HAS_WAITERS | RW_WRITE_WANTED; need_wait = RW_WRITE_LOCKED | RW_THREAD; queue = TS_WRITER_Q; @@ -333,15 +401,13 @@ rw_vector_enter(krwlock_t *rw, const krw LOCKSTAT_ENTER(lsflag); - KPREEMPT_DISABLE(curlwp); for (owner = rw->rw_owner;;) { /* * Read the lock owner field. If the need-to-wait * indicator is clear, then try to acquire the lock. */ if ((owner & need_wait) == 0) { - next = rw_cas(rw, owner, (owner + incr) & - ~RW_WRITE_WANTED); + next = rw_cas(rw, owner, (owner + incr) & mask); if (__predict_true(next == owner)) { /* Got it! */ RW_MEMBAR_ENTER(); @@ -355,15 +421,34 @@ rw_vector_enter(krwlock_t *rw, const krw owner = next; continue; } - if (__predict_false(RW_OWNER(rw) == curthread)) { + if (__predict_false(RW_OWNER(rw) == (uintptr_t)l)) { rw_abort(__func__, __LINE__, rw, "locking against myself"); } + /* - * If the lock owner is running on another CPU, and - * there are no existing waiters, then spin. + * If the lock owner is running on another CPU, and there + * are no existing waiters, then spin. + * + * If trying to acquire a write lock, and the lock is + * currently read held, after a brief wait set the write + * wanted bit to block out new readers and try to avoid + * starvation. When the hold is acquired, we'll clear the + * WRITE_WANTED flag to give readers a chance again. With + * luck this should nudge things in the direction of + * interleaving readers and writers when there is high + * contention. */ - if (rw_oncpu(owner)) { + if ((owner & RW_SPIN) != 0 && rw_must_sleep(rw, l)) { + next = rw_cas(rw, owner, owner & ~RW_SPIN); + if (next != owner) { + owner = next; + continue; + } + owner &= ~RW_SPIN; + } + if ((owner & RW_SPIN) != 0) { + RW_ASSERT(rw, (owner & RW_HAS_WAITERS) == 0); LOCKSTAT_START_TIMER(lsflag, spintime); u_int count = SPINLOCK_BACKOFF_MIN; do { @@ -371,7 +456,17 @@ rw_vector_enter(krwlock_t *rw, const krw SPINLOCK_BACKOFF(count); KPREEMPT_DISABLE(curlwp); owner = rw->rw_owner; - } while (rw_oncpu(owner)); + if ((owner & need_wait) == 0) + break; + if (count < SPINLOCK_BACKOFF_MAX) + continue; + if (op == RW_WRITER && + (owner & RW_WRITE_LOCKED) == 0 && + (owner & RW_WRITE_WANTED) == 0) { + owner = rw_cas(rw, owner, + owner | RW_WRITE_WANTED); + } + } while ((owner & RW_SPIN) != 0); LOCKSTAT_STOP_TIMER(lsflag, spintime); LOCKSTAT_COUNT(spincnt, 1); if ((owner & need_wait) == 0) @@ -385,17 +480,17 @@ rw_vector_enter(krwlock_t *rw, const krw ts = turnstile_lookup(rw); /* - * Mark the rwlock as having waiters. If the set fails, - * then we may not need to sleep and should spin again. - * Reload rw_owner because turnstile_lookup() may have - * spun on the turnstile chain lock. + * Mark the rwlock as having waiters, and disable spinning. + * If the set fails, then we may not need to sleep and + * should spin again. Reload rw_owner now that we own + * the turnstile chain lock. */ owner = rw->rw_owner; - if ((owner & need_wait) == 0 || rw_oncpu(owner)) { + if ((owner & need_wait) == 0) { turnstile_exit(rw); continue; } - next = rw_cas(rw, owner, owner | set_wait); + next = rw_cas(rw, owner, (owner | set_wait) & ~RW_SPIN); if (__predict_false(next != owner)) { turnstile_exit(rw); owner = next; @@ -411,43 +506,98 @@ rw_vector_enter(krwlock_t *rw, const krw * No need for a memory barrier because of context switch. * If not handed the lock, then spin again. */ - if (op == RW_READER || (rw->rw_owner & RW_THREAD) == curthread) + if (op == RW_READER) break; - owner = rw->rw_owner; + if ((owner & RW_THREAD) == (uintptr_t)l) + break; } KPREEMPT_ENABLE(curlwp); LOCKSTAT_EVENT_RA(lsflag, rw, LB_RWLOCK | (op == RW_WRITER ? LB_SLEEP1 : LB_SLEEP2), slpcnt, slptime, - (l->l_rwcallsite != 0 ? l->l_rwcallsite : - (uintptr_t)__builtin_return_address(0))); + (l->l_rwcallsite != 0 ? l->l_rwcallsite : ra)); LOCKSTAT_EVENT_RA(lsflag, rw, LB_RWLOCK | LB_SPIN, spincnt, spintime, - (l->l_rwcallsite != 0 ? l->l_rwcallsite : - (uintptr_t)__builtin_return_address(0))); + (l->l_rwcallsite != 0 ? l->l_rwcallsite : ra)); LOCKSTAT_EXIT(lsflag); - RW_ASSERT(rw, (op != RW_READER && RW_OWNER(rw) == curthread) || + RW_ASSERT(rw, (op != RW_READER && RW_OWNER(rw) == (uintptr_t)l) || (op == RW_READER && RW_COUNT(rw) != 0)); RW_LOCKED(rw, op); } /* - * rw_vector_exit: + * rw_enter: * - * Release a rwlock. + * The fast path for acquiring a lock that considers only the + * uncontended case. Falls back to rw_vector_enter(). */ void +rw_enter(krwlock_t *rw, const krw_t op) +{ + uintptr_t owner, incr, need_wait, next, mask; + lwp_t *l; + + RW_ASSERT(rw, !cpu_intr_p()); + RW_ASSERT(rw, curlwp != NULL); + RW_WANTLOCK(rw, op); + + l = curlwp; + KPREEMPT_DISABLE(l); + mask = rw_enter_lwp(rw, l); + + /* + * We play a slight trick here. If we're a reader, we want + * increment the read count. If we're a writer, we want to + * set the owner field and the WRITE_LOCKED bit. + * + * In the latter case, we expect those bits to be zero, + * therefore we can use an add operation to set them, which + * means an add operation for both cases. + */ + if (op == RW_READER) { + incr = RW_READ_INCR; + need_wait = RW_WRITE_LOCKED | RW_WRITE_WANTED; + } else { + RW_ASSERT(rw, op == RW_WRITER); + incr = (uintptr_t)l | RW_WRITE_LOCKED; + need_wait = RW_WRITE_LOCKED | RW_THREAD; + } + + /* + * Read the lock owner field. If the need-to-wait + * indicator is clear, then try to acquire the lock. + */ + owner = rw->rw_owner; + if (__predict_true((owner & need_wait) == 0)) { + next = rw_cas(rw, owner, (owner + incr) & mask); + if (__predict_true(next == owner)) { + /* Got it! */ + RW_LOCKED(rw, op); + KPREEMPT_ENABLE(l); + RW_MEMBAR_ENTER(); + return; + } + } + + rw_vector_enter(rw, op, mask, (uintptr_t)__builtin_return_address(0)); +} + +/* + * rw_vector_exit: + * + * The slow path for releasing a rwlock, that considers all conditions. + * Marked __noinline to prevent the compiler pulling it into rw_enter(). + */ +static void __noinline rw_vector_exit(krwlock_t *rw) { - uintptr_t curthread, owner, decr, newown, next; + uintptr_t owner, decr, newown, next; turnstile_t *ts; int rcnt, wcnt; lwp_t *l; - l = curlwp; - curthread = (uintptr_t)l; - RW_ASSERT(rw, curthread != 0); + RW_ASSERT(rw, kpreempt_disabled()); /* * Again, we use a trick. Since we used an add operation to @@ -455,13 +605,12 @@ rw_vector_exit(krwlock_t *rw) * them, which makes the read-release and write-release path * the same. */ + l = curlwp; owner = rw->rw_owner; if (__predict_false((owner & RW_WRITE_LOCKED) != 0)) { - RW_UNLOCKED(rw, RW_WRITER); - RW_ASSERT(rw, RW_OWNER(rw) == curthread); - decr = curthread | RW_WRITE_LOCKED; + RW_ASSERT(rw, RW_OWNER(rw) == (uintptr_t)l); + decr = (uintptr_t)l | RW_WRITE_LOCKED; } else { - RW_UNLOCKED(rw, RW_READER); RW_ASSERT(rw, RW_COUNT(rw) != 0); decr = RW_READ_INCR; } @@ -476,12 +625,21 @@ rw_vector_exit(krwlock_t *rw) newown = (owner - decr); if ((newown & (RW_THREAD | RW_HAS_WAITERS)) == RW_HAS_WAITERS) break; + /* Want spinning enabled if lock is becoming free. */ + if ((newown & RW_THREAD) == 0) + newown |= RW_SPIN; next = rw_cas(rw, owner, newown); - if (__predict_true(next == owner)) + if (__predict_true(next == owner)) { + rw_exit_lwp(rw, l); + KPREEMPT_ENABLE(l); return; + } owner = next; } + /* If there are waiters, there can't be spinners. */ + RW_ASSERT(rw, (newown & RW_SPIN) == 0); + /* * Grab the turnstile chain lock. This gets the interlock * on the sleep queue. Once we have that, we can adjust the @@ -494,6 +652,7 @@ rw_vector_exit(krwlock_t *rw) wcnt = TS_WAITERS(ts, TS_WRITER_Q); rcnt = TS_WAITERS(ts, TS_READER_Q); + RW_ASSERT(rw, wcnt + rcnt > 0); /* * Give the lock away. @@ -506,23 +665,21 @@ rw_vector_exit(krwlock_t *rw) * set WRITE_WANTED to block out new readers, and let them * do the work of acquiring the lock in rw_vector_enter(). */ - if (rcnt == 0 || decr == RW_READ_INCR) { - RW_ASSERT(rw, wcnt != 0); + if (wcnt > 0 && (rcnt == 0 || decr == RW_READ_INCR)) { RW_ASSERT(rw, (owner & RW_WRITE_WANTED) != 0); if (rcnt != 0) { /* Give the lock to the longest waiting writer. */ - l = TS_FIRST(ts, TS_WRITER_Q); - newown = (uintptr_t)l | (owner & RW_NODEBUG); + lwp_t *l2 = TS_FIRST(ts, TS_WRITER_Q); + newown = (uintptr_t)l2 | (owner & RW_NODEBUG); newown |= RW_WRITE_LOCKED | RW_HAS_WAITERS; if (wcnt > 1) newown |= RW_WRITE_WANTED; rw_swap(rw, owner, newown); - turnstile_wakeup(ts, TS_WRITER_Q, 1, l); + turnstile_wakeup(ts, TS_WRITER_Q, 1, l2); } else { /* Wake all writers and let them fight it out. */ - newown = owner & RW_NODEBUG; - newown |= RW_WRITE_WANTED; + newown = (owner & RW_NODEBUG) | RW_SPIN; rw_swap(rw, owner, newown); turnstile_wakeup(ts, TS_WRITER_Q, wcnt, NULL); } @@ -543,37 +700,94 @@ rw_vector_exit(krwlock_t *rw) rw_swap(rw, owner, newown); turnstile_wakeup(ts, TS_READER_Q, rcnt, NULL); } + rw_exit_lwp(rw, l); + KPREEMPT_ENABLE(l); } /* - * rw_vector_tryenter: + * rw_exit: + * + * The fast path for releasing a lock that considers only the + * uncontended case. Falls back to rw_vector_exit(). + */ +void +rw_exit(krwlock_t *rw) +{ + uintptr_t owner, decr, newown, next; + lwp_t *l; + + RW_ASSERT(rw, curlwp != NULL); + + /* + * Again, we use a trick. Since we used an add operation to + * set the required lock bits, we can use a subtract to clear + * them, which makes the read-release and write-release path + * the same. + */ + l = curlwp; + owner = rw->rw_owner; + if ((owner & RW_WRITE_LOCKED) != 0) { + RW_UNLOCKED(rw, RW_WRITER); + RW_ASSERT(rw, RW_OWNER(rw) == (uintptr_t)l); + decr = (uintptr_t)l | RW_WRITE_LOCKED; + } else { + RW_UNLOCKED(rw, RW_READER); + RW_ASSERT(rw, RW_COUNT(rw) != 0); + decr = RW_READ_INCR; + } + + /* Now try to release it. */ + RW_MEMBAR_EXIT(); + KPREEMPT_DISABLE(l); + newown = (owner - decr); + if (__predict_true((newown & (RW_THREAD | RW_HAS_WAITERS)) != + RW_HAS_WAITERS)) { + /* Want spinning (re-)enabled if lock is becoming free. */ + if ((newown & RW_THREAD) == 0) + newown |= RW_SPIN; + next = rw_cas(rw, owner, newown); + if (__predict_true(next == owner)) { + rw_exit_lwp(rw, l); + KPREEMPT_ENABLE(l); + return; + } + } + rw_vector_exit(rw); +} + +/* + * rw_tryenter: * * Try to acquire a rwlock. */ int -rw_vector_tryenter(krwlock_t *rw, const krw_t op) +rw_tryenter(krwlock_t *rw, const krw_t op) { - uintptr_t curthread, owner, incr, need_wait, next; + uintptr_t owner, incr, need_wait, next, mask; lwp_t *l; - l = curlwp; - curthread = (uintptr_t)l; + RW_ASSERT(rw, curlwp != NULL); - RW_ASSERT(rw, curthread != 0); + l = curlwp; + KPREEMPT_DISABLE(l); + mask = rw_enter_lwp(rw, l); if (op == RW_READER) { incr = RW_READ_INCR; need_wait = RW_WRITE_LOCKED | RW_WRITE_WANTED; } else { RW_ASSERT(rw, op == RW_WRITER); - incr = curthread | RW_WRITE_LOCKED; + incr = (uintptr_t)l | RW_WRITE_LOCKED; need_wait = RW_WRITE_LOCKED | RW_THREAD; } for (owner = rw->rw_owner;; owner = next) { - if (__predict_false((owner & need_wait) != 0)) + if (__predict_false((owner & need_wait) != 0)) { + rw_exit_lwp(rw, l); + KPREEMPT_ENABLE(l); return 0; - next = rw_cas(rw, owner, owner + incr); + } + next = rw_cas(rw, owner, (owner + incr) & mask); if (__predict_true(next == owner)) { /* Got it! */ break; @@ -582,9 +796,10 @@ rw_vector_tryenter(krwlock_t *rw, const RW_WANTLOCK(rw, op); RW_LOCKED(rw, op); - RW_ASSERT(rw, (op != RW_READER && RW_OWNER(rw) == curthread) || + RW_ASSERT(rw, (op != RW_READER && RW_OWNER(rw) == (uintptr_t)l) || (op == RW_READER && RW_COUNT(rw) != 0)); + KPREEMPT_ENABLE(l); RW_MEMBAR_ENTER(); return 1; } @@ -597,20 +812,14 @@ rw_vector_tryenter(krwlock_t *rw, const void rw_downgrade(krwlock_t *rw) { - uintptr_t owner, curthread, newown, next; + uintptr_t owner, newown, next; turnstile_t *ts; int rcnt, wcnt; - lwp_t *l; - l = curlwp; - curthread = (uintptr_t)l; - RW_ASSERT(rw, curthread != 0); + RW_ASSERT(rw, curlwp != NULL); RW_ASSERT(rw, (rw->rw_owner & RW_WRITE_LOCKED) != 0); - RW_ASSERT(rw, RW_OWNER(rw) == curthread); + RW_ASSERT(rw, RW_OWNER(rw) == (uintptr_t)curlwp); RW_UNLOCKED(rw, RW_WRITER); -#if !defined(DIAGNOSTIC) - __USE(curthread); -#endif RW_MEMBAR_PRODUCER(); @@ -649,7 +858,8 @@ rw_downgrade(krwlock_t *rw) /* * If there are no readers, just preserve the * waiters bits, swap us down to one read hold and - * return. + * return. Don't set the spin bit as nobody's + * running yet. */ RW_ASSERT(rw, wcnt != 0); RW_ASSERT(rw, (rw->rw_owner & RW_WRITE_WANTED) != 0); @@ -667,7 +877,8 @@ rw_downgrade(krwlock_t *rw) * Give the lock to all blocked readers. We may * retain one read hold if downgrading. If there is * a writer waiting, new readers will be blocked - * out. + * out. Don't set the spin bit as nobody's running + * yet. */ newown = owner & RW_NODEBUG; newown += (rcnt << RW_READ_COUNT_SHIFT) + RW_READ_INCR; @@ -699,16 +910,15 @@ rw_downgrade(krwlock_t *rw) int rw_tryupgrade(krwlock_t *rw) { - uintptr_t owner, curthread, newown, next; + uintptr_t owner, newown, next; struct lwp *l; - l = curlwp; - curthread = (uintptr_t)l; - RW_ASSERT(rw, curthread != 0); + RW_ASSERT(rw, curlwp != NULL); RW_ASSERT(rw, rw_read_held(rw)); + l = curlwp; for (owner = RW_READ_INCR;; owner = next) { - newown = curthread | RW_WRITE_LOCKED | (owner & ~RW_THREAD); + newown = (uintptr_t)l | RW_WRITE_LOCKED | (owner & ~RW_THREAD); next = rw_cas(rw, owner, newown); if (__predict_true(next == owner)) { RW_MEMBAR_PRODUCER(); @@ -725,7 +935,7 @@ rw_tryupgrade(krwlock_t *rw) RW_WANTLOCK(rw, RW_WRITER); RW_LOCKED(rw, RW_WRITER); RW_ASSERT(rw, rw->rw_owner & RW_WRITE_LOCKED); - RW_ASSERT(rw, RW_OWNER(rw) == curthread); + RW_ASSERT(rw, RW_OWNER(rw) == (uintptr_t)l); return 1; } @@ -817,22 +1027,14 @@ rw_owner(wchan_t obj) /* * rw_owner_running: * - * Return true if a RW lock is unheld, or write held and the owner is - * running on a CPU. For the pagedaemon. + * Return true if a RW lock is unheld, or held and the owner is running + * on a CPU. For the pagedaemon only - do not document or use in other + * code. */ bool rw_owner_running(const krwlock_t *rw) { -#ifdef MULTIPROCESSOR - uintptr_t owner; - bool rv; + uintptr_t owner = rw->rw_owner; - kpreempt_disable(); - owner = rw->rw_owner; - rv = (owner & RW_THREAD) == 0 || rw_oncpu(owner); - kpreempt_enable(); - return rv; -#else - return rw_owner(rw) == curlwp; -#endif + return (owner & RW_THREAD) == 0 || (owner & RW_SPIN) != 0; } Index: kern/kern_synch.c =================================================================== RCS file: /cvsroot/src/sys/kern/kern_synch.c,v retrieving revision 1.342 diff -u -p -r1.342 kern_synch.c --- kern/kern_synch.c 23 Feb 2020 16:27:09 -0000 1.342 +++ kern/kern_synch.c 27 Feb 2020 22:36:11 -0000 @@ -75,6 +75,7 @@ __KERNEL_RCSID(0, "$NetBSD: kern_synch.c #include "opt_dtrace.h" #define __MUTEX_PRIVATE +#define __RWLOCK_PRIVATE #include #include @@ -714,6 +715,9 @@ mi_switch(lwp_t *l) /* We're down to only one lock, so do debug checks. */ LOCKDEBUG_BARRIER(l->l_mutex, 1); + /* Disable spinning on any R/W locks that we hold. */ + rw_switch(); + /* Count the context switch. */ CPU_COUNT(CPU_COUNT_NSWTCH, 1); l->l_ncsw++; Index: sys/lwp.h =================================================================== RCS file: /cvsroot/src/sys/sys/lwp.h,v retrieving revision 1.202 diff -u -p -r1.202 lwp.h --- sys/lwp.h 15 Feb 2020 18:12:15 -0000 1.202 +++ sys/lwp.h 27 Feb 2020 22:36:12 -0000 @@ -186,6 +186,7 @@ struct lwp { u_short l_exlocks; /* !: lockdebug: excl. locks held */ u_short l_psrefs; /* !: count of psref held */ u_short l_blcnt; /* !: count of kernel_lock held */ + struct krwlock *l_rwlocks[8]; /* !: tracks first N held rwlocks */ volatile int l_nopreempt; /* !: don't preempt me! */ volatile u_int l_dopreempt; /* s: kernel preemption pending */ int l_pflag; /* !: LWP private flags */ Index: sys/rwlock.h =================================================================== RCS file: /cvsroot/src/sys/sys/rwlock.h,v retrieving revision 1.16 diff -u -p -r1.16 rwlock.h --- sys/rwlock.h 22 Feb 2020 21:24:45 -0000 1.16 +++ sys/rwlock.h 27 Feb 2020 22:36:12 -0000 @@ -71,6 +71,7 @@ typedef struct krwlock krwlock_t; #define RW_HAS_WAITERS 0x01UL /* lock has waiters */ #define RW_WRITE_WANTED 0x02UL /* >= 1 waiter is a writer */ #define RW_WRITE_LOCKED 0x04UL /* lock is currently write locked */ +#define RW_SPIN 0x08UL #define RW_NODEBUG 0x10UL /* LOCKDEBUG disabled */ #define RW_READ_COUNT_SHIFT 5 @@ -80,11 +81,9 @@ typedef struct krwlock krwlock_t; #define RW_COUNT(rw) ((rw)->rw_owner & RW_THREAD) #define RW_FLAGS(rw) ((rw)->rw_owner & ~RW_THREAD) -void rw_vector_enter(krwlock_t *, const krw_t); -void rw_vector_exit(krwlock_t *); -int rw_vector_tryenter(krwlock_t *, const krw_t); void _rw_init(krwlock_t *, uintptr_t); bool rw_owner_running(const krwlock_t *); +void rw_switch(void); #endif /* __RWLOCK_PRIVATE */ struct krwlock {