Index: sys/arch/aarch64/aarch64/copyinout.S
===================================================================
RCS file: /cvsroot/src/sys/arch/aarch64/aarch64/copyinout.S,v
retrieving revision 1.9
diff -u -p -r1.9 copyinout.S
--- sys/arch/aarch64/aarch64/copyinout.S	14 Sep 2018 13:47:14 -0000	1.9
+++ sys/arch/aarch64/aarch64/copyinout.S	16 Jun 2020 22:56:38 -0000
@@ -59,8 +59,7 @@ RCSID("$NetBSD: copyinout.S,v 1.9 2018/0
 
 	.macro exit_cpu_onfault
 	/* curlwp->l_md.md_onfault = NULL */
-	mrs	x0, tpidr_el1			/* curcpu */
-	ldr	x0, [x0, #CI_CURLWP]		/* x0 = curlwp */
+	mrs	x0, tpidr_el1			/* x0 = curlwp */
 	str	xzr, [x0, #L_MD_ONFAULT]	/* lwp->l_md_onfault = NULL */
 9:
 	add	sp, sp, #FB_T_SIZE		/* pop stack */
Index: sys/arch/aarch64/aarch64/cpu_machdep.c
===================================================================
RCS file: /cvsroot/src/sys/arch/aarch64/aarch64/cpu_machdep.c,v
retrieving revision 1.10
diff -u -p -r1.10 cpu_machdep.c
--- sys/arch/aarch64/aarch64/cpu_machdep.c	21 May 2020 05:41:40 -0000	1.10
+++ sys/arch/aarch64/aarch64/cpu_machdep.c	16 Jun 2020 22:56:38 -0000
@@ -261,7 +261,7 @@ cpu_need_resched(struct cpu_info *ci, st
 		intr_ipi_send(ci->ci_kcpuset, IPI_AST);
 #endif
 	} else {
-		setsoftast(ci);	/* force call to ast() */
+		l->l_md.md_astpending = 1;
 	}
 }
 
@@ -272,7 +272,22 @@ cpu_need_proftick(struct lwp *l)
 	KASSERT(l->l_cpu == curcpu());
 
 	l->l_pflag |= LP_OWEUPC;
-	setsoftast(l->l_cpu);
+	l->l_md.md_astpending = 1;
+}
+
+void
+cpu_signotify(struct lwp *l)
+{
+
+	KASSERT(kpreempt_disabled());
+
+	if (l->l_cpu != curcpu()) {
+#ifdef MULTIPROCESSOR
+		intr_ipi_send(l->l_cpu->ci_kcpuset, IPI_AST);
+#endif
+	} else {
+		l->l_md.md_astpending = 1;
+	}
 }
 
 #ifdef __HAVE_PREEMPTION
Index: sys/arch/aarch64/aarch64/cpuswitch.S
===================================================================
RCS file: /cvsroot/src/sys/arch/aarch64/aarch64/cpuswitch.S,v
retrieving revision 1.21
diff -u -p -r1.21 cpuswitch.S
--- sys/arch/aarch64/aarch64/cpuswitch.S	23 May 2020 18:08:59 -0000	1.21
+++ sys/arch/aarch64/aarch64/cpuswitch.S	16 Jun 2020 22:56:38 -0000
@@ -1,7 +1,7 @@
 /* $NetBSD: cpuswitch.S,v 1.21 2020/05/23 18:08:59 ryo Exp $ */
 
 /*-
- * Copyright (c) 2014 The NetBSD Foundation, Inc.
+ * Copyright (c) 2014, 2020 The NetBSD Foundation, Inc.
  * All rights reserved.
  *
  * This code is derived from software contributed to The NetBSD Foundation
@@ -116,8 +116,10 @@ ENTRY_NP(cpu_switchto)
 1:
 #endif
 
-	mrs	x3, tpidr_el1
+	msr	tpidr_el1, x1		/* switch curlwp to new lwp */
+	ldr	x3, [x1, #L_CPU]
 	str	x1, [x3, #CI_CURLWP]	/* switch curlwp to new lwp */
+	dmb	st			/* see comments in kern_mutex.c */
 	ENABLE_INTERRUPT
 
 	/*
@@ -161,15 +163,17 @@ ENTRY_NP(cpu_switchto_softint)
 	stp	x27, x28, [sp, #TF_X27]
 	stp	x29, x2, [sp, #TF_X29]	/* tf->lr = softint_cleanup; */
 
-	mrs	x3, tpidr_el1		/* x3 := curcpu() */
+	mrs	x19, tpidr_el1		/* x19 := curcpu()->ci_curlwp */
 	DISABLE_INTERRUPT
-	ldr	x19, [x3, #CI_CURLWP]	/* x19 := curcpu()->ci_curlwp */
+	ldr	x3, [x19, #L_CPU]	/* x3 := curlwp->l_cpu */
 	mov	x4, sp
 	mrs	x5, cpacr_el1
 	ldr	x6, [x19, #L_PCB]	/* x6 = lwp_getpcb(curlwp) */
 	str	x4, [x6, #PCB_TF]
 	str	x5, [x19, #L_MD_CPACR]
+	msr	tpidr_el1, x0		/* curlwp = softlwp; */
 	str	x0, [x3, #CI_CURLWP]	/* curcpu()->ci_curlwp = softlwp; */
+					/* no need for memory barrier here */
 
 #ifdef ARMV83_PAC
 	/* Switch the PAC key. */
@@ -200,9 +204,11 @@ ENTRY_NP(cpu_switchto_softint)
 	mov	x0, x19			/* x0 := pinned_lwp */
 	bl	_C_LABEL(softint_dispatch)
 
-	mrs	x3, tpidr_el1
 	DISABLE_INTERRUPT
+	msr	tpidr_el1, x19		/* curlwp = pinned_lwp */
+	ldr	x3, [x19, #L_CPU]	/* x3 = curlwp->l_cpu */
 	str	x19, [x3, #CI_CURLWP]	/* curcpu()->ci_curlwp := x19 */
+	dmb	st			/* see comments in kern_mutex.c */
 	ldr	x6, [x19, #L_PCB]	/* x6 = lwp_getpcb(curlwp) */
 	ldr	x4, [x6, #PCB_TF]	/* x4 := pinned_lwp->l_addr->pcb_tf */
 #ifdef DDB
@@ -245,7 +251,8 @@ END(cpu_switchto_softint)
 ENTRY_NP(softint_cleanup)
 	mov	lr, x20			/* restore original lr */
 
-	mrs	x3, tpidr_el1		/* curcpu() */
+	mrs	x3, tpidr_el1		/* curlwp */
+	ldr	x3, [x3, #L_CPU]	/* curcpu */
 	ldr	w2, [x3, #CI_MTX_COUNT]	/* ->ci_mtx_count */
 	add	w2, w2, #1
 	str	w2, [x3, #CI_MTX_COUNT]
@@ -361,15 +368,13 @@ ENTRY_NP(el0_trap)
 #endif
 ENTRY_NP(el0_trap_exit)
 1:
-	/* while (curcpu()->ci_astpending & __BIT(0)) { */
+	/* while (curlwp->l_md.md_astpending != 0) { */
 	mrs	x8, tpidr_el1
-	ldr	w9, [x8, #CI_ASTPENDING]
-	tbz	w9, #0, 9f
+	ldr	w9, [x8, #L_MD_ASTPENDING]
+	cbz	w9, 9f
 
-	/*  atomic_and_uint(&curcpu()->ci_astpending, ~__BIT(0)); */
-	mov	w1, #~__BIT(0)
-	add	x0, x8, #CI_ASTPENDING
-	bl	_C_LABEL(atomic_and_uint);
+	/* curlwp->l_md.md_astpending = 0; */
+	str	xzr, [x8, #L_MD_ASTPENDING]
 
 	/*  trap_doast(tf); */
 	ENABLE_INTERRUPT
@@ -381,8 +386,7 @@ ENTRY_NP(el0_trap_exit)
 9:
 	DISABLE_INTERRUPT		/* make sure I|F marked */
 
-	mrs	x8, tpidr_el1
-	ldr	x9, [x8, #CI_CURLWP]
+	mrs	x9, tpidr_el1
 	ldr	x23, [x9, #L_MD_CPACR]
 	msr	cpacr_el1, x23		/* FP unit EL0 handover */
 	isb				/* necessary? */
@@ -444,8 +448,7 @@ END(cpu_Debugger)
  * int cpu_set_onfault(struct faultbuf *fb)
  */
 ENTRY_NP(cpu_set_onfault)
-	mrs	x3, tpidr_el1
-	ldr	x2, [x3, #CI_CURLWP]	/* curlwp = curcpu()->ci_curlwp */
+	mrs	x2, tpidr_el1		/* curlwp */
 	str	x0, [x2, #L_MD_ONFAULT] /* l_md.md_onfault = fb */
 
 	stp	x19, x20, [x0, #(FB_X19 * 8)]
Index: sys/arch/aarch64/aarch64/db_machdep.c
===================================================================
RCS file: /cvsroot/src/sys/arch/aarch64/aarch64/db_machdep.c,v
retrieving revision 1.24
diff -u -p -r1.24 db_machdep.c
--- sys/arch/aarch64/aarch64/db_machdep.c	22 May 2020 19:29:26 -0000	1.24
+++ sys/arch/aarch64/aarch64/db_machdep.c	16 Jun 2020 22:56:38 -0000
@@ -315,8 +315,6 @@ show_cpuinfo(struct cpu_info *ci)
 	    &ci->ci_cpl, cpuid, cpuinfobuf.ci_cpl);
 	db_printf("%p cpu[%lu].ci_softints     = 0x%08x\n",
 	    &ci->ci_softints, cpuid, cpuinfobuf.ci_softints);
-	db_printf("%p cpu[%lu].ci_astpending   = 0x%08x\n",
-	    &ci->ci_astpending, cpuid, cpuinfobuf.ci_astpending);
 	db_printf("%p cpu[%lu].ci_intr_depth   = %u\n",
 	    &ci->ci_intr_depth, cpuid, cpuinfobuf.ci_intr_depth);
 	db_printf("%p cpu[%lu].ci_biglock_count = %u\n",
Index: sys/arch/aarch64/aarch64/fusu.S
===================================================================
RCS file: /cvsroot/src/sys/arch/aarch64/aarch64/fusu.S,v
retrieving revision 1.6
diff -u -p -r1.6 fusu.S
--- sys/arch/aarch64/aarch64/fusu.S	6 Apr 2019 03:06:24 -0000	1.6
+++ sys/arch/aarch64/aarch64/fusu.S	16 Jun 2020 22:56:38 -0000
@@ -51,8 +51,7 @@ RCSID("$NetBSD: fusu.S,v 1.6 2019/04/06 
 
 	.macro exit_cpu_onfault
 	/* curlwp->l_md.md_onfault = NULL */
-	mrs	x1, tpidr_el1			/* curcpu */
-	ldr	x1, [x1, #CI_CURLWP]		/* x1 = curlwp */
+	mrs	x1, tpidr_el1			/* x1 = curlwp */
 	str	xzr, [x1, #L_MD_ONFAULT]	/* lwp->l_md_onfault = NULL */
 9:
 	add	sp, sp, #FB_T_SIZE		/* pop stack */
Index: sys/arch/aarch64/aarch64/genassym.cf
===================================================================
RCS file: /cvsroot/src/sys/arch/aarch64/aarch64/genassym.cf,v
retrieving revision 1.26
diff -u -p -r1.26 genassym.cf
--- sys/arch/aarch64/aarch64/genassym.cf	28 May 2020 04:51:44 -0000	1.26
+++ sys/arch/aarch64/aarch64/genassym.cf	16 Jun 2020 22:56:38 -0000
@@ -151,6 +151,7 @@ define	L_MD_FLAGS		offsetof(struct lwp, 
 define	L_MD_UTF		offsetof(struct lwp, l_md.md_utf)
 define	L_MD_CPACR		offsetof(struct lwp, l_md.md_cpacr)
 define	L_MD_ONFAULT		offsetof(struct lwp, l_md.md_onfault)
+define	L_MD_ASTPENDING		offsetof(struct lwp, l_md.md_astpending)
 define	L_MD_IA_KERN		offsetof(struct lwp, l_md.md_ia_kern)
 define	L_MD_IA_USER		offsetof(struct lwp, l_md.md_ia_user)
 define	L_MD_IB_USER		offsetof(struct lwp, l_md.md_ib_user)
@@ -285,7 +286,6 @@ define	CI_CPL			offsetof(struct cpu_info
 define	CI_CPUID		offsetof(struct cpu_info, ci_cpuid)
 define	CI_MIDR			offsetof(struct cpu_info, ci_id.ac_midr)
 define	CI_MPIDR		offsetof(struct cpu_info, ci_id.ac_mpidr)
-define	CI_ASTPENDING		offsetof(struct cpu_info, ci_astpending)
 define	CI_WANT_RESCHED		offsetof(struct cpu_info, ci_want_resched)
 define	CI_INTR_DEPTH		offsetof(struct cpu_info, ci_intr_depth)
 define	CI_MTX_COUNT		offsetof(struct cpu_info, ci_mtx_count)
Index: sys/arch/aarch64/aarch64/idle_machdep.S
===================================================================
RCS file: /cvsroot/src/sys/arch/aarch64/aarch64/idle_machdep.S,v
retrieving revision 1.5
diff -u -p -r1.5 idle_machdep.S
--- sys/arch/aarch64/aarch64/idle_machdep.S	27 Jan 2019 04:48:09 -0000	1.5
+++ sys/arch/aarch64/aarch64/idle_machdep.S	16 Jun 2020 22:56:38 -0000
@@ -74,7 +74,8 @@ ENTRY(cpu_idle)
 	adr	x0, 1f
 	str	x0, [sp, #TF_PC]		/* CLKF_PC refer to tf_pc */
 
-	mrs	x1, tpidr_el1			/* get curcpu() */
+	mrs	x1, tpidr_el1			/* get curlwp */
+	ldr	x1, [x1, #L_CPU]		/* get curcpu */
 	ldr	w28, [x1, #CI_INTR_DEPTH]	/* w28 = ci->ci_intr_depth */
 	add	w2, w28, #1			/* w2 = intr_depth + 1 */
 
@@ -86,7 +87,8 @@ ENTRY(cpu_idle)
 	str	w2, [x1, #CI_INTR_DEPTH]	/* ci->ci_intr_depth++ */
 	bl	ARM_IRQ_HANDLER			/* irqhandler(trapframe) */
 1:
-	mrs	x1, tpidr_el1			/* get curcpu() */
+	mrs	x1, tpidr_el1			/* get curlwp */
+	ldr	x1, [x1, #L_CPU]		/* get curcpu */
 	str	w28, [x1, #CI_INTR_DEPTH]	/* ci->ci_intr_depth = old */
 
 #if defined(__HAVE_FAST_SOFTINTS) && !defined(__HAVE_PIC_FAST_SOFTINTS)
Index: sys/arch/aarch64/aarch64/locore.S
===================================================================
RCS file: /cvsroot/src/sys/arch/aarch64/aarch64/locore.S,v
retrieving revision 1.64
diff -u -p -r1.64 locore.S
--- sys/arch/aarch64/aarch64/locore.S	28 May 2020 04:51:44 -0000	1.64
+++ sys/arch/aarch64/aarch64/locore.S	16 Jun 2020 22:56:38 -0000
@@ -174,9 +174,9 @@ vstart:
 	msr	tpidrro_el0, xzr
 
 	/* set curcpu() */
-	adrl	x0, cpu_info_store	/* cpu_info_store is cpu_info[0] */
-	msr	tpidr_el1, x0		/* curcpu is cpu_info[0] */
-	DPRINTREG("curcpu           = ", x0);
+	adrl	x0, lwp0		/* curlwp is lwp0 */
+	msr	tpidr_el1, x0
+	DPRINTREG("curlwp           = ", x0);
 
 	/* init PAC if supported */
 	mov	x0, #1
@@ -189,8 +189,8 @@ vstart:
 1:
 
 	/* set topology information */
-	mrs	x0, tpidr_el1		/* curcpu */
-	mrs     x1, mpidr_el1
+	adrl	x0, cpu_info_store	/* curcpu */
+	mrs	x1, mpidr_el1
 	mov	x2, #0
 	bl	arm_cpu_topology_set
 
@@ -453,7 +453,7 @@ mp_vstart:
 	mul	x0, x27, x0
 	adrl	x1, _C_LABEL(cpu_info_store)
 	add	x0, x0, x1		/* x0 = &cpu_info_store[cpuindex] */
-	msr	tpidr_el1, x0		/* tpidr_el1 = curcpu() = x0 */
+	msr	tpidr_el1, x0		/* tpidr_el1 = curcpu = x0 */
 
 	mrs	x1, midr_el1
 	str	x1, [x0, #CI_MIDR]	/* curcpu()->ci_cpuid = midr_el1 */
@@ -503,9 +503,9 @@ mp_vstart:
 9:
 
 	/* fill my cpu_info */
-	mrs	x0, tpidr_el1		/* curcpu() */
-
-	ldr	x1, [x0, #CI_IDLELWP]	/* x1 = curcpu()->ci_data.cpu_idlelwp */
+	mrs	x0, tpidr_el1		/* curcpu */
+	ldr	x1, [x0, #CI_IDLELWP]	/* x0 = curcpu()->ci_idlelwp */
+	msr	tpidr_el1, x1		/* tpidr_el1 = curlwp = x1 */
 	str	x1, [x0, #CI_CURLWP]	/* curlwp is idlelwp */
 
 	/* get my stack from lwp */
@@ -524,7 +524,8 @@ mp_vstart:
 1:
 
 	mov	fp, xzr			/* trace back starts here */
-	mrs	x0, tpidr_el1		/* curcpu() */
+	mrs	x0, tpidr_el1		/* curlwp */
+	ldr	x0, [x0, #L_CPU]	/* curlwp->l_cpu */
 	bl	_C_LABEL(cpu_hatch)
 	mov	x0, xzr
 	b	_C_LABEL(idle_loop)	/* never to return */
Index: sys/arch/aarch64/aarch64/vectors.S
===================================================================
RCS file: /cvsroot/src/sys/arch/aarch64/aarch64/vectors.S,v
retrieving revision 1.17
diff -u -p -r1.17 vectors.S
--- sys/arch/aarch64/aarch64/vectors.S	23 May 2020 18:08:59 -0000	1.17
+++ sys/arch/aarch64/aarch64/vectors.S	16 Jun 2020 22:56:38 -0000
@@ -81,8 +81,7 @@ ENTRY_NBTI(\func)
 
 	.if \el == 0
 	/* curlwp->l_private = tpidr{,ro}_el0 */
-	mrs	x1, tpidr_el1		/* curcpu() */
-	ldr	x1, [x1, #CI_CURLWP]	/* x1 = curcpu()->ci_curlwp */
+	mrs	x1, tpidr_el1		/* x1 = curlwp */
 	mrs	x0, tpidr\tpidr\()_el0
 	str	x0, [x1, #L_PRIVATE]	/* curlwp->l_private = tpidr{,ro}_el0 */
 
Index: sys/arch/aarch64/include/cpu.h
===================================================================
RCS file: /cvsroot/src/sys/arch/aarch64/include/cpu.h,v
retrieving revision 1.22
diff -u -p -r1.22 cpu.h
--- sys/arch/aarch64/include/cpu.h	10 Mar 2020 01:17:33 -0000	1.22
+++ sys/arch/aarch64/include/cpu.h	16 Jun 2020 22:56:38 -0000
@@ -1,7 +1,7 @@
 /* $NetBSD: cpu.h,v 1.22 2020/03/10 01:17:33 christos Exp $ */
 
 /*-
- * Copyright (c) 2014 The NetBSD Foundation, Inc.
+ * Copyright (c) 2014, 2020 The NetBSD Foundation, Inc.
  * All rights reserved.
  *
  * This code is derived from software contributed to The NetBSD Foundation
@@ -74,19 +74,33 @@ struct cpu_info {
 	struct cpu_data ci_data;
 	device_t ci_dev;
 	cpuid_t ci_cpuid;
-	struct lwp *ci_curlwp;
+
+	/*
+	 * the following are in their own cache line, as they are stored to
+	 * regularly by remote CPUs; when they were mixed with other fields
+	 * we observed frequent cache misses.
+	 */
+	int ci_want_resched __aligned(COHERENCY_UNIT);
+	/* XXX pending IPIs? */
+
+	/*
+	 * this is stored frequently, and is fetched by remote CPUs.
+	 */
+	struct lwp *ci_curlwp __aligned(COHERENCY_UNIT);
 	struct lwp *ci_onproc;
-	struct lwp *ci_softlwps[SOFTINT_COUNT];
+
+	/*
+	 * largely CPU-private.
+	 */
+	struct lwp *ci_softlwps[SOFTINT_COUNT] __aligned(COHERENCY_UNIT);
 
 	uint64_t ci_lastintr;
 
 	int ci_mtx_oldspl;
 	int ci_mtx_count;
 
-	int ci_want_resched;
 	int ci_cpl;
 	volatile u_int ci_softints;
-	volatile u_int ci_astpending;
 	volatile u_int ci_intr_depth;
 
 	/* event counters */
@@ -113,18 +127,23 @@ struct cpu_info {
 } __aligned(COHERENCY_UNIT);
 
 #ifdef _KERNEL
-static inline struct cpu_info *
-curcpu(void)
+static inline struct lwp * __attribute__ ((const))
+aarch64_curlwp(void)
 {
-	struct cpu_info *ci;
-	__asm __volatile ("mrs %0, tpidr_el1" : "=r"(ci));
-	return ci;
+	struct lwp *l;
+	__asm("mrs %0, tpidr_el1" : "=r"(l));
+	return l;
 }
-#define curlwp			(curcpu()->ci_curlwp)
 
-#define setsoftast(ci)		atomic_or_uint(&(ci)->ci_astpending, __BIT(0))
-#define cpu_signotify(l)	setsoftast((l)->l_cpu)
+/* forward declaration; defined in sys/lwp.h. */
+static __inline struct cpu_info *lwp_getcpu(struct lwp *);
+
+#define	curcpu()		(lwp_getcpu(aarch64_curlwp()))
+#define	setsoftast(ci)		(cpu_signotify((ci)->ci_onproc))
+#undef curlwp
+#define	curlwp			(aarch64_curlwp())
 
+void	cpu_signotify(struct lwp *l);
 void	cpu_need_proftick(struct lwp *l);
 
 void	cpu_hatch(struct cpu_info *);
@@ -147,6 +166,7 @@ extern struct cpu_info cpu_info_store[];
 	cii = 0, __USE(cii), ci = curcpu(); ci != NULL; ci = NULL
 #endif /* MULTIPROCESSOR */
 
+#define	LWP0_CPU_INFO	(&cpu_info_store[0])
 
 static inline void
 cpu_dosoftints(void)
Index: sys/arch/aarch64/include/proc.h
===================================================================
RCS file: /cvsroot/src/sys/arch/aarch64/include/proc.h,v
retrieving revision 1.7
diff -u -p -r1.7 proc.h
--- sys/arch/aarch64/include/proc.h	23 May 2020 18:08:59 -0000	1.7
+++ sys/arch/aarch64/include/proc.h	16 Jun 2020 22:56:38 -0000
@@ -43,6 +43,7 @@ struct mdlwp {
 	struct trapframe *md_utf;
 	uint64_t md_cpacr;
 	uint32_t md_flags;
+	volatile uint32_t md_astpending;
 
 	uint64_t md_ia_kern[2]; /* APIAKey{Lo,Hi}_EL1 used in the kernel */
 	uint64_t md_ia_user[2]; /* APIAKey{Lo,Hi}_EL1 used in user-process */