From b752fd75083fe3ad3e9d95c4cb543fafd7b3e58c Mon Sep 17 00:00:00 2001
From: Taylor R Campbell <riastradh@NetBSD.org>
Date: Sun, 27 Feb 2022 19:43:11 +0000
Subject: [PATCH 1/3] mips/cavium: Insert appropriate membars around IPIs.
---
sys/arch/mips/cavium/octeon_intr.c | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/sys/arch/mips/cavium/octeon_intr.c b/sys/arch/mips/cavium/octeon_intr.c
index a4afc88bef80..68b3b1768a89 100644
--- a/sys/arch/mips/cavium/octeon_intr.c
+++ b/sys/arch/mips/cavium/octeon_intr.c
@@ -545,6 +545,7 @@ octeon_ipi_intr(void *arg)
ipi_mask &= mips3_ld(cpu->cpu_mbox_set);
if (ipi_mask == 0)
return 0;
+ membar_enter();
mips3_sd(cpu->cpu_mbox_clr, ipi_mask);
@@ -563,8 +564,9 @@ octeon_ipi_intr(void *arg)
#endif
/* if the request is clear, it was previously processed */
- if ((ci->ci_request_ipis & ipi_mask) == 0)
+ if ((atomic_load_relaxed(&ci->ci_request_ipis) & ipi_mask) == 0)
return 0;
+ membar_enter();
atomic_or_64(&ci->ci_active_ipis, ipi_mask);
atomic_and_64(&ci->ci_request_ipis, ~ipi_mask);
@@ -597,8 +599,10 @@ octeon_send_ipi(struct cpu_info *ci, int req)
const u_int ipi_shift = ipi_prio[req] == IPL_SCHED ? 16 : 0;
const uint32_t ipi_mask = __BIT(req + ipi_shift);
+ membar_exit();
atomic_or_64(&ci->ci_request_ipis, ipi_mask);
+ membar_exit();
mips3_sd(cpu->cpu_mbox_set, ipi_mask);
return 0;
From b3513665c8bfe69c2440810b12457fe94e58834c Mon Sep 17 00:00:00 2001
From: Taylor R Campbell <riastradh@NetBSD.org>
Date: Wed, 2 Mar 2022 18:14:07 +0000
Subject: [PATCH 2/3] mips/cavium: Fix membars around establishing interrupt
handlers.
---
sys/arch/mips/cavium/octeon_intr.c | 25 ++++++++++++++++++-------
1 file changed, 18 insertions(+), 7 deletions(-)
diff --git a/sys/arch/mips/cavium/octeon_intr.c b/sys/arch/mips/cavium/octeon_intr.c
index 68b3b1768a89..f8c4342c8a28 100644
--- a/sys/arch/mips/cavium/octeon_intr.c
+++ b/sys/arch/mips/cavium/octeon_intr.c
@@ -54,6 +54,7 @@ __KERNEL_RCSID(0, "$NetBSD: octeon_intr.c,v 1.24 2020/08/18 07:41:41 skrll Exp $
#include <sys/kernel.h>
#include <sys/kmem.h>
#include <sys/atomic.h>
+#include <sys/xcall.h>
#include <lib/libkern/libkern.h>
@@ -353,6 +354,14 @@ octeon_intr_establish(int irq, int ipl, int (*func)(void *), void *arg)
ih->ih_irq = irq;
ih->ih_ipl = ipl;
+ /*
+ * Make sure the initialization is visible on all CPUs before
+ * we expose it in octciu_intrs. This way we don't need to
+ * issue any membar for a load-acquire when handling the
+ * interrupt.
+ */
+ xc_barrier(0);
+
mutex_enter(&octeon_intr_lock);
/*
@@ -361,8 +370,7 @@ octeon_intr_establish(int irq, int ipl, int (*func)(void *), void *arg)
KASSERTMSG(octciu_intrs[irq] == NULL, "irq %d in use! (%p)",
irq, octciu_intrs[irq]);
- octciu_intrs[irq] = ih;
- membar_producer();
+ atomic_store_relaxed(&octciu_intrs[irq], ih);
/*
* Now enable it.
@@ -462,13 +470,15 @@ octeon_intr_disestablish(void *cookie)
break;
}
- /*
- * Now remove it since we shouldn't get interrupts for it.
- */
- octciu_intrs[irq] = NULL;
+ atomic_store_relaxed(&octciu_intrs[irq], NULL);
mutex_exit(&octeon_intr_lock);
+ /*
+ * Wait until the interrupt handler is no longer running on all
+ * CPUs before freeing ih and returning.
+ */
+ xc_barrier(0);
kmem_free(ih, sizeof(*ih));
}
@@ -507,7 +517,8 @@ octeon_iointr(int ipl, vaddr_t pc, uint32_t ipending)
const int irq = (bank * 64) + bit;
hwpend[bank] &= ~__BIT(bit);
- struct octeon_intrhand * const ih = octciu_intrs[irq];
+ struct octeon_intrhand * const ih =
+ atomic_load_relaxed(&octciu_intrs[irq]);
cpu->cpu_intr_evs[irq].ev_count++;
if (__predict_true(ih != NULL)) {
#ifdef MULTIPROCESSOR
From 09bdffd156b248451e9c4e733338fd2386d8aadb Mon Sep 17 00:00:00 2001
From: Taylor R Campbell <riastradh@NetBSD.org>
Date: Wed, 2 Mar 2022 18:18:27 +0000
Subject: [PATCH 3/3] mips/rmixl: Insert appropriate membars around IPIs.
---
sys/arch/mips/rmi/rmixl_intr.c | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/sys/arch/mips/rmi/rmixl_intr.c b/sys/arch/mips/rmi/rmixl_intr.c
index 3e9b49731bdf..c37f13508097 100644
--- a/sys/arch/mips/rmi/rmixl_intr.c
+++ b/sys/arch/mips/rmi/rmixl_intr.c
@@ -967,6 +967,7 @@ rmixl_send_ipi(struct cpu_info *ci, int tag)
| (RMIXL_INTRVEC_IPI + tag);
mutex_enter(&rmixl_ipi_lock);
+ membar_exit();
atomic_or_64(&ci->ci_request_ipis, req);
RMIXL_PICREG_WRITE(RMIXL_PIC_IPIBASE, r);
mutex_exit(&rmixl_ipi_lock);
@@ -984,8 +985,9 @@ rmixl_ipi_intr(void *arg)
KASSERT((uintptr_t)arg < NIPIS);
/* if the request is clear, it was previously processed */
- if ((ci->ci_request_ipis & ipi_mask) == 0)
+ if ((atomic_load_relaxed(&ci->ci_request_ipis) & ipi_mask) == 0)
return 0;
+ membar_enter();
atomic_or_64(&ci->ci_active_ipis, ipi_mask);
atomic_and_64(&ci->ci_request_ipis, ~ipi_mask);