Index: sparc/cpu.c
===================================================================
RCS file: /cvsroot/src/sys/arch/sparc/sparc/cpu.c,v
retrieving revision 1.248
diff -p -u -u -r1.248 cpu.c
--- sparc/cpu.c	25 Jul 2014 17:21:32 -0000	1.248
+++ sparc/cpu.c	6 Dec 2017 20:16:39 -0000
@@ -183,7 +183,7 @@ int go_smp_cpus = 0;	/* non-primary CPUs
  * This must be locked around all message transactions to ensure only
  * one CPU is generating them.
  */
-static kmutex_t xpmsg_mutex;
+kmutex_t xpmsg_mutex;
 
 #endif /* MULTIPROCESSOR */
 
@@ -367,6 +367,10 @@ cpu_init_evcnt(struct cpu_info *cpi)
 			     NULL, cpu_name(cpi), "IPI mutex_trylock fail");
 	evcnt_attach_dynamic(&cpi->ci_xpmsg_mutex_fail_call, EVCNT_TYPE_MISC,
 			     NULL, cpu_name(cpi), "IPI mutex_trylock fail/call");
+	evcnt_attach_dynamic(&cpi->ci_xpmsg_mutex_not_held, EVCNT_TYPE_MISC,
+			     NULL, cpu_name(cpi), "IPI with mutex not held");
+	evcnt_attach_dynamic(&cpi->ci_xpmsg_bogus, EVCNT_TYPE_MISC,
+			     NULL, cpu_name(cpi), "bogus IPI");
 
 	/*
 	 * These are the per-cpu per-IPL hard & soft interrupt counters.
@@ -653,6 +657,8 @@ xcall(xcall_func_t func, xcall_trap_t tr
 	char *bufp = errbuf;
 	size_t bufsz = sizeof errbuf, wrsz;
 
+	if (is_noop) return;
+
 	mybit = (1 << cpuinfo.ci_cpuid);
 	callself = func && (cpuset & mybit) != 0;
 	cpuset &= ~mybit;
@@ -714,7 +720,10 @@ xcall(xcall_func_t func, xcall_trap_t tr
 		if ((cpuset & (1 << n)) == 0)
 			continue;
 
-		cpi->msg.tag = XPMSG_FUNC;
+		/*
+		 * Write msg.tag last - if another CPU is polling above it may
+		 * end up seeing an incomplete message. Not likely but still.
+		 */ 
 		cpi->msg.complete = 0;
 		p = &cpi->msg.u.xpmsg_func;
 		p->func = func;
@@ -722,6 +731,9 @@ xcall(xcall_func_t func, xcall_trap_t tr
 		p->arg0 = arg0;
 		p->arg1 = arg1;
 		p->arg2 = arg2;
+		__insn_barrier();
+		cpi->msg.tag = XPMSG_FUNC;
+		__insn_barrier();
 		/* Fast cross calls use interrupt level 14 */
 		raise_ipi(cpi,13+fasttrap);/*xcall_cookie->pil*/
 	}
@@ -737,7 +749,7 @@ xcall(xcall_func_t func, xcall_trap_t tr
 	 * have completed (bailing if it takes "too long", being loud about
 	 * this in the process).
 	 */
-	done = is_noop;
+	done = 0;
 	i = 1000000;	/* time-out, not too long, but still an _AGE_ */
 	while (!done) {
 		if (--i < 0) {
@@ -774,7 +786,7 @@ xcall(xcall_func_t func, xcall_trap_t tr
 
 	if (i >= 0 || debug_xcall == 0) {
 		if (i < 0)
-			printf_nolog("%s\n", errbuf);
+			aprint_error("%s\n", errbuf);
 		mutex_spin_exit(&xpmsg_mutex);
 		return;
 	}
Index: sparc/cpuvar.h
===================================================================
RCS file: /cvsroot/src/sys/arch/sparc/sparc/cpuvar.h,v
retrieving revision 1.92
diff -p -u -u -r1.92 cpuvar.h
--- sparc/cpuvar.h	16 Nov 2013 23:54:01 -0000	1.92
+++ sparc/cpuvar.h	6 Dec 2017 20:16:39 -0000
@@ -171,7 +171,7 @@ struct cpu_info {
 	 * the pending register to avoid a hardware bug.
 	 */
 #define raise_ipi(cpi,lvl)	do {			\
-	int x;						\
+	volatile int x;					\
 	(cpi)->intreg_4m->pi_set = PINTR_SINTRLEV(lvl);	\
 	x = (cpi)->intreg_4m->pi_pend; __USE(x);	\
 } while (0)
@@ -340,6 +340,8 @@ struct cpu_info {
 	struct evcnt ci_savefpstate_null;
 	struct evcnt ci_xpmsg_mutex_fail;
 	struct evcnt ci_xpmsg_mutex_fail_call;
+	struct evcnt ci_xpmsg_mutex_not_held;
+	struct evcnt ci_xpmsg_bogus;
 	struct evcnt ci_intrcnt[16];
 	struct evcnt ci_sintrcnt[16];
 };
Index: sparc/intr.c
===================================================================
RCS file: /cvsroot/src/sys/arch/sparc/sparc/intr.c,v
retrieving revision 1.118
diff -p -u -u -r1.118 intr.c
--- sparc/intr.c	16 Nov 2013 23:54:01 -0000	1.118
+++ sparc/intr.c	6 Dec 2017 20:16:39 -0000
@@ -76,6 +76,8 @@ static int intr_biglock_wrapper(void *);
 void *xcall_cookie;
 #endif
 
+extern kmutex_t xpmsg_mutex;
+
 void	strayintr(struct clockframe *);
 #ifdef DIAGNOSTIC
 void	bogusintr(struct clockframe *);
@@ -241,7 +243,7 @@ nmi_hard(void)
 			DELAY(1);
 			if (n-- > 0)
 				continue;
-			printf("nmi_hard: SMP botch.");
+			printf("nmi_hard: SMP botch.\n");
 			break;
 		}
 	}
@@ -364,6 +366,27 @@ xcallintr(void *v)
 	if (v != xcallintr)
 		cpuinfo.ci_sintrcnt[13].ev_count++;
 
+	if (mutex_owned(&xpmsg_mutex) == 0) {
+		cpuinfo.ci_xpmsg_mutex_not_held.ev_count++;
+#ifdef DEBUG
+		printf("%s: mutex not held\n", __func__);
+#endif
+		cpuinfo.msg.complete = 1;
+		kpreempt_enable();
+		return;
+	}
+
+	if (cpuinfo.msg.complete != 0) {
+		cpuinfo.ci_xpmsg_bogus.ev_count++;
+#ifdef DEBUG
+		volatile struct xpmsg_func *p = &cpuinfo.msg.u.xpmsg_func;
+		printf("%s: bogus message %08x %08x %08x %08x\n", __func__,
+		    cpuinfo.msg.tag, (uint32_t)p->func, p->arg0, p->arg1);
+#endif
+		kpreempt_enable();
+		return;
+	}
+
 	/* notyet - cpuinfo.msg.received = 1; */
 	switch (cpuinfo.msg.tag) {
 	case XPMSG_FUNC: