commit 818c6e1e8638b4bda654d6c8a055a6ac564a4aca Author: Ryota Ozaki Date: Tue Jan 23 14:10:45 2018 +0900 Multiple softint xcall diff --git a/sys/kern/subr_xcall.c b/sys/kern/subr_xcall.c index 8e4a95dc661..1dacde2cedc 100644 --- a/sys/kern/subr_xcall.c +++ b/sys/kern/subr_xcall.c @@ -98,6 +98,7 @@ typedef struct { void * xc_arg2; uint64_t xc_headp; uint64_t xc_donep; + unsigned int xc_ipl; } xc_state_t; /* Bit indicating high (1) or low (0) priority. */ @@ -108,7 +109,7 @@ static xc_state_t xc_low_pri __cacheline_aligned; /* High priority xcall structures. */ static xc_state_t xc_high_pri __cacheline_aligned; -static void * xc_sih __cacheline_aligned; +static void * xc_sihs[4] __cacheline_aligned; /* Event counters. */ static struct evcnt xc_unicast_ev __cacheline_aligned; @@ -117,9 +118,24 @@ static struct evcnt xc_broadcast_ev __cacheline_aligned; static void xc_init(void); static void xc_thread(void *); -static inline uint64_t xc_highpri(xcfunc_t, void *, void *, struct cpu_info *); +static inline uint64_t xc_highpri(xcfunc_t, void *, void *, struct cpu_info *, + unsigned int); static inline uint64_t xc_lowpri(xcfunc_t, void *, void *, struct cpu_info *); +/* The internal form of IPL */ +#define XC_IPL_MASK 0xff00 +/* + * Assign 0 to XC_IPL_SOFTSERIAL to treat the default value (just XC_HIGHPRI) as + * IPL_SOFTSERIAL. + */ +#define XC_IPL_SOFTSERIAL 0 +#define XC_IPL_SOFTBIO 1 +#define XC_IPL_SOFTCLOCK 2 +#define XC_IPL_SOFTNET 3 +#define XC_IPL_MAX XC_IPL_SOFTNET + +CTASSERT(XC_IPL_MAX <= __arraycount(xc_sihs)); + /* * xc_init: * @@ -137,9 +153,19 @@ xc_init(void) memset(xchi, 0, sizeof(xc_state_t)); mutex_init(&xchi->xc_lock, MUTEX_DEFAULT, IPL_SOFTSERIAL); cv_init(&xchi->xc_busy, "xchicv"); - xc_sih = softint_establish(SOFTINT_SERIAL | SOFTINT_MPSAFE, - xc__highpri_intr, NULL); - KASSERT(xc_sih != NULL); + +#define SETUP_SOFTINT(xipl, sipl) do { \ + xc_sihs[(xipl)] = softint_establish( (sipl) | SOFTINT_MPSAFE,\ + xc__highpri_intr, NULL); \ + KASSERT(xc_sihs[(xipl)] != NULL); \ + } while (0) + + SETUP_SOFTINT(XC_IPL_SOFTSERIAL, SOFTINT_SERIAL); + SETUP_SOFTINT(XC_IPL_SOFTBIO, SOFTINT_BIO); + SETUP_SOFTINT(XC_IPL_SOFTCLOCK, SOFTINT_CLOCK); + SETUP_SOFTINT(XC_IPL_SOFTNET, SOFTINT_NET); + +#undef SETUP_SOFTINT evcnt_attach_dynamic(&xc_unicast_ev, EVCNT_TYPE_MISC, NULL, "crosscall", "unicast"); @@ -147,6 +173,38 @@ xc_init(void) "crosscall", "broadcast"); } +/* + * Encode an IPL to a form that can be embedded into flags of xc_broadcast + * or xc_unicast. + */ +unsigned int +xc_encode_ipl(int ipl) +{ + + switch (ipl) { + case IPL_SOFTSERIAL: + return __SHIFTIN(XC_IPL_SOFTSERIAL, XC_IPL_MASK); + case IPL_SOFTBIO: + return __SHIFTIN(XC_IPL_SOFTBIO, XC_IPL_MASK); + case IPL_SOFTCLOCK: + return __SHIFTIN(XC_IPL_SOFTCLOCK, XC_IPL_MASK); + case IPL_SOFTNET: + return __SHIFTIN(XC_IPL_SOFTNET, XC_IPL_MASK); + } + + panic("Invalid IPL: %d", ipl); +} + +/* + * Extract an XC_IPL from flags of xc_broadcast or xc_unicast. + */ +static inline unsigned int +xc_extract_ipl(unsigned int flags) +{ + + return __SHIFTOUT(flags, XC_IPL_MASK); +} + /* * xc_init_cpu: * @@ -176,13 +234,14 @@ xc_init_cpu(struct cpu_info *ci) * Trigger a call on all CPUs in the system. */ uint64_t -xc_broadcast(u_int flags, xcfunc_t func, void *arg1, void *arg2) +xc_broadcast(unsigned int flags, xcfunc_t func, void *arg1, void *arg2) { KASSERT(!cpu_intr_p() && !cpu_softintr_p()); if ((flags & XC_HIGHPRI) != 0) { - return xc_highpri(func, arg1, arg2, NULL); + int ipl = xc_extract_ipl(flags); + return xc_highpri(func, arg1, arg2, NULL, ipl); } else { return xc_lowpri(func, arg1, arg2, NULL); } @@ -194,7 +253,7 @@ xc_broadcast(u_int flags, xcfunc_t func, void *arg1, void *arg2) * Trigger a call on one CPU. */ uint64_t -xc_unicast(u_int flags, xcfunc_t func, void *arg1, void *arg2, +xc_unicast(unsigned int flags, xcfunc_t func, void *arg1, void *arg2, struct cpu_info *ci) { @@ -202,7 +261,8 @@ xc_unicast(u_int flags, xcfunc_t func, void *arg1, void *arg2, KASSERT(!cpu_intr_p() && !cpu_softintr_p()); if ((flags & XC_HIGHPRI) != 0) { - return xc_highpri(func, arg1, arg2, ci); + int ipl = xc_extract_ipl(flags); + return xc_highpri(func, arg1, arg2, ci, ipl); } else { return xc_lowpri(func, arg1, arg2, ci); } @@ -329,8 +389,12 @@ xc_thread(void *cookie) void xc_ipi_handler(void) { + xc_state_t *xc = & xc_high_pri; + + KASSERT(xc->xc_ipl < __arraycount(xc_sihs)); + /* Executes xc__highpri_intr() via software interrupt. */ - softint_schedule(xc_sih); + softint_schedule(xc_sihs[xc->xc_ipl]); } /* @@ -376,7 +440,8 @@ xc__highpri_intr(void *dummy) * Trigger a high priority call on one or more CPUs. */ static inline uint64_t -xc_highpri(xcfunc_t func, void *arg1, void *arg2, struct cpu_info *ci) +xc_highpri(xcfunc_t func, void *arg1, void *arg2, struct cpu_info *ci, + unsigned int ipl) { xc_state_t *xc = &xc_high_pri; uint64_t where; @@ -389,6 +454,7 @@ xc_highpri(xcfunc_t func, void *arg1, void *arg2, struct cpu_info *ci) xc->xc_arg1 = arg1; xc->xc_arg2 = arg2; xc->xc_headp += (ci ? 1 : ncpu); + xc->xc_ipl = ipl; where = xc->xc_headp; mutex_exit(&xc->xc_lock); diff --git a/sys/sys/xcall.h b/sys/sys/xcall.h index d0c40d85d3e..086dc7d9390 100644 --- a/sys/sys/xcall.h +++ b/sys/sys/xcall.h @@ -33,7 +33,8 @@ #define _SYS_XCALL_H_ #ifdef _KERNEL -#define XC_HIGHPRI 0x01 /* high priority */ +#define XC_HIGHPRI 0x01 /* high priority */ +#define XC_HIGHPRI_IPL(ipl) (XC_HIGHPRI | xc_encode_ipl(ipl)) typedef void (*xcfunc_t)(void *, void *); @@ -49,6 +50,8 @@ uint64_t xc_broadcast(u_int, xcfunc_t, void *, void *); uint64_t xc_unicast(u_int, xcfunc_t, void *, void *, struct cpu_info *); void xc_wait(uint64_t); +unsigned int xc_encode_ipl(int); + #endif /* _KERNEL */ #endif /* _SYS_XCALL_H_ */