From dbad4b770d7b7eee4dc60415ece783706a509d9b Mon Sep 17 00:00:00 2001 From: Taylor R Campbell Date: Wed, 29 Jan 2020 19:49:15 +0000 Subject: [PATCH 1/4] New functions percpu_setup and percpu_teardown. These let you sleep for allocation or draining users before deallocation when setting up a percpu -- currently we have many abuses of percpu_foreach in tree for that purpose. --- distrib/sets/lists/comp/mi | 6 +++ share/man/man9/Makefile | 4 +- share/man/man9/percpu.9 | 93 ++++++++++++++++++++++++++++++++++---- sys/kern/subr_percpu.c | 55 ++++++++++++++++++++++ sys/sys/percpu.h | 2 + 5 files changed, 151 insertions(+), 9 deletions(-) diff --git a/distrib/sets/lists/comp/mi b/distrib/sets/lists/comp/mi index 039edf3ffa87..1e0eaaa92b62 100644 --- a/distrib/sets/lists/comp/mi +++ b/distrib/sets/lists/comp/mi @@ -11576,6 +11576,8 @@ ./usr/share/man/cat9/percpu_init.0 comp-obsolete obsolete ./usr/share/man/cat9/percpu_init_cpu.0 comp-obsolete obsolete ./usr/share/man/cat9/percpu_putref.0 comp-sys-catman .cat +./usr/share/man/cat9/percpu_setup.0 comp-sys-catman .cat +./usr/share/man/cat9/percpu_teardown.0 comp-sys-catman .cat ./usr/share/man/cat9/pfil.0 comp-sys-catman .cat ./usr/share/man/cat9/pfil_add_hook.0 comp-sys-catman .cat ./usr/share/man/cat9/pfil_add_ihook.0 comp-sys-catman .cat @@ -19505,6 +19507,8 @@ ./usr/share/man/html9/percpu_init.html comp-obsolete obsolete ./usr/share/man/html9/percpu_init_cpu.html comp-obsolete obsolete ./usr/share/man/html9/percpu_putref.html comp-sys-htmlman html +./usr/share/man/html9/percpu_setup.html comp-sys-htmlman html +./usr/share/man/html9/percpu_teardown.html comp-sys-htmlman html ./usr/share/man/html9/pfil.html comp-sys-htmlman html ./usr/share/man/html9/pfil_add_hook.html comp-sys-htmlman html ./usr/share/man/html9/pfil_add_ihook.html comp-sys-htmlman html @@ -27593,6 +27597,8 @@ ./usr/share/man/man9/percpu_init.9 comp-obsolete obsolete ./usr/share/man/man9/percpu_init_cpu.9 comp-obsolete obsolete ./usr/share/man/man9/percpu_putref.9 comp-sys-man .man +./usr/share/man/man9/percpu_setup.9 comp-sys-man .man +./usr/share/man/man9/percpu_teardown.9 comp-sys-man .man ./usr/share/man/man9/pfil.9 comp-sys-man .man ./usr/share/man/man9/pfil_add_hook.9 comp-sys-man .man ./usr/share/man/man9/pfil_add_ihook.9 comp-sys-man .man diff --git a/share/man/man9/Makefile b/share/man/man9/Makefile index fb83c94971ec..378a411e02ef 100644 --- a/share/man/man9/Makefile +++ b/share/man/man9/Makefile @@ -666,7 +666,9 @@ MLINKS+=percpu.9 percpu_alloc.9 \ percpu.9 percpu_free.9 \ percpu.9 percpu_getref.9 \ percpu.9 percpu_putref.9 \ - percpu.9 percpu_foreach.9 + percpu.9 percpu_foreach.9 \ + percpu.9 percpu_setup.9 \ + percpu.9 percpu_teardown.9 MLINKS+=pfil.9 pfil_hook_get.9 \ pfil.9 pfil_add_hook.9 \ pfil.9 pfil_remove_hook.9 \ diff --git a/share/man/man9/percpu.9 b/share/man/man9/percpu.9 index 79ce894fd732..975c249d8354 100644 --- a/share/man/man9/percpu.9 +++ b/share/man/man9/percpu.9 @@ -27,7 +27,7 @@ .\" ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE .\" POSSIBILITY OF SUCH DAMAGE. .\" -.Dd May 31, 2017 +.Dd January 29, 2020 .Dt PERCPU 9 .Os .Sh NAME @@ -36,7 +36,9 @@ .Nm percpu_free , .Nm percpu_getref , .Nm percpu_putref , -.Nm percpu_foreach +.Nm percpu_foreach , +.Nm percpu_setup , +.Nm percpu_teardown .Nd per-CPU storage allocator .Sh SYNOPSIS .In sys/percpu.h @@ -51,6 +53,10 @@ .Fn percpu_putref "percpu_t *pc" .Ft void .Fn percpu_foreach "percpu_t *pc" "percpu_callback_t cb" "void *arg" +.Ft void +.Fn percpu_setup "percpu_t *pc" "size_t size" "percpu_callback_t cb" "void *arg" +.Ft void +.Fn percpu_teardown "percpu_t *pc" "size_t size" "percpu_callback_t cb" "void *arg" .Sh DESCRIPTION The machine-independent .Nm @@ -111,6 +117,13 @@ Follow each .Fn percpu_getref call with a matching call to .Fn percpu_putref . +.Pp +Caller +.Em MUST NOT +sleep after +.Fn percpu_getref , +not even on an adaptive lock, before +.Fn percpu_putref . .It Fn percpu_putref "pc" Indicate that the thread is finished with the pointer returned by the matching @@ -118,9 +131,9 @@ call to .Fn percpu_getref . Re-enables preemption. .It Fn percpu_foreach "pc" "cb" "arg" -On each CPU, for +For each CPU, with .Fa ci -the corresponding +being the corresponding .Vt "struct cpu_info *" and .Fa "p" @@ -132,12 +145,76 @@ run .Fa "arg" .Fa "ci" .Fc . -Call this in thread context. +The call to +.Fa cb +runs in the current thread; use +.Xr xcall 9 +for cross-calls to run logic on other CPUs. +.Pp +Must be used in thread context. +.Fa cb +.Em MUST NOT +sleep except on adaptive locks, and should be fast. +Do not rely on any particular order of iteration over the CPUs. +.It Fn percpu_setup "pc" "size" "cb" "arg" +For each CPU, with +.Fa ci +being the corresponding +.Vt struct cpu_info * +and +.Fa p +a pointer to a temporary buffer of +.Fa size +bytes, run +.Fo "(*cb)" +.Fa p +.Fa arg +.Fa ci +.Fc ; +then copy the content of the temporary buffer to that CPU's per-CPU +storage for +.Fa pc . +.Pp +Must be used in thread context. .Fa cb -should be non-blocking and fast. -Do not rely on +.Em MAY +sleep, e.g. to allocate memory. +Do not rely on any particular order of iteration over the CPUs. +The size of +.Fa pc +from +.Fn percpu_alloc +must match +.Fa size . +.It Fn percpu_teardown "pc" "size" "cb" "arg" +For each CPU, with +.Fa ci +being the corresponding +.Vt struct cpu_info * +and +.Fa p +a pointer to a temporary buffer of +.Fa size +bytes initialized with that CPU's per-CPU storage for +.Fa pc , +run +.Fo "(*cb)" +.Fa p +.Fa arg +.Fa ci +.Fc . +.Pp +Must be used in thread context. .Fa cb -to be run on the CPUs in any particular order. +.Em MAY +sleep, e.g. to wait for users to drain before deallocating memory. +Do not rely on any particular order of iteration over the CPUs. +The size of +.Fa pc +from +.Fn percpu_alloc +must match +.Fa size . .El .Sh CODE REFERENCES The diff --git a/sys/kern/subr_percpu.c b/sys/kern/subr_percpu.c index bcb5ccacf5ac..4a36fa3cab77 100644 --- a/sys/kern/subr_percpu.c +++ b/sys/kern/subr_percpu.c @@ -368,3 +368,58 @@ percpu_foreach(percpu_t *pc, percpu_callback_t cb, void *arg) } percpu_traverse_exit(); } + +/* + * percpu_setup: call the specified callback to set up each CPU. + * + * => may sleep. + * => called in thread context. + * => caller should not rely on the cpu iteration order. + * => the callback may sleep to allocate. + */ +void +percpu_setup(percpu_t *percpu, size_t size, percpu_callback_t cb, + void *cookie) +{ + void *buf; + CPU_INFO_ITERATOR cii; + struct cpu_info *ci; + + buf = kmem_alloc(size, KM_SLEEP); + for (CPU_INFO_FOREACH(cii, ci)) { + memset(buf, 0, size); + (*cb)(buf, cookie, ci); + percpu_traverse_enter(); + memcpy(percpu_getptr_remote(percpu, ci), buf, size); + percpu_traverse_exit(); + } + kmem_free(buf, size); +} + +/* + * percpu_teardown: call the specified callback to tear down each CPU. + * + * => may sleep. + * => called in thread context. + * => caller should not rely on the cpu iteration order. + * => the callback may sleep to allocate. + */ +void +percpu_teardown(struct percpu *percpu, size_t size, percpu_callback_t cb, + void *cookie) +{ + void *buf; + CPU_INFO_ITERATOR cii; + struct cpu_info *ci; + + buf = kmem_alloc(size, KM_SLEEP); + for (CPU_INFO_FOREACH(cii, ci)) { + percpu_traverse_enter(); + memcpy(buf, percpu_getptr_remote(percpu, ci), size); + explicit_memset(percpu_getptr_remote(percpu, ci), 0, size); + percpu_traverse_exit(); + (*cb)(buf, cookie, ci); + } + explicit_memset(buf, 0, size); /* paranoia */ + kmem_free(buf, size); +} diff --git a/sys/sys/percpu.h b/sys/sys/percpu.h index 8513af5deebb..e3630ac8a474 100644 --- a/sys/sys/percpu.h +++ b/sys/sys/percpu.h @@ -40,6 +40,8 @@ void percpu_putref(percpu_t *); typedef void (*percpu_callback_t)(void *, void *, struct cpu_info *); void percpu_foreach(percpu_t *, percpu_callback_t, void *); +void percpu_setup(percpu_t *, size_t, percpu_callback_t, void *); +void percpu_teardown(percpu_t *, size_t, percpu_callback_t, void *); /* low-level api; don't use unless necessary */ void percpu_traverse_enter(void); From 9f0e5c6b50563db910aa563641f22e477f1e5f81 Mon Sep 17 00:00:00 2001 From: Taylor R Campbell Date: Wed, 29 Jan 2020 21:51:33 +0000 Subject: [PATCH 2/4] Switch arm pic allocation from percpu_foreach to percpu_setup. --- sys/arch/arm/pic/pic.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sys/arch/arm/pic/pic.c b/sys/arch/arm/pic/pic.c index 38959b588a5c..8be1bb0b8d8f 100644 --- a/sys/arch/arm/pic/pic.c +++ b/sys/arch/arm/pic/pic.c @@ -707,7 +707,8 @@ pic_add(struct pic_softc *pic, int irqbase) /* * Now allocate the per-cpu evcnts. */ - percpu_foreach(pic->pic_percpu, pic_percpu_allocate, pic); + percpu_setup(pic->pic_percpu, sizeof(struct pic_percpu), + pic_percpu_allocate, pic); pic->pic_sources = &pic_sources[sourcebase]; pic->pic_irqbase = irqbase; From 6bf1793cccd8419d876843e713e0d59cf40128ba Mon Sep 17 00:00:00 2001 From: Taylor R Campbell Date: Wed, 29 Jan 2020 21:54:21 +0000 Subject: [PATCH 3/4] Switch percpu_foreach to percpu_setup/destroy in sys/net. Can't sleep for allocation in percpu_foreach. --- sys/net/if.c | 6 ++++-- sys/net/if_l2tp.c | 6 ++++-- sys/net/route.c | 3 ++- sys/netinet/wqinput.c | 3 ++- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/sys/net/if.c b/sys/net/if.c index 1b9df2a16df0..d4852e085588 100644 --- a/sys/net/if.c +++ b/sys/net/if.c @@ -2918,7 +2918,8 @@ if_tunnel_alloc_ro_percpu(void) percpu_t *ro_percpu; ro_percpu = percpu_alloc(sizeof(struct tunnel_ro)); - percpu_foreach(ro_percpu, if_tunnel_ro_init_pc, NULL); + percpu_setup(ro_percpu, sizeof(struct tunnel_ro), + if_tunnel_ro_init_pc, NULL); return ro_percpu; } @@ -2938,7 +2939,8 @@ void if_tunnel_free_ro_percpu(percpu_t *ro_percpu) { - percpu_foreach(ro_percpu, if_tunnel_ro_fini_pc, NULL); + percpu_teardown(ro_percpu, sizeof(struct tunnel_ro), + if_tunnel_ro_fini_pc, NULL); percpu_free(ro_percpu, sizeof(struct tunnel_ro)); } diff --git a/sys/net/if_l2tp.c b/sys/net/if_l2tp.c index e71d78fd406a..347c623af043 100644 --- a/sys/net/if_l2tp.c +++ b/sys/net/if_l2tp.c @@ -268,7 +268,8 @@ l2tp_clone_create(struct if_clone *ifc, int unit) sc->l2tp_ro_percpu = if_tunnel_alloc_ro_percpu(); sc->l2tp_ifq_percpu = percpu_alloc(sizeof(struct ifqueue *)); - percpu_foreach(sc->l2tp_ifq_percpu, l2tp_ifq_init_pc, NULL); + percpu_setup(sc->l2tp_ifq_percpu, sizeof(struct ifqueue *), + l2tp_ifq_init_pc, NULL); sc->l2tp_si = softint_establish(si_flags, l2tpintr_softint, sc); mutex_enter(&l2tp_softcs.lock); @@ -367,7 +368,8 @@ l2tp_clone_destroy(struct ifnet *ifp) mutex_exit(&sc->l2tp_lock); softint_disestablish(sc->l2tp_si); - percpu_foreach(sc->l2tp_ifq_percpu, l2tp_ifq_fini_pc, NULL); + percpu_teardown(sc->l2tp_ifq_percpu, sizeof(struct ifqueue *), + l2tp_ifq_fini_pc, NULL); percpu_free(sc->l2tp_ifq_percpu, sizeof(struct ifqueue *)); mutex_enter(&l2tp_softcs.lock); diff --git a/sys/net/route.c b/sys/net/route.c index ef19e0f837ca..6546ccae00c4 100644 --- a/sys/net/route.c +++ b/sys/net/route.c @@ -2234,7 +2234,8 @@ rtcache_percpu_alloc(void) percpu_t *pc; pc = percpu_alloc(sizeof(struct route *)); - percpu_foreach(pc, rtcache_percpu_init_cpu, NULL); + percpu_setup(pc, sizeof(struct route *), rtcache_percpu_init_cpu, + NULL); return pc; } diff --git a/sys/netinet/wqinput.c b/sys/netinet/wqinput.c index d05d25d4eda4..695b077e2cf7 100644 --- a/sys/netinet/wqinput.c +++ b/sys/netinet/wqinput.c @@ -189,7 +189,8 @@ wqinput_create(const char *name, void (*func)(struct mbuf *, int, int)) pool_init(&wqi->wqi_work_pool, sizeof(struct wqinput_work), 0, 0, 0, name, NULL, IPL_SOFTNET); wqi->wqi_worklists = percpu_alloc(sizeof(struct wqinput_worklist *)); - percpu_foreach(wqi->wqi_worklists, wqinput_percpu_init_cpu, NULL); + percpu_setup(wqi->wqi_worklists, sizeof(struct wqinput_worklist *), + wqinput_percpu_init_cpu, NULL); wqi->wqi_input = func; wqinput_sysctl_setup(name, wqi); From 076a412068db4ebafde7b055013b470f9955d99b Mon Sep 17 00:00:00 2001 From: Taylor R Campbell Date: Thu, 30 Jan 2020 02:11:30 +0000 Subject: [PATCH 4/4] Switch percpu_foreach to percpu_setup in opencrypto. Can't sleep for allocation in percpu_foreach. --- sys/opencrypto/crypto.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sys/opencrypto/crypto.c b/sys/opencrypto/crypto.c index 0685ebfc7636..d5940ecf6ab0 100644 --- a/sys/opencrypto/crypto.c +++ b/sys/opencrypto/crypto.c @@ -563,7 +563,8 @@ crypto_init0(void) coherency_unit, 0, 0, "cryptkop", NULL, IPL_NET, NULL, NULL, NULL); crypto_crp_qs_percpu = percpu_alloc(sizeof(struct crypto_crp_qs)); - percpu_foreach(crypto_crp_qs_percpu, crypto_crp_qs_init_pc, NULL); + percpu_setup(crypto_crp_qs_percpu, sizeof(struct crypto_crp_qs), + crypto_crp_qs_init_pc, NULL); crypto_crp_ret_qs_init();