diff --git a/sys/kern/subr_psref.c b/sys/kern/subr_psref.c index c3f76ab0e74..4c985f3dbac 100644 --- a/sys/kern/subr_psref.c +++ b/sys/kern/subr_psref.c @@ -74,12 +74,11 @@ __KERNEL_RCSID(0, "$NetBSD: subr_psref.c,v 1.7 2017/06/01 02:45:13 chs Exp $"); #include #include #include +#include #include #include #include -LIST_HEAD(psref_head, psref); - static bool _psref_held(const struct psref_target *, struct psref_class *, bool); @@ -103,7 +102,7 @@ struct psref_class { * Not exposed by the API. */ struct psref_cpu { - struct psref_head pcpu_head; + struct percpu_list_head pcpu_head; }; /* @@ -135,7 +134,7 @@ psref_cpu_drained_p(void *p, void *cookie, struct cpu_info *ci __unused) const struct psref_cpu *pcpu = p; bool *retp = cookie; - if (!LIST_EMPTY(&pcpu->pcpu_head)) + if (!PERCPU_LIST_EMPTY(&pcpu->pcpu_head)) *retp = false; } @@ -194,7 +193,7 @@ psref_check_duplication(struct psref_cpu *pcpu, struct psref *psref, bool found = false; struct psref *_psref; - LIST_FOREACH(_psref, &pcpu->pcpu_head, psref_entry) { + PERCPU_LIST_FOREACH(_psref, &pcpu->pcpu_head, struct psref, psref_entry) { if (_psref == psref && _psref->psref_target == target) { found = true; @@ -226,7 +225,6 @@ void psref_acquire(struct psref *psref, const struct psref_target *target, struct psref_class *class) { - struct psref_cpu *pcpu; int s; KASSERTMSG((kpreempt_disabled() || cpu_softintr_p() || @@ -242,21 +240,25 @@ psref_acquire(struct psref *psref, const struct psref_target *target, /* Block interrupts and acquire the current CPU's reference list. */ s = splraiseipl(class->prc_iplcookie); - pcpu = percpu_getref(class->prc_percpu); - #ifdef DEBUG + struct psref_cpu *pcpu = percpu_getref(class->prc_percpu); + /* Sanity-check if the target is already acquired with the same psref. */ psref_check_duplication(pcpu, psref, target); #endif /* Record our reference. */ - LIST_INSERT_HEAD(&pcpu->pcpu_head, psref, psref_entry); psref->psref_target = target; psref->psref_lwp = curlwp; psref->psref_cpu = curcpu(); + percpu_list_entry_init(&psref->psref_entry); + PERCPU_LIST_INSERT_HEAD(class->prc_percpu, struct psref_cpu, pcpu_head, + psref, psref_entry); /* Release the CPU list and restore interrupts. */ +#ifdef DEBUG percpu_putref(class->prc_percpu); +#endif splx(s); } @@ -302,7 +304,8 @@ psref_release(struct psref *psref, const struct psref_target *target, * (as does blocking interrupts). */ s = splraiseipl(class->prc_iplcookie); - LIST_REMOVE(psref, psref_entry); + PERCPU_LIST_REMOVE(class->prc_percpu, struct psref_cpu, pcpu_head, + psref, psref_entry); splx(s); /* If someone is waiting for users to drain, notify 'em. */ @@ -325,7 +328,6 @@ void psref_copy(struct psref *pto, const struct psref *pfrom, struct psref_class *class) { - struct psref_cpu *pcpu; int s; KASSERTMSG((kpreempt_disabled() || cpu_softintr_p() || @@ -350,16 +352,16 @@ psref_copy(struct psref *pto, const struct psref *pfrom, /* Block interrupts and acquire the current CPU's reference list. */ s = splraiseipl(class->prc_iplcookie); - pcpu = percpu_getref(class->prc_percpu); /* Record the new reference. */ - LIST_INSERT_HEAD(&pcpu->pcpu_head, pto, psref_entry); pto->psref_target = pfrom->psref_target; pto->psref_lwp = curlwp; pto->psref_cpu = curcpu(); + percpu_list_entry_init(&pto->psref_entry); + PERCPU_LIST_INSERT_HEAD(class->prc_percpu, struct psref_cpu, pcpu_head, + pto, psref_entry); /* Release the CPU list and restore interrupts. */ - percpu_putref(class->prc_percpu); splx(s); } @@ -474,7 +476,7 @@ _psref_held(const struct psref_target *target, struct psref_class *class, pcpu = percpu_getref(class->prc_percpu); /* Search through all the references on this CPU. */ - LIST_FOREACH(psref, &pcpu->pcpu_head, psref_entry) { + PERCPU_LIST_FOREACH(psref, &pcpu->pcpu_head, struct psref, psref_entry) { /* Sanity-check the reference's CPU. */ KASSERTMSG((psref->psref_cpu == curcpu()), "passive reference transferred from CPU %u to CPU %u", diff --git a/sys/sys/percpu_list.h b/sys/sys/percpu_list.h new file mode 100644 index 00000000000..4e099806fa1 --- /dev/null +++ b/sys/sys/percpu_list.h @@ -0,0 +1,259 @@ +/* $NetBSD$ */ + +/*- + * Copyright (c) 2017 Internet Initiative Japan Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef _SYS_PERCPU_LIST_H_ +#define _SYS_PERCPU_LIST_H_ + +#include +#include +#include + +struct percpu_list_entry; + +/* must be in percpu_alloc()'ed structure */ +struct percpu_list_head { + struct percpu_list_entry *pclh_first; +}; + +struct percpu_list_entry { + struct percpu_list_entry **pcle_prevp; + struct percpu_list_entry *pcle_next; +}; + +/* mark for prevp which means the element is first */ +#define _PERCPU_LIST_FIRST ((void *)1ul) + +static inline void +percpu_list_entry_init(struct percpu_list_entry *entry) +{ + + entry->pcle_next = NULL; + entry->pcle_prevp = NULL; +} + +static inline struct percpu_list_entry * +percpu_list_first(const struct percpu_list_head *head) +{ + + return head->pclh_first; +} + +static inline void * +_percpu_list_first_container(const struct percpu_list_head *head, + const ptrdiff_t offset) +{ + struct percpu_list_entry *first; + + /* percpu_getref'ed? */ + KASSERT(kpreempt_disabled()); + + first = head->pclh_first; + return (first == NULL ? NULL : (char *)first - offset); +} + +static inline struct percpu_list_entry * +percpu_list_next(const struct percpu_list_entry *entry) +{ + + return entry->pcle_next; +} + +static inline void * +_percpu_list_next_container(const struct percpu_list_entry *entry, + const ptrdiff_t offset) +{ + struct percpu_list_entry *next = entry->pcle_next; + + return (next == NULL ? NULL : (char *)next - offset); +} + +static inline void +percpu_list_insert_head(struct percpu *percpu, const ptrdiff_t hoffset, + struct percpu_list_entry *new) +{ + void *pcpu; + struct percpu_list_head *head; + + KASSERT(new->pcle_next == NULL); + KASSERT(new->pcle_prevp == NULL); + + pcpu = percpu_getref(percpu); + + head = (struct percpu_list_head *)((char *)pcpu - hoffset); + new->pcle_prevp = _PERCPU_LIST_FIRST; + new->pcle_next = head->pclh_first; + if (head->pclh_first != NULL) + head->pclh_first->pcle_prevp = &new->pcle_next; + head->pclh_first = new; + + percpu_putref(percpu); +} + +static inline void +percpu_list_insert_before(struct percpu *percpu, const ptrdiff_t hoffset, + struct percpu_list_entry *entry, + struct percpu_list_entry *new) +{ + + KASSERT(new->pcle_next == NULL); + KASSERT(new->pcle_prevp == NULL); + + /* + * Don't dereference prevp of first element. That can cause panic when + * percpu_cpu_enlarge() runs. See PR kern/52515. + */ + if (__predict_false(entry->pcle_prevp == _PERCPU_LIST_FIRST)) { + void *pcpu; + struct percpu_list_head *head; + + pcpu = percpu_getref(percpu); + + head = (struct percpu_list_head *)((char *)pcpu - hoffset); + new->pcle_prevp = _PERCPU_LIST_FIRST; + new->pcle_next = entry; + head->pclh_first = new; + entry->pcle_prevp = &new->pcle_next; + + percpu_putref(percpu); + } else { + new->pcle_prevp = entry->pcle_prevp; + new->pcle_next = entry; + *entry->pcle_prevp = new; + entry->pcle_prevp = &new->pcle_next; + } +} + +static inline void +percpu_list_insert_after(struct percpu *percpu __unused, + const ptrdiff_t hoffset __unused, + struct percpu_list_entry *entry, + struct percpu_list_entry *new) +{ + + KASSERT(new->pcle_next == NULL); + KASSERT(new->pcle_prevp == NULL); + + new->pcle_prevp = &entry->pcle_next; + new->pcle_next = entry->pcle_next; + if (new->pcle_next != NULL) + new->pcle_next->pcle_prevp = &new->pcle_next; + entry->pcle_next = new; +} + +static inline void +percpu_list_remove(struct percpu *percpu, const ptrdiff_t hoffset, + struct percpu_list_entry *entry) +{ + + /* + * Don't dereference prevp of first element. See PR kern/52515. + */ + if (__predict_false(entry->pcle_prevp == _PERCPU_LIST_FIRST)) { + void *pcpu; + struct percpu_list_head *head; + + pcpu = percpu_getref(percpu); + + head = (struct percpu_list_head *)((char *)pcpu - hoffset); + if (entry->pcle_next != NULL) + entry->pcle_next->pcle_prevp = _PERCPU_LIST_FIRST; + head->pclh_first = entry->pcle_next; + entry->pcle_prevp = NULL; + + percpu_putref(percpu); + } else { + if (entry->pcle_next != NULL) + entry->pcle_next->pcle_prevp = entry->pcle_prevp; + *entry->pcle_prevp = entry->pcle_next; + entry->pcle_prevp = NULL; + } +} + +static inline bool +percpu_list_empty(const struct percpu_list_head *head) +{ + + /* percpu_getref'ed? */ + KASSERT(kpreempt_disabled()); + + return head->pclh_first == NULL; +} + +/* + * Type-safe macros for convenience. + */ + +#ifdef __COVERITY__ +#define _PERCPU_LIST_VALIDATE_PTRS(P, Q) 0 +#define _PERCPU_LIST_VALIDATE_CONTAINER(P, T, F) 0 +#else +#define _PERCPU_LIST_VALIDATE_PTRS(P, Q) \ + (0 * sizeof((P) - (Q)) * sizeof(*(P)) * sizeof(*(Q))) +#define _PERCPU_LIST_VALIDATE_CONTAINER(P, T, F) \ + (0 * sizeof((P) - &((T *)(((char *)(P)) - offsetof(T, F)))->F)) +#endif + +#define PERCPU_LIST_ENTRY_INITIALIZER \ + { .pcle_next = NULL, .pcle_prevp = NULL } + +#define PERCPU_LIST_ENTRY_INIT(elem, field) \ + percpu_list_entry_init(&(elem)->field) + +#define PERCPU_LIST_EMPTY(head) \ + percpu_list_empty((head)) + +#define PERCPU_LIST_INSERT_HEAD(percpu, ptype, hfield, elem, efield) \ + percpu_list_insert_head((percpu), offsetof(ptype, hfield), \ + &(elem)->efield) +#define PERCPU_LIST_INSERT_BEFORE(percpu, ptype, hfield, elem, new, efield) \ + percpu_list_insert_before(percpu, offsetof(ptype, hfield), \ + &(elem)->efield + _PERCPU_LIST_VALIDATE_PTRS(elem, new), \ + &(new)->efield) +#define PERCPU_LIST_INSERT_AFTER(percpu, ptype, hfield, elem, new, efield) \ + percpu_list_insert_after(percpu, offsetof(ptype, hfield), \ + &(elem)->efield + _PERCPU_LIST_VALIDATE_PTRS(elem, new), \ + &(new)->efield) +#define PERCPU_LIST_REMOVE(percpu, ptype, hfield, elem, efield) \ + percpu_list_remove(percpu, offsetof(ptype, hfield), \ + &(elem)->efield) + +#define PERCPU_LIST_FIRST(head, type, field) \ + ((type *)(_percpu_list_first_container((head), \ + offsetof(type, field))) \ + + _PERCPU_LIST_VALIDATE_CONTAINER(percpu_list_first(head), \ + type, field)) +#define PERCPU_LIST_NEXT(value, type, field) \ + ((type *)(_percpu_list_next_container(&(value)->field, \ + offsetof(type, field))) \ + + _PERCPU_LIST_VALIDATE_CONTAINER(percpu_list_next(&(value)->field), \ + type, field)) +#define PERCPU_LIST_FOREACH(value, head, type, field) \ + for ((value) = PERCPU_LIST_FIRST((head), type, field); \ + (value) != NULL; \ + (value) = PERCPU_LIST_NEXT((value), type, field)) +#endif /* _SYS_PERCPU_LIST_H_ */ diff --git a/sys/sys/psref.h b/sys/sys/psref.h index 88db6dbb603..51ca2d297e9 100644 --- a/sys/sys/psref.h +++ b/sys/sys/psref.h @@ -33,7 +33,7 @@ #define _SYS_PSREF_H #include -#include +#include struct cpu_info; struct lwp; @@ -69,7 +69,7 @@ struct psref_target { * written only on the local CPU. */ struct psref { - LIST_ENTRY(psref) psref_entry; + struct percpu_list_entry psref_entry; const struct psref_target *psref_target; struct lwp *psref_lwp; struct cpu_info *psref_cpu;