/* $NetBSD$ */ /*- * Copyright (c) 2015 The NetBSD Foundation, Inc. * All rights reserved. * * This code is derived from software contributed to The NetBSD Foundation * by Taylor R. Campbell. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #include __KERNEL_RCSID(0, "$NetBSD$"); #include #include #include #include #include #include #include #ifdef __HAVE_CPU_COUNTER #include #endif #include #define ENTROPY_MAXDEBT 32 #define ENTROPY_MAXEXTRACT 32 CTASSERT(ENTROPY_MAXEXTRACT <= ENTPOOL_OUTRATE_BYTES); /* * Per-CPU entropy state. */ struct entropy_cpu { struct entpool ec_pool; /* entropy pool */ unsigned ec_credit; /* pending credit */ bool ec_locked; /* lock against hardintr: * hardintr drops samples if * locked, so we need not * actually block hardintrs */ uint64_t ec_bytesdropped; uint64_t ec_bytesentered; }; /* * Global state. */ struct percpu *entropy_percpu __read_mostly; /* struct entropy_cpu */ void *entropy_softint_cookie __read_mostly; struct { kmutex_t lock; struct entpool pool; unsigned debt; kcondvar_t cv; LIST_HEAD(krndsource) sources; rndsave_t *seed; bool initialized; } entropy_global __cacheline_aligned; /* * Parameters. */ bool entropy_collection = ENTROPY_COLLECTION; bool entropy_depletion = ENTROPY_DEPLETION; bool entropy_hwrngtest = ENTROPY_HWRNGTEST; /* * XXX entropy_depletion is not supported by any cryptography theory * and defends mainly against bogeymen. * * XXX rngtest is a nice side channel for storing what's going into the * entropy pool on the kernel stack and in extra buffers, and exposing * it to variable-time operations. */ static const struct sysctlnode *entropy_sysctlnode; static struct sysctllog *entropy_sysctllog; /* * entropy_timer: Cycle counter, time counter, or anything that changes * a wee bit unpredictably. */ static inline uint32_t entropy_timer(void) { struct bintime bt; uint32_t v; #ifdef __HAVE_CPU_COUNTER if (__predict_true(cpu_hascounter())) return cpu_counter32(); #endif if (__predict_false(cold)) return 0; binuptime(&bt); v = bt->bt_frac; v ^= bt->bt_frac >> 32; v ^= bt->bt_sec; v ^= bt->bt_sec >> 32; return v; } /* * entropy_init: Initialize the entropy subsystem. Returns 0 on * success, error on failure. */ int entropy_init(void) { uint32_t extra[11]; unsigned i = 0; int error; /* Grab some cycle counts early at boot. */ extra[i++] = entropy_timer(); /* Allocate the per-CPU state. */ entropy_percpu = percpu_alloc(sizeof(struct entropy_cpu)); if (entropy_percpu == NULL) { error = ENOMEM; goto fail0; } extra[i++] = entropy_timer(); /* Establish the softint. */ entropy_softintcookie = softint_establish(SOFTINT_CLOCK|SOFTINT_MPSAFE, &entropy_softintr, NULL); if (entropy_softintcookie == NULL) { error = ENOMEM; goto fail1; } extra[i++] = entropy_timer(); /* Create the sysctl directory. */ sysctl_createv(&entropy_sysctllog, 0, NULL, &entropy_sysctlnode, CTLFLAG_PERMANENT, CTLTYPE_NODE, "entropy", SYSCTL_DESCR("Entropy (random number sources) options"), NULL, 0, NULL, 0, CTL_KERN, CTL_CREATE, CTL_EOL); if (entropy_sysctlnode == NULL) goto fail2; extra[i++] = entropy_timer(); /* Create the sysctl knobs. */ /* XXX These probably shouldn't be writable at securelevel>0. */ sysctl_createv(&entropy_sysctllog, 0, &entropy_sysctlnode, NULL, CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_BOOL, "collect", SYSCTL_DESCR("Automatically collect entropy from hardware"), NULL, 0, &entropy_collection, 0, CTL_CREATE, CTL_EOL); extra[i++] = entropy_timer(); sysctl_createv(&entropy_sysctllog, 0, &entropy_sysctlnode, NULL, CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_BOOL, "deplete", SYSCTL_DESCR("Voodoo: `deplete' entropy pool when observed"), NULL, 0, &entropy_depletion, 0, CTL_CREATE, CTL_EOL); extra[i++] = entropy_timer(); sysctl_createv(&entropy_sysctllog, 0, &entropy_sysctlnode, NULL, CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_BOOL, "hwrngtest", SYSCTL_DESCR("Statistically test output of hardware RNG devices"), NULL, 0, &entropy_hwrngtest, 0, CTL_CREATE, CTL_EOL); extra[i++] = entropy_timer(); /* Initialize the global state. */ mutex_init(&entropy_global.lock, MUTEX_DEFAULT, IPL_SOFTCLOCK); extra[i++] = entropy_timer(); entropy_global.debt = ENTROPY_MAXDEBT; extra[i++] = entropy_timer(); cv_init(&entropy_global.cv, "entropy"); extra[i++] = entropy_timer(); LIST_INIT(&entropy_global.sources); extra[i++] = entropy_timer(); /* Enter the cycle counts we grabbed into the global pool. */ KASSERT(i == __arraycount(extra)); entpool_enter(&entropy_global.pool, extra, sizeof extra); /* If the boot loader already provided a seed, use it. */ if (entropy_global.seed == NULL) { rnd_printf("no entropy seed\n"); } else { rndsave_t *const seed = entropy_global.seed; entpool_enter(&entropy_global.pool, seed->data, sizeof(seed->data)); entropy_global.debt -= MIN(entropy_global.debt, seed->entropy/NBBY); explicit_memset(seed, 0, sizeof(*seed)); } /* Success! */ entropy_global.initialized = true; return 0; fail2: softint_disestablish(entropy_softintcookie); fail1: percpu_free(entropy_percpu, sizeof(struct entropy_cpu)); fail0: KASSERT(error); return error; } /* * entropy_seed: Provide initial entropy to the system. To be called * via the bootloader before entropy_init. */ void entropy_seed(void *seed, size_t len) { SHA1_CTX ctx; uint8_t digest[SHA1_DIGEST_LENGTH]; if (len != sizeof(*seed)) { rnd_printf("invalid seed length: %zu\n", len); return; } SHA1Init(&ctx); SHA1Update(&ctx, &seed->entropy, sizeof(seed->entropy)); SHA1Update(&ctx, &seed->data, sizeof(seed->data)); SHA1Final(digest, &ctx); if (consttime_memcmp(digest, seed->digest, sizeof(digest)) != 0) { rnd_printf("invalid seed checksum\n"); return; } if (entropy_global.initialized) { mutex_enter(&entropy_global.lock); if (entropy_global.seed) { rnd_printf("already seeded\n"); return; } entropy_global.seed = seed; entpool_enter(&entropy_global.pool, seed->data, sizeof(seed->data)); entropy_global.debt -= MIN(entropy_global.debt, seed->entropy/NBBY); if (entropy_global.debt == 0) cv_broadcast(&entropy_global.cv, &entropy_global.lock); explicit_memset(seed, 0, sizeof(*seed)); mutex_exit(&entropy_global.lock); } else { if (entropy_global.seed) { rnd_printf("already seeded\n"); return; } entropy_global.seed = seed; } } /* * entropy_bootrequest: Request entropy from all sources at boot, once * config is complete and interrupts are running. */ void entropy_bootrequest(void) { KASSERT(entropy_global.initialized); /* * Simply request enough to satisfy the maximum entropy debt. * This is harmless overkill if the bootloader provided a seed. */ rndsources_request(ENTROPY_MAXDEBT); } /* * entropy_credit: If entrate is nonzero, it is a lower bound on * the estimated bits of data per bit of entropy in a len-byte sample * recently fed into the system entropy pool. In that case, credit * len/entrate bytes of entropy to the entropy debt. */ static inline void entropy_credit(size_t len, unsigned entrate) { /* * If there's no entropy debt, or the entropy rate of the * source is too low, do nothing. */ if (__predict_true(entropy_global.debt == 0 && !entropy_depletion)) return; if ((entrate == 0) || (len/entrate == 0)) return; /* * Otherwise, credit to the entropy debt and wake waiters if we * reduced it to zero. */ mutex_enter(&entropy_global.lock); entropy_global.debt -= MIN(entropy_global.debt, len/entrate); if (entropy_global.debt == 0) /* XXX How to make sure /dev/urandom gets it ASAP? */ cv_broadcast(&entropy_global.cv); mutex_exit(&entropy_global.lock); } /* * entropy_enter: Enter len bytes of data from buf into the system's * entropy pool, permuting the state as necessary when the internal * buffer fills up. If entrate is nonzero, it is a lower bound on the * estimated bits of data per bit of entropy in buf. In that case, * credit len/entrate bytes of entropy to the entropy debt. */ void entropy_enter(const void *buf, size_t len, unsigned entrate) { const unsigned rate = ENTPOOL_INRATE_BYTES; struct entropy_cpu *ec; int s; KASSERT(entropy_global.initialized); KASSERTMSG(!cpu_intr_p(), "use entropy_enter_intr from interrupt context"); /* Acquire the per-CPU state, blocking hard and soft interrupts. */ ec = percpu_getref(entropy_percpu); s = splsoftclock(); KASSERT(!ec->ec_locked); ec->ec_locked = true; /* Enter into the per-CPU pool. */ entpool_enter(&ec->ec_pool, buf, len); /* Release the per-CPU state. */ KASSERT(ec->ec_locked); ec->ec_locked = false; splx(s); percpu_putref(entropy_percpu); /* Credit any entropy. */ entropy_credit(len, entrate); } /* * entropy_extract: Gather entropy on all CPUs into the global pool * with a cross-call and extract an output from it. Flags may have: * * ENTROPY_WAIT Wait for entropy if not available yet. * ENTROPY_SIG Allow interruption by a signal during wait. * * Return zero on success, or error on failure: * * EWOULDBLOCK No entropy and ENTROPY_WAIT not set. * EINTR/ERESTART No entropy, ENTROPY_SIG set, and interrupted. * * Always fill buf, even on error. */ int entropy_extract(void *buf, size_t len, int flags) { uint32_t extra[5]; uint64_t ticket; bool waited; uint8_t wait; int error; ASSERT_SLEEPABLE(); KASSERT(entropy_global.initialized); KASSERT(len <= ENTROPY_MAXEXTRACT); if (!entropy_depletion) waited = false; /* Grab CPU number and cycle counter to mix extra into the pool. */ extra[i++] = cpu_number(); extra[i++] = entropy_timer(); /* * First, wait until there is enough entropy in the system, and * lock extraction. */ mutex_enter(&entropy_global.lock); error = 0; while (0 < entropy_global.debt) { /* Request entropy from sources. */ mutex_exit(&entropy_global.lock); rndsources_request(len); mutex_enter(&entropy_global.lock); /* If we got enough already, we're done. */ if (entropy_global.debt == 0) { KASSERT(error == 0); break; } /* * Otherwise, wait for some to come in. If not doing * entropy depletion, note that we will have already * waited if we exit the loop. */ if (!entropy_depletion) waited = true; if (!ISSET(flags, ENTROPY_WAIT)) { error = EWOULDBLOCK; break; } if (ISSET(flags, ENTROPY_SIG)) { error = cv_wait_sig(&entropy_global.cv, &entropy_global.lock); if (error) break; } else { cv_wait(&entropy_global.cv, &entropy_global.lock); } } mutex_exit(&entropy_global.lock); /* Gather entropy on all CPUs with a broadcast xcall. */ ticket = xc_broadcast(0, &entropy_extract_xc, NULL, NULL); extra[i++] = entropy_timer(); xc_wait(ticket); extra[i++] = entropy_timer(); /* * Mix the extra into the pool, extract the requested data, and * unlock extraction. */ mutex_enter(&entropy_global.lock); extra[i++] = entropy_timer(); KASSERT(i == __arraycount(extra)); entpool_enter(&entropy_global.pool, extra, sizeof extra); entpool_extract(&entropy_global.pool, buf, len); if (entropy_depletion) { /* * If we're depleting entropy, deplete it. * * We always deplete it in full rather than partially * in an attempt to thwart iterative guessing attacks: * if the actual entropy is less than the estimated * entropy, it may be feasible for an attacker to guess * the state by querying the entropy pool and trying * guesses offline. In that case, if we only accrue a * little more entropy before giving more out, the * attacker must guess only that little more entropy in * order to keep up. * * If, instead, we always gather another 256 bits of * entropy, and somehow cease to be wrong in our * estimate of how much entropy is coming in (...), * the attacker will have a hard time keeping up. */ entropy_global.debt = ENTROPY_MAXDEBT; } else { /* * If we don't deplete entropy, and we didn't wait for * boot entropy, get an extra byte to decide whether to * artificially wait and, if so, for how long. */ if (!waited) { KASSERT(error == 0); entpool_extract(&entropy_global.pool, &wait, sizeof wait); } } mutex_exit(&entropy_global.lock); if (!entropy_depletion) { /* * If we're not depleting entropy, then after the boot * entropy is available (if !waited), deliberately wait * anyway in half the invocations (if wait & 1). The * number of ticks to wait is distributed uniformly in * [0, 128). This keeps both blocking and nonblocking * code paths exercised. */ if (!waited) { KASSERT(error == 0); if (wait & 1) { if (!ISSET(flags, ENTROPY_WAIT)) return EWOULDBLOCK; error = kpause("entropy", ISSET(flags, ENTROPY_SIG), wait >> 1 /* timeout in ticks */, NULL); if (error) return error; } } } return error; } /* * entropy_extract_xc: Extract output from the local CPU's entropy pool * and enter it into the global pool. */ static void entropy_extract_xc(void *arg1 __unused, void *arg2 __unused) { struct entropy_cpu *ec; uint8_t buf[ENTPOOL_OUTRATE_BYTES]; uint32_t extra[7]; unsigned i; int s; /* Grab CPU number and cycle counter to mix extra into the pool. */ extra[i++] = cpu_number(); extra[i++] = entropy_timer(); /* Acquire the per-CPU state, blocking hard and soft interrupts. */ ec = percpu_getref(entropy_percpu); s = splsoftclock(); KASSERT(!ec->ec_locked); ec->ec_locked = true; extra[i++] = entropy_timer(); /* Extract the data. */ entpool_extract(&ec->ec_pool, buf, sizeof buf); extra[i++] = entropy_timer(); /* Release the per-CPU state. */ KASSERT(!ec->ec_locked); ec->ec_locked = false; splx(s); percpu_putref(entropy_percpu); extra[i++] = entropy_timer(); /* Enter the data and extra into the global pool. */ mutex_enter(&entropy_global.lock); extra[i++] = entropy_timer(); entpool_enter(&entropy_global.pool, buf, sizeof buf); explicit_memset(buf, 0, sizeof buf); extra[i++] = entropy_timer(); KASSERT(i == __arraycount(extra)); entpool_enter(&entropy_global.pool, extra, sizeof extra); explicit_memset(extra, 0, sizeof extra); mutex_exit(&entropy_global.lock); } /* * entropy_enter_intr: Enter up to len bytes of data from buf into the * system's entropy pool. If entrate is nonzero, it is a lower bound * on the estimated bits of data per bit of entropy in buf. In that * case, credit n/entrate bytes of entropy to the entropy debt, where n * is the number of bytes actually used from buf. Schedule a softint * to stir the entropy pool if we filled the input buffer. Return the * number of bytes actually used. * * Using this in thread context will work, but you might as well use * entropy_enter in that case to contribute all the entropy. */ size_t entropy_enter_intr(const void *buf, size_t len, unsigned entrate) { const unsigned rate = ENTPOOL_INRATE_BYTES; struct entropy_cpu *ec; size_t nused = 0; int s; KASSERT(entropy_global.initialized); /* * Acquire the per-CPU state. If someone is in the middle of * using it, drop the sample. Otherwise, block higher-priority * interrupts. */ ec = percpu_getref(entropy_percpu); if (ec->ec_locked) goto out; ec->ec_locked = true; /* Enter as much as we can into the per-CPU pool. */ nused = entpool_enter_nostir(&ec->ec_pool, buf, len); /* If not all got used, need to stir. */ /* XXX What if all got used, but we filled the buffer? Plugh. */ if (nused < len) softint_schedule(entropy_softintcookie); /* If there's entropy debt and we provided any data, credit it. */ if (__predict_false(entropy_depletion || 0 < entropy_global.debt) && 0 < entrate && 0 < nused/entrate) { softint_schedule(entropy_softintcookie); ec->ec_credit += MIN(ENTROPY_MAXDEBT - ec->ec_credit, nused/entrate); } out: ec->ec_bytesdropped += len - nused; ec->ec_bytesentered += nused; /* Release the per-CPU state. */ KASSERT(ec->ec_locked); ec->ec_locked = false; percpu_putref(entropy_percpu); return nused; } /* * entropy_softintr: Softint handler for processing interrupt entropy. * Stir the pool and credit the entropy if the interrupt provided any. */ static void entropy_softintr(void *cookie __unused) { struct entropy_cpu *ec; unsigned credit; /* Grab the per-CPU entropy state and lock out interrupts. */ ec = percpu_getref(entropy_percpu); KASSERT(!ec->ec_locked); ec->ec_locked = true; /* Stir the pool. */ entpool_stir(&ec->ec_pool); /* Find the amount of entropy to credit. */ credit = ec->ec_credit; ec->ec_credit = 0; /* Unlock interrupts and release the per-CPU state. */ KASSERT(ec->ec_locked); ec->ec_locked = false; percpu_putref(entropy_percpu); /* Credit to the entropy debt. */ entropy_credit(credit, 1); } /* * Entropy sources. Usage: * * - Allocate and zero a struct krndsource. * - Optionally, set a callback with rndsource_setcb. * - Attach it with rnd_attach_source. * - Use rnd_add_data with it. * - Detach it with rnd_detach_source. * - Free the struct krndsource. */ /* * rndsource_setcb: Set the request callback for the entropy source rs, * if it can provide entropy on demand. Must be done before * rnd_attach_source. */ void rndsource_setcb(struct krndsource *rs, void (*request)(size_t, void *), void *requestarg) { rs->rs_request = request; rs->rs_requestarg = requestarg; } /* * rnd_attach_source: Attach the entropy source rs. Must be done after * rndsource_setcb, if any, and before any calls to rnd_add_data. */ void rnd_attach_source(struct krndsource *rs, const char *name, int type, int flags) { uint32_t extra[8]; unsigned i = 0; uint8_t *cotest = NULL; rngtest_t *rngtest = NULL; extra[i++] = cpu_number(); extra[i++] = entropy_timer(); flags |= RND_FLAG_NO_ESTIMATE; switch (type) { case RND_TYPE_NET: flags |= RND_FLAG_NO_COLLECT; break; case RND_TYPE_RNG: cotest = kmem_alloc(HWRNG_COTEST_BYTES, KM_SLEEP); rngtest = kmem_alloc(sizeof(*rngtest), KM_SLEEP); strlcpy(rngtest->rt_name, name, sizeof(rngtest->rt_name)); break; } extra[i++] = entropy_timer(); refcount_init(&rs->rs_refcount); strlcpy(rs->rs_name, name, sizeof(rs->rs_name)); rs->rs_type = type; rs->rs_flags = flags; rs->rs_bad = 0; rs->rs_bits = 0; rs->rs_rngtest = rngtest; rs->rs_rngtest_resid = (rngtest == NULL ? 0 : FIPS140_RNG_TEST_BYTES); rs->rs_cotest = cotest; rs->rs_cotest_resid = (cotest == NULL ? 0 : HWRNG_COTEST_BYTES); extra[i++] = entropy_timer(); mutex_enter(&entropy_global.lock); extra[i++] = entropy_timer(); LIST_INSERT_HEAD(&entropy_global.sources, rs, rs_list); extra[i++] = entropy_timer(); mutex_exit(&entropy_global.lock); extra[i++] = entropy_timer(); rndsource_request(rs); extra[i++] = entropy_timer(); KASSERT(i == __arraycount(extra)); entropy_enter(extra, sizeof extra, 0); } /* * rnd_detach_source: Detach the entropy source rs. May sleep waiting * for users to drain. Further use is not allowed. */ void rnd_detach_source(struct krndsource *rs) { /* We may wait for other users drain. */ ASSERT_SLEEPABLE(); /* Remove it from the list and wait for users to drain. */ mutex_enter(&entropy_global.lock); LIST_REMOVE(rs, rs_list); refcount_dec_drain(&rs->rs_refcount, &entropy_global.lock, &entropy_global.cv); mutex_exit(&entropy_global.lock); /* Free the rngtest if there was one. */ if (rs->rs_rngtest != NULL) { explicit_memset(rs->rs_rngtest, 0, sizeof(*rs->rs_rngtest)); kmem_free(rs->rs_rngtest, sizeof(*rs->rs_rngtest)); rs->rs_rngtest = NULL; /* paranoia */ } } /* * rndsources_request: Request n bytes of entropy from all sources in * the system. OK if we overdo it. */ static void rndsources_request(size_t n) { struct krndsource *rs, *next; /* Walk the list of sources under the global entropy state lock. */ mutex_enter(&entropy_global.lock); LIST_FOREACH_SAFE(rs, &entropy_global.sources, rs_list, next) { /* Try to acquire a reference. If we can't, skip it. */ if (refcount_inc(&rs->rs_refcount) != 0) continue; /* Drop the lock while we call the callback. */ mutex_exit(&entropy_global.lock); rndsource_request(rs, n); mutex_enter(&entropy_global.lock); /* * Release the reference. If we held the last one, * notify rnd_detach_source. We use * refcount_dec_local, not refcount_dec_lock or * refcount_dec_broadcast, because all access is * serialized under the lock already. */ if (refcount_dec_local(&rs->rs_refcount)) cv_broadcast(&entropy_global.cv); } mutex_exit(&entropy_global.lock); } /* * rndsource_request: Request that rs try to contribute n bytes to the * entropy pool. */ static void rndsource_request(struct krndsource *rs, size_t n) { size_t nreq = n; KASSERT(refcount_referenced_p(&rs->rs_refcount)); if (rs->rs_type == RND_TYPE_RNG) { /* * Hardware entropy sources enter data in units of * COTEST_BYTES, and must first enter * FIPS140_RNG_TEST_BYTES for testing. * * XXX If entropy_hwrngtest is switched from off to on, * there is a window in which a request here will fail * to ask for enough bytes. */ if (entropy_hwrngtest && rs->rs_rngtest_resid) { if (nreq > SIZE_MAX - FIPS140_RNG_TEST_BYTES) nreq = SIZE_MAX; else nreq += FIPS140_RNG_TEST_BYTES; } if (entropy_hwrngtest) { if (nreq > SIZE_MAX - COTEST_BYTES) nreq = SIZE_MAX; else nreq = roundup(nreq, COTEST_BYTES); } } /* Call the callback with the new requested number of bytes. */ (*rs->rs_request)(nreq, rs->rs_requestarg); } /* * rnd_add_uint32: Enter 32 bits of data from an entropy source into * the pool. * * If rs is NULL, may not be called from interrupt context. * * If rs is non-NULL, may be called from any context. May drop data if * called from interrupt context. */ void rnd_add_uint32(struct krndsource *rs, uint32_t value) { rnd_add_data(rs, &value, sizeof value, 0); } /* * rnd_add_data: Enter data from an entropy source into the pool. * * If rs is NULL, may not be called from interrupt context. * * If rs is non-NULL, may be called from any context. May drop data if * called from interrupt context. */ void rnd_add_data(struct krndsource *rs, const void *buf, uint32_t len, uint32_t entropybits) { uint32_t extra[3]; unsigned i = 0; unsigned entrate; size_t nused; /* If there's no rndsource, just enter the data and time now. */ if (rs == NULL) { extra[i++] = cpu_number(); extra[i++] = entropy_timer(); entrate = NBBY*(len/entropybits); entropy_enter(buf, len, entrate); extra[i++] = entropy_timer(); KASSERT(i == __arraycount(extra)); entropy_enter(extra, sizeof extra, 0); return; } /* * Skip if: * - we're not collecting entropy, or * - this source has been marked bad, or * - the operator doesn't want to collect entropy from this, or * - neither data nor timings are being collected from this. */ if (!entropy_collection || rs->rs_bad || ISSET(rs->rs_flags, RND_FLAG_NO_COLLECT) || !ISSET(rs->rs_flags, RND_FLAG_COLLECT_VALUE|RND_FLAG_COLLECT_TIME)) return; /* Hardware RNG devices get special treatment. */ if (rs->rs_type == RND_TYPE_RNG && entropy_hwrngtest) { hwrng_add_data(rs, buf, len, entrate); return; } /* Grab CPU number and cycle counter to mix extra into the pool. */ extra[i++] = cpu_number(); extra[i++] = entropy_timer(); /* If we are collecting data, enter them. */ if (ISSET(rs->rs_flags, RND_FLAG_COLLECT_VALUE)) { entrate = NBBY*(len/entropybits); if (cpu_intr_p()) { nused = entropy_enter_intr(buf, len, entrate); } else { entropy_enter(buf, len, entrate); nused = len; } if (0 < entrate) rs->rs_bits += NBBY*nused/entrate; } extra[i++] = entropy_timer(); /* If we are collecting timings, enter them. */ KASSERT(i == __arraycount(extra)); if (ISSET(rs->rs_flags, RND_FLAG_COLLECT_TIME)) { if (cpu_intr_p()) (void)entropy_enter_intr(extra, sizeof extra, 0); else entropy_enter(extra, sizeof extra, 0); } } /* * hwrng_add_data: Subject hardware RNGs to statistical tests before * allowing anything from them into the entropy pool. */ static void hwrng_add_data(struct krndsource *rs, const void *buf, size_t len, uint32_t entropybits) { const unsigned entrate = NBBY*(len/entropybits); uint32_t extra = entropy_timer(); const uint8_t *p = buf; size_t n = len; size_t s, m; int r; KASSERT(rs->rs_cotest != NULL); KASSERT(rs->rs_rngtest != NULL); /* * Continuous output test: gather buffers of HWRNG_COTEST_BYTES * and make sure the first half doesn't match the second half * each time we fill the buffer before contributing the buffer * to the entropy pool. */ KASSERT(0 < rs->rs_cotest_resid); while (rs->rs_cotest_resid <= n) { /* Fill the cotest buffer and check it. */ s = HWRNG_COTEST_BYTES - rs->rs_cotest_resid; memcpy(rs->rs_cotest + s, p, rs->rs_cotest_resid); p += rs->rs_cotest_resid; n -= rs->rs_cotest_resid; m = HWRNG_COTEST_BYTES/2; if (consttime_memcmp(rs->rs_cotest, rs->rs_cotest + m, m) == 0) { rnd_printf("%s: continuous output test failed\n", rs->rs_name); rs->rs_bad = 1; return; } /* * We filled a buffer and passed the continuous-output * test. If we haven't applied a FIPS-140 statistical * test, contribute it to the rngtest buffer. */ if (0 < rs->rs_rngtest_resid) { s = FIPS140_RNG_TEST_BYTES - rs->rs_rngtest_resid; m = MIN(rs->rs_rngtest_resid, HWRNG_COTEST_BYTES); memcpy(rs->rs_rngtest->rt_b + s, rs->rs_cotest, m); rs->rs_rngtest_resid -= m; /* If we filled the rngtest buffer, test it. */ if (rs->rs_rngtest_resid == 0) { if (rngtest(rs->rs_rngtest)) { rnd_printf("%s" ": failed statistical test", rs->rs_name); rs->rs_bad = 1; return; } } } /* Enter the data and/or timing as requested. */ if (ISSET(rs->rs_flags, RND_FLAG_COLLECT_VALUE)) { /* * Assume the hardware has a constant entropy * rate and compute it from the current sample. */ m = HWRNG_COTEST_BYTES; if (cpu_intr_p()) (void)entropy_enter_intr(rs->rs_cotest, m, entrate); else entropy_enter(rs->rs_cotest, m, entrate); } if (ISSET(rs->rs_flags, RND_FLAG_COLLECT_TIME)) { if (cpu_intr_p()) (void)entropy_enter_intr(&extra, sizeof extra, 0); else entropy_enter(&extra, sizeof extra, 0); extra = entropy_timer(); } /* Don't let the inputs to the entropy pool hang around. */ explicit_memset(rs->rs_cotest, 0, HWRNG_COTEST_BYTES); rs->rs_cotest_resid = HWRNG_COTEST_BYTES; } /* * Can't fill the cotest buffer. Put what we have left in it * and be done. */ s = HWRNG_COTEST_BYTES - rs->rs_cotest_resid; memcpy(rs->rs_cotest + s, p, n); rs->rs_cotest_resid -= n; KASSERT(0 < rs->rs_cotest_resid); }