Since all changes of crng_init now go through credit_init_bits(), we can
fix a long standing race in which two concurrent callers of
credit_init_bits() have the new bit count >= some threshold, but are
doing so with crng_init as a lower threshold, checked outside of a lock,
resulting in crng_reseed() or similar being called twice.
In order to fix this, we can use the original cmpxchg value of the bit
count, and only change crng_init when the bit count transitions from
below a threshold to meeting the threshold.
Cc: Dominik Brodowski <[email protected]>
Signed-off-by: Jason A. Donenfeld <[email protected]>
---
drivers/char/random.c | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/drivers/char/random.c b/drivers/char/random.c
index a1a76487ea35..79409cf27a25 100644
--- a/drivers/char/random.c
+++ b/drivers/char/random.c
@@ -823,7 +823,7 @@ static void extract_entropy(void *buf, size_t nbytes)
static void credit_init_bits(size_t nbits)
{
- unsigned int init_bits, orig, add;
+ unsigned int new, orig, add;
unsigned long flags;
if (crng_ready() || !nbits)
@@ -833,12 +833,12 @@ static void credit_init_bits(size_t nbits)
do {
orig = READ_ONCE(input_pool.init_bits);
- init_bits = min_t(unsigned int, POOL_BITS, orig + add);
- } while (cmpxchg(&input_pool.init_bits, orig, init_bits) != orig);
+ new = min_t(unsigned int, POOL_BITS, orig + add);
+ } while (cmpxchg(&input_pool.init_bits, orig, new) != orig);
- if (!crng_ready() && init_bits >= POOL_READY_BITS)
+ if (orig < POOL_READY_BITS && new >= POOL_READY_BITS)
crng_reseed();
- else if (unlikely(crng_init == CRNG_EMPTY && init_bits >= POOL_EARLY_BITS)) {
+ else if (orig < POOL_EARLY_BITS && new >= POOL_EARLY_BITS) {
spin_lock_irqsave(&base_crng.lock, flags);
if (crng_init == CRNG_EMPTY) {
extract_entropy(base_crng.key, sizeof(base_crng.key));
--
2.35.1
Initialization happens once -- by way of credit_init_bits() -- and then
it never happens again. Therefore, it doesn't need to be in
crng_reseed(), which is a hot path that is called multiple times. It
also doesn't make sense to have there, as initialization activity is
better associated with initialization routines.
After the prior commit, crng_reseed() now won't be called by multiple
concurrent callers, which means that we can safely move the
"finialize_init" logic into crng_init_bits() unconditionally.
Cc: Dominik Brodowski <[email protected]>
Signed-off-by: Jason A. Donenfeld <[email protected]>
---
drivers/char/random.c | 43 +++++++++++++++++++------------------------
1 file changed, 19 insertions(+), 24 deletions(-)
diff --git a/drivers/char/random.c b/drivers/char/random.c
index 79409cf27a25..1598bb40376e 100644
--- a/drivers/char/random.c
+++ b/drivers/char/random.c
@@ -266,7 +266,6 @@ static void crng_reseed(void)
unsigned long flags;
unsigned long next_gen;
u8 key[CHACHA_KEY_SIZE];
- bool finalize_init = false;
extract_entropy(key, sizeof(key));
@@ -283,28 +282,9 @@ static void crng_reseed(void)
++next_gen;
WRITE_ONCE(base_crng.generation, next_gen);
WRITE_ONCE(base_crng.birth, jiffies);
- if (!crng_ready()) {
- crng_init = CRNG_READY;
- finalize_init = true;
- }
+ crng_init = CRNG_READY;
spin_unlock_irqrestore(&base_crng.lock, flags);
memzero_explicit(key, sizeof(key));
- if (finalize_init) {
- process_random_ready_list();
- wake_up_interruptible(&crng_init_wait);
- kill_fasync(&fasync, SIGIO, POLL_IN);
- pr_notice("crng init done\n");
- if (unseeded_warning.missed) {
- pr_notice("%d get_random_xx warning(s) missed due to ratelimiting\n",
- unseeded_warning.missed);
- unseeded_warning.missed = 0;
- }
- if (urandom_warning.missed) {
- pr_notice("%d urandom warning(s) missed due to ratelimiting\n",
- urandom_warning.missed);
- urandom_warning.missed = 0;
- }
- }
}
/*
@@ -836,10 +816,25 @@ static void credit_init_bits(size_t nbits)
new = min_t(unsigned int, POOL_BITS, orig + add);
} while (cmpxchg(&input_pool.init_bits, orig, new) != orig);
- if (orig < POOL_READY_BITS && new >= POOL_READY_BITS)
- crng_reseed();
- else if (orig < POOL_EARLY_BITS && new >= POOL_EARLY_BITS) {
+ if (orig < POOL_READY_BITS && new >= POOL_READY_BITS) {
+ crng_reseed(); /* Sets crng_init to CRNG_READY under base_crng.lock. */
+ process_random_ready_list();
+ wake_up_interruptible(&crng_init_wait);
+ kill_fasync(&fasync, SIGIO, POLL_IN);
+ pr_notice("crng init done\n");
+ if (unseeded_warning.missed) {
+ pr_notice("%d get_random_xx warning(s) missed due to ratelimiting\n",
+ unseeded_warning.missed);
+ unseeded_warning.missed = 0;
+ }
+ if (urandom_warning.missed) {
+ pr_notice("%d urandom warning(s) missed due to ratelimiting\n",
+ urandom_warning.missed);
+ urandom_warning.missed = 0;
+ }
+ } else if (orig < POOL_EARLY_BITS && new >= POOL_EARLY_BITS) {
spin_lock_irqsave(&base_crng.lock, flags);
+ /* Check if crng_init is CRNG_EMPTY, to avoid race with crng_reseed(). */
if (crng_init == CRNG_EMPTY) {
extract_entropy(base_crng.key, sizeof(base_crng.key));
crng_init = CRNG_EARLY;
--
2.35.1
Am Mon, May 09, 2022 at 02:14:09PM +0200 schrieb Jason A. Donenfeld:
> Initialization happens once -- by way of credit_init_bits() -- and then
> it never happens again. Therefore, it doesn't need to be in
> crng_reseed(), which is a hot path that is called multiple times. It
> also doesn't make sense to have there, as initialization activity is
> better associated with initialization routines.
>
> After the prior commit, crng_reseed() now won't be called by multiple
> concurrent callers, which means that we can safely move the
> "finialize_init" logic into crng_init_bits() unconditionally.
>
> Cc: Dominik Brodowski <[email protected]>
> Signed-off-by: Jason A. Donenfeld <[email protected]>
> ---
> drivers/char/random.c | 43 +++++++++++++++++++------------------------
> 1 file changed, 19 insertions(+), 24 deletions(-)
>
> diff --git a/drivers/char/random.c b/drivers/char/random.c
> index 79409cf27a25..1598bb40376e 100644
> --- a/drivers/char/random.c
> +++ b/drivers/char/random.c
> @@ -266,7 +266,6 @@ static void crng_reseed(void)
> unsigned long flags;
> unsigned long next_gen;
> u8 key[CHACHA_KEY_SIZE];
> - bool finalize_init = false;
>
> extract_entropy(key, sizeof(key));
>
> @@ -283,28 +282,9 @@ static void crng_reseed(void)
> ++next_gen;
> WRITE_ONCE(base_crng.generation, next_gen);
> WRITE_ONCE(base_crng.birth, jiffies);
> - if (!crng_ready()) {
> - crng_init = CRNG_READY;
> - finalize_init = true;
> - }
> + crng_init = CRNG_READY;
Why unconditionally (you revert that bit in the static branch patch and make
it conditional again; so I see no reason for that here)?
Otherwise, looks good:
Reviewed-by: Dominik Brodowski <[email protected]>
Thanks,
Dominik
Hi David,
On Fri, May 13, 2022 at 1:38 PM David Laight <[email protected]> wrote:
>
> From: Jason A. Donenfeld
> > Sent: 13 May 2022 11:22
> >
> > On Fri, May 13, 2022 at 08:24:19AM +0200, Dominik Brodowski wrote:
> > > > - if (!crng_ready()) {
> > > > - crng_init = CRNG_READY;
> > > > - finalize_init = true;
> > > > - }
> > > > + crng_init = CRNG_READY;
> > >
> > > Why unconditionally
> >
> > To avoid a useless branch.
>
> Are you now dirtying a cache line that would
> otherwise be clean?
Fair enough. I'll keep the branch.
Jason
Hi Dominik,
On Fri, May 13, 2022 at 08:24:19AM +0200, Dominik Brodowski wrote:
> > - if (!crng_ready()) {
> > - crng_init = CRNG_READY;
> > - finalize_init = true;
> > - }
> > + crng_init = CRNG_READY;
>
> Why unconditionally
To avoid a useless branch.
> (you revert that bit in the static branch patch and make
> it conditional again; so I see no reason for that here)?
With the static branch patch, I can totally remove the branch and the
store all together, so we get the best of both worlds.
Jason
Am Mon, May 09, 2022 at 02:14:08PM +0200 schrieb Jason A. Donenfeld:
> Since all changes of crng_init now go through credit_init_bits(), we can
> fix a long standing race in which two concurrent callers of
> credit_init_bits() have the new bit count >= some threshold, but are
> doing so with crng_init as a lower threshold, checked outside of a lock,
> resulting in crng_reseed() or similar being called twice.
Sidenote: crng_reseed() did manage quite fine if called twice in short
order.
> In order to fix this, we can use the original cmpxchg value of the bit
> count, and only change crng_init when the bit count transitions from
> below a threshold to meeting the threshold.
>
> Cc: Dominik Brodowski <[email protected]>
> Signed-off-by: Jason A. Donenfeld <[email protected]>
Reviewed-by: Dominik Brodowski <[email protected]>
Thanks,
Dominik
Hi Dominik,
On Fri, May 13, 2022 at 08:23:40AM +0200, Dominik Brodowski wrote:
> Am Mon, May 09, 2022 at 02:14:08PM +0200 schrieb Jason A. Donenfeld:
> > Since all changes of crng_init now go through credit_init_bits(), we can
> > fix a long standing race in which two concurrent callers of
> > credit_init_bits() have the new bit count >= some threshold, but are
> > doing so with crng_init as a lower threshold, checked outside of a lock,
> > resulting in crng_reseed() or similar being called twice.
>
> Sidenote: crng_reseed() did manage quite fine if called twice in short
> order.
With regards to crng_finialize, it did, but not with regards to
prematurely emptying patches and all that. IOW, buggy but not that bad.
>
> > In order to fix this, we can use the original cmpxchg value of the bit
> > count, and only change crng_init when the bit count transitions from
> > below a threshold to meeting the threshold.
> >
> > Cc: Dominik Brodowski <[email protected]>
> > Signed-off-by: Jason A. Donenfeld <[email protected]>
>
> Reviewed-by: Dominik Brodowski <[email protected]>
>
> Thanks,
> Dominik
Jason
From: Jason A. Donenfeld
> Sent: 13 May 2022 11:22
>
> On Fri, May 13, 2022 at 08:24:19AM +0200, Dominik Brodowski wrote:
> > > - if (!crng_ready()) {
> > > - crng_init = CRNG_READY;
> > > - finalize_init = true;
> > > - }
> > > + crng_init = CRNG_READY;
> >
> > Why unconditionally
>
> To avoid a useless branch.
Are you now dirtying a cache line that would
otherwise be clean?
David
-
Registered Address Lakeside, Bramley Road, Mount Farm, Milton Keynes, MK1 1PT, UK
Registration No: 1397386 (Wales)
Initialization happens once -- by way of credit_init_bits() -- and then
it never happens again. Therefore, it doesn't need to be in
crng_reseed(), which is a hot path that is called multiple times. It
also doesn't make sense to have there, as initialization activity is
better associated with initialization routines.
After the prior commit, crng_reseed() now won't be called by multiple
concurrent callers, which means that we can safely move the
"finialize_init" logic into crng_init_bits() unconditionally.
Reviewed-by: Dominik Brodowski <[email protected]>
Signed-off-by: Jason A. Donenfeld <[email protected]>
---
Changes v1->v2:
- Keep crng_init = CRNG_READY conditional to avoid dirtying cache line.
drivers/char/random.c | 42 +++++++++++++++++++-----------------------
1 file changed, 19 insertions(+), 23 deletions(-)
diff --git a/drivers/char/random.c b/drivers/char/random.c
index 3646eac353b7..8cea19024936 100644
--- a/drivers/char/random.c
+++ b/drivers/char/random.c
@@ -265,7 +265,6 @@ static void crng_reseed(void)
unsigned long flags;
unsigned long next_gen;
u8 key[CHACHA_KEY_SIZE];
- bool finalize_init = false;
extract_entropy(key, sizeof(key));
@@ -282,28 +281,10 @@ static void crng_reseed(void)
++next_gen;
WRITE_ONCE(base_crng.generation, next_gen);
WRITE_ONCE(base_crng.birth, jiffies);
- if (!crng_ready()) {
+ if (!crng_ready())
crng_init = CRNG_READY;
- finalize_init = true;
- }
spin_unlock_irqrestore(&base_crng.lock, flags);
memzero_explicit(key, sizeof(key));
- if (finalize_init) {
- process_random_ready_list();
- wake_up_interruptible(&crng_init_wait);
- kill_fasync(&fasync, SIGIO, POLL_IN);
- pr_notice("crng init done\n");
- if (unseeded_warning.missed) {
- pr_notice("%d get_random_xx warning(s) missed due to ratelimiting\n",
- unseeded_warning.missed);
- unseeded_warning.missed = 0;
- }
- if (urandom_warning.missed) {
- pr_notice("%d urandom warning(s) missed due to ratelimiting\n",
- urandom_warning.missed);
- urandom_warning.missed = 0;
- }
- }
}
/*
@@ -835,10 +816,25 @@ static void credit_init_bits(size_t nbits)
new = min_t(unsigned int, POOL_BITS, orig + add);
} while (cmpxchg(&input_pool.init_bits, orig, new) != orig);
- if (orig < POOL_READY_BITS && new >= POOL_READY_BITS)
- crng_reseed();
- else if (orig < POOL_EARLY_BITS && new >= POOL_EARLY_BITS) {
+ if (orig < POOL_READY_BITS && new >= POOL_READY_BITS) {
+ crng_reseed(); /* Sets crng_init to CRNG_READY under base_crng.lock. */
+ process_random_ready_list();
+ wake_up_interruptible(&crng_init_wait);
+ kill_fasync(&fasync, SIGIO, POLL_IN);
+ pr_notice("crng init done\n");
+ if (unseeded_warning.missed) {
+ pr_notice("%d get_random_xx warning(s) missed due to ratelimiting\n",
+ unseeded_warning.missed);
+ unseeded_warning.missed = 0;
+ }
+ if (urandom_warning.missed) {
+ pr_notice("%d urandom warning(s) missed due to ratelimiting\n",
+ urandom_warning.missed);
+ urandom_warning.missed = 0;
+ }
+ } else if (orig < POOL_EARLY_BITS && new >= POOL_EARLY_BITS) {
spin_lock_irqsave(&base_crng.lock, flags);
+ /* Check if crng_init is CRNG_EMPTY, to avoid race with crng_reseed(). */
if (crng_init == CRNG_EMPTY) {
extract_entropy(base_crng.key, sizeof(base_crng.key));
crng_init = CRNG_EARLY;
--
2.35.1