Re: incorrect large buffer handling in md5.c/sha1.c/sha256.c/sha512.c

[Paul Eggert]

On 05/18/2012 01:29 PM, Serge Belyshev wrote:

> this part does exactly the same: increment ctx->total [1] by one, where
> larger increment may be needed.

OK, I see.  But this bug can only happen on unusual hosts, right?
I.e., hosts where size_t is wider than 64 bits, or where
uint64_t does not exist.  Anyway, the bug can easily be fixed even
if it's only on unusual hosts, so I pushed this further patch.

diff --git a/ChangeLog b/ChangeLog
index fbe9c1d..7dfccd7 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -9,6 +9,11 @@
        * lib/sha1.c (sha1_process_block):
        * lib/sha256.c (sha256_process_block):
        Don't assume the buffer length is less than 2**32.
+       * lib/sha512.c (sha512_process_block): Likewise.
+       Here, the bug is present only in the rare case where the host does
+       not support uint64_t or where size_t is wider than 64 bits.
+       Use u64size to work around the problems.
+       * lib/u64.h (u64size): New macro.
 
 2012-05-15  Pádraig Brady  <address@hidden>
 
diff --git a/lib/sha512.c b/lib/sha512.c
index 0c0779c..cf62f20 100644
--- a/lib/sha512.c
+++ b/lib/sha512.c
@@ -485,13 +485,15 @@ sha512_process_block (const void *buffer, size_t len, 
struct sha512_ctx *ctx)
   u64 f = ctx->state[5];
   u64 g = ctx->state[6];
   u64 h = ctx->state[7];
+  u64 lolen = u64size (len);
 
   /* First increment the byte count.  FIPS PUB 180-2 specifies the possible
      length of the file up to 2^128 bits.  Here we only compute the
      number of bytes.  Do a double word increment.  */
-  ctx->total[0] = u64plus (ctx->total[0], u64lo (len));
-  if (u64lt (ctx->total[0], u64lo (len)))
-    ctx->total[1] = u64plus (ctx->total[1], u64lo (1));
+  ctx->total[0] = u64plus (ctx->total[0], lolen);
+  ctx->total[1] = u64plus (ctx->total[1],
+                           u64plus (u64size (len >> 31 >> 31 >> 2),
+                                    u64lo (u64lt (ctx->total[0], lolen))));
 
 #define S0(x) u64xor (u64rol(x, 63), u64xor (u64rol (x, 56), u64shr (x, 7)))
 #define S1(x) u64xor (u64rol (x, 45), u64xor (u64rol (x, 3), u64shr (x, 6)))
diff --git a/lib/u64.h b/lib/u64.h
index dadd6d7..f5ec9eb 100644
--- a/lib/u64.h
+++ b/lib/u64.h
@@ -30,6 +30,7 @@ typedef uint64_t u64;
 # define u64hilo(hi, lo) ((u64) (((u64) (hi) << 32) + (lo)))
 # define u64init(hi, lo) u64hilo (hi, lo)
 # define u64lo(x) ((u64) (x))
+# define u64size(x) u64lo (x)
 # define u64lt(x, y) ((x) < (y))
 # define u64and(x, y) ((x) & (y))
 # define u64or(x, y) ((x) | (y))
@@ -72,6 +73,16 @@ u64lo (uint32_t lo)
   return r;
 }
 
+/* Return a u64 value representing SIZE.  */
+static inline u64
+u64size (size_t size)
+{
+  u64 r;
+  r.hi = size >> 31 >> 1;
+  r.lo = size;
+  return r;
+}
+
 /* Return X < Y.  */
 static inline int
 u64lt (u64 x, u64 y)