raw: audit fixes — DEFLATE row bytes in u64, checked is_packed12, cap invariant documented (#2138)

The reviewer audit of the first commit found the DEFLATE arm computed row_bytes in
wrapping usize before the gate, so on wasm32 a hostile chunk header (width 2^29+1,
16-bit) could wrap cw * cpp * bits to a small value and slip a garbage chunk past the
new gate. row_bytes is now computed in u64 (the gate caps at MAX_SAMPLES, so the usize
cast is lossless on 32-bit too), checked_pixel_count takes u64 directly so no call site
can truncate before the check, is_packed12 uses a checked_mul chain instead of comparing
against a possibly-wrapped length, and the arw decode head documents the invariant the
cap provides: with w*h <= 2^30 every downstream usize product stays inside range on
32-bit targets.

Behaviour is unchanged for every input the previous commit accepted; the only shrunk
set is chunks that passed because wrapping made row_bytes small.

crates/raw/src/lib.rs

@@ -72,13 +72,12 @@ use serde::{Deserialize, Serialize};
 pub const MAX_SAMPLES: usize = 1 << 30;
 
 /// Checked product of input-derived dimensions, in u64 so it cannot wrap
-/// on either 32-bit (wasm32) or 64-bit targets.
+/// on either 32-bit (wasm32) or 64-bit targets. The arguments are u64 too, so callers can feed
+/// unchecked products of header values (row bytes, sample counts) without truncating on 32-bit.
 ///
 /// Returns `None` on overflow, zero, or when the product exceeds [`MAX_SAMPLES`].
-pub(crate) fn checked_pixel_count(width: usize, height: usize) -> Option<u64> {
-    let w = width as u64;
-    let h = height as u64;
-    w.checked_mul(h).filter(|&n| n > 0 && n <= MAX_SAMPLES as u64)
+pub(crate) fn checked_pixel_count(width: u64, height: u64) -> Option<u64> {
+    width.checked_mul(height).filter(|&n| n > 0 && n <= MAX_SAMPLES as u64)
 }
 
 /// Errors from raw decoding.
@@ -776,7 +775,7 @@ impl RawImage {
             return Err(RawError::Unsupported("2 samples per pixel".into()));
         }
         if samples {
-            let expected = crate::checked_pixel_count(self.width, self.height)
+            let expected = crate::checked_pixel_count(self.width as u64, self.height as u64)
                 .and_then(|n| n.checked_mul(self.cpp as u64))
                 .ok_or_else(|| RawError::Corrupt("data length mismatch".into()))?;
             let actual = u64::try_from(self.data.len()).map_err(|_| RawError::Corrupt("data length mismatch".into()))?;
@@ -1443,7 +1442,9 @@ mod checked_arithmetic_tests {
         assert_eq!(checked_pixel_count(65536, 65536), None);
         assert_eq!(checked_pixel_count(65536, 65537), None);
         // u32::MAX × 2 wraps on 32-bit usize.
-        assert_eq!(checked_pixel_count(u32::MAX as usize, 2), None);
+        assert_eq!(checked_pixel_count(u64::from(u32::MAX), 2), None);
+        // inputs wider than a 32-bit usize reach the gate untruncated (2^40 × 2^40 overflows u64).
+        assert_eq!(checked_pixel_count(1 << 40, 1 << 40), None);
     }
 
     #[test]

crates/raw/src/tiffraw.rs

@@ -192,8 +192,11 @@ fn decode_chunk(data: &[u8], info: &ImageInfo, c: &Chunk, order: ByteOrder, pack
         }
         comp::ADOBE_DEFLATE | comp::DEFLATE => {
             let bytes_per = if float { bits.div_ceil(8) as usize } else { 0 };
-            let row_bytes = if float { cw * cpp * bytes_per } else { (cw * cpp * bits as usize).div_ceil(8) };
-            let inflate_len = crate::checked_pixel_count(row_bytes, ch).ok_or_else(|| RawError::Corrupt("bad TIFF chunk dimensions".into()))?;
+            // row bytes in u64: a wrapping `cw · cpp · …` in usize could come out small on 32-bit targets and slip past the gate
+            let row_bytes = if float { cw as u64 * cpp as u64 * bytes_per as u64 } else { (cw as u64 * cpp as u64 * bits as u64).div_ceil(8) };
+            // the gate caps at MAX_SAMPLES (≤ 2^30), so the `usize` cast below is lossless on 32-bit targets too
+            let inflate_len =
+                crate::checked_pixel_count(row_bytes, ch as u64).ok_or_else(|| RawError::Corrupt("bad TIFF chunk dimensions".into()))?;
             let raw = inflate(src, inflate_len as usize)?;
             unpack_chunk(&raw, order, packing, cw, ch, cpp, bits, float, info.predictor)
         }

crates/raw/src/vendor/arw.rs

@@ -245,7 +245,9 @@ fn tag2010_wb(model: &str, block: &[u8], order: lightcraft_tiff::ByteOrder) -> O
 /// DSLR-A900 sample this is the whole strip (36 917 760 = 6080 × 4048 × 1.5) and it ends at the end of the file.
 fn is_packed12(info: &ImageInfo, strip_count: usize, strip_len: u64) -> bool {
     let (w, h) = (info.width as u64, info.height as u64);
-    info.bits() == 12 && strip_count == 1 && w % 2 == 0 && w > 0 && h > 0 && strip_len == w * h * 3 / 2
+    // checked_mul chain: a wrapping `w · h · 3` must refuse the strip, not compare against a wrapped length
+    let packed_len = w.checked_mul(h).and_then(|n| n.checked_mul(3)).map(|n| n / 2);
+    info.bits() == 12 && strip_count == 1 && w % 2 == 0 && w > 0 && h > 0 && packed_len == Some(strip_len)
 }
 
 /// Unpack plain 12-bit samples stored least-significant-bit first: two pixels per three bytes, the first being
@@ -621,7 +623,8 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode) -> Result<RawImage> {
     let raw = raw_ifd(&tiff).ok_or_else(|| RawError::Unsupported("ARW without a CFA image IFD (old ARW or SR2)".into()))?;
     let info = raw.image()?;
     let (w, h) = (info.width as usize, info.height as usize);
-    let samples = crate::checked_pixel_count(w, h).ok_or_else(|| RawError::Limit("image too large"))?;
+    // the cap (≤ 2^30) keeps every downstream usize product (w·h·2, w·h·3/2, y·w) inside range on 32-bit targets
+    let samples = crate::checked_pixel_count(w as u64, h as u64).ok_or(RawError::Limit("image too large"))?;
     let bits = info.bits() as u32;
     let linear_rgb = info.compression == 7 && info.photometric == photometric::YCBCR;
     let chunks = info.chunks(bytes.len() as u64);