Compose: key the vector-mask cache by mask content, not a temporary's address (#4426) (#4428)

RenderVectorMasks keyed compiled masks by the address of the VectorMask it
was given. hiding_mask/effects_source pass temporary clones, whose addresses
get reused, so a later layer received an earlier layer's compiled mask.

Co-authored-by: Claude Opus 5.5 <[email protected]>

crates/compose/src/advanced_tests.rs

@@ -426,6 +426,26 @@ fn vector_mask_hides_effects() {
     }
 }
 
+/// The per-render vector-mask cache must key masks by content: `hiding_mask` and
+/// `effects_source` hand it temporary clones whose addresses get reused (#4426).
+#[test]
+fn vector_mask_cache_does_not_mix_up_layers() {
+    use photocraft_doc::{Path, Subpath, VectorMask};
+    for depth in DEPTHS {
+        let mut d = doc(depth, true);
+        for (name, color, x0, x1) in [("a", BLUE, -4.0, 6.0), ("b", GREEN, 6.0, 16.0)] {
+            let mut l = solid(&d, name, full(&d), color);
+            l.effects.items = vec![inside_stroke()];
+            l.advanced.vector_mask_hides_effects = true;
+            let path = Path::new(vec![Subpath::polygon(&[(x0, -4.0), (x1, -4.0), (x1, 12.0), (x0, 12.0)])]);
+            l.vector_mask = Some(VectorMask::new(path));
+            d.layers.push(l);
+        }
+        check(&d, 2, 4, BLUE, "layer A inside its own mask");
+        check(&d, 9, 4, GREEN, "layer B inside its own mask");
+    }
+}
+
 #[test]
 fn defaults_take_the_plain_paths() {
     let d = doc(SampleType::U8, true);

crates/compose/src/lib.rs

@@ -451,18 +451,27 @@ fn render_reduced_in_bands(doc: &Document, b: Rect, w: u32, h: u32, damage: Opti
     Buffer { rect: out, px: acc }
 }
 
+type VectorMaskSlot = std::sync::Arc<std::sync::OnceLock<photocraft_vector::CompiledVectorMask>>;
+
 #[derive(Default)]
 struct RenderVectorMasks {
-    masks: std::sync::Mutex<std::collections::HashMap<usize, std::sync::Arc<std::sync::OnceLock<photocraft_vector::CompiledVectorMask>>>>,
+    /// Compiled masks by content. Callers pass temporary clones (see `advanced.rs`), so an
+    /// address cannot identify a mask: a dropped clone's address is reused by the next one.
+    masks: std::sync::Mutex<Vec<(photocraft_doc::VectorMask, VectorMaskSlot)>>,
 }
 
 impl RenderVectorMasks {
     fn values(&self, mask: &photocraft_doc::VectorMask, rect: Rect) -> Vec<f32> {
-        // Addresses identify immutable mask instances only for this render's lifetime.
-        let key = std::ptr::from_ref(mask) as usize;
         let slot = {
             let mut masks = self.masks.lock().unwrap_or_else(|e| e.into_inner());
-            masks.entry(key).or_default().clone()
+            match masks.iter().find(|(m, _)| m == mask) {
+                Some((_, slot)) => slot.clone(),
+                None => {
+                    let slot = VectorMaskSlot::default();
+                    masks.push((mask.clone(), slot.clone()));
+                    slot
+                }
+            }
         };
         // Compilation is sequential. Rendering can enter Rayon and must happen after
         // initialization, so waiting tiles cannot re-enter a slot still being built.