feat: implement layer mask editing support with scratch buffer and coordinate normalization

hcie-engine-api/src/layer_property_ops.rs

@@ -459,4 +459,102 @@ impl Engine {
         self.below_cache_active_idx = None;
         self.below_cache_dirty = true;
     }
+
+    /// Check if a layer has an active layer mask.
+    pub fn has_layer_mask(&self, layer_id: u64) -> bool {
+        if let Some(idx) = self.document.layer_index_by_id(layer_id) {
+            self.document.layers.get(idx).map(|l| l.mask_pixels.is_some()).unwrap_or(false)
+        } else {
+            false
+        }
+    }
+
+    /// Add a white (opaque/reveal) layer mask to the specified layer if not present.
+    pub fn add_layer_mask(&mut self, layer_id: u64) {
+        if let Some(idx) = self.document.layer_index_by_id(layer_id) {
+            if let Some(layer) = self.document.layers.get_mut(idx) {
+                let mask_size = (layer.width * layer.height) as usize;
+                layer.mask_pixels = Some(vec![255u8; mask_size]);
+                layer.mask_bounds = Some([0, 0, layer.height as i32, layer.width as i32]);
+                layer.mask_default_color = 255;
+                layer.dirty = true;
+                self.document.composite_dirty = true;
+                self.document.modified = true;
+                self.below_cache_dirty = true;
+                self.document.composite_dirty = true;
+            }
+        }
+    }
+
+    /// Remove the layer mask from the specified layer.
+    pub fn remove_layer_mask(&mut self, layer_id: u64) {
+        if let Some(idx) = self.document.layer_index_by_id(layer_id) {
+            if let Some(layer) = self.document.layers.get_mut(idx) {
+                layer.mask_pixels = None;
+                layer.mask_bounds = None;
+                layer.dirty = true;
+                self.document.composite_dirty = true;
+                self.document.modified = true;
+                self.below_cache_dirty = true;
+                self.document.composite_dirty = true;
+            }
+        }
+    }
+
+    /// Toggle the existence of a layer mask on the specified layer.
+    pub fn toggle_layer_mask(&mut self, layer_id: u64) {
+        if self.has_layer_mask(layer_id) {
+            self.remove_layer_mask(layer_id);
+        } else {
+            self.add_layer_mask(layer_id);
+        }
+    }
+
+    /// Returns `true` if current drawing operations target the layer mask.
+    pub fn is_editing_mask(&self) -> bool {
+        self.editing_mask
+    }
+
+    /// Set whether current drawing operations target the active layer's mask.
+    pub fn set_editing_mask(&mut self, editing: bool) {
+        self.editing_mask = editing;
+    }
+
+    /// Toggle mask editing mode.
+    pub fn toggle_editing_mask(&mut self) {
+        self.editing_mask = !self.editing_mask;
+    }
+
+    /// Returns a reference to the mask pixel bytes for the given layer.
+    pub fn get_layer_mask_pixels(&self, layer_id: u64) -> Option<&[u8]> {
+        if let Some(idx) = self.document.layer_index_by_id(layer_id) {
+            self.document.layers.get(idx).and_then(|l| l.mask_pixels.as_deref())
+        } else {
+            None
+        }
+    }
+
+    /// Returns mask bounds [top, left, bottom, right] for the given layer if set.
+    pub fn get_layer_mask_bounds(&self, layer_id: u64) -> Option<[i32; 4]> {
+        if let Some(idx) = self.document.layer_index_by_id(layer_id) {
+            self.document.layers.get(idx).and_then(|l| l.mask_bounds)
+        } else {
+            None
+        }
+    }
+
+    /// Update the mask pixel bytes and bounds for the given layer.
+    pub fn set_layer_mask_pixels(&mut self, layer_id: u64, mask: Vec<u8>, bounds: Option<[i32; 4]>) {
+        if let Some(idx) = self.document.layer_index_by_id(layer_id) {
+            if let Some(layer) = self.document.layers.get_mut(idx) {
+                layer.mask_pixels = Some(mask);
+                layer.mask_bounds = bounds;
+                layer.dirty = true;
+                self.document.composite_dirty = true;
+                self.document.modified = true;
+                self.below_cache_dirty = true;
+                self.document.composite_dirty = true;
+            }
+        }
+    }
 }

hcie-engine-api/src/lib.rs

@@ -348,12 +348,14 @@ pub struct Engine {
     /// At `end_stroke()`, the buffer is moved into the background thread for
     /// sub-rect extraction, then returned via the pending_history channel.
     stroke_before_buf: Option<Vec<u8>>,
+    mask_before_buf: Option<Vec<u8>>,
     tile_layers: Vec<Option<TiledLayer>>,
     svg_sources: std::collections::HashMap<u64, Vec<u8>>,
     filter_preview_original: Option<Vec<u8>>,
     stroke_effects_backup: Option<Vec<hcie_protocol::effects::LayerEffect>>,
     stroke_styles_backup: Option<Vec<hcie_protocol::LayerStyle>>,
     pub is_eraser: bool,
+    pub editing_mask: bool,
     /// Pooled stroke mask buffer. Zeroed on `begin_stroke()` and `end_stroke()`
     /// but **never dropped** between strokes. Reused if size matches, reallocated
     /// only when canvas dimensions change. Avoids ~8MB alloc/stroke on 4K.
@@ -363,6 +365,8 @@ pub struct Engine {
     /// on 4K canvases. The pixel-bridge extracts dirty rect from this buffer
     /// without allocating its own buffer.
     composite_scratch: Option<Vec<u8>>,
+    /// Pooled RGBA scratch buffer for layer mask editing passes.
+    pub mask_rgba_scratch: Option<Vec<u8>>,
     /// Accumulated stroke extent `[x0, y0, x1, y1]` for sub-rect snapshot.
     /// Tracks the union of all brush areas painted during the current stroke.
     /// Independent from `document.dirty_bounds` — survives composite emits.
@@ -540,14 +544,17 @@ impl Engine {
             cyclic_speed: 0.5,
             stroke_before: None,
             stroke_before_buf: None,
+            mask_before_buf: None,
             tile_layers: Vec::new(),
             svg_sources: std::collections::HashMap::new(),
             filter_preview_original: None,
             stroke_effects_backup: None,
             stroke_styles_backup: None,
             is_eraser: false,
+            editing_mask: false,
             active_stroke_mask: None,
             composite_scratch: None,
+            mask_rgba_scratch: None,
             last_stroke_bounds: None,
             below_cache: None,
             below_cache_active_idx: None,

hcie-engine-api/src/stroke_brush.rs

@@ -105,6 +105,167 @@ impl Engine {
         tip
     }
 
+    /// Helper method to execute a drawing action either on active layer pixels (when editing_mask is false)
+    /// or on the active layer mask (when editing_mask is true), converting between RGBA and greyscale mask.
+    /// Ensures active layer's mask is expanded to full canvas dimensions if present.
+    pub fn ensure_full_canvas_mask(&mut self, layer_id: u64) {
+        let cw = self.document.canvas_width;
+        let ch = self.document.canvas_height;
+        if let Some(layer) = self.document.get_layer_by_id_mut(layer_id) {
+            let full_size = (cw * ch) as usize;
+            if let Some(ref mask) = layer.mask_pixels {
+                if mask.len() != full_size {
+                    let mut full_mask = vec![layer.mask_default_color; full_size];
+                    if let Some(mb) = layer.mask_bounds {
+                        let (m_top, m_left, m_bottom, m_right) =
+                            (mb[0].max(0) as u32, mb[1].max(0) as u32, mb[2].max(0) as u32, mb[3].max(0) as u32);
+                        let mw = (m_right - m_left) as usize;
+                        let mh = (m_bottom - m_top) as usize;
+                        if mw > 0 && mh > 0 && mask.len() == mw * mh {
+                            for y in 0..mh {
+                                let gy = m_top as usize + y;
+                                if gy >= ch as usize { break; }
+                                for x in 0..mw {
+                                    let gx = m_left as usize + x;
+                                    if gx >= cw as usize { break; }
+                                    full_mask[gy * cw as usize + gx] = mask[y * mw + x];
+                                }
+                            }
+                        }
+                    }
+                    layer.mask_pixels = Some(full_mask);
+                    layer.mask_bounds = Some([0, 0, ch as i32, cw as i32]);
+                }
+            }
+        }
+    }
+
+    pub fn draw_target_pixels_or_mask<F>(&mut self, layer_id: u64, f: F)
+    where
+        F: FnOnce(
+            &mut [u8],
+            u32,
+            u32,
+            Option<&[u8]>,
+            Option<&mut [u8]>,
+            Option<&[u8]>,
+            Option<&mut Vec<(f32, f32)>>,
+        ),
+    {
+        let is_editing = self.editing_mask;
+        if is_editing {
+            self.ensure_full_canvas_mask(layer_id);
+        }
+
+        if !is_editing {
+            let mask_ref = self.cached_selection_mask.as_deref();
+            let active_stroke_mask = self.active_stroke_mask.as_deref_mut();
+            let stroke_before_buf = self.stroke_before_buf.as_deref();
+            let sketch_hist = &mut self.sketch_history;
+
+            if let Some(layer) = self.document.get_layer_by_id_mut(layer_id) {
+                let w = layer.width;
+                let h = layer.height;
+                f(
+                    &mut layer.pixels,
+                    w,
+                    h,
+                    mask_ref,
+                    active_stroke_mask,
+                    stroke_before_buf,
+                    Some(sketch_hist),
+                );
+                layer.dirty = true;
+            }
+        } else {
+            let cw = self.document.canvas_width as usize;
+            let ch = self.document.canvas_height as usize;
+            let mask_len = cw * ch;
+            if mask_len == 0 { return; }
+
+            if let Some(layer) = self.document.get_layer_by_id_mut(layer_id) {
+                if layer.mask_pixels.is_none() {
+                    layer.mask_pixels = Some(vec![255u8; mask_len]);
+                    layer.mask_bounds = Some([0, 0, ch as i32, cw as i32]);
+                    layer.mask_default_color = 255;
+                }
+            }
+
+            let (rx0, ry0, rx1, ry1) = match self.last_stroke_bounds {
+                Some([x0, y0, x1, y1]) => (
+                    (x0.saturating_sub(15) as usize).min(cw),
+                    (y0.saturating_sub(15) as usize).min(ch),
+                    ((x1 + 15) as usize).min(cw),
+                    ((y1 + 15) as usize).min(ch),
+                ),
+                None => (0, 0, cw, ch),
+            };
+
+            let needed_len = mask_len * 4;
+            let mut scratch = self.mask_rgba_scratch.take().unwrap_or_default();
+            if scratch.len() != needed_len {
+                scratch.resize(needed_len, 0);
+            }
+
+            if let Some(layer) = self.document.get_layer_by_id(layer_id) {
+                if let Some(ref mask) = layer.mask_pixels {
+                    for y in ry0..ry1 {
+                        let row_off = y * cw;
+                        for x in rx0..rx1 {
+                            let idx = row_off + x;
+                            let v = mask[idx];
+                            let off = idx * 4;
+                            scratch[off] = v;
+                            scratch[off + 1] = v;
+                            scratch[off + 2] = v;
+                            scratch[off + 3] = 255;
+                        }
+                    }
+                }
+            }
+
+            {
+                let mask_ref = self.cached_selection_mask.as_deref();
+                let active_stroke_mask = self.active_stroke_mask.as_deref_mut();
+                let stroke_before_buf = self.stroke_before_buf.as_deref();
+                let sketch_hist = &mut self.sketch_history;
+                f(
+                    &mut scratch,
+                    cw as u32,
+                    ch as u32,
+                    mask_ref,
+                    active_stroke_mask,
+                    stroke_before_buf,
+                    Some(sketch_hist),
+                );
+            }
+
+            if let Some(layer) = self.document.get_layer_by_id_mut(layer_id) {
+                if let Some(ref mut mask) = layer.mask_pixels {
+                    for y in ry0..ry1 {
+                        let row_off = y * cw;
+                        for x in rx0..rx1 {
+                            let idx = row_off + x;
+                            let off = idx * 4;
+                            let r = scratch[off] as u32;
+                            let g = scratch[off + 1] as u32;
+                            let b = scratch[off + 2] as u32;
+                            let a = scratch[off + 3] as u32;
+                            let gray = (r * 299 + g * 587 + b * 114) / 1000;
+                            mask[idx] = ((gray * a) / 255) as u8;
+                        }
+                    }
+                }
+                layer.dirty = true;
+            }
+
+            self.mask_rgba_scratch = Some(scratch);
+        }
+        self.below_cache_dirty = true;
+        self.document.composite_dirty = true;
+        self.document.modified = true;
+    }
+
     pub fn stroke_to(&mut self, layer_id: u64, x: f32, y: f32, pressure: f32) {
         let w = self.document.canvas_width as f32;
         let h = self.document.canvas_height as f32;
@@ -115,59 +276,66 @@ impl Engine {
         }
         let tip = self.brush_tip_with_time_dynamics(x, y);
         let color = self.apply_cyclic_color(self.current_color);
-        let mask_ref = self.cached_selection_mask.as_deref();
 
         if Self::is_specialized_style(tip.style) {
             if let Some((lx, ly, lp)) = self.last_stroke_pos {
-                if let Some(layer) = self.document.get_layer_by_id_mut(layer_id) {
-                    // If click without movement, draw a single dab at current position
-                    let points_vec = if (lx - x).abs() < 0.001 && (ly - y).abs() < 0.001 {
-                        vec![(x, y, pressure)]
-                    } else {
-                        vec![(lx, ly, lp), (x, y, pressure)]
-                    };
-                    draw_specialized_stroke(
-                        &mut layer.pixels,
-                        layer.width,
-                        layer.height,
-                        &points_vec,
-                        tip.style,
-                        tip.size,
-                        tip.hardness,
-                        color,
-                        tip.opacity,
-                        tip.spacing,
-                        self.is_eraser,
-                        if tip.style == BrushStyle::Sketch {
-                            Some(&mut self.sketch_history)
+                let points_vec = if (lx - x).abs() < 0.001 && (ly - y).abs() < 0.001 {
+                    vec![(x, y, pressure)]
+                } else {
+                    vec![(lx, ly, lp), (x, y, pressure)]
+                };
+                let is_eraser = self.is_eraser;
+
+                let dirty_r = Self::brush_dirty_radius(&tip);
+                self.document.expand_dirty(lx as u32, ly as u32, dirty_r);
+                self.document.expand_dirty(x as u32, y as u32, dirty_r);
+                self.expand_stroke_bounds(lx as u32, ly as u32, dirty_r);
+                self.expand_stroke_bounds(x as u32, y as u32, dirty_r);
+
+                self.draw_target_pixels_or_mask(
+                    layer_id,
+                    |pixels, lw, lh, mask_ref, active_stroke_mask, stroke_before_buf, sketch_history| {
+                        let sketch_hist = if tip.style == BrushStyle::Sketch {
+                            sketch_history
                         } else {
                             None
-                        },
-                        tip.spray_particle_size,
-                        tip.spray_density,
-                        mask_ref,
-                        self.active_stroke_mask.as_deref_mut(),
-                        self.stroke_before_buf.as_deref(),
-                        tip.color_variant,
-                        tip.variant_amount,
-                        tip.density,
-                        tip.jitter_amount,
-                        tip.scatter_amount,
-                        tip.angle,
-                        tip.roundness,
-                        tip.rotation_random,
-                        tip.drawing_angle,
-                        false,
-                    );
-                    layer.dirty = true;
+                        };
+                        draw_specialized_stroke(
+                            pixels,
+                            lw,
+                            lh,
+                            &points_vec,
+                            tip.style,
+                            tip.size,
+                            tip.hardness,
+                            color,
+                            tip.opacity,
+                            tip.spacing,
+                            is_eraser,
+                            sketch_hist,
+                            tip.spray_particle_size,
+                            tip.spray_density,
+                            mask_ref,
+                            active_stroke_mask,
+                            stroke_before_buf,
+                            tip.color_variant,
+                            tip.variant_amount,
+                            tip.density,
+                            tip.jitter_amount,
+                            tip.scatter_amount,
+                            tip.angle,
+                            tip.roundness,
+                            tip.rotation_random,
+                            tip.drawing_angle,
+                            false,
+                        );
+                    },
+                );
+
+                if let Some(layer) = self.document.get_layer_by_id_mut(layer_id) {
                     if !layer.effects.is_empty() || !layer.styles.is_empty() {
                         *layer.effects_cache.lock().unwrap() = None;
                     }
-                    let dirty_r = Self::brush_dirty_radius(&tip);
-                    self.document.expand_dirty(lx as u32, ly as u32, dirty_r);
-                    self.document.expand_dirty(x as u32, y as u32, dirty_r);
-                    self.expand_stroke_bounds(lx as u32, ly as u32, dirty_r);
-                    self.expand_stroke_bounds(x as u32, y as u32, dirty_r);
                 }
                 self.document.composite_dirty = true;
                 self.document.modified = true;
@@ -176,32 +344,60 @@ impl Engine {
                 return;
             }
         } else {
-            if let Some(layer) = self.document.get_layer_by_id_mut(layer_id) {
-                draw_brush_stroke(
-                    layer,
-                    &[(x, y, pressure)],
-                    color,
-                    &tip,
-                    self.is_eraser,
-                    mask_ref,
-                    self.active_stroke_mask.as_deref_mut(),
-                    self.stroke_before_buf.as_deref(),
+            let is_eraser = self.is_eraser;
+
+            let dirty_r = tip.size.max(2.0);
+            if let Some((lx, ly, _)) = self.last_stroke_pos {
+                self.document.expand_dirty(lx as u32, ly as u32, dirty_r);
+            }
+            self.document.expand_dirty(x as u32, y as u32, dirty_r);
+            if let Some((lx, ly, _)) = self.last_stroke_pos {
+                self.expand_stroke_bounds(lx as u32, ly as u32, dirty_r);
+            }
+            self.expand_stroke_bounds(x as u32, y as u32, dirty_r);
+
+            if self.editing_mask {
+                self.draw_target_pixels_or_mask(
+                    layer_id,
+                    |pixels, lw, lh, mask_ref, active_stroke_mask, stroke_before_buf, _sketch_history| {
+                        let mut temp_layer = hcie_protocol::Layer::from_rgba("temp", lw, lh, pixels.to_vec());
+                        draw_brush_stroke(
+                            &mut temp_layer,
+                            &[(x, y, pressure)],
+                            color,
+                            &tip,
+                            is_eraser,
+                            mask_ref,
+                            active_stroke_mask,
+                            stroke_before_buf,
+                        );
+                        pixels.copy_from_slice(&temp_layer.pixels);
+                    },
                 );
-                layer.dirty = true;
+            } else {
+                let mask_ref = self.cached_selection_mask.as_deref();
+                let active_stroke_mask = self.active_stroke_mask.as_deref_mut();
+                let stroke_before_buf = self.stroke_before_buf.as_deref();
+                if let Some(layer) = self.document.get_layer_by_id_mut(layer_id) {
+                    draw_brush_stroke(
+                        layer,
+                        &[(x, y, pressure)],
+                        color,
+                        &tip,
+                        is_eraser,
+                        mask_ref,
+                        active_stroke_mask,
+                        stroke_before_buf,
+                    );
+                }
+            }
+
+            if let Some(layer) = self.document.get_layer_by_id_mut(layer_id) {
                 if !layer.effects.is_empty() || !layer.styles.is_empty() {
                     layer
                         .effects_dirty
                         .store(true, std::sync::atomic::Ordering::Release);
                 }
-                let dirty_r = tip.size.max(2.0);
-                if let Some((lx, ly, _)) = self.last_stroke_pos {
-                    self.document.expand_dirty(lx as u32, ly as u32, dirty_r);
-                }
-                self.document.expand_dirty(x as u32, y as u32, dirty_r);
-                if let Some((lx, ly, _)) = self.last_stroke_pos {
-                    self.expand_stroke_bounds(lx as u32, ly as u32, dirty_r);
-                }
-                self.expand_stroke_bounds(x as u32, y as u32, dirty_r);
             }
             self.document.composite_dirty = true;
             self.document.modified = true;

hcie-engine-api/src/stroke_cache.rs

@@ -59,6 +59,10 @@ pub struct PendingHistoryItem {
     pub return_before: Option<Vec<u8>>,
     /// Full-layer after buffer returned from background thread for pool reuse.
     pub return_after: Option<Vec<u8>>,
+    /// Grayscale before mask buffer returned from background thread for pool reuse.
+    pub return_mask_before: Option<Vec<u8>>,
+    /// Indicates if this snapshot represents a mask edit rather than a pixel edit.
+    pub is_mask: bool,
 }
 
 impl Engine {
@@ -98,8 +102,33 @@ impl Engine {
                     self.stroke_before_buf = Some(buf);
                 }
             }
+            if let Some(buf) = item.return_mask_before {
+                if self
+                    .mask_before_buf
+                    .as_ref()
+                    .map_or(true, |b| b.len() != buf.len())
+                {
+                    self.mask_before_buf = Some(buf);
+                }
+            }
 
-            if let Some(bounds) = item.bounds {
+            if item.is_mask {
+                if let Some(bounds) = item.bounds {
+                    self.document.push_mask_draw_snapshot_subrect(
+                        item.layer_idx,
+                        item.before_pixels,
+                        item.after_pixels,
+                        (bounds[0], bounds[1], bounds[2], bounds[3]),
+                        item.description,
+                    );
+                } else {
+                    self.document.push_mask_draw_snapshot(
+                        item.layer_idx,
+                        item.before_pixels,
+                        item.description,
+                    );
+                }
+            } else if let Some(bounds) = item.bounds {
                 self.document.push_draw_snapshot_subrect(
                     item.layer_idx,
                     item.before_pixels,
@@ -191,12 +220,71 @@ impl Engine {
 
             // Pool the "before" pixel buffer: reuse if size matches, otherwise allocate.
             // This avoids a ~33MB allocation per stroke on 4K canvases.
-            match &mut self.stroke_before_buf {
-                Some(buf) if buf.len() == layer_pixels => {
-                    buf.copy_from_slice(&layer.pixels);
+            if self.editing_mask {
+                let mut rgba_buf = match self.stroke_before_buf.take() {
+                    Some(mut b) if b.len() == layer_pixels => {
+                        b.fill(255);
+                        b
+                    }
+                    _ => vec![255u8; layer_pixels],
+                };
+                let mut gray_buf = match self.mask_before_buf.take() {
+                    Some(mut b) if b.len() == layer_size => {
+                        b.fill(255);
+                        b
+                    }
+                    _ => vec![255u8; layer_size],
+                };
+
+                let ch = layer.height as usize;
+                let cw = layer.width as usize;
+                let full_size = cw * ch;
+
+                if let Some(ref mut mask) = layer.mask_pixels {
+                    if mask.len() != full_size {
+                        let mut full_mask = vec![layer.mask_default_color; full_size];
+                        if let Some(mb) = layer.mask_bounds {
+                            let (m_top, m_left, m_bottom, m_right) =
+                                (mb[0].max(0) as u32, mb[1].max(0) as u32, mb[2].max(0) as u32, mb[3].max(0) as u32);
+                            let mw = (m_right - m_left) as usize;
+                            let mh = (m_bottom - m_top) as usize;
+                            if mw > 0 && mh > 0 && mask.len() == mw * mh {
+                                for y in 0..mh {
+                                    let gy = m_top as usize + y;
+                                    if gy >= ch { break; }
+                                    for x in 0..mw {
+                                        let gx = m_left as usize + x;
+                                        if gx >= cw { break; }
+                                        full_mask[gy * cw + gx] = mask[y * mw + x];
+                                    }
+                                }
+                            }
+                        }
+                        *mask = full_mask;
+                        layer.mask_bounds = Some([0, 0, ch as i32, cw as i32]);
+                    }
                 }
-                _ => {
-                    self.stroke_before_buf = Some(layer.pixels.clone());
+
+                if let Some(ref mask) = layer.mask_pixels {
+                    gray_buf.copy_from_slice(mask);
+                    for (i, &v) in mask.iter().enumerate() {
+                        let off = i * 4;
+                        rgba_buf[off] = v;
+                        rgba_buf[off + 1] = v;
+                        rgba_buf[off + 2] = v;
+                        rgba_buf[off + 3] = 255;
+                    }
+                }
+                self.stroke_before_buf = Some(rgba_buf);
+                self.mask_before_buf = Some(gray_buf);
+            } else {
+                match &mut self.stroke_before_buf {
+                    Some(buf) if buf.len() == layer_pixels => {
+                        buf.copy_from_slice(&layer.pixels);
+                    }
+                    _ => {
+                        self.stroke_before_buf = Some(layer.pixels.clone());
+                    }
                 }
             }
             let before_shapes = if let LayerData::Vector { shapes } = &layer.data {
@@ -254,17 +342,35 @@ impl Engine {
                         layer_pixels,
                         self.stroke_before_buf.as_ref().map(|b| b.len())
                     );
-                    let before = match self.stroke_before_buf.take() {
-                        Some(buf) if buf.len() == layer_pixels => buf,
-                        _ => vec![0u8; layer_pixels],
+                    let mut mask_before = None;
+                    let before = if self.editing_mask {
+                        let gray_len = lw as usize * layer.height as usize;
+                        let mb = match self.mask_before_buf.take() {
+                            Some(buf) if buf.len() == gray_len => buf,
+                            _ => vec![255u8; gray_len],
+                        };
+                        mask_before = Some(mb.clone());
+                        mb
+                    } else {
+                        match self.stroke_before_buf.take() {
+                            Some(buf) if buf.len() == layer_pixels => buf,
+                            _ => vec![0u8; layer_pixels],
+                        }
                     };
                     log::debug!("[end_stroke] before.len()={}, lw={}", before.len(), lw);
 
                     // Layer 3: Zero-copy after-snapshot via raw pointer.
                     // SAFETY: After end_stroke(), no mutations happen to layer.pixels
                     // until the next begin_stroke(). The background thread only reads.
-                    let after_ptr = SendPtr::new(layer.pixels.as_ptr());
-                    let after_len = layer.pixels.len();
+                    let (after_ptr, after_len) = if self.editing_mask {
+                        if layer.mask_pixels.is_none() {
+                            layer.mask_pixels = Some(vec![255; (layer.width * layer.height) as usize]);
+                        }
+                        let mask = layer.mask_pixels.as_ref().unwrap();
+                        (SendPtr::new(mask.as_ptr()), mask.len())
+                    } else {
+                        (SendPtr::new(layer.pixels.as_ptr()), layer.pixels.len())
+                    };
 
                     let after_shapes = if let LayerData::Vector { shapes } = &layer.data {
                         Some(shapes.clone())
@@ -275,6 +381,7 @@ impl Engine {
                     let style = self.current_tip.style;
                     let bounds = self.last_stroke_bounds;
                     let pending_history = self.pending_history.clone();
+                    let is_mask = self.editing_mask;
 
                     std::thread::spawn(move || {
                         let t_start = std::time::Instant::now();
@@ -290,25 +397,33 @@ impl Engine {
                             before_shapes,
                             after_shapes,
                             description: format!(
-                                "Brush Stroke ({})",
+                                "{} ({})",
+                                if is_mask { "Mask Edit" } else { "Brush Stroke" },
                                 crate::stroke_brush::brush_style_label(style)
                             ),
                             return_before: None,
                             return_after: None,
+                            return_mask_before: mask_before,
+                            is_mask,
                         };
 
                         if let Some([sx0, sy0, sx1, sy1]) = bounds {
                             if sx0 < sx1 && sy0 < sy1 {
                                 let rw = sx1 - sx0;
                                 let rh = sy1 - sy0;
-                                let rect_size = (rw * rh * 4) as usize;
+                                let rect_size = if is_mask {
+                                    (rw * rh) as usize
+                                } else {
+                                    (rw * rh * 4) as usize
+                                };
                                 log::debug!("[end_stroke_bg] bounds=[{},{},{},{}], rw={}, rh={}, rect_size={}, before.len()={}, after.len()={}, lw={}", sx0, sy0, sx1, sy1, rw, rh, rect_size, before.len(), after_slice.len(), lw);
                                 let mut before_rect = vec![0u8; rect_size];
                                 let mut after_rect = vec![0u8; rect_size];
+                                let bpp = if is_mask { 1 } else { 4 };
                                 for row in 0..rh {
-                                    let src_start = (((sy0 + row) * lw + sx0) * 4) as usize;
-                                    let dst_start = (row * rw * 4) as usize;
-                                    let len = (rw * 4) as usize;
+                                    let src_start = (((sy0 + row) * lw + sx0) * bpp) as usize;
+                                    let dst_start = (row * rw * bpp) as usize;
+                                    let len = (rw * bpp) as usize;
                                     before_rect[dst_start..dst_start + len]
                                         .copy_from_slice(&before[src_start..src_start + len]);
                                     after_rect[dst_start..dst_start + len]
@@ -320,7 +435,9 @@ impl Engine {
                                     item.after_pixels = after_rect;
                                     item.bounds = Some([sx0, sy0, sx1, sy1]);
                                     // Return before buffer to pool (after is a borrowed pointer)
-                                    item.return_before = Some(before);
+                                    if !is_mask {
+                                        item.return_before = Some(before);
+                                    }
                                     pending_history.lock().unwrap().push(item);
                                 }
                                 log::trace!(

hcie-engine-api/tests/layer_mask_test.rs

@@ -0,0 +1,33 @@
+use hcie_engine_api::Engine;
+
+#[test]
+fn test_layer_mask_api_and_editing() {
+    let mut engine = Engine::new(100, 100);
+    let layer_id = engine.active_layer_id();
+
+    // Initially layer has no mask
+    assert!(!engine.has_layer_mask(layer_id));
+    assert!(engine.get_layer_mask_pixels(layer_id).is_none());
+
+    // Add layer mask
+    engine.add_layer_mask(layer_id);
+    assert!(engine.has_layer_mask(layer_id));
+    let mask = engine.get_layer_mask_pixels(layer_id).expect("Mask pixels must exist");
+    assert_eq!(mask.len(), 100 * 100);
+    assert_eq!(mask[0], 255);
+
+    // Toggle mask editing mode
+    assert!(!engine.is_editing_mask());
+    engine.set_editing_mask(true);
+    assert!(engine.is_editing_mask());
+
+    // Set custom mask pixels
+    let custom_mask = vec![128u8; 100 * 100];
+    engine.set_layer_mask_pixels(layer_id, custom_mask.clone(), Some([0, 0, 100, 100]));
+    assert_eq!(engine.get_layer_mask_pixels(layer_id), Some(custom_mask.as_slice()));
+
+    // Remove layer mask
+    engine.remove_layer_mask(layer_id);
+    assert!(!engine.has_layer_mask(layer_id));
+    assert!(engine.get_layer_mask_pixels(layer_id).is_none());
+}