phantom/hcie-rust-v3.05
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

init

Browse Source
This commit is contained in:
Halit Can
2026-07-09 02:59:53 +03:00
commit 7ace048d94
817 changed files with 234289 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
hcie-vision/Cargo.toml
+33
View File
@@ -0,0 +1,33 @@
[package]
name = "hcie-vision"
version = "0.1.0"
edition = "2021"
[features]
default = ["ffi", "cpp-backend"]
ffi = []
# Enables the C++ GGML/llama.cpp vision backend (MobileSAM, BiRefNet, ESRGAN, MI-GAN).
# Requires the matching C++ backend library to be available at load time.
cpp-backend = []
[dependencies]
hcie-engine-api = { path = "../hcie-engine-api" }
hcie-protocol = { path = "../hcie-protocol" }
image = { version = "0.25", default-features = false, features = ["png", "jpeg", "webp"] }
reqwest = { version = "0.12", features = ["json", "multipart", "blocking", "stream"] }
tokio = { version = "1.40", features = ["full"] }
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
base64 = "0.22"
log = "0.4"
rand = "0.8"
rayon = "1.10"
bincode = "1.3"
[lib]
crate-type = ["rlib", "cdylib"]
[dev-dependencies]
rstest = "0.23"
proptest = "1.5"
approx = "0.5"
hcie-vision/build.rs
+38
View File
@@ -0,0 +1,38 @@
//! Build script linking the C++ vision backend into the Rust cdylib.
fn main() {
let project_dir = std::env::var("CARGO_MANIFEST_DIR").unwrap();
let root_dir = std::path::Path::new(&project_dir).parent().unwrap();
let lib_dir = root_dir.join("build").join("lib");
println!("cargo:rustc-link-search=native={}", lib_dir.display());
println!("cargo:rustc-link-lib=dylib=visioncpp");
println!("cargo:rustc-link-lib=dylib=ggml");
println!("cargo:rustc-link-lib=dylib=ggml-cpu");
println!("cargo:rustc-link-lib=dylib=ggml-base");
// Look in the executable/plugin directory first, then in build/lib.
println!("cargo:rustc-link-arg=-Wl,-rpath,$ORIGIN");
println!("cargo:rustc-link-arg=-Wl,-rpath,{}", lib_dir.display());
// Copy runtime dependencies next to the built plugin so both the
// cargo target directory and the plugins/ staging directory work without
// extra LD_LIBRARY_PATH setup.
let target_dir = std::path::Path::new(&project_dir)
.join("..").join("..").join("target")
.join(std::env::var("PROFILE").unwrap());
if target_dir.exists() {
let libs = [
"libvisioncpp.so", "libvisioncpp.so.0", "libvisioncpp.so.0.3.1",
"libggml.so", "libggml.so.0", "libggml.so.0.9.9",
"libggml-cpu.so", "libggml-cpu.so.0", "libggml-cpu.so.0.9.9",
"libggml-base.so", "libggml-base.so.0", "libggml-base.so.0.9.9",
];
for lib in &libs {
let src = lib_dir.join(lib);
let dst = target_dir.join(lib);
if src.exists() {
std::fs::copy(&src, &dst).ok();
}
}
}
}
hcie-vision/src/comfy.rs
+1238
View File
File diff suppressed because it is too large Load Diff
hcie-vision/src/ffi.rs
+296
View File
@@ -0,0 +1,296 @@
use std::ffi::CStr;
use std::os::raw::c_char;
use crate::inpaint::{criminisi_inpaint_with_radius, build_polygon_mask, build_stroke_mask};
use crate::vision::{run_background_removal_task, run_object_segment_task, run_super_resolution_task};
// ── C++ backend FFI declarations (brought over from v3) ─────────────────────
// These are only compiled when the `cpp-backend` feature is enabled. They
// mirror the `vision.cpp` C ABI used by MobileSAM / BiRefNet / ESRGAN /
// MI-GAN via GGML/LLama.cpp. The Rust cdylib does not need to define these
// symbols itself; they are resolved at load time against the matching C++
// backend library, or left unresolved when `cpp-backend` is disabled.
#[cfg(feature = "cpp-backend")]
use std::os::raw::{c_void};
#[cfg(feature = "cpp-backend")]
#[repr(C)]
#[derive(Debug, Copy, Clone)]
pub struct VispImage {
pub width: i32,
pub height: i32,
pub stride: i32,
pub format: i32,
pub data: *mut c_void,
}
#[cfg(feature = "cpp-backend")]
#[repr(C)]
#[derive(Debug, Copy, Clone)]
pub struct VispImageView {
pub width: i32,
pub height: i32,
pub stride: i32,
pub format: i32,
pub data: *const c_void,
}
#[cfg(feature = "cpp-backend")]
#[repr(C)]
pub struct BackendDevice(c_void);
#[cfg(feature = "cpp-backend")]
#[repr(C)]
pub struct AnyModel(c_void);
#[cfg(feature = "cpp-backend")]
#[repr(C)]
pub struct ImageData(c_void);
#[cfg(feature = "cpp-backend")]
#[repr(i32)]
#[derive(Debug, Copy, Clone)]
pub enum ModelFamily {
Sam = 0,
Birefnet = 1,
DepthAnything = 2,
Migan = 3,
Esrgan = 4,
}
#[cfg(feature = "cpp-backend")]
#[repr(i32)]
#[derive(Debug, Copy, Clone)]
pub enum ImageFormat {
RgbaU8 = 0,
BgraU8 = 1,
ArgbU8 = 2,
RgbU8 = 3,
AlphaU8 = 4,
RgbaF32 = 5,
RgbF32 = 6,
AlphaF32 = 7,
}
#[cfg(feature = "cpp-backend")]
extern "C" {
pub fn visp_get_last_error() -> *const c_char;
pub fn visp_image_destroy(img: *mut ImageData);
pub fn visp_device_init(device_type: i32, out_device: *mut *mut BackendDevice) -> i32;
pub fn visp_device_destroy(device: *mut BackendDevice);
pub fn visp_model_load(filepath: *const c_char, dev: *const BackendDevice, arch: i32, out: *mut *mut AnyModel) -> i32;
pub fn visp_model_destroy(model: *mut AnyModel, arch: i32);
pub fn visp_model_compute(
model: *mut AnyModel,
family: i32,
inputs: *const VispImageView,
n_inputs: i32,
args: *const i32,
n_args: i32,
out_image: *mut VispImage,
out_data: *mut *mut ImageData,
) -> i32;
}
// ── Rust-to-C exports used by hcie-engine-api via dynamic loading ───────────
use std::sync::atomic::{AtomicU8, Ordering};
/// Global vision backend preference (0 = CPU, 1 = GPU). When the C++ backend
/// feature is enabled this is forwarded to `visp_device_init` as the device
/// type. When disabled it is still logged so operators can confirm which
/// backend would have been used.
static VISION_BACKEND: AtomicU8 = AtomicU8::new(0);
#[no_mangle]
pub unsafe extern "C" fn hcie_vision_set_backend_c(backend: u8) {
VISION_BACKEND.store(backend, Ordering::Relaxed);
}
/// Returns the currently configured backend code (0 = CPU, 1 = GPU).
pub(crate) fn vision_backend_code() -> u8 {
VISION_BACKEND.load(Ordering::Relaxed)
}
fn return_boxed_bytes(bytes: Vec<u8>, out_ptr: *mut *mut u8, out_len: *mut usize) {
unsafe {
let boxed = bytes.into_boxed_slice();
let len = boxed.len();
let raw = Box::into_raw(boxed) as *mut u8;
*out_ptr = raw;
*out_len = len;
}
}
#[no_mangle]
pub unsafe extern "C" fn free_buffer_c(ptr: *mut u8, len: usize) {
if !ptr.is_null() {
let _ = Box::from_raw(std::slice::from_raw_parts_mut(ptr, len) as *mut [u8]);
}
}
#[no_mangle]
pub unsafe extern "C" fn hcie_vision_background_removal_c(
img_pixels: *const u8,
img_len: usize,
width: u32,
height: u32,
out_buffer: *mut *mut u8,
out_len: *mut usize,
) -> bool {
if img_pixels.is_null() || out_buffer.is_null() || out_len.is_null() || img_len == 0 {
return false;
}
let img_bytes = std::slice::from_raw_parts(img_pixels, img_len).to_vec();
match run_background_removal_task("", &img_bytes, width, height) {
Ok(result) => {
return_boxed_bytes(result, out_buffer, out_len);
true
}
Err(_) => false,
}
}
#[no_mangle]
pub unsafe extern "C" fn hcie_vision_object_segment_c(
img_pixels: *const u8,
img_len: usize,
width: u32,
height: u32,
point_x: u32,
point_y: u32,
bbox_min_x: u32,
bbox_min_y: u32,
bbox_max_x: u32,
bbox_max_y: u32,
model_path_ptr: *const std::os::raw::c_char,
out_buffer: *mut *mut u8,
out_len: *mut usize,
) -> bool {
if img_pixels.is_null() || out_buffer.is_null() || out_len.is_null() || img_len == 0 {
return false;
}
let img_bytes = std::slice::from_raw_parts(img_pixels, img_len).to_vec();
// u32::MAX is the sentinel for "not provided"
let has_bbox = !(bbox_min_x == u32::MAX && bbox_min_y == u32::MAX && bbox_max_x == u32::MAX && bbox_max_y == u32::MAX);
let has_point = !(point_x == u32::MAX && point_y == u32::MAX);
let bbox = if has_bbox { Some((bbox_min_x, bbox_min_y, bbox_max_x, bbox_max_y)) } else { None };
let point = if has_point { Some((point_x, point_y)) } else { None };
let model_path = if model_path_ptr.is_null() {
String::new()
} else {
CStr::from_ptr(model_path_ptr).to_string_lossy().into_owned()
};
match run_object_segment_task(&model_path, &img_bytes, width, height, point, bbox) {
Ok(result) => {
return_boxed_bytes(result, out_buffer, out_len);
true
}
Err(_) => false,
}
}
#[no_mangle]
pub unsafe extern "C" fn hcie_vision_super_resolution_c(
img_pixels: *const u8,
img_len: usize,
width: u32,
height: u32,
out_buffer: *mut *mut u8,
out_len: *mut usize,
) -> bool {
if img_pixels.is_null() || out_buffer.is_null() || out_len.is_null() || img_len == 0 {
return false;
}
let img_bytes = std::slice::from_raw_parts(img_pixels, img_len).to_vec();
match run_super_resolution_task("", &img_bytes, width, height) {
Ok(result) => {
return_boxed_bytes(result, out_buffer, out_len);
true
}
Err(_) => false,
}
}
#[no_mangle]
pub unsafe extern "C" fn hcie_vision_inpaint_c(
pixels: *const u8,
pixels_len: usize,
width: u32,
height: u32,
hole_mask: *const u8,
hole_len: usize,
radius: i32,
) -> bool {
if pixels.is_null() || hole_mask.is_null() || pixels_len == 0 || hole_len == 0 {
return false;
}
let px_slice = std::slice::from_raw_parts_mut(pixels as *mut u8, pixels_len);
let hole_slice = std::slice::from_raw_parts(hole_mask, hole_len);
let hole_bool: Vec<bool> = hole_slice.iter().map(|&b| b != 0).collect();
let mut layer = hcie_protocol::Layer::from_rgba("inpaint", width, height, px_slice.to_vec());
criminisi_inpaint_with_radius(&mut layer, &hole_bool, radius);
px_slice.copy_from_slice(&layer.pixels);
true
}
#[no_mangle]
pub unsafe extern "C" fn hcie_vision_build_polygon_mask_c(
points_json: *const c_char,
width: u32,
height: u32,
out_buffer: *mut *mut u8,
out_len: *mut usize,
) -> bool {
if points_json.is_null() || out_buffer.is_null() || out_len.is_null() {
return false;
}
let json_str = match CStr::from_ptr(points_json).to_str() {
Ok(s) => s,
Err(_) => return false,
};
let points: Vec<[u32; 2]> = match serde_json::from_str(json_str) {
Ok(p) => p,
Err(_) => return false,
};
let pts: Vec<(u32, u32)> = points.iter().map(|p| (p[0], p[1])).collect();
let mask = build_polygon_mask(&pts, width, height);
let bytes: Vec<u8> = mask.iter().map(|&b| if b { 1u8 } else { 0u8 }).collect();
return_boxed_bytes(bytes, out_buffer, out_len);
true
}
#[no_mangle]
pub unsafe extern "C" fn hcie_vision_build_stroke_mask_c(
points_json: *const c_char,
selection_ptr: *const u8,
selection_len: usize,
radius: f32,
width: u32,
height: u32,
out_buffer: *mut *mut u8,
out_len: *mut usize,
) -> bool {
if points_json.is_null() || out_buffer.is_null() || out_len.is_null() {
return false;
}
let json_str = match CStr::from_ptr(points_json).to_str() {
Ok(s) => s,
Err(_) => return false,
};
let points: Vec<[u32; 2]> = match serde_json::from_str(json_str) {
Ok(p) => p,
Err(_) => return false,
};
let pts: Vec<(u32, u32)> = points.iter().map(|p| (p[0], p[1])).collect();
let sel = if selection_ptr.is_null() || selection_len == 0 {
None
} else {
Some(std::slice::from_raw_parts(selection_ptr, selection_len))
};
let mask = build_stroke_mask(&pts, sel, radius, width, height);
let bytes: Vec<u8> = mask.iter().map(|&b| if b { 1u8 } else { 0u8 }).collect();
return_boxed_bytes(bytes, out_buffer, out_len);
true
}
hcie-vision/src/inpaint.rs
+747
View File
@@ -0,0 +1,747 @@
use hcie_protocol::Layer;
const ALPHA: f32 = 255.0;
const CANDIDATES: usize = 1000; // Optimized for speed/quality balance
const MAX_TRIES: usize = CANDIDATES * 10;
// ── Public API ───────────────────────────────────────────────────────────────
/// Fill every pixel where `hole[y*w+x] == true` in `layer`
/// using Criminisi exemplar-based inpainting with default patch radius (4).
#[allow(dead_code)]
pub fn criminisi_inpaint(layer: &mut Layer, hole: &[bool]) {
criminisi_inpaint_with_radius(layer, hole, 2);
}
/// Fill every pixel where `hole[y*w+x] == true` in `layer`
/// using Criminisi exemplar-based inpainting with custom patch radius.
pub fn criminisi_inpaint_with_radius(layer: &mut Layer, hole: &[bool], radius: i32) {
let w = layer.width as i32;
let h = layer.height as i32;
let n = (w * h) as usize;
if hole.len() != n || n == 0 {
return;
}
// Safety check: Is the layer empty?
if layer.pixels.iter().all(|&p| p == 0) {
log::warn!("[inpaint] Layer is entirely empty/transparent. Inpainting will have no source texture.");
}
let mut pix = layer.pixels.clone();
let mut mask = hole.to_vec();
let mut conf = vec![1.0_f32; n];
for i in 0..n {
if mask[i] {
conf[i] = 0.0;
}
}
let patch_sz = ((2 * radius + 1) * (2 * radius + 1)) as f32;
// STEP 1: compute hole bounding box once — front stays inside it forever
let (bx0, by0, bx1, by1) = {
let (mut x0, mut y0, mut x1, mut y1) = (w, h, 0i32, 0i32);
for y in 0..h {
for x in 0..w {
if mask[(y * w + x) as usize] {
x0 = x0.min(x);
y0 = y0.min(y);
x1 = x1.max(x);
y1 = y1.max(y);
}
}
}
if x0 > x1 {
return;
}
(x0, y0, x1, y1)
};
let mut rng = Lcg::default();
// STEP 2: main fill loop
loop {
let front = build_front(&mask, w, h, bx0, by0, bx1, by1);
if front.is_empty() {
break;
}
// STEP 3: find highest-priority patch centre on the front
#[cfg(not(target_arch = "wasm32"))]
let best_patch = {
use rayon::prelude::*;
front.par_iter().map(|&(x, y)| {
let pr = conf_term(x, y, &mask, &conf, w, h, radius, patch_sz)
* data_term(x, y, &pix, &mask, w, h).max(1e-3);
(x, y, pr)
}).max_by(|a, b| a.2.partial_cmp(&b.2).unwrap_or(std::cmp::Ordering::Equal))
};
#[cfg(target_arch = "wasm32")]
let best_patch = {
front.iter().map(|&(x, y)| {
let pr = conf_term(x, y, &mask, &conf, w, h, radius, patch_sz)
* data_term(x, y, &pix, &mask, w, h).max(1e-3);
(x, y, pr)
}).max_by(|a, b| a.2.partial_cmp(&b.2).unwrap_or(std::cmp::Ordering::Equal))
};
let (bx, by) = if let Some((x, y, _)) = best_patch {
(x, y)
} else {
front[0]
};
let c_val = conf_term(bx, by, &mask, &conf, w, h, radius, patch_sz);
// STEP 4: find best matching known patch
match find_source(bx, by, &pix, &mask, w, h, &mut rng, radius) {
Some((sx, sy)) => {
// STEP 5: copy hole pixels from source patch, update confidence
for dy in -radius..=radius {
for dx in -radius..=radius {
let tx = bx + dx;
let ty = by + dy;
if tx < 0 || ty < 0 || tx >= w || ty >= h {
continue;
}
let ti = (ty * w + tx) as usize;
if !mask[ti] {
continue;
}
let sx2 = (sx + dx).clamp(0, w - 1);
let sy2 = (sy + dy).clamp(0, h - 1);
let si = (sy2 * w + sx2) as usize;
pix[ti * 4] = pix[si * 4];
pix[ti * 4 + 1] = pix[si * 4 + 1];
pix[ti * 4 + 2] = pix[si * 4 + 2];
pix[ti * 4 + 3] = pix[si * 4 + 3];
conf[ti] = c_val;
mask[ti] = false;
}
}
}
None => {
// No valid source — fill with average boundary color to prevent transparency holes
let c_avg = boundary_avg(bx, by, &pix, &mask, w, h, radius);
for dy in -radius..=radius { for dx in -radius..=radius {
let tx = bx + dx; let ty = by + dy;
if tx < 0 || ty < 0 || tx >= w || ty >= h { continue; }
let ti = (ty * w + tx) as usize;
if !mask[ti] { continue; }
pix[ti*4] = c_avg[0];
pix[ti*4+1] = c_avg[1];
pix[ti*4+2] = c_avg[2];
pix[ti*4+3] = 255;
mask[ti] = false;
}}
}
}
}
// STEP 6: Poisson Refinement Pass (Task #071)
// Subtler refinement to avoid "blob" look.
log::warn!("[inpaint] Starting subtle Poisson refinement...");
poisson_refine(&mut pix, hole, w, h, 5); // Fewer iterations
layer.pixels = pix;
layer.dirty = true;
log::warn!("[inpaint] Finished successfully.");
}
/// [CRITICAL_LOGIC_START]
/// Poisson Refinement Pass: Fixes color seams and artifacts in filled regions.
fn poisson_refine(pix: &mut [u8], hole: &[bool], w: i32, h: i32, iterations: usize) {
let n = (w * h) as usize;
let mut r = vec![0.0f32; n];
let mut g = vec![0.0f32; n];
let mut b = vec![0.0f32; n];
let mut guidance_r = vec![0.0f32; n];
let mut guidance_g = vec![0.0f32; n];
let mut guidance_b = vec![0.0f32; n];
// 1. Initialize and compute guidance (Laplacian of the inpainted result)
for i in 0..n {
r[i] = pix[i * 4] as f32;
g[i] = pix[i * 4 + 1] as f32;
b[i] = pix[i * 4 + 2] as f32;
}
for y in 1..h - 1 {
for x in 1..w - 1 {
let i = (y * w + x) as usize;
if hole[i] {
// Laplacian guidance field from inpainted result
let up = i - w as usize; let down = i + w as usize;
let left = i - 1; let right = i + 1;
guidance_r[i] = 4.0 * r[i] - (r[up] + r[down] + r[left] + r[right]);
guidance_g[i] = 4.0 * g[i] - (g[up] + g[down] + g[left] + g[right]);
guidance_b[i] = 4.0 * b[i] - (b[up] + b[down] + b[left] + b[right]);
}
}
}
// 2. Iterative Gauss-Seidel Solver (Poisson Smoothing)
for _ in 0..iterations {
for y in 1..h - 1 {
for x in 1..w - 1 {
let i = (y * w + x) as usize;
if hole[i] {
let up = i - w as usize; let down = i + w as usize;
let left = i - 1; let right = i + 1;
// Relax color values while keeping boundary conditions
// Dirichlet boundaries are the pixels outside the 'hole'
r[i] = (r[up] + r[down] + r[left] + r[right] + guidance_r[i]) / 4.0;
g[i] = (g[up] + g[down] + g[left] + g[right] + guidance_g[i]) / 4.0;
b[i] = (b[up] + b[down] + b[left] + b[right] + guidance_b[i]) / 4.0;
}
}
}
}
// 3. Write back with subtle blend (Task #071)
for i in 0..n {
if hole[i] {
let pr = pix[i * 4] as f32;
let pg = pix[i * 4 + 1] as f32;
let pb = pix[i * 4 + 2] as f32;
// 15% refine strength (Subtle to prevent "blob" look)
let strength = 0.15;
pix[i * 4] = (pr * (1.0 - strength) + r[i] * strength).clamp(0.0, 255.0) as u8;
pix[i * 4 + 1] = (pg * (1.0 - strength) + g[i] * strength).clamp(0.0, 255.0) as u8;
pix[i * 4 + 2] = (pb * (1.0 - strength) + b[i] * strength).clamp(0.0, 255.0) as u8;
}
}
}
// [CRITICAL_LOGIC_END]
// ── Fill-front ───────────────────────────────────────────────────────────────
fn build_front(
mask: &[bool],
w: i32,
h: i32,
x0: i32,
y0: i32,
x1: i32,
y1: i32,
) -> Vec<(i32, i32)> {
let mut front = Vec::new();
for y in y0.max(0)..=y1.min(h - 1) {
for x in x0.max(0)..=x1.min(w - 1) {
if !mask[(y * w + x) as usize] {
continue;
}
let on_front = [(-1i32, 0i32), (1, 0), (0, -1), (0, 1)]
.iter()
.any(|&(dx, dy)| {
let nx = x + dx;
let ny = y + dy;
nx >= 0 && ny >= 0 && nx < w && ny < h && !mask[(ny * w + nx) as usize]
});
if on_front {
front.push((x, y));
}
}
}
front
}
// ── Priority terms ───────────────────────────────────────────────────────────
/// C(p) = Σ conf[known pixels in Ψ_p] / |Ψ_p|
#[allow(clippy::too_many_arguments)]
fn conf_term(
bx: i32,
by: i32,
mask: &[bool],
conf: &[f32],
w: i32,
h: i32,
radius: i32,
patch_sz: f32,
) -> f32 {
let mut sum = 0.0_f32;
for dy in -radius..=radius {
for dx in -radius..=radius {
let x = bx + dx;
let y = by + dy;
if x < 0 || y < 0 || x >= w || y >= h {
continue;
}
let i = (y * w + x) as usize;
if !mask[i] {
sum += conf[i];
}
}
}
sum / patch_sz
}
/// D(p) = |∇I⊥_p · n_p| / α — isophote strength along fill-front normal.
fn data_term(bx: i32, by: i32, pix: &[u8], mask: &[bool], w: i32, h: i32) -> f32 {
let lum = |x: i32, y: i32| -> Option<f32> {
if x < 0 || y < 0 || x >= w || y >= h {
return None;
}
let i = (y * w + x) as usize;
if mask[i] {
return None;
}
let p = i * 4;
Some(pix[p] as f32 * 0.299 + pix[p + 1] as f32 * 0.587 + pix[p + 2] as f32 * 0.114)
};
// Find maximum gradient magnitude in known 3x3 neighborhood
let mut max_grad_x = 0.0_f32;
let mut max_grad_y = 0.0_f32;
let mut max_grad_mag = -1.0_f32;
for dy in -1..=1 {
for dx in -1..=1 {
let cx = bx + dx;
let cy = by + dy;
if let Some(center) = lum(cx, cy) {
let r = lum(cx + 1, cy).unwrap_or(center);
let l = lum(cx - 1, cy).unwrap_or(center);
let d = lum(cx, cy + 1).unwrap_or(center);
let u = lum(cx, cy - 1).unwrap_or(center);
let gx = (r - l) * 0.5;
let gy = (d - u) * 0.5;
let mag = gx * gx + gy * gy;
if mag > max_grad_mag {
max_grad_mag = mag;
max_grad_x = gx;
max_grad_y = gy;
}
}
}
}
let (ix, iy) = (-max_grad_y, max_grad_x); // isophote ⊥ gradient
let m = |x: i32, y: i32| -> f32 {
if x < 0 || y < 0 || x >= w || y >= h {
return 0.0;
}
if mask[(y * w + x) as usize] {
1.0
} else {
0.0
}
};
let a = m(bx + 1, by) - m(bx - 1, by);
let b = m(bx, by + 1) - m(bx, by - 1);
let len = (a * a + b * b).sqrt().max(1e-8);
(ix * (a / len) + iy * (b / len)).abs() / ALPHA
}
// ── Source-patch search ──────────────────────────────────────────────────────
#[allow(clippy::too_many_arguments)]
fn find_source(
bx: i32,
by: i32,
pix: &[u8],
mask: &[bool],
w: i32,
h: i32,
rng: &mut Lcg,
radius: i32,
) -> Option<(i32, i32)> {
let range_x = (w - 2 * radius) as u64;
let range_y = (h - 2 * radius) as u64;
if range_x == 0 || range_y == 0 {
return None;
}
let mut best: Option<(i32, i32)> = None;
let mut best_ssd = f32::MAX;
let (mut hits, mut tries) = (0usize, 0usize);
// Phase 1: Global random search
let mut candidates_list = Vec::with_capacity(CANDIDATES);
while hits < CANDIDATES && tries < MAX_TRIES {
tries += 1;
let sx = radius + rng.next_bounded(range_x) as i32;
let sy = radius + rng.next_bounded(range_y) as i32;
if !patch_all_known(sx, sy, mask, w, h, radius) {
continue;
}
hits += 1;
candidates_list.push((sx, sy));
}
#[cfg(not(target_arch = "wasm32"))]
{
use rayon::prelude::*;
if let Some((sx, sy, ssd)) = candidates_list
.into_par_iter()
.map(|(sx, sy)| {
let ssd = patch_ssd(bx, by, sx, sy, pix, mask, w, h, radius);
(sx, sy, ssd)
})
.min_by(|a, b| a.2.partial_cmp(&b.2).unwrap_or(std::cmp::Ordering::Equal))
{
best = Some((sx, sy));
best_ssd = ssd;
}
}
#[cfg(target_arch = "wasm32")]
{
for (sx, sy) in candidates_list {
let ssd = patch_ssd(bx, by, sx, sy, pix, mask, w, h, radius);
if ssd < best_ssd {
best_ssd = ssd;
best = Some((sx, sy));
}
}
}
// Phase 2: Local search refinement (Hill climbing)
if let Some((mut sx, mut sy)) = best {
for _ in 0..5 {
let mut improved = false;
for dy in -1..=1 {
for dx in -1..=1 {
if dx == 0 && dy == 0 { continue; }
let nx = sx + dx;
let ny = sy + dy;
if patch_all_known(nx, ny, mask, w, h, radius) {
let ssd = patch_ssd(bx, by, nx, ny, pix, mask, w, h, radius);
if ssd < best_ssd {
best_ssd = ssd;
sx = nx; sy = ny;
improved = true;
}
}
}
}
if !improved { break; }
}
best = Some((sx, sy));
}
// Phase 3: Deterministic fallback if nothing found
if best.is_none() {
// Try a coarser grid search to find ANY valid patch
let step = radius.max(2);
for y in (radius..h-radius).step_by(step as usize) {
for x in (radius..w-radius).step_by(step as usize) {
if patch_all_known(x, y, mask, w, h, radius) {
let ssd = patch_ssd(bx, by, x, y, pix, mask, w, h, radius);
if ssd < best_ssd {
best_ssd = ssd;
best = Some((x, y));
}
}
}
}
}
best
}
/// Returns true if every pixel in the patch is known (not in hole).
fn patch_all_known(cx: i32, cy: i32, mask: &[bool], w: i32, h: i32, radius: i32) -> bool {
for dy in -radius..=radius {
for dx in -radius..=radius {
let x = cx + dx;
let y = cy + dy;
if x < 0 || y < 0 || x >= w || y >= h {
return false;
}
if mask[(y * w + x) as usize] {
return false;
}
}
}
true
}
/// Normalised SSD over known pixels in the destination patch.
#[allow(clippy::too_many_arguments)]
fn patch_ssd(
bx: i32,
by: i32,
sx: i32,
sy: i32,
pix: &[u8],
mask: &[bool],
w: i32,
h: i32,
radius: i32,
) -> f32 {
let mut ssd = 0.0_f32;
let mut cnt = 0u32;
for dy in -radius..=radius {
for dx in -radius..=radius {
let tx = bx + dx;
let ty = by + dy;
if tx < 0 || ty < 0 || tx >= w || ty >= h {
continue;
}
let ti = (ty * w + tx) as usize;
if mask[ti] {
continue;
}
let tp = ti * 4;
let si = ((sy + dy) * w + (sx + dx)) as usize * 4;
let dr = pix[tp] as f32 - pix[si] as f32;
let dg = pix[tp + 1] as f32 - pix[si + 1] as f32;
let db = pix[tp + 2] as f32 - pix[si + 2] as f32;
ssd += dr * dr + dg * dg + db * db;
cnt += 1;
}
}
if cnt > 0 {
ssd / cnt as f32
} else {
f32::MAX
}
}
/// Returns average color of known pixels in the patch neighborhood.
fn boundary_avg(bx: i32, by: i32, pix: &[u8], mask: &[bool], w: i32, h: i32, radius: i32) -> [u8; 4] {
let mut r = 0.0f32; let mut g = 0.0f32; let mut b = 0.0f32; let mut count = 0.0f32;
for dy in -radius..=radius {
for dx in -radius..=radius {
let x = bx + dx; let y = by + dy;
if x >= 0 && x < w && y >= 0 && y < h {
let i = (y * w + x) as usize;
if !mask[i] {
r += pix[i * 4] as f32;
g += pix[i * 4 + 1] as f32;
b += pix[i * 4 + 2] as f32;
count += 1.0;
}
}
}
}
if count > 0.0 {
[(r / count) as u8, (g / count) as u8, (b / count) as u8, 255]
} else {
[128, 128, 128, 255]
}
}
// ── Minimal LCG PRNG ─────────────────────────────────────────────────────────
struct Lcg {
state: u64,
}
impl Default for Lcg {
fn default() -> Self {
Self {
state: 0xDEAD_BEEF_CAFE_F00D,
}
}
}
impl Lcg {
fn next(&mut self) -> u64 {
self.state = self
.state
.wrapping_mul(6_364_136_223_846_793_005)
.wrapping_add(1_442_695_040_888_963_407);
self.state
}
fn next_bounded(&mut self, bound: u64) -> u64 {
if bound == 0 {
return 0;
}
self.next() % bound
}
}
/// [Task #070] Red-eye Removal Tool
pub fn draw_redeye_removal(layer: &mut Layer, mask: Option<&[u8]>, points: &[(u32, u32)], size: f32, opacity: f32) {
if points.is_empty() { return; }
let r = (size / 2.0).max(1.0);
let w = layer.width;
let h = layer.height;
// 1. Build mask from collected points
let mut redeye_mask = vec![false; (w * h) as usize];
for &(px, py) in points {
let cx = px as i32;
let cy = py as i32;
let ri = r as i32;
for dy in -ri..=ri {
for dx in -ri..=ri {
if dx*dx + dy*dy <= ri*ri {
let tx = cx + dx;
let ty = cy + dy;
if tx >= 0 && tx < w as i32 && ty >= 0 && ty < h as i32 {
redeye_mask[(ty as u32 * w + tx as u32) as usize] = true;
}
}
}
}
}
for y in 0..h {
for x in 0..w {
let idx_pixel = (y * w + x) as usize;
if !redeye_mask[idx_pixel] { continue; }
// Global mask check
let mask_val = if let Some(m) = mask {
*m.get(idx_pixel).unwrap_or(&0)
} else {
255
};
if mask_val == 0 { continue; }
let idx = idx_pixel * 4;
let pr = layer.pixels[idx];
let pg = layer.pixels[idx+1];
let pb = layer.pixels[idx+2];
// Detection: R must be dominant — slightly relaxed threshold (1.2 instead of 1.5)
let avg_gb = (pg as f32 + pb as f32) / 2.0;
if pr as f32 > avg_gb * 1.2 && pr > 50 && pr > pg && pr > pb {
// Target: Desaturate and darken
let target_v = avg_gb.min(pr as f32 * 0.5); // Ensure it darkens
let target_r = target_v * 0.9;
// For a stroke, feathering is applied at the edges of the stroke mask.
// We'll approximate feathering based on distance to non-masked areas.
// Simple version: just apply directly since brush itself has an anti-aliased feel.
let blend = opacity * (mask_val as f32 / 255.0);
layer.pixels[idx] = (pr as f32 * (1.0 - blend) + target_r * blend).round() as u8;
layer.pixels[idx+1] = (pg as f32 * (1.0 - blend) + target_v * blend).round() as u8;
layer.pixels[idx+2] = (pb as f32 * (1.0 - blend) + target_v * blend).round() as u8;
}
}
}
layer.dirty = true;
}
/// Belirtilen noktaları birleştirip içini dolduran bir maske oluşturur (Lasso tarzı).
pub fn build_polygon_mask(points: &[(u32, u32)], w: u32, h: u32) -> Vec<bool> {
let mut mask = vec![false; (w * h) as usize];
if points.len() < 3 {
return mask;
}
// 1. Bounding Box bul (Performans için)
let mut min_x = w;
let mut max_x = 0;
let mut min_y = h;
let mut max_y = 0;
for &(x, y) in points {
if x < min_x { min_x = x; }
if x > max_x { max_x = x; }
if y < min_y { min_y = y; }
if y > max_y { max_y = y; }
}
// 2. Önce hattın kendisini boya (Sızıntı olmasın)
for &(x, y) in points {
if x < w && y < h {
mask[(y * w + x) as usize] = true;
}
}
// 3. Her piksel için Point-in-Polygon testi (Sadece kutu içinde)
for y in min_y..=max_y {
if y >= h { continue; }
for x in min_x..=max_x {
if x >= w { continue; }
let mut inside = false;
let mut j = points.len() - 1;
for i in 0..points.len() {
let (pi_x, pi_y) = (points[i].0 as i32, points[i].1 as i32);
let (pj_x, pj_y) = (points[j].0 as i32, points[j].1 as i32);
let (x_i, y_i) = (x as i32, y as i32);
if ((pi_y > y_i) != (pj_y > y_i)) &&
(x_i < (pj_x - pi_x) * (y_i - pi_y) / (pj_y - pi_y) + pi_x) {
inside = !inside;
}
j = i;
}
if inside {
mask[(y * w + x) as usize] = true;
}
}
}
mask
}
/// Builds a boolean hole mask from stroke points + optional global selection mask.
pub fn build_stroke_mask(points: &[(u32, u32)], sel: Option<&[u8]>, r: f32, w: u32, h: u32) -> Vec<bool> {
let mut hole = vec![false; (w * h) as usize];
let ri = r as i32;
for &(px, py) in points {
let (cx, cy) = (px as i32, py as i32);
for dy in -ri..=ri { for dx in -ri..=ri {
if dx*dx + dy*dy <= ri*ri {
let tx = cx + dx; let ty = cy + dy;
if tx >= 0 && tx < w as i32 && ty >= 0 && ty < h as i32 {
hole[(ty as u32 * w + tx as u32) as usize] = true;
}
}
}}
}
if let Some(m) = sel {
for i in 0..hole.len() { if m[i] == 0 { hole[i] = false; } }
}
hole
}
/// [Task #071] Spot Removal — exemplar-based inpainting (Criminisi 2004).
pub fn draw_spot_removal(layer: &mut Layer, mask: Option<&[u8]>, points: &[(u32, u32)], size: f32) {
if points.is_empty() { return; }
let r = (size / 2.0).max(1.0);
let hole = build_stroke_mask(points, mask, r, layer.width, layer.height);
if hole.iter().all(|&b| !b) { return; }
criminisi_inpaint_with_radius(layer, &hole, 4); // Standard radius
}
/// [Task #071] Smart Patch — exemplar-based inpainting (Criminisi 2004).
/// Supports both stroke-based and global-selection-based hole definition.
pub fn draw_smart_patch_v1(layer: &mut Layer, mask: Option<&[u8]>, points: &[(u32, u32)], size: f32, tool_is_ai_removal: bool) {
let w = layer.width;
let h = layer.height;
let r = (size / 2.0).max(1.0);
// Build hole: If it's AI removal, treat points as a Lasso (Polygon).
// Otherwise (Standard Smart Patch), treat as a Brush (Stroke).
let hole = if tool_is_ai_removal && points.len() > 2 {
build_polygon_mask(points, w, h)
} else {
build_stroke_mask(points, mask, r, w, h)
};
if hole.iter().all(|&b| !b) { return; }
log::warn!("[draw_smart_patch_v1] Starting inpaint on {}x{} layer with radius 4 (polygon: {})",
w, h, tool_is_ai_removal);
criminisi_inpaint_with_radius(layer, &hole, 4);
}
/// Task wrapper for inpaint functionality (used by hcie-app)
#[allow(clippy::too_many_arguments)]
pub fn run_inpaint_task(
model_path: &str,
img_bytes: &[u8],
_width: u32,
_height: u32,
_seed: &[u8],
_prompt: Option<String>,
_point: Option<(u32, u32)>,
_bbox: Option<(u32, u32, u32, u32)>,
) -> Result<Vec<u8>, String> {
log::info!("[Vision] Running inpainting with model: {}", model_path);
// Placeholder: Return the original image for now
Ok(img_bytes.to_vec())
}
hcie-vision/src/lib.rs
+16
View File
@@ -0,0 +1,16 @@
#![allow(dead_code)]
pub mod types;
pub mod comfy;
pub mod inpaint;
pub mod smart_patch;
pub mod vision;
#[cfg(feature = "ffi")]
pub mod ffi;
pub use types::ComfyAssets;
pub use comfy::ComfyManager;
pub use vision::VisionManager;
pub use inpaint::{criminisi_inpaint, criminisi_inpaint_with_radius, build_polygon_mask, build_stroke_mask, draw_redeye_removal, draw_spot_removal, draw_smart_patch_v1, run_inpaint_task};
pub use smart_patch::{draw_smart_patch, PatchParams, PatchBlendMode, SolverConfig};
pub use vision::{run_background_removal_task, run_object_segment_task, run_super_resolution_task};
hcie-vision/src/smart_patch.rs
+462
View File
@@ -0,0 +1,462 @@
#![allow(dead_code)]
//! Smart Patch Tool — Poisson seamless cloning for raster layers.
//!
//! Bu modül, bir kaynak bölgeyi (source patch) hedef bir katmana
//! (target layer) gradient alanını koruyarak yerleştirir. Klasik
//! Pérez et al. (2003) "Poisson Image Editing" makalesindeki
//! yaklaşımı takip eder:
//!
//! ```text
//! Δf = div(v) Ω içinde
//! f = f* ∂Ω üzerinde
//! ```
//!
//! Burada `v` kılavuz vektör alanıdır (kaynağın gradyanı veya
//! karışık gradyan), `f*` hedefin sınır değerleridir. Ayrık
//! formda her iç piksel için:
//!
//! ```text
//! |N_p| f_p - Σ_{q ∈ N_p ∩ Ω} f_q
//! = Σ_{q ∈ N_p ∩ ∂Ω} f*_q + Σ_{q ∈ N_p} v_{pq}
//! ```
//!
//! Bu seyrek lineer sistemi Gauss-Seidel + SOR ile çözüyoruz;
//! büyük küçük patch'ler için pratik ve allocation-lite.
// ---------------------------------------------------------------------------
// Dış tipler — gerçek editörde bunlar ayrı modüllerden geliyor olur.
// Burada self-contained bir kesit sunuyoruz.
// ---------------------------------------------------------------------------
use hcie_protocol::Layer;
// Layer yardımcı metodları (smart_patch'e özel)
trait LayerExt {
fn patch_idx(&self, x: i32, y: i32) -> usize;
}
impl LayerExt for Layer {
#[inline]
fn patch_idx(&self, x: i32, y: i32) -> usize {
(y as usize * self.width as usize + x as usize) * 4
}
}
/// Araç durumu — UI'dan gelen parametreler burada tutulur.
pub struct PatchParams {
/// Kaynak katman (snapshot). Hedef katmana klonlanacak görüntü.
pub source: Layer,
/// Kaynaktan alınacak bölgenin sol-üst köşesi.
pub source_origin: (i32, i32),
/// Bu bölgenin hedefteki yerleşim sol-üst köşesi.
pub dest_origin: (i32, i32),
/// Klonlanacak bölgenin genişlik × yükseklik.
pub patch_size: (u32, u32),
/// Karıştırma modu.
pub blend_mode: PatchBlendMode,
/// SOR için çözücü ayarları.
pub solver: SolverConfig,
}
/// Kılavuz vektör alanı stratejisi (Poisson'a özel).
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum PatchBlendMode {
/// v = ∇g (kaynağın gradyanı). Düz "seamless clone".
#[allow(dead_code)]
Normal,
/// Her komşulukta |∇f*| vs |∇g| karşılaştırıp büyüğünü seçer.
Mixed,
}
#[derive(Clone, Copy, Debug)]
pub struct SolverConfig {
pub max_iters: u32,
pub tolerance: f32,
/// Successive Over-Relaxation faktörü (1.0 = düz Gauss-Seidel).
/// 1.7–1.95 arası değerler yamuk olmayan bölgelerde hızlandırır.
pub omega: f32,
}
impl Default for SolverConfig {
fn default() -> Self {
Self {
max_iters: 800,
tolerance: 1e-3,
omega: 1.85,
}
}
}
// ---------------------------------------------------------------------------
// Ana giriş noktası
// ---------------------------------------------------------------------------
/// [Task #071] Smart Patch Tool (Poisson Blending)
pub fn draw_smart_patch(layer: &mut Layer, mask: Option<&[u8]>, params: &mut PatchParams) {
let (pw, ph) = params.patch_size;
if pw == 0 || ph == 0 {
return;
}
// 1) Patch bölgesi için ikili maskeyi hazırla. Caller maske verdiyse
// onu kullan, vermediyse dikdörtgeni komple iç bölge kabul et.
let region = match mask {
Some(m) => Mask::from_user(m, pw, ph),
None => Mask::full(pw, ph),
};
// 2) Hedef ve kaynak dikdörtgenlerini katman sınırlarına kırp.
let clip = match Clip::compute(layer, &params.source, params.source_origin, params.dest_origin, pw, ph) {
Some(c) => c,
None => return, // tamamen dışarıda
};
// 3) Her kanal için Poisson sistemini ayrı ayrı çöz.
// Alfa kanalını olduğu gibi yumuşak geçir — gradyan karışımı RGB'de.
let mut channels: [ChannelBuffer; 3] = [
ChannelBuffer::new(clip.width, clip.height),
ChannelBuffer::new(clip.width, clip.height),
ChannelBuffer::new(clip.width, clip.height),
];
for (c, buf) in channels.iter_mut().enumerate() {
buf.load(layer, &params.source, &clip, &region, c);
solve_poisson(buf, &region, &clip, params.blend_mode, &params.solver);
}
// 4) Sonucu katmana geri yaz. Alfa için kaynak alfa ile
// çarpımsal bir karışım uyguluyoruz ki patch yumuşakça otursun.
write_back(layer, &params.source, &clip, &region, &channels);
}
// ---------------------------------------------------------------------------
// Maske
// ---------------------------------------------------------------------------
/// 0 = dış, 1 = iç (çözülecek piksel), 2 = iç ama hedef katman sınırında.
struct Mask {
w: u32,
h: u32,
data: Vec<u8>,
}
impl Mask {
fn full(w: u32, h: u32) -> Self {
Self { w, h, data: vec![1; (w * h) as usize] }
}
fn from_user(m: &[u8], w: u32, h: u32) -> Self {
let expected = (w * h) as usize;
let mut data = vec![0u8; expected];
let n = m.len().min(expected);
for i in 0..n {
data[i] = if m[i] >= 128 { 1 } else { 0 };
}
Self { w, h, data }
}
#[inline]
fn get(&self, x: i32, y: i32) -> u8 {
if x < 0 || y < 0 || (x as u32) >= self.w || (y as u32) >= self.h {
0
} else {
self.data[(y as u32 * self.w + x as u32) as usize]
}
}
#[inline]
fn is_interior(&self, x: i32, y: i32) -> bool {
self.get(x, y) != 0
}
}
// ---------------------------------------------------------------------------
// Kırpma: kaynak ve hedef geometrisini ortak bir çalışma bölgesine indirger
// ---------------------------------------------------------------------------
struct Clip {
/// Çalışma bölgesinin boyutu (patch üzerinde kırpılmış).
width: u32,
height: u32,
/// Hedef katmanda çalışma bölgesinin sol-üstü.
dst_x: i32,
dst_y: i32,
/// Kaynak katmanda çalışma bölgesinin sol-üstü.
src_x: i32,
src_y: i32,
/// Maske içi başlangıç ofseti (kırpıldıysa).
mask_x: i32,
mask_y: i32,
}
impl Clip {
fn compute(
dst: &Layer,
src: &Layer,
src_origin: (i32, i32),
dst_origin: (i32, i32),
pw: u32,
ph: u32,
) -> Option<Self> {
let (sox, soy) = src_origin;
let (dox, doy) = dst_origin;
// Hedef için kırpma
let dx0 = dox.max(0);
let dy0 = doy.max(0);
let dx1 = (dox + pw as i32).min(dst.width as i32);
let dy1 = (doy + ph as i32).min(dst.height as i32);
// Kaynak için kırpma — aynı offset korunmalı
let shift_x = dx0 - dox;
let shift_y = dy0 - doy;
let sx0 = sox + shift_x;
let sy0 = soy + shift_y;
let mut w = (dx1 - dx0).min(src.width as i32 - sx0);
let mut h = (dy1 - dy0).min(src.height as i32 - sy0);
if sx0 < 0 {
w -= -sx0;
}
if sy0 < 0 {
h -= -sy0;
}
let sx0 = sx0.max(0);
let sy0 = sy0.max(0);
let dx0 = dx0 + (sx0 - (sox + shift_x));
let dy0 = dy0 + (sy0 - (soy + shift_y));
if w <= 0 || h <= 0 {
return None;
}
Some(Clip {
width: w as u32,
height: h as u32,
dst_x: dx0,
dst_y: dy0,
src_x: sx0,
src_y: sy0,
mask_x: shift_x.max(0),
mask_y: shift_y.max(0),
})
}
}
// ---------------------------------------------------------------------------
// Kanal bazlı çözücü bufferı
// ---------------------------------------------------------------------------
struct ChannelBuffer {
w: u32,
h: u32,
/// Hedeften okunmuş başlangıç değeri (aynı zamanda sınır değeri).
dst: Vec<f32>,
/// Kaynaktan okunmuş değer (gradyan hesabı için).
src: Vec<f32>,
/// Çözülmekte olan alan (iteratif güncellenir).
f: Vec<f32>,
}
impl ChannelBuffer {
fn new(w: u32, h: u32) -> Self {
let n = (w * h) as usize;
Self { w, h, dst: vec![0.0; n], src: vec![0.0; n], f: vec![0.0; n] }
}
fn load(&mut self, layer: &Layer, source: &Layer, clip: &Clip, _region: &Mask, channel: usize) {
for y in 0..clip.height as i32 {
for x in 0..clip.width as i32 {
let dst_px = layer.patch_idx(clip.dst_x + x, clip.dst_y + y);
let src_px = source.patch_idx(clip.src_x + x, clip.src_y + y);
let i = (y as u32 * self.w + x as u32) as usize;
self.dst[i] = layer.pixels[dst_px + channel] as f32;
self.src[i] = source.pixels[src_px + channel] as f32;
self.f[i] = self.dst[i];
}
}
}
}
// ---------------------------------------------------------------------------
// Çözücü: Gauss-Seidel + SOR, mixed gradient opsiyonlu
// ---------------------------------------------------------------------------
fn solve_poisson(buf: &mut ChannelBuffer, region: &Mask, clip: &Clip, mode: PatchBlendMode, cfg: &SolverConfig) {
let w = buf.w as i32;
let h = buf.h as i32;
let omega = cfg.omega.clamp(1.0, 1.99);
for iter in 0..cfg.max_iters {
let mut max_delta = 0.0f32;
for y in 0..h {
for x in 0..w {
let mx = x + clip.mask_x;
let my = y + clip.mask_y;
if !region.is_interior(mx, my) {
continue;
}
let i = (y as u32 * buf.w + x as u32) as usize;
let mut sum_boundary_or_interior = 0.0f32;
let mut sum_guidance = 0.0f32;
let mut n_count = 0.0f32;
let neighbors = [(-1, 0), (1, 0), (0, -1), (0, 1)];
for (dx, dy) in neighbors {
let nx = x + dx;
let ny = y + dy;
if nx < 0 || ny < 0 || nx >= w || ny >= h {
sum_boundary_or_interior += buf.f[i];
n_count += 1.0;
continue;
}
let j = (ny as u32 * buf.w + nx as u32) as usize;
n_count += 1.0;
let nmx = nx + clip.mask_x;
let nmy = ny + clip.mask_y;
if region.is_interior(nmx, nmy) {
sum_boundary_or_interior += buf.f[j];
} else {
sum_boundary_or_interior += buf.dst[j];
}
let g_pq = buf.src[i] - buf.src[j];
let f_pq = buf.dst[i] - buf.dst[j];
let v_pq = match mode {
PatchBlendMode::Normal => g_pq,
PatchBlendMode::Mixed => {
if g_pq.abs() >= f_pq.abs() { g_pq } else { f_pq }
}
};
sum_guidance += v_pq;
}
if n_count == 0.0 {
continue;
}
let new_val = (sum_boundary_or_interior + sum_guidance) / n_count;
let old_val = buf.f[i];
let relaxed = old_val + omega * (new_val - old_val);
let delta = (relaxed - old_val).abs();
if delta > max_delta {
max_delta = delta;
}
buf.f[i] = relaxed;
}
}
if max_delta < cfg.tolerance * 255.0 {
let _ = iter;
break;
}
}
}
// ---------------------------------------------------------------------------
// Geri yazma
// ---------------------------------------------------------------------------
fn write_back(layer: &mut Layer, source: &Layer, clip: &Clip, region: &Mask, ch: &[ChannelBuffer; 3]) {
for y in 0..clip.height as i32 {
for x in 0..clip.width as i32 {
let mx = x + clip.mask_x;
let my = y + clip.mask_y;
if !region.is_interior(mx, my) {
continue;
}
let i = (y as u32 * ch[0].w + x as u32) as usize;
let dst_px = layer.patch_idx(clip.dst_x + x, clip.dst_y + y);
let src_px = source.patch_idx(clip.src_x + x, clip.src_y + y);
for (c, buf) in ch.iter().enumerate().take(3) {
layer.pixels[dst_px + c] = clamp_u8(buf.f[i]);
}
let sa = source.pixels[src_px + 3] as f32 / 255.0;
let da = layer.pixels[dst_px + 3] as f32 / 255.0;
let out_a = da + sa * (1.0 - da);
layer.pixels[dst_px + 3] = clamp_u8(out_a * 255.0);
}
}
}
#[inline]
fn clamp_u8(v: f32) -> u8 {
if v.is_nan() {
return 0;
}
v.round().clamp(0.0, 255.0) as u8
}
// ---------------------------------------------------------------------------
// Testler — sağlık kontrolü
// ---------------------------------------------------------------------------
#[cfg(test)]
mod tests {
use super::*;
fn solid_layer(w: u32, h: u32, rgba: [u8; 4]) -> Layer {
let mut layer = Layer::new_transparent("test", w, h);
for px in layer.pixels.chunks_exact_mut(4) {
px.copy_from_slice(&rgba);
}
layer
}
#[test]
fn identity_patch_on_uniform_regions_is_stable() {
let mut target = solid_layer(16, 16, [120, 120, 120, 255]);
let source = solid_layer(16, 16, [120, 120, 120, 255]);
let mut st = PatchParams {
source,
source_origin: (4, 4),
dest_origin: (8, 8),
patch_size: (6, 6),
blend_mode: PatchBlendMode::Normal,
solver: SolverConfig { max_iters: 200, tolerance: 1e-3, omega: 1.7 },
};
draw_smart_patch(&mut target, None, &mut st);
for px in target.pixels.chunks_exact(4) {
assert!((px[0] as i32 - 120).abs() <= 1);
}
}
#[test]
fn patch_interior_moves_toward_source_gradient() {
let mut target = solid_layer(32, 32, [200, 200, 200, 255]);
let mut source = solid_layer(32, 32, [0, 0, 0, 255]);
for y in 0..32 {
for x in 0..32 {
let v = (y * 8) as u8;
let i = ((y * 32 + x) * 4) as usize;
source.pixels[i] = v;
source.pixels[i + 1] = v;
source.pixels[i + 2] = v;
}
}
let mut st = PatchParams {
source,
source_origin: (8, 8),
dest_origin: (12, 12),
patch_size: (8, 8),
blend_mode: PatchBlendMode::Normal,
solver: SolverConfig::default(),
};
draw_smart_patch(&mut target, None, &mut st);
let outside = ((11 * 32 + 12) * 4) as usize;
assert_eq!(target.pixels[outside], 200);
let top = ((12 * 32 + 15) * 4) as usize;
let bot = ((19 * 32 + 15) * 4) as usize;
assert!(target.pixels[bot] as i32 > target.pixels[top] as i32);
}
}
hcie-vision/src/types.rs
+75
View File
@@ -0,0 +1,75 @@
use serde::{Deserialize, Serialize};
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
pub struct ComfyAssets {
pub models: Vec<String>,
pub clips: Vec<String>,
pub has_inpaint_nodes: bool,
pub has_color_match: bool,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct VisionModelConfig {
pub task: VisionTask,
pub model_path: String,
pub backend: VisionBackend,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
pub enum VisionTask {
#[default]
Segmentation,
BackgroundRemoval,
Inpainting,
SuperResolution,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
pub enum VisionBackend {
#[default]
Cpu,
Gpu,
}
/// Poisson smart patch blend mode
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum PatchBlendMode {
Normal,
Mixed,
}
impl Default for PatchBlendMode {
fn default() -> Self { Self::Mixed }
}
/// Poisson solver configuration
#[derive(Clone, Copy, Debug)]
pub struct SolverConfig {
pub max_iters: u32,
pub tolerance: f32,
pub omega: f32,
}
impl Default for SolverConfig {
fn default() -> Self {
Self { max_iters: 800, tolerance: 1e-3, omega: 1.85 }
}
}
/// Smart Patch parameters
pub struct PatchParams {
pub source: SourceLayer,
pub source_origin: (i32, i32),
pub dest_origin: (i32, i32),
pub patch_size: (u32, u32),
pub blend_mode: PatchBlendMode,
pub solver: SolverConfig,
}
/// Lightweight source layer copy (pixels + dims only) for smart_patch
#[derive(Clone)]
pub struct SourceLayer {
pub pixels: Vec<u8>,
pub width: u32,
pub height: u32,
}
hcie-vision/src/vision.rs
+259
View File
@@ -0,0 +1,259 @@
#![allow(dead_code, unreachable_patterns, unused_variables, unused_assignments, unexpected_cfgs)]
use std::collections::VecDeque;
#[cfg(feature = "cpp-backend")]
use crate::ffi;
#[cfg(feature = "cpp-backend")]
use std::ffi::CString;
#[cfg(feature = "cpp-backend")]
use std::os::raw::c_void;
/// Bridge to the vision.cpp C++ library or native Rust equivalent.
pub struct VisionManager {
// This would hold the pointers to the loaded GGML contexts
}
impl Default for VisionManager {
fn default() -> Self {
Self::new()
}
}
impl VisionManager {
pub fn new() -> Self {
Self {}
}
}
/// High-level background removal task
pub fn run_background_removal_task(
model_path: &str,
_img_bytes: &[u8],
width: u32,
height: u32,
) -> Result<Vec<u8>, String> {
log::info!("[Vision] Running background removal with model: {}", model_path);
// Placeholder: Return a dummy mask (all white)
let mask = vec![255u8; (width * height) as usize];
Ok(mask)
}
/// High-level object segment task — uses MobileSAM via GGML FFI, falls back to flood fill.
pub fn run_object_segment_task(
model_path: &str,
img_bytes: &[u8],
width: u32,
height: u32,
point: Option<(u32, u32)>,
bbox: Option<(u32, u32, u32, u32)>,
) -> Result<Vec<u8>, String> {
let backend = crate::ffi::vision_backend_code();
log::info!("[Vision] SAM: model='{}' {}x{} point={:?} bbox={:?} backend={}", model_path, width, height, point, bbox, if backend == 0 { "CPU" } else { "GPU" });
#[cfg(feature = "cpp-backend")]
if !model_path.is_empty() && std::path::Path::new(model_path).exists() {
match run_sam_ggml(model_path, img_bytes, width, height, point, bbox, backend) {
Ok(mask) => return Ok(mask),
Err(e) => log::warn!("[Vision] SAM GGML failed: {}. Falling back.", e),
}
}
// Fallback: flood fill from point, rectangle from bbox
sam_fallback(img_bytes, width, height, point, bbox)
}
#[cfg(feature = "cpp-backend")]
fn run_sam_ggml(
model_path: &str,
img_bytes: &[u8],
width: u32,
height: u32,
point: Option<(u32, u32)>,
bbox: Option<(u32, u32, u32, u32)>,
backend_code: u8,
) -> Result<Vec<u8>, String> {
// Convert RGBA → RGB (SAM expects RGB)
let mut rgb: Vec<u8> = Vec::with_capacity((width * height * 3) as usize);
for chunk in img_bytes.chunks(4) {
rgb.extend_from_slice(&chunk[..3]);
}
unsafe {
// 1. Init device
let mut device_ptr: *mut ffi::BackendDevice = std::ptr::null_mut();
let ok = ffi::visp_device_init(backend_code as i32, &mut device_ptr);
if ok == 0 {
return Err(format!("visp_device_init: {}", get_visp_error()));
}
let _device_guard = DeviceGuard(device_ptr);
// 2. Load model (arch=0 = Sam)
let path_cstr = CString::new(model_path).map_err(|e| e.to_string())?;
let mut model_ptr: *mut ffi::AnyModel = std::ptr::null_mut();
let ok = ffi::visp_model_load(path_cstr.as_ptr(), device_ptr, 0, &mut model_ptr);
if ok == 0 {
return Err(format!("visp_model_load: {}", get_visp_error()));
}
log::info!("[Vision] SAM model loaded");
let _model_guard = ModelGuard(model_ptr, 0);
// 3. Build image view (RgbU8 = 3)
let image_view = ffi::VispImageView {
width: width as i32,
height: height as i32,
stride: (width * 3) as i32,
format: 3, // RgbU8
data: rgb.as_ptr() as *const c_void,
};
// 4. Build prompt args
let args: Vec<i32> = if let Some((x, y)) = point {
log::info!("[Vision] SAM prompt: point ({},{})", x, y);
vec![x as i32, y as i32]
} else if let Some((bx, by, bw, bh)) = bbox {
log::info!("[Vision] SAM prompt: bbox ({},{},{},{})", bx, by, bx+bw, by+bh);
vec![bx as i32, by as i32, (bx + bw) as i32, (by + bh) as i32]
} else {
log::info!("[Vision] SAM prompt: center fallback");
vec![(width / 2) as i32, (height / 2) as i32]
};
// 5. Run compute
let mut out_image = ffi::VispImage {
width: 0, height: 0, stride: 0, format: 0,
data: std::ptr::null_mut(),
};
let mut out_data: *mut ffi::ImageData = std::ptr::null_mut();
let ok = ffi::visp_model_compute(
model_ptr, 0, // Sam
&image_view, 1,
args.as_ptr(), args.len() as i32,
&mut out_image, &mut out_data,
);
if ok == 0 {
return Err(format!("visp_model_compute: {}", get_visp_error()));
}
log::info!("[Vision] SAM output: {}x{} format={}", out_image.width, out_image.height, out_image.format);
let out_w = out_image.width as usize;
let out_h = out_image.height as usize;
// 6. Extract alpha mask from output
// format 4 = AlphaU8 (single channel), format 0 = RgbaU8 (take alpha)
let raw_mask: Vec<u8> = match out_image.format {
4 => { // AlphaU8
std::slice::from_raw_parts(out_image.data as *const u8, out_w * out_h).to_vec()
}
0 => { // RgbaU8 — extract alpha channel
let rgba = std::slice::from_raw_parts(out_image.data as *const u8, out_w * out_h * 4);
rgba.chunks(4).map(|c| c[3]).collect()
}
_ => {
log::warn!("[Vision] SAM unknown output format {}, reading as single channel", out_image.format);
std::slice::from_raw_parts(out_image.data as *const u8, out_w * out_h).to_vec()
}
};
ffi::visp_image_destroy(out_data);
// 7. Resize to canvas size if SAM output is smaller (e.g. 256px mask → canvas size)
let mask = if out_w != width as usize || out_h != height as usize {
log::info!("[Vision] SAM resizing mask {}x{} → {}x{}", out_w, out_h, width, height);
let mut resized = vec![0u8; (width * height) as usize];
for y in 0..height as usize {
for x in 0..width as usize {
let sx = x * out_w / width as usize;
let sy = y * out_h / height as usize;
resized[y * width as usize + x] = raw_mask[sy * out_w + sx];
}
}
resized
} else {
raw_mask
};
Ok(mask)
}
}
#[cfg(feature = "cpp-backend")]
unsafe fn get_visp_error() -> String {
let ptr = ffi::visp_get_last_error();
if ptr.is_null() { return "unknown error".to_string(); }
std::ffi::CStr::from_ptr(ptr).to_string_lossy().into_owned()
}
// RAII guards so device/model are freed even on early return
#[cfg(feature = "cpp-backend")]
struct DeviceGuard(*mut ffi::BackendDevice);
#[cfg(feature = "cpp-backend")]
impl Drop for DeviceGuard {
fn drop(&mut self) { unsafe { if !self.0.is_null() { ffi::visp_device_destroy(self.0); } } }
}
#[cfg(feature = "cpp-backend")]
struct ModelGuard(*mut ffi::AnyModel, i32);
#[cfg(feature = "cpp-backend")]
impl Drop for ModelGuard {
fn drop(&mut self) { unsafe { if !self.0.is_null() { ffi::visp_model_destroy(self.0, self.1); } } }
}
fn sam_fallback(
img_bytes: &[u8],
width: u32,
height: u32,
point: Option<(u32, u32)>,
bbox: Option<(u32, u32, u32, u32)>,
) -> Result<Vec<u8>, String> {
let mut mask = vec![0u8; (width * height) as usize];
if let Some((bx, by, bw, bh)) = bbox {
for y in by..by.saturating_add(bh).min(height) {
for x in bx..bx.saturating_add(bw).min(width) {
mask[(y * width + x) as usize] = 255;
}
}
} else if let Some((cx, cy)) = point {
if cx < width && cy < height {
let start_idx = (cy * width + cx) as usize * 4;
if start_idx + 3 < img_bytes.len() {
let tc = [img_bytes[start_idx], img_bytes[start_idx+1], img_bytes[start_idx+2]];
let mut queue = VecDeque::new();
queue.push_back((cx, cy));
mask[(cy * width + cx) as usize] = 255;
let mut count = 0usize;
while let Some((x, y)) = queue.pop_front() {
count += 1;
if count > 100_000 { break; }
for (dx, dy) in [(0i32,1i32),(0,-1),(1,0),(-1,0)] {
let nx = x as i32 + dx;
let ny = y as i32 + dy;
if nx >= 0 && nx < width as i32 && ny >= 0 && ny < height as i32 {
let ni = (ny as u32 * width + nx as u32) as usize;
if mask[ni] == 0 {
let ii = ni * 4;
let diff = (img_bytes[ii] as i32 - tc[0] as i32).abs()
+ (img_bytes[ii+1] as i32 - tc[1] as i32).abs()
+ (img_bytes[ii+2] as i32 - tc[2] as i32).abs();
if diff < 96 { mask[ni] = 255; queue.push_back((nx as u32, ny as u32)); }
}
}
}
}
}
}
}
Ok(mask)
}
/// High-level super resolution task
pub fn run_super_resolution_task(
model_path: &str,
_img_bytes: &[u8],
width: u32,
height: u32,
) -> Result<Vec<u8>, String> {
log::info!("[Vision] Running super resolution with model: {}", model_path);
// Placeholder: Return a dummy mask (all white)
let mask = vec![255u8; (width * height) as usize];
Ok(mask)
}