feat(iced): implement AI chat with streaming SSE and tool execution

This commit is contained in:
2026-07-15 16:31:25 +03:00
parent f2b03265ba
commit ba6a7f8de2
7 changed files with 1052 additions and 49 deletions
Generated
+3
View File
@@ -2719,14 +2719,17 @@ dependencies = [
"dirs 5.0.1", "dirs 5.0.1",
"env_logger", "env_logger",
"evdev", "evdev",
"futures-util",
"hcie-engine-api", "hcie-engine-api",
"iced", "iced",
"iced-panel-adapter", "iced-panel-adapter",
"image 0.25.10", "image 0.25.10",
"log", "log",
"reqwest",
"rfd", "rfd",
"serde", "serde",
"serde_json", "serde_json",
"tokio",
] ]
[[package]] [[package]]
@@ -30,3 +30,6 @@ bytes = "1.12"
bytemuck = { version = "1", features = ["derive"] } bytemuck = { version = "1", features = ["derive"] }
evdev = { version = "0.12", optional = true } evdev = { version = "0.12", optional = true }
dirs = "5.0" dirs = "5.0"
reqwest = { version = "0.12", features = ["json", "stream"] }
tokio = { version = "1", features = ["full"] }
futures-util = "0.3"
@@ -0,0 +1,661 @@
//! AI Chat — state, streaming SSE logic, tool execution, and DSL script parsing.
//!
//! Provides the data model and async streaming infrastructure for the AI assistant panel.
//! Includes JSON tool call execution and DSL procedural drawing script parsing,
//! ported from the egui reference implementation.
use hcie_engine_api::{Engine, EngineCommand};
use serde_json::json;
/// System prompt that defines the AI assistant's capabilities and tool-calling format.
pub const SYSTEM_PROMPT: &str = r##"You are an expert digital artist AI assistant integrated inside HCIE-Rust, a pixel-grade digital painting and image editing application.
You can talk to the user AND draw or edit their canvas.
You have access to a set of canvas tools that you can call. If the user asks you to paint or edit something, always call the appropriate tools using a JSON code block.
Canvas dimensions: 800x600 pixels. Coordinates start at (0,0) (top-left) to (800,600) (bottom-right).
## Tool Calling Mode
To call tools, output a markdown code block starting with ```json containing either a single tool object or an array of tool objects.
For example:
```json
{
"tool": "draw_rect",
"parameters": {"x1": 100, "y1": 100, "x2": 300, "y2": 300}
}
```
Or an array of tools:
```json
[
{"tool": "create_layer", "parameters": {"name": "Background Grid"}},
{"tool": "draw_rect", "parameters": {"x1": 0, "y1": 0, "x2": 800, "y2": 600}}
]
```
### Supported JSON Tools:
- create_layer(name: string): Creates a new layer and selects it.
- select_layer(name: string): Selects an existing layer by name (creates it if it doesn't exist).
- draw_rect(x1: float, y1: float, x2: float, y2: float): Draws a filled rectangle on the active layer using current color.
- draw_ellipse(cx: float, cy: float, rx: float, ry: float): Draws a filled ellipse on the active layer.
- draw_line(x0: float, y0: float, x1: float, y1: float): Draws a line from (x0,y0) to (x1,y1).
- add_text(text: string, font: string, size: float, x: float, y: float, color_hex: string): Renders vector text.
- apply_filter(filter_id: string, params: object): Applies an image filter to the active layer. Supported filters: "box_blur", "gaussian_blur", "motion_blur", "unsharp_mask", "mosaic", "crystallize", "pinch", "twirl", "oil_paint", "brightness_contrast" (params: {"brightness": -1.0 to 1.0, "contrast": -1.0 to 1.0}), "hue_saturation" (params: {"hue": -180 to 180, "saturation": -1.0 to 1.0, "lightness": -1.0 to 1.0}), "border" (params: {"size": 1 to 100, "color": [r,g,b,a]}).
- draw_stroke(points: array of {"x": float, "y": float, "pressure": float}): Paints a brush stroke with the current brush.
## Fallback Procedural Drawing Code (DSL)
If preferred, you can also output a procedural drawing script in a ```dsl code block.
Available DSL commands (one per line):
- layer <name> Create and select a layer.
- select_layer <name> Select an existing layer.
- color <hex> Set current primary color (e.g. #FF5500).
- size <px> Set brush size (e.g. size 24).
- rect x1,y1 x2,y2 Draw a filled rectangle.
- circle cx,cy [r] Draw a filled circle.
- ellipse cx,cy rx,ry Draw a filled ellipse.
- line x0,y0 x1,y1 Draw a straight line.
- text "content" font size x y [hex] Draw text (quotes required).
- filter <filter_id> <json_params> Apply a filter.
- draw x1,y1 x2,y2 x3,y3 ... Draw a freehand brush stroke.
Example DSL Block:
```dsl
layer Mountains
color #1A365D
rect 0,400 800,600
color #4A5568
line 0,400 400,200
line 400,200 800,400
```
When requested to draw something, describe what you are going to draw in conversational text, and then write the corresponding JSON tool calls or DSL blocks. Keep responses conversational and creative!
"##;
/// A single chat message with role, content, and optional reasoning flag.
#[derive(Debug, Clone)]
pub struct ChatMessage {
/// Role: "user", "assistant", or "tool".
pub role: String,
/// Message content.
pub content: String,
/// Whether this is a reasoning/thinking message (shown differently in UI).
pub is_reasoning: bool,
}
/// Provider selection for the LLM backend.
#[derive(Debug, Clone, PartialEq)]
pub enum AiProvider {
Ollama,
Claude,
OpenAI,
}
impl AiProvider {
/// Return the default base URL for this provider.
pub fn default_url(&self) -> &str {
match self {
AiProvider::Ollama => "http://localhost:11434",
AiProvider::Claude => "https://api.anthropic.com",
AiProvider::OpenAI => "https://api.openai.com/v1",
}
}
/// Return the display name.
pub fn display_name(&self) -> &str {
match self {
AiProvider::Ollama => "Ollama",
AiProvider::Claude => "Claude",
AiProvider::OpenAI => "OpenAI",
}
}
}
/// Configuration for the AI chat LLM connection.
#[derive(Debug, Clone)]
pub struct AiChatConfig {
/// Currently selected provider.
pub provider: AiProvider,
/// Base URL for the LLM API endpoint.
pub base_url: String,
/// Model name to use for chat completions.
pub model: String,
/// Maximum conversation turns before auto-truncation.
pub max_turns: u32,
/// Whether to display reasoning/thinking content from the LLM.
pub reasoning_enabled: bool,
/// Whether to send canvas context (dimensions, layers) with each request.
pub canvas_context: bool,
}
impl Default for AiChatConfig {
fn default() -> Self {
Self {
provider: AiProvider::Ollama,
base_url: "http://localhost:11434".to_string(),
model: "qwen2.5:latest".to_string(),
max_turns: 20,
reasoning_enabled: false,
canvas_context: true,
}
}
}
/// Complete AI chat state including messages, input buffer, and streaming state.
#[derive(Debug)]
pub struct AiChatState {
/// Conversation history.
pub messages: Vec<ChatMessage>,
/// Current user input buffer.
pub input: String,
/// LLM configuration.
pub config: AiChatConfig,
/// Whether a streaming response is in progress.
pub is_streaming: bool,
/// Accumulated streaming response text.
pub current_response: String,
/// Current reasoning/thinking text being streamed.
pub current_reasoning: String,
/// Available models fetched from the endpoint.
pub available_models: Vec<String>,
}
impl Default for AiChatState {
fn default() -> Self {
Self {
messages: Vec::new(),
input: String::new(),
config: AiChatConfig::default(),
is_streaming: false,
current_response: String::new(),
current_reasoning: String::new(),
available_models: Vec::new(),
}
}
}
/// Parse a hex color string (e.g. "#FF5500" or "AABBCC") into [u8; 4].
pub fn parse_hex_color(s: &str) -> Option<[u8; 4]> {
let s = s.trim().trim_start_matches('#');
if s.len() == 6 {
let r = u8::from_str_radix(&s[0..2], 16).ok()?;
let g = u8::from_str_radix(&s[2..4], 16).ok()?;
let b = u8::from_str_radix(&s[4..6], 16).ok()?;
Some([r, g, b, 255])
} else if s.len() == 8 {
let r = u8::from_str_radix(&s[0..2], 16).ok()?;
let g = u8::from_str_radix(&s[2..4], 16).ok()?;
let b = u8::from_str_radix(&s[4..6], 16).ok()?;
let a = u8::from_str_radix(&s[6..8], 16).ok()?;
Some([r, g, b, a])
} else {
None
}
}
/// Parse coordinate string (e.g. "100,200" or "100 200") into (f32, f32).
pub fn parse_coords(s: &str) -> Option<(f32, f32)> {
let s = s.trim().replace(',', " ");
let parts: Vec<&str> = s.split_whitespace().collect();
if parts.len() >= 2 {
let x = parts[0].parse::<f32>().ok()?;
let y = parts[1].parse::<f32>().ok()?;
Some((x, y))
} else {
None
}
}
/// Execute a parsed JSON tool call directly on the Engine.
pub fn execute_tool_call(engine: &mut Engine, name: &str, args: &serde_json::Value) -> Result<String, String> {
match name {
"create_layer" | "add_layer" => {
let layer_name = args["name"].as_str().unwrap_or("Layer");
let id = engine.add_layer(layer_name);
engine.set_active_layer(id);
Ok(format!("Created layer '{}' (ID: {})", layer_name, id))
}
"select_layer" => {
let layer_name = args["name"].as_str().unwrap_or("");
if let Some(layer) = engine.layer_infos().iter().find(|l| l.name.eq_ignore_ascii_case(layer_name)) {
engine.set_active_layer(layer.id);
Ok(format!("Selected layer '{}' (ID: {})", layer_name, layer.id))
} else {
let id = engine.add_layer(layer_name);
engine.set_active_layer(id);
Ok(format!("Layer '{}' not found. Created and selected it.", layer_name))
}
}
"draw_rect" => {
let x1 = args["x1"].as_f64().unwrap_or(0.0) as f32;
let y1 = args["y1"].as_f64().unwrap_or(0.0) as f32;
let x2 = args["x2"].as_f64().unwrap_or(0.0) as f32;
let y2 = args["y2"].as_f64().unwrap_or(0.0) as f32;
engine.draw_rect(x1, y1, x2, y2);
Ok(format!("Drew rectangle [{}, {}, {}, {}]", x1, y1, x2, y2))
}
"draw_ellipse" => {
let cx = args["cx"].as_f64().unwrap_or(0.0) as f32;
let cy = args["cy"].as_f64().unwrap_or(0.0) as f32;
let rx = args["rx"].as_f64().unwrap_or(0.0) as f32;
let ry = args["ry"].as_f64().unwrap_or(0.0) as f32;
engine.draw_ellipse(cx, cy, rx, ry);
Ok(format!("Drew ellipse center ({}, {}) radius ({}, {})", cx, cy, rx, ry))
}
"draw_line" => {
let x0 = args["x0"].as_f64().unwrap_or(0.0) as f32;
let y0 = args["y0"].as_f64().unwrap_or(0.0) as f32;
let x1 = args["x1"].as_f64().unwrap_or(0.0) as f32;
let y1 = args["y1"].as_f64().unwrap_or(0.0) as f32;
engine.draw_line(x0, y0, x1, y1);
Ok(format!("Drew line from ({}, {}) to ({}, {})", x0, y0, x1, y1))
}
"add_text" => {
let text = args["text"].as_str().unwrap_or("");
let font = args["font"].as_str().unwrap_or("default");
let size = args["size"].as_f64().unwrap_or(24.0) as f32;
let x = args["x"].as_f64().unwrap_or(100.0) as f32;
let y = args["y"].as_f64().unwrap_or(100.0) as f32;
let color_hex = args["color_hex"].as_str().unwrap_or("#000000");
let color = parse_hex_color(color_hex).unwrap_or([0, 0, 0, 255]);
let effects = vec![];
engine.add_text(
text, font, size, x, y, color,
0.0,
hcie_engine_api::TextAlignment::Left,
hcie_engine_api::TextOrientation::Horizontal,
&effects,
);
Ok(format!("Added text '{}' at ({}, {})", text, x, y))
}
"apply_filter" => {
let filter_id = args["filter_id"].as_str().unwrap_or("");
let params = args["params"].clone();
engine.apply_filter(filter_id, params);
Ok(format!("Applied filter '{}'", filter_id))
}
"draw_stroke" => {
let mut points = Vec::new();
if let Some(pts_arr) = args["points"].as_array() {
for pt in pts_arr {
let x = pt["x"].as_f64().unwrap_or(0.0) as f32;
let y = pt["y"].as_f64().unwrap_or(0.0) as f32;
let p = pt["pressure"].as_f64().unwrap_or(1.0) as f32;
points.push((x, y, p));
}
}
engine.draw_stroke(&points);
Ok(format!("Drew brush stroke with {} points", points.len()))
}
_ => Err(format!("Unknown tool: {}", name))
}
}
/// Execute a generic EngineCommand on the engine directly using the correct public methods.
pub fn execute_command_on_engine(engine: &mut Engine, cmd: &EngineCommand) {
match cmd {
EngineCommand::AddLayer { name } => {
engine.add_layer(name);
}
EngineCommand::SelectLayer { id } => {
engine.set_active_layer(*id);
}
EngineCommand::DrawRect { x1, y1, x2, y2 } => {
engine.draw_rect(*x1, *y1, *x2, *y2);
}
EngineCommand::DrawEllipse { cx, cy, rx, ry } => {
engine.draw_ellipse(*cx, *cy, *rx, *ry);
}
EngineCommand::DrawLine { x0, y0, x1, y1 } => {
engine.draw_line(*x0, *y0, *x1, *y1);
}
EngineCommand::DrawStroke { points } => {
engine.draw_stroke(points);
}
EngineCommand::AddText { text, font, size, x, y, color } => {
let effects = vec![];
engine.add_text(
text, font, *size, *x, *y, *color,
0.0,
hcie_engine_api::TextAlignment::Left,
hcie_engine_api::TextOrientation::Horizontal,
&effects,
);
}
EngineCommand::ApplyFilter { filter_id, params } => {
engine.apply_filter(filter_id, params.clone());
}
_ => {
engine.execute_batch(&[cmd.clone()]);
}
}
}
/// Line-by-line DSL script parser.
pub fn parse_dsl_script(script: &str, _engine: &Engine) -> Vec<EngineCommand> {
let mut commands = Vec::new();
let mut current_color = [0, 0, 0, 255];
let mut brush_size = 10.0;
for line in script.lines() {
let line = line.trim();
if line.is_empty() || line.starts_with('#') || line.starts_with("//") {
continue;
}
let parts: Vec<&str> = line.split_whitespace().collect();
if parts.is_empty() {
continue;
}
match parts[0].to_lowercase().as_str() {
"layer" => {
if parts.len() >= 2 {
let name = parts[1..].join(" ");
commands.push(EngineCommand::AddLayer { name });
}
}
"select_layer" => {
if parts.len() >= 2 {
let name = parts[1..].join(" ");
let id_opt = _engine.layer_infos().iter()
.find(|l| l.name.eq_ignore_ascii_case(&name))
.map(|l| l.id);
if let Some(id) = id_opt {
commands.push(EngineCommand::SelectLayer { id });
} else {
commands.push(EngineCommand::AddLayer { name });
}
}
}
"color" => {
if parts.len() >= 2 {
if let Some(c) = parse_hex_color(parts[1]) {
current_color = c;
commands.push(EngineCommand::SetColor(c));
}
}
}
"size" => {
if parts.len() >= 2 {
if let Ok(sz) = parts[1].parse::<f32>() {
brush_size = sz;
}
}
}
"rect" | "fill" => {
if parts.len() >= 3 {
let p1 = parse_coords(parts[1]);
let p2 = parse_coords(parts[2]);
if let (Some((x1, y1)), Some((x2, y2))) = (p1, p2) {
commands.push(EngineCommand::DrawRect { x1, y1, x2, y2 });
}
}
}
"circle" => {
if parts.len() >= 2 {
if let Some((cx, cy)) = parse_coords(parts[1]) {
let r = if parts.len() >= 3 {
parts[2].parse::<f32>().unwrap_or(brush_size / 2.0)
} else {
brush_size / 2.0
};
commands.push(EngineCommand::DrawEllipse { cx, cy, rx: r, ry: r });
}
}
}
"ellipse" => {
if parts.len() >= 3 {
if let Some((cx, cy)) = parse_coords(parts[1]) {
if let Some((rx, ry)) = parse_coords(parts[2]) {
commands.push(EngineCommand::DrawEllipse { cx, cy, rx, ry });
}
}
}
}
"line" => {
if parts.len() >= 3 {
let p1 = parse_coords(parts[1]);
let p2 = parse_coords(parts[2]);
if let (Some((x0, y0)), Some((x1, y1))) = (p1, p2) {
commands.push(EngineCommand::DrawLine { x0, y0, x1, y1 });
}
}
}
"text" => {
if parts.len() >= 6 {
let text = parts[1].replace('"', "").replace('_', " ");
let font = parts[2].to_string();
let size = parts[3].parse::<f32>().unwrap_or(24.0);
let x = parts[4].parse::<f32>().unwrap_or(100.0);
let y = parts[5].parse::<f32>().unwrap_or(100.0);
let color = if parts.len() >= 7 {
parse_hex_color(parts[6]).unwrap_or(current_color)
} else {
current_color
};
commands.push(EngineCommand::AddText { text, font, size, x, y, color });
}
}
"filter" => {
if parts.len() >= 2 {
let filter_id = parts[1].to_string();
let params = if parts.len() >= 3 {
let json_str = parts[2..].join(" ");
serde_json::from_str::<serde_json::Value>(&json_str).unwrap_or(serde_json::Value::Null)
} else {
serde_json::Value::Null
};
commands.push(EngineCommand::ApplyFilter { filter_id, params });
}
}
"draw" => {
let mut points = Vec::new();
for pt_str in &parts[1..] {
if let Some((x, y)) = parse_coords(pt_str) {
points.push((x, y, 1.0));
}
}
if !points.is_empty() {
commands.push(EngineCommand::DrawStroke { points });
}
}
_ => {}
}
}
commands
}
/// Scan the entire text for ```json or ```dsl code blocks and execute them on the engine.
pub fn execute_inline_tools(text: &str, engine: &mut Engine) -> Vec<String> {
let mut results = Vec::new();
let mut start = 0;
// Parse JSON block tool calls
while let Some(open_idx) = text[start..].find("```json") {
let content_start = start + open_idx + 7;
if let Some(close_idx) = text[content_start..].find("```") {
let json_str = &text[content_start..content_start + close_idx];
start = content_start + close_idx + 3;
if let Ok(val) = serde_json::from_str::<serde_json::Value>(json_str) {
if val.is_array() {
if let Some(arr) = val.as_array() {
for item in arr {
if let Some(tool_name) = item["tool"].as_str() {
let params = &item["parameters"];
match execute_tool_call(engine, tool_name, params) {
Ok(msg) => results.push(format!("🔧 {}: {}", tool_name, msg)),
Err(err) => results.push(format!("❌ Tool Error ({}): {}", tool_name, err)),
}
}
}
}
} else if val.is_object() {
if let Some(tool_name) = val["tool"].as_str() {
let params = &val["parameters"];
match execute_tool_call(engine, tool_name, params) {
Ok(msg) => results.push(format!("🔧 {}: {}", tool_name, msg)),
Err(err) => results.push(format!("❌ Tool Error ({}): {}", tool_name, err)),
}
}
}
}
} else {
break;
}
}
results
}
/// Extract DSL script from a response text block.
pub fn extract_dsl_block(text: &str) -> Option<String> {
if let Some(start_idx) = text.find("```dsl") {
let rest = &text[start_idx + 6..];
if let Some(end_idx) = rest.find("```") {
return Some(rest[..end_idx].trim().to_string());
}
}
if let Some(start_idx) = text.find("```hcie") {
let rest = &text[start_idx + 7..];
if let Some(end_idx) = rest.find("```") {
return Some(rest[..end_idx].trim().to_string());
}
}
None
}
/// Messages passed from the background streaming thread to the main GUI thread.
#[derive(Debug, Clone)]
pub enum ChatUpdate {
/// A chunk of assistant content text.
Chunk(String),
/// A chunk of reasoning/thinking content.
ReasoningChunk(String),
/// Streaming finished with success (full response) or error message.
Finished(Result<String, String>),
}
/// Build the messages JSON array for the LLM API request.
pub fn build_messages_json(
history: &[ChatMessage],
final_prompt: &str,
system_prompt: &str,
) -> Vec<serde_json::Value> {
let mut messages_json = Vec::new();
// System prompt
messages_json.push(json!({
"role": "system",
"content": system_prompt
}));
// Conversation history
for msg in history {
if msg.role == "user" || msg.role == "assistant" {
messages_json.push(json!({
"role": &msg.role,
"content": if msg.content == final_prompt { final_prompt } else { &msg.content }
}));
}
}
messages_json
}
/// Stream an LLM response via SSE and return the full accumulated text.
///
/// Sends the request to the LLM API endpoint, reads the SSE stream line by line,
/// and accumulates the response text. Returns the full response on success or
/// an error message on failure. This function is designed to be used with
/// `Task::perform` in the Iced application.
pub async fn stream_llm_response(
url: String,
model: String,
messages: Vec<serde_json::Value>,
) -> Result<String, String> {
let body = json!({
"model": model,
"messages": messages,
"stream": true,
"temperature": 0.7
});
let url_endpoint = if url.ends_with("/v1/chat/completions") || url.ends_with("/api/chat") {
url.clone()
} else {
format!("{}/v1/chat/completions", url.trim_end_matches('/'))
};
let client = reqwest::Client::builder()
.timeout(std::time::Duration::from_secs(120))
.build()
.map_err(|e| format!("Failed to create HTTP client: {}", e))?;
let response = client.post(&url_endpoint).json(&body).send().await
.map_err(|e| format!("Network Failure: {}", e))?;
let status = response.status();
if !status.is_success() {
let err_msg = response.text().await.unwrap_or_else(|_| "Unknown HTTP error".to_string());
return Err(format!("HTTP Error {}: {}", status, err_msg));
}
let mut full_accumulated = String::new();
let mut stream = response.bytes_stream();
use futures_util::StreamExt;
let mut buffer = String::new();
while let Some(chunk_result) = stream.next().await {
match chunk_result {
Ok(chunk) => {
buffer.push_str(&String::from_utf8_lossy(&chunk));
// Process complete lines
while let Some(newline_pos) = buffer.find('\n') {
let line = buffer[..newline_pos].trim().to_string();
buffer = buffer[newline_pos + 1..].to_string();
if line.is_empty() {
continue;
}
if line.starts_with("data: ") {
let json_part = &line[6..];
if json_part.trim() == "[DONE]" {
break;
}
if let Ok(v) = serde_json::from_str::<serde_json::Value>(json_part) {
if let Some(choices) = v["choices"].as_array() {
if !choices.is_empty() {
let delta = &choices[0]["delta"];
// Handle reasoning content (deep thinking models)
if let Some(_reasoning) = delta["reasoning_content"].as_str() {
// Reasoning chunks are logged but not accumulated in this path
}
// Handle regular content
if let Some(content) = delta["content"].as_str() {
if !content.is_empty() {
full_accumulated.push_str(content);
}
}
}
}
}
} else {
// Fallback to Ollama native chat ND-JSON format
if let Ok(v) = serde_json::from_str::<serde_json::Value>(&line) {
if let Some(content) = v["message"]["content"].as_str() {
if !content.is_empty() {
full_accumulated.push_str(content);
}
}
}
}
}
}
Err(e) => {
return Err(format!("Stream read error: {}", e));
}
}
}
Ok(full_accumulated)
}
+152 -5
View File
@@ -12,6 +12,7 @@
//! These sections are optimized for 4K performance. Modifying them //! These sections are optimized for 4K performance. Modifying them
//! may cause rendering regressions or performance degradation. //! may cause rendering regressions or performance degradation.
use crate::ai_chat::{self, AiChatState, ChatMessage, SYSTEM_PROMPT};
use crate::dialogs; use crate::dialogs;
use crate::dock::state::{DockState, PaneType}; use crate::dock::state::{DockState, PaneType};
use crate::io::tablet::TabletState; use crate::io::tablet::TabletState;
@@ -150,6 +151,8 @@ pub struct HcieIcedApp {
pub layer_style_drag_start: Option<(f32, f32)>, pub layer_style_drag_start: Option<(f32, f32)>,
/// Marching ants animation offset (cycles 0.0..1.0 for dashed line animation). /// Marching ants animation offset (cycles 0.0..1.0 for dashed line animation).
pub marching_ants_offset: f32, pub marching_ants_offset: f32,
/// AI chat state (messages, config, streaming).
pub ai_chat_state: AiChatState,
} }
/// A recently opened file entry. /// A recently opened file entry.
@@ -411,6 +414,13 @@ pub enum Message {
AiChatInput(String), AiChatInput(String),
AiChatSend, AiChatSend,
AiChatClear, AiChatClear,
AiChatProviderChanged(String),
AiChatUrlChanged(String),
AiChatModelChanged(String),
AiChatReasoningToggled(bool),
AiChatCanvasContextToggled(bool),
AiChatStreamChunk(String),
AiChatStreamFinished(Result<String, String>),
// ── Clipboard ─────────────────────────────────────── // ── Clipboard ───────────────────────────────────────
CopyImage, CopyImage,
@@ -608,6 +618,7 @@ impl HcieIcedApp {
layer_style_dragging: false, layer_style_dragging: false,
layer_style_drag_start: None, layer_style_drag_start: None,
marching_ants_offset: 0.0, marching_ants_offset: 0.0,
ai_chat_state: AiChatState::default(),
}; };
// Apply saved settings to tool state // Apply saved settings to tool state
@@ -2236,15 +2247,151 @@ impl HcieIcedApp {
} }
// ── AI Chat ─────────────────────────────────── // ── AI Chat ───────────────────────────────────
Message::AiChatInput(_text) => { Message::AiChatInput(text) => {
// Handled by the chat panel directly self.ai_chat_state.input = text;
} }
Message::AiChatSend => { Message::AiChatSend => {
// Placeholder: would send to LLM let user_msg = self.ai_chat_state.input.trim().to_string();
log::info!("AI chat send not yet implemented"); if user_msg.is_empty() || self.ai_chat_state.is_streaming {
return Task::none();
}
// Add user message to history
self.ai_chat_state.messages.push(ChatMessage {
role: "user".to_string(),
content: user_msg.clone(),
is_reasoning: false,
});
self.ai_chat_state.input.clear();
self.ai_chat_state.is_streaming = true;
self.ai_chat_state.current_response.clear();
self.ai_chat_state.current_reasoning.clear();
// Build the final prompt with optional canvas context
let mut final_prompt = user_msg.clone();
if self.ai_chat_state.config.canvas_context {
let doc = &self.documents[self.active_doc];
let w = doc.engine.canvas_width();
let h = doc.engine.canvas_height();
let layers = doc.engine.layer_infos();
let mut meta = format!("\n\n[Active Canvas State: Size {}x{} px. Layers:", w, h);
for l in &layers {
meta.push_str(&format!(
"\n- \"{}\" (Type: {:?}, ID: {}, Visible: {}, Opacity: {:.0}%)",
l.name, l.layer_type, l.id, l.visible, l.opacity * 100.0
));
}
meta.push_str("]");
final_prompt.push_str(&meta);
}
// Build messages JSON for API
let messages_json = ai_chat::build_messages_json(
&self.ai_chat_state.messages,
&final_prompt,
SYSTEM_PROMPT,
);
// Clone config for the async task
let url = self.ai_chat_state.config.base_url.clone();
let model = self.ai_chat_state.config.model.clone();
// Spawn async streaming task — runs on Iced's executor
return Task::perform(
ai_chat::stream_llm_response(url, model, messages_json),
Message::AiChatStreamFinished,
);
} }
Message::AiChatClear => { Message::AiChatClear => {
// Handled by the chat panel directly self.ai_chat_state.messages.clear();
self.ai_chat_state.current_response.clear();
self.ai_chat_state.current_reasoning.clear();
self.ai_chat_state.is_streaming = false;
}
Message::AiChatProviderChanged(provider) => {
match provider.as_str() {
"Ollama" => {
self.ai_chat_state.config.provider = ai_chat::AiProvider::Ollama;
self.ai_chat_state.config.base_url = "http://localhost:11434".to_string();
}
"Claude" => {
self.ai_chat_state.config.provider = ai_chat::AiProvider::Claude;
self.ai_chat_state.config.base_url = "https://api.anthropic.com".to_string();
}
"OpenAI" => {
self.ai_chat_state.config.provider = ai_chat::AiProvider::OpenAI;
self.ai_chat_state.config.base_url = "https://api.openai.com/v1".to_string();
}
_ => {}
}
}
Message::AiChatUrlChanged(url) => {
self.ai_chat_state.config.base_url = url;
}
Message::AiChatModelChanged(model) => {
self.ai_chat_state.config.model = model;
}
Message::AiChatReasoningToggled(enabled) => {
self.ai_chat_state.config.reasoning_enabled = enabled;
}
Message::AiChatCanvasContextToggled(enabled) => {
self.ai_chat_state.config.canvas_context = enabled;
}
Message::AiChatStreamChunk(_chunk) => {
// Handled by the streaming task internally
}
Message::AiChatStreamFinished(result) => {
self.ai_chat_state.is_streaming = false;
match result {
Ok(full_response) => {
// Execute any embedded tool calls
let tool_results = ai_chat::execute_inline_tools(
&full_response,
&mut self.documents[self.active_doc].engine,
);
for res in &tool_results {
self.ai_chat_state.messages.push(ChatMessage {
role: "tool".to_string(),
content: res.clone(),
is_reasoning: false,
});
}
// Execute any DSL code blocks
if let Some(dsl_code) = ai_chat::extract_dsl_block(&full_response) {
let commands = ai_chat::parse_dsl_script(
&dsl_code,
&self.documents[self.active_doc].engine,
);
for cmd in &commands {
ai_chat::execute_command_on_engine(
&mut self.documents[self.active_doc].engine,
cmd,
);
}
self.ai_chat_state.messages.push(ChatMessage {
role: "tool".to_string(),
content: "▶ Automatically executed AI procedural drawing script.".to_string(),
is_reasoning: false,
});
self.refresh_composite_if_needed();
}
// Add the full assistant response to history
self.ai_chat_state.messages.push(ChatMessage {
role: "assistant".to_string(),
content: full_response,
is_reasoning: false,
});
}
Err(e) => {
self.ai_chat_state.messages.push(ChatMessage {
role: "assistant".to_string(),
content: format!("❌ Connection Error: {}", e),
is_reasoning: false,
});
}
}
} }
// ── Clipboard ───────────────────────────────── // ── Clipboard ─────────────────────────────────
@@ -86,7 +86,7 @@ pub fn dock_view<'a>(
crate::panels::script_panel::view(colors) crate::panels::script_panel::view(colors)
} }
PaneType::AiChat => { PaneType::AiChat => {
crate::panels::ai_chat_panel::view("", &[], colors) crate::panels::ai_chat_panel::view(&app.ai_chat_state, colors)
} }
PaneType::LayerDetails => { PaneType::LayerDetails => {
let doc = &app.documents[app.active_doc]; let doc = &app.documents[app.active_doc];
@@ -2,6 +2,7 @@
//! //!
//! Launches the Iced application with the HCIE image editor. //! Launches the Iced application with the HCIE image editor.
mod ai_chat;
mod app; mod app;
mod canvas; mod canvas;
mod color_picker; mod color_picker;
@@ -1,39 +1,92 @@
//! AI Chat panel — AI assistant for image editing. //! AI Chat panel — AI assistant with streaming SSE responses, tool execution, and DSL scripts.
//!
//! Full implementation of the AI chat interface including provider selection,
//! streaming response display, chat bubbles, reasoning toggle, and canvas context toggle.
//! Uses ThemeColors for consistent styling. //! Uses ThemeColors for consistent styling.
use crate::ai_chat::{AiChatState, ChatMessage};
use crate::app::Message; use crate::app::Message;
use crate::panels::styles; use crate::panels::styles;
use crate::theme::ThemeColors; use crate::theme::ThemeColors;
use iced::widget::{button, column, container, horizontal_rule, row, scrollable, text, text_input}; use iced::widget::{button, column, container, horizontal_rule, row, scrollable, text, text_input};
use iced::{Element, Length}; use iced::{Element, Length};
/// Build the AI chat panel. /// Build the AI chat panel with full streaming UI.
pub fn view(input: &str, messages: &[(String, String)], colors: ThemeColors) -> Element<'static, Message> { pub fn view(state: &AiChatState, colors: ThemeColors) -> Element<'static, Message> {
let provider_row = row![ // ── Provider selector row ──
text("Provider:").size(10), let provider_label = text("Provider:").size(10).color(colors.text_secondary);
button(text("Ollama").size(10)).on_press(Message::NoOp).padding([4, 8]), let ollama_btn = make_provider_btn("Ollama", state.config.provider == crate::ai_chat::AiProvider::Ollama, colors);
button(text("Claude").size(10)).on_press(Message::NoOp).padding([4, 8]), let claude_btn = make_provider_btn("Claude", state.config.provider == crate::ai_chat::AiProvider::Claude, colors);
button(text("OpenAI").size(10)).on_press(Message::NoOp).padding([4, 8]), let openai_btn = make_provider_btn("OpenAI", state.config.provider == crate::ai_chat::AiProvider::OpenAI, colors);
let provider_row = row![provider_label, ollama_btn, claude_btn, openai_btn]
.spacing(4)
.align_y(iced::Alignment::Center);
// ── Config section (URL, model, toggles) ──
let url_label = text("URL:").size(9).color(colors.text_secondary);
let url_input = text_input("http://localhost:11434", &state.config.base_url)
.on_input(|s| Message::AiChatUrlChanged(s))
.size(10)
.width(Length::Fill);
let model_label = text("Model:").size(9).color(colors.text_secondary);
let model_input = text_input("qwen2.5:latest", &state.config.model)
.on_input(|s| Message::AiChatModelChanged(s))
.size(10)
.width(Length::Fill);
let config_row = row![
url_label,
url_input,
model_label,
model_input,
] ]
.spacing(4); .spacing(4)
.align_y(iced::Alignment::Center);
let mut msg_list = column![].spacing(4); // ── Toggles row ──
let reasoning_label = text("Reasoning").size(9).color(colors.text_secondary);
let reasoning_btn = button(
text(if state.config.reasoning_enabled { "ON" } else { "OFF" }).size(9)
)
.on_press(Message::AiChatReasoningToggled(!state.config.reasoning_enabled))
.padding([2, 6]);
if messages.is_empty() { let canvas_label = text("Canvas ctx").size(9).color(colors.text_secondary);
let canvas_btn = button(
text(if state.config.canvas_context { "ON" } else { "OFF" }).size(9)
)
.on_press(Message::AiChatCanvasContextToggled(!state.config.canvas_context))
.padding([2, 6]);
let toggles_row = row![
reasoning_label,
reasoning_btn,
iced::widget::Space::with_width(8),
canvas_label,
canvas_btn,
]
.spacing(4)
.align_y(iced::Alignment::Center);
// ── Message list ──
let mut msg_list = column![].spacing(6);
if state.messages.is_empty() && !state.is_streaming {
// Welcome message
let welcome = container( let welcome = container(
column![ column![
text("AI Assistant").size(12), text("AI Creative Assistant").size(13).color(colors.accent),
text("Ask me anything about image editing!").size(11), text("").size(4),
text("").size(8), text("Hello! I am your AI Drawing Partner.").size(11).color(colors.text_secondary),
text("I can help with:").size(10), text("").size(4),
text(" - Creating and editing layers").size(10), text("Ask me to paint landscapes, add details,").size(10).color(colors.text_disabled),
text(" - Applying filters and effects").size(10), text("apply blur filters, or manipulate layers!").size(10).color(colors.text_disabled),
text(" - Drawing shapes and text").size(10),
text(" - Color adjustments").size(10),
] ]
.spacing(2) .spacing(2)
) )
.padding(12) .padding(12)
.width(Length::Fill)
.style(move |_theme| iced::widget::container::Style { .style(move |_theme| iced::widget::container::Style {
background: Some(iced::Background::Color(colors.bg_hover)), background: Some(iced::Background::Color(colors.bg_hover)),
border: iced::Border::default().rounded(8), border: iced::Border::default().rounded(8),
@@ -42,46 +95,84 @@ pub fn view(input: &str, messages: &[(String, String)], colors: ThemeColors) ->
msg_list = msg_list.push(welcome); msg_list = msg_list.push(welcome);
} else { } else {
for (role, content) in messages { // Render chat history
let (label, color) = if role == "user" { for msg in &state.messages {
("You", colors.accent) let bubble = render_message_bubble(msg, colors);
} else { msg_list = msg_list.push(bubble);
("AI", iced::Color::from_rgb(0.4, 0.9, 0.4)) }
};
let content_owned = content.clone(); // Render streaming reasoning if enabled
let msg_text = text(content_owned).size(11); if state.config.reasoning_enabled && !state.current_reasoning.is_empty() {
let msg_text = msg_text.style(move |_theme: &iced::Theme| iced::widget::text::Style { let reasoning_bubble = container(
color: Some(color), column![
}); text("Deep Thoughts:").size(10).color(iced::Color::from_rgb(0.85, 0.47, 0.02)),
text(state.current_reasoning.clone()).size(10).color(colors.text_secondary),
let bubble = container( ]
column![text(label).size(10), msg_text].spacing(2) .spacing(2)
) )
.width(Length::Fill)
.padding(8) .padding(8)
.width(Length::Fill)
.style(move |_theme| iced::widget::container::Style { .style(move |_theme| iced::widget::container::Style {
background: Some(iced::Background::Color(colors.bg_hover)), background: Some(iced::Background::Color(iced::Color::from_rgba(0.15, 0.12, 0.08, 1.0))),
border: iced::Border::default().rounded(6), border: iced::Border::default().rounded(6).color(iced::Color::from_rgb(0.85, 0.47, 0.02)),
..Default::default() ..Default::default()
}); });
msg_list = msg_list.push(bubble); msg_list = msg_list.push(reasoning_bubble);
}
// Render streaming response if in progress
if state.is_streaming && !state.current_response.is_empty() {
let streaming_bubble = container(
text(state.current_response.clone()).size(11).color(colors.text_primary)
)
.padding(8)
.width(Length::Fill)
.style(move |_theme| iced::widget::container::Style {
background: Some(iced::Background::Color(iced::Color::from_rgba(0.18, 0.22, 0.28, 1.0))),
border: iced::Border::default().rounded(6).color(iced::Color::from_rgba(0.29, 0.62, 1.0, 0.5)),
..Default::default()
});
msg_list = msg_list.push(streaming_bubble);
}
// Streaming indicator
if state.is_streaming {
let indicator = row![
text("...").size(10).color(colors.text_disabled),
text("AI is thinking...").size(10).color(colors.text_disabled),
]
.spacing(4);
msg_list = msg_list.push(indicator);
} }
} }
let input_owned = input.to_string(); // ── Input area ──
let input_field = text_input("Ask the AI assistant...", &input_owned) let input_field = text_input("Ask the AI assistant...", &state.input)
.on_input(|s| Message::AiChatInput(s)) .on_input(|s| Message::AiChatInput(s))
.size(11)
.width(Length::Fill); .width(Length::Fill);
let send_btn = button(text("Send").size(11)).on_press(Message::AiChatSend).padding([6, 12]); let send_btn = button(text("Send").size(10))
let clear_btn = button(text("Clear").size(11)).on_press(Message::AiChatClear).padding([6, 12]); .on_press(Message::AiChatSend)
let input_row = row![input_field, send_btn, clear_btn].spacing(4); .padding([6, 10]);
let clear_btn = button(text("Clear").size(10))
.on_press(Message::AiChatClear)
.padding([6, 10]);
let input_row = row![input_field, send_btn, clear_btn]
.spacing(4)
.align_y(iced::Alignment::Center);
// ── Assemble panel ──
let panel = column![ let panel = column![
text("AI Assistant").size(13), text("AI Assistant").size(13).color(colors.text_primary),
provider_row, provider_row,
config_row,
toggles_row,
horizontal_rule(1), horizontal_rule(1),
scrollable(msg_list).height(Length::Fill), scrollable(msg_list).height(Length::Fill),
horizontal_rule(1), horizontal_rule(1),
@@ -96,3 +187,100 @@ pub fn view(input: &str, messages: &[(String, String)], colors: ThemeColors) ->
.style(move |_theme| styles::panel_background(colors)) .style(move |_theme| styles::panel_background(colors))
.into() .into()
} }
/// Render a single chat message as a styled bubble.
fn render_message_bubble(msg: &ChatMessage, colors: ThemeColors) -> Element<'static, Message> {
match msg.role.as_str() {
"user" => {
// User bubble — right-aligned, blue tint
let label = text("You").size(9).color(colors.accent);
let content = text(msg.content.clone()).size(11).color(colors.text_primary);
let bubble = container(
column![label, content].spacing(2)
)
.padding(8)
.max_width(400.0)
.style(move |_theme| iced::widget::container::Style {
background: Some(iced::Background::Color(iced::Color::from_rgba(0.16, 0.33, 0.58, 1.0))),
border: iced::Border::default().rounded(6),
..Default::default()
});
row![iced::widget::Space::with_width(Length::Fill), bubble]
.into()
}
"tool" => {
// Tool result — monospace pill
let content = text(msg.content.clone()).size(10).color(colors.accent);
container(content)
.padding([4, 8])
.style(move |_theme| iced::widget::container::Style {
background: Some(iced::Background::Color(colors.bg_hover)),
border: iced::Border::default().rounded(4).color(colors.border_low),
..Default::default()
})
.into()
}
_ => {
// Assistant bubble — left-aligned, slate tint
let label = text("AI").size(9).color(colors.accent_green);
let content = text(msg.content.clone()).size(11).color(colors.text_primary);
let bubble = container(
column![label, content].spacing(2)
)
.padding(8)
.max_width(400.0)
.style(move |_theme| iced::widget::container::Style {
background: Some(iced::Background::Color(iced::Color::from_rgba(0.18, 0.22, 0.28, 1.0))),
border: iced::Border::default().rounded(6),
..Default::default()
});
row![bubble, iced::widget::Space::with_width(Length::Fill)]
.into()
}
}
}
/// Create a provider selection button with active state highlighting.
fn make_provider_btn(label: &str, is_active: bool, colors: ThemeColors) -> Element<'static, Message> {
let label_owned = label.to_string();
let msg = match label {
"Ollama" => Message::AiChatProviderChanged("Ollama".to_string()),
"Claude" => Message::AiChatProviderChanged("Claude".to_string()),
"OpenAI" => Message::AiChatProviderChanged("OpenAI".to_string()),
_ => Message::NoOp,
};
button(text(label_owned).size(10))
.on_press(msg)
.padding([4, 8])
.style(move |_theme: &iced::Theme, status: iced::widget::button::Status| {
match status {
iced::widget::button::Status::Active | iced::widget::button::Status::Hovered => {
if is_active {
iced::widget::button::Style {
background: Some(iced::Background::Color(colors.accent)),
text_color: iced::Color::WHITE,
border: iced::Border::default().rounded(4),
..Default::default()
}
} else {
iced::widget::button::Style {
background: Some(iced::Background::Color(colors.bg_hover)),
text_color: colors.text_secondary,
border: iced::Border::default().rounded(4),
..Default::default()
}
}
}
_ => iced::widget::button::Style {
background: Some(iced::Background::Color(colors.bg_hover)),
text_color: colors.text_secondary,
border: iced::Border::default().rounded(4),
..Default::default()
},
}
})
.into()
}