GeneralBots/generalbots
1
0
Fork 0
Code Activity Actions 5

refactor: unify BASIC compilation into BasicCompiler only, runtime uses ScriptService::run() on pre-compiled .ast
Some checks failed
BotServer CI/CD / build (push) Has been cancelled
Details

Browse source 
- Move all preprocessing transforms (convert_multiword_keywords, preprocess_llm_keyword,
  convert_while_wend_syntax, predeclare_variables) into BasicCompiler::preprocess_basic
  so .ast files are fully preprocessed by Drive Monitor
- Replace ScriptService compile/compile_preprocessed/compile_tool_script with
  single run(ast_content) that does engine.compile() + eval_ast_with_scope()
- Remove .bas fallback in tool_executor and start.bas paths - .ast only
- Remove dead code: preprocess_basic_script, normalize_variables_to_lowercase,
  convert_save_for_tools, parse_save_parts, normalize_word
- Fix: USE KB 'cartas' in tool .ast now correctly converted to USE_KB('cartas')
  during compilation, ensuring KB context injection works after tool execution
- Fix: add trace import in llm/mod.rs
This commit is contained in:
Rodrigo Rodriguez (Pragmatismo) 2026-04-13 14:05:55 -03:00
parent 723407cfd6
commit f8b47d1ac2
8 changed files with 152 additions and 931 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
src/auto_task/intent_classifier.rs
View file

@ -721,8 +721,8 @@ END ON
context_vars.insert("original_intent".to_string(), compiled.original_intent.clone()); context_vars.insert("original_intent".to_string(), compiled.original_intent.clone());
script_service.inject_config_variables(context_vars); script_service.inject_config_variables(context_vars);
// Compile and execute the BASIC program // Compile and execute dynamically generated BASIC program
let ast = match script_service.compile(basic_program) { let ast = match script_service.engine.compile(basic_program) {
Ok(ast) => ast, Ok(ast) => ast,
Err(e) => { Err(e) => {
let error_msg = format!("Failed to compile BASIC program: {}", e); let error_msg = format!("Failed to compile BASIC program: {}", e);
@ -746,7 +746,7 @@ END ON
} }
}; };
let execution_result = script_service.run(&ast); let execution_result: Result<rhai::Dynamic, Box<rhai::EvalAltResult>> = script_service.engine.eval_ast_with_scope(&mut script_service.scope, &ast);
match execution_result { match execution_result {
Ok(result) => { Ok(result) => {

9
src/basic/compiler/mod.rs
View file

@ -448,6 +448,9 @@ impl BasicCompiler {
source.to_string() source.to_string()
}; };
let source = source.as_str(); let source = source.as_str();
// Preprocess LLM keyword to add WITH OPTIMIZE FOR "speed" syntax
let source = crate::basic::ScriptService::preprocess_llm_keyword(source);
let mut has_schedule = false; let mut has_schedule = false;
let script_name = Path::new(source_path) let script_name = Path::new(source_path)
.file_stem() .file_stem()
@ -615,6 +618,12 @@ impl BasicCompiler {
}; };
// Convert BEGIN TALK and BEGIN MAIL blocks to Rhai code // Convert BEGIN TALK and BEGIN MAIL blocks to Rhai code
let result = crate::basic::compiler::blocks::convert_begin_blocks(&result); let result = crate::basic::compiler::blocks::convert_begin_blocks(&result);
// Convert ALL multi-word keywords to underscore versions (e.g., "USE KB" → "USE_KB")
let result = crate::basic::ScriptService::convert_multiword_keywords(&result);
// Convert WHILE...WEND to Rhai while { } blocks BEFORE if/then conversion
let result = crate::basic::ScriptService::convert_while_wend_syntax(&result);
// Pre-declare all variables at outer scope so assignments inside blocks work correctly
let result = crate::basic::ScriptService::predeclare_variables(&result);
// Convert IF ... THEN / END IF to if ... { } // Convert IF ... THEN / END IF to if ... { }
let result = crate::basic::ScriptService::convert_if_then_syntax(&result); let result = crate::basic::ScriptService::convert_if_then_syntax(&result);
// Convert SELECT ... CASE / END SELECT to match expressions // Convert SELECT ... CASE / END SELECT to match expressions

764
src/basic/mod.rs
View file

@ -6,7 +6,6 @@ use crate::basic::keywords::switch_case::switch_keyword;
use crate::core::shared::models::UserSession; use crate::core::shared::models::UserSession;
use crate::core::shared::state::AppState; use crate::core::shared::state::AppState;
use diesel::prelude::*; use diesel::prelude::*;
use log::trace;
use rhai::{Dynamic, Engine, EvalAltResult, Scope}; use rhai::{Dynamic, Engine, EvalAltResult, Scope};
use std::collections::HashMap; use std::collections::HashMap;
use std::sync::Arc; use std::sync::Arc;
@ -47,7 +46,6 @@ use self::keywords::http_operations::register_http_operations;
use self::keywords::last::last_keyword; use self::keywords::last::last_keyword;
#[cfg(feature = "automation")] #[cfg(feature = "automation")]
use self::keywords::on_form_submit::on_form_submit_keyword; use self::keywords::on_form_submit::on_form_submit_keyword;
use self::keywords::switch_case::preprocess_switch;
use self::keywords::use_tool::use_tool_keyword; use self::keywords::use_tool::use_tool_keyword;
use self::keywords::use_website::{clear_websites_keyword, register_use_website_function}; use self::keywords::use_website::{clear_websites_keyword, register_use_website_function};
use self::keywords::web_data::register_web_data_keywords; use self::keywords::web_data::register_web_data_keywords;
@ -321,334 +319,20 @@ impl ScriptService {
} }
} }
} }
fn preprocess_basic_script(&self, script: &str) -> Result<String, String> { /// Run a pre-compiled .ast script (loaded from Drive).
let _ = self; // silence unused self warning - kept for API consistency /// Compilation happens only in BasicCompiler (Drive Monitor).
let script = preprocess_switch(script); /// Runtime only compiles the already-preprocessed Rhai source and executes it.
pub fn run(&mut self, ast_content: &str) -> Result<Dynamic, Box<EvalAltResult>> {
// Preprocess LLM keyword to add WITH OPTIMIZE FOR "speed" syntax let ast = match self.engine.compile(ast_content) {
// This is needed because Rhai's custom syntax requires the full syntax Ok(ast) => ast,
let script = Self::preprocess_llm_keyword(&script); Err(e) => return Err(Box::new(e.into())),
// Convert ALL multi-word keywords to underscore versions (e.g., "USE WEBSITE" → "USE_WEBSITE")
// This avoids Rhai custom syntax conflicts and makes the system more secure
let script = Self::convert_multiword_keywords(&script);
let script = Self::normalize_variables_to_lowercase(&script);
let mut result = String::new();
let mut for_stack: Vec<usize> = Vec::new();
let mut current_indent = 0;
for line in script.lines() {
let trimmed = line.trim();
if trimmed.is_empty() || trimmed.starts_with("//") || trimmed.starts_with('\'') {
continue;
}
if trimmed.starts_with("FOR EACH") {
for_stack.push(current_indent);
result.push_str(&" ".repeat(current_indent));
result.push_str(trimmed);
result.push_str("{\n");
current_indent += 4;
result.push_str(&" ".repeat(current_indent));
result.push('\n');
continue;
}
if trimmed.starts_with("NEXT") {
if let Some(expected_indent) = for_stack.pop() {
if (current_indent - 4) != expected_indent {
return Err("NEXT without matching FOR EACH (indentation mismatch)".to_string());
}
current_indent -= 4;
result.push_str(&" ".repeat(current_indent));
result.push_str("}\n");
result.push_str(&" ".repeat(current_indent));
result.push_str(trimmed);
result.push(';');
result.push('\n');
continue;
}
log::error!("NEXT without matching FOR EACH");
return Err("NEXT without matching FOR EACH".to_string());
}
if trimmed == "EXIT FOR" {
result.push_str(&" ".repeat(current_indent));
result.push_str(trimmed);
result.push('\n');
continue;
}
result.push_str(&" ".repeat(current_indent));
let basic_commands = [
"SET",
"CREATE",
"PRINT",
"FOR",
"FIND",
"GET",
"EXIT",
"IF",
"THEN",
"ELSE",
"END IF",
"WHILE",
"WEND",
"DO",
"LOOP",
"HEAR",
"TALK",
"SET CONTEXT",
"SET USER",
"GET BOT MEMORY",
"SET BOT MEMORY",
"IMAGE",
"VIDEO",
"AUDIO",
"SEE",
"SEND FILE",
"SWITCH",
"CASE",
"DEFAULT",
"END SWITCH",
"USE KB",
"CLEAR KB",
"USE TOOL",
"CLEAR TOOLS",
"ADD SUGGESTION",
"CLEAR SUGGESTIONS",
"INSTR",
"IS_NUMERIC",
"IS NUMERIC",
"POST",
"PUT",
"PATCH",
"DELETE",
"SET HEADER",
"CLEAR HEADERS",
"GRAPHQL",
"SOAP",
"SAVE",
"INSERT",
"UPDATE",
"DELETE",
"MERGE",
"FILL",
"MAP",
"FILTER",
"AGGREGATE",
"JOIN",
"PIVOT",
"GROUP BY",
"READ",
"WRITE",
"COPY",
"MOVE",
"LIST",
"COMPRESS",
"EXTRACT",
"UPLOAD",
"DOWNLOAD",
"GENERATE PDF",
"MERGE PDF",
"WEBHOOK",
"POST TO",
"POST TO INSTAGRAM",
"POST TO FACEBOOK",
"POST TO LINKEDIN",
"POST TO TWITTER",
"GET INSTAGRAM METRICS",
"GET FACEBOOK METRICS",
"GET LINKEDIN METRICS",
"GET TWITTER METRICS",
"DELETE POST",
"SEND MAIL",
"SEND TEMPLATE",
"CREATE TEMPLATE",
"GET TEMPLATE",
"ON ERROR RESUME NEXT",
"ON ERROR GOTO",
"CLEAR ERROR",
"ERROR MESSAGE",
"ON FORM SUBMIT",
"SCORE LEAD",
"GET LEAD SCORE",
"QUALIFY LEAD",
"UPDATE LEAD SCORE",
"AI SCORE LEAD",
"ABS",
"ROUND",
"INT",
"FIX",
"FLOOR",
"CEIL",
"MAX",
"MIN",
"MOD",
"RANDOM",
"RND",
"SGN",
"SQR",
"SQRT",
"LOG",
"EXP",
"POW",
"SIN",
"COS",
"TAN",
"SUM",
"AVG",
"NOW",
"TODAY",
"DATE",
"TIME",
"YEAR",
"MONTH",
"DAY",
"HOUR",
"MINUTE",
"SECOND",
"WEEKDAY",
"DATEADD",
"DATEDIFF",
"FORMAT_DATE",
"ISDATE",
"VAL",
"STR",
"CINT",
"CDBL",
"CSTR",
"ISNULL",
"ISEMPTY",
"TYPEOF",
"ISARRAY",
"ISOBJECT",
"ISSTRING",
"ISNUMBER",
"NVL",
"IIF",
"ARRAY",
"UBOUND",
"LBOUND",
"COUNT",
"SORT",
"UNIQUE",
"CONTAINS",
"INDEX_OF",
"PUSH",
"POP",
"SHIFT",
"REVERSE",
"SLICE",
"SPLIT",
"CONCAT",
"FLATTEN",
"RANGE",
"THROW",
"ERROR",
"IS_ERROR",
"ASSERT",
"LOG_ERROR",
"LOG_WARN",
"LOG_INFO",
];
let is_basic_command = basic_commands.iter().any(|&cmd| trimmed.starts_with(cmd));
let is_control_flow = trimmed.starts_with("IF")
|| trimmed.starts_with("ELSE")
|| trimmed.starts_with("END IF");
result.push_str(trimmed);
let needs_semicolon = is_basic_command
|| !for_stack.is_empty()
|| is_control_flow
|| (!trimmed.ends_with(';') && !trimmed.ends_with('{') && !trimmed.ends_with('}'));
if needs_semicolon {
result.push(';');
}
result.push('\n');
}
if !for_stack.is_empty() {
return Err("Unclosed FOR EACH loop".to_string());
}
Ok(result)
}
pub fn compile(&self, script: &str) -> Result<rhai::AST, Box<EvalAltResult>> {
let processed_script = match self.preprocess_basic_script(script) {
Ok(s) => s,
Err(e) => return Err(Box::new(EvalAltResult::ErrorRuntime(Dynamic::from(e), rhai::Position::NONE))),
}; };
trace!("Processed Script:\n{}", processed_script); self.engine.eval_ast_with_scope(&mut self.scope, &ast)
match self.engine.compile(&processed_script) {
Ok(ast) => Ok(ast),
Err(parse_error) => Err(Box::new(parse_error.into())),
}
}
/// Compile preprocessed script content (from .ast file) - skips preprocessing
pub fn compile_preprocessed(&self, script: &str) -> Result<rhai::AST, Box<EvalAltResult>> {
trace!("Compiling preprocessed script directly");
match self.engine.compile(script) {
Ok(ast) => Ok(ast),
Err(parse_error) => Err(Box::new(parse_error.into())),
}
}
/// Compile a tool script (.bas file with PARAM/DESCRIPTION metadata lines)
/// Filters out tool metadata before compiling
pub fn compile_tool_script(&self, script: &str) -> Result<rhai::AST, Box<EvalAltResult>> {
// Filter out PARAM, DESCRIPTION, comment, and empty lines (tool metadata)
let executable_script: String = script
.lines()
.filter(|line| {
let trimmed = line.trim();
// Keep lines that are NOT PARAM, DESCRIPTION, comments, or empty
!(trimmed.starts_with("PARAM ") ||
trimmed.starts_with("PARAM\t") ||
trimmed.starts_with("DESCRIPTION ") ||
trimmed.starts_with("DESCRIPTION\t") ||
trimmed.starts_with("REM ") ||
trimmed.starts_with("REM\t") ||
trimmed == "REM" || // bare REM line
trimmed.starts_with('\'') || // BASIC comment lines
trimmed.starts_with('#') || // Hash comment lines
trimmed.is_empty())
})
.collect::<Vec<&str>>()
.join("\n");
trace!("Filtered tool metadata: {} -> {} chars", script.len(), executable_script.len());
// Apply minimal preprocessing for tools (skip variable normalization to avoid breaking multi-line strings)
let script = preprocess_switch(&executable_script);
let script = Self::convert_multiword_keywords(&script);
// Skip normalize_variables_to_lowercase for tools - it breaks multi-line strings
trace!("Preprocessed tool script for Rhai compilation");
// Convert SAVE statements with field lists to map-based SAVE (simplified version for tools)
let script = Self::convert_save_for_tools(&script);
// Convert BEGIN TALK and BEGIN MAIL blocks to single calls
let script = crate::basic::compiler::blocks::convert_begin_blocks(&script);
// Convert WHILE...WEND to Rhai while { } blocks BEFORE if/then conversion
let script = Self::convert_while_wend_syntax(&script);
// Pre-declare all variables at outer scope so assignments inside blocks work correctly.
// In Rhai, a plain `x = val` inside a block updates the outer variable -
// but only if `x` was declared outside with `let`.
let script = Self::predeclare_variables(&script);
// Convert IF ... THEN / END IF to if ... { }
let script = Self::convert_if_then_syntax(&script);
// Convert SELECT ... CASE / END SELECT to match expressions
let script = Self::convert_select_case_syntax(&script);
// Convert BASIC keywords to lowercase (but preserve variable casing)
let script = Self::convert_keywords_to_lowercase(&script);
// Save to file for debugging
if let Err(e) = std::fs::write("/tmp/tool_preprocessed.bas", &script) {
log::warn!("Failed to write preprocessed script: {}", e);
}
match self.engine.compile(&script) {
Ok(ast) => Ok(ast),
Err(parse_error) => Err(Box::new(parse_error.into())),
}
} }
/// Pre-declare all BASIC variables at the top of the script with `let var = ();`. /// Pre-declare all BASIC variables at the top of the script with `let var = ();`.
/// This allows assignments inside loops/if-blocks to update outer-scope variables in Rhai. /// This allows assignments inside loops/if-blocks to update outer-scope variables in Rhai.
fn predeclare_variables(script: &str) -> String { pub(crate) fn predeclare_variables(script: &str) -> String {
use std::collections::BTreeSet; use std::collections::BTreeSet;
let reserved: std::collections::HashSet<&str> = [ let reserved: std::collections::HashSet<&str> = [
"if", "else", "while", "for", "loop", "return", "break", "continue", "if", "else", "while", "for", "loop", "return", "break", "continue",
@ -701,114 +385,23 @@ impl ScriptService {
declarations.push_str(script); declarations.push_str(script);
declarations declarations
} }
pub fn run(&mut self, ast: &rhai::AST) -> Result<Dynamic, Box<EvalAltResult>> {
self.engine.eval_ast_with_scope(&mut self.scope, ast)
}
/// Execute a BASIC script asynchronously /// Execute a pre-compiled .ast script asynchronously
pub async fn execute_script( pub async fn execute_script(
state: Arc<AppState>, state: Arc<AppState>,
user: UserSession, user: UserSession,
script: &str, ast_content: &str,
) -> Result<String, String> { ) -> Result<String, String> {
let mut script_service = Self::new(state.clone(), user.clone()); let mut script_service = Self::new(state.clone(), user.clone());
script_service.load_bot_config_params(&state, user.bot_id); script_service.load_bot_config_params(&state, user.bot_id);
match script_service.compile(script) { match script_service.run(ast_content) {
Ok(ast) => { Ok(result) => Ok(result.to_string()),
match script_service.run(&ast) { Err(e) => Err(format!("Script error: {}", e)),
Ok(result) => Ok(result.to_string()),
Err(e) => Err(format!("Execution error: {}", e)),
}
}
Err(e) => Err(format!("Compilation error: {}", e)),
} }
} }
/// Convert SAVE statements for tool compilation (simplified, no DB lookup) /// Pre-declare all BASIC variables at the top of the script with `let var = ();`.
/// SAVE "table", var1, var2, ... -> let __data__ = #{var1: var1, var2: var2, ...}; SAVE "table", __data__
fn convert_save_for_tools(script: &str) -> String {
let mut result = String::new();
let mut save_counter = 0;
for line in script.lines() {
let trimmed = line.trim();
// Check if this is a SAVE statement
if trimmed.to_uppercase().starts_with("SAVE ") {
// Parse SAVE statement
// Format: SAVE "table", value1, value2, ...
let content = &trimmed[4..].trim();
// Simple parse by splitting on commas (outside quotes)
let parts = Self::parse_save_parts(content);
// If more than 2 parts, convert to map-based SAVE
if parts.len() > 2 {
let table_name = parts[0].trim_matches('"');
let values: Vec<&str> = parts.iter().skip(1).map(|s| s.trim()).collect();
// Build map with variable names as keys
let map_pairs: Vec<String> = values.iter().map(|v| format!("{}: {}", v, v)).collect();
let map_expr = format!("#{{{}}}", map_pairs.join(", "));
let data_var = format!("__save_data_{}__", save_counter);
save_counter += 1;
let converted = format!("let {} = {};\nINSERT \"{}\", {};", data_var, map_expr, table_name, data_var);
result.push_str(&converted);
result.push('\n');
continue;
}
}
result.push_str(line);
result.push('\n');
}
result
}
/// Parse SAVE statement parts (handles quoted strings)
fn parse_save_parts(s: &str) -> Vec<String> {
let mut parts = Vec::new();
let mut current = String::new();
let mut in_quotes = false;
let mut chars = s.chars().peekable();
while let Some(c) = chars.next() {
match c {
'"' if !in_quotes => {
in_quotes = true;
current.push(c);
}
'"' if in_quotes => {
in_quotes = false;
current.push(c);
}
',' if !in_quotes => {
parts.push(current.trim().to_string());
current = String::new();
// Skip whitespace after comma
while let Some(&next_c) = chars.peek() {
if next_c.is_whitespace() {
chars.next();
} else {
break;
}
}
}
_ => current.push(c),
}
}
if !current.is_empty() {
parts.push(current.trim().to_string());
}
parts
}
/// Set a variable in the script scope (for tool parameters)
pub fn set_variable(&mut self, name: &str, value: &str) -> Result<(), Box<dyn std::error::Error>> { pub fn set_variable(&mut self, name: &str, value: &str) -> Result<(), Box<dyn std::error::Error>> {
use rhai::Dynamic; use rhai::Dynamic;
self.scope.set_or_push(name, Dynamic::from(value.to_string())); self.scope.set_or_push(name, Dynamic::from(value.to_string()));
@ -1682,312 +1275,6 @@ impl ScriptService {
result result
} }
fn normalize_variables_to_lowercase(script: &str) -> String {
use regex::Regex;
let mut result = String::new();
let keywords = [
"SET",
"CREATE",
"PRINT",
"FOR",
"FIND",
"GET",
"EXIT",
"IF",
"THEN",
"ELSE",
"END",
"WHILE",
"WEND",
"DO",
"LOOP",
"HEAR",
"TALK",
"NEXT",
"FUNCTION",
"SUB",
"CALL",
"RETURN",
"DIM",
"AS",
"NEW",
"ARRAY",
"OBJECT",
"LET",
"REM",
"AND",
"OR",
"NOT",
"TRUE",
"FALSE",
"NULL",
"SWITCH",
"CASE",
"DEFAULT",
"USE",
"KB",
"TOOL",
"CLEAR",
"ADD",
"SUGGESTION",
"SUGGESTIONS",
"TOOLS",
"CONTEXT",
"USER",
"BOT",
"MEMORY",
"IMAGE",
"VIDEO",
"AUDIO",
"SEE",
"SEND",
"FILE",
"POST",
"PUT",
"PATCH",
"DELETE",
"SAVE",
"INSERT",
"UPDATE",
"MERGE",
"FILL",
"MAP",
"FILTER",
"AGGREGATE",
"JOIN",
"PIVOT",
"GROUP",
"BY",
"READ",
"WRITE",
"COPY",
"MOVE",
"LIST",
"COMPRESS",
"EXTRACT",
"UPLOAD",
"DOWNLOAD",
"GENERATE",
"PDF",
"WEBHOOK",
"TEMPLATE",
"FORM",
"SUBMIT",
"SCORE",
"LEAD",
"QUALIFY",
"AI",
"ABS",
"ROUND",
"INT",
"FIX",
"FLOOR",
"CEIL",
"MAX",
"MIN",
"MOD",
"RANDOM",
"RND",
"SGN",
"SQR",
"SQRT",
"LOG",
"EXP",
"POW",
"SIN",
"COS",
"TAN",
"SUM",
"AVG",
"NOW",
"TODAY",
"DATE",
"TIME",
"YEAR",
"MONTH",
"DAY",
"HOUR",
"MINUTE",
"SECOND",
"WEEKDAY",
"DATEADD",
"DATEDIFF",
"FORMAT",
"ISDATE",
"VAL",
"STR",
"CINT",
"CDBL",
"CSTR",
"ISNULL",
"ISEMPTY",
"TYPEOF",
"ISARRAY",
"ISOBJECT",
"ISSTRING",
"ISNUMBER",
"NVL",
"IIF",
"UBOUND",
"LBOUND",
"COUNT",
"SORT",
"UNIQUE",
"CONTAINS",
"INDEX",
"OF",
"PUSH",
"POP",
"SHIFT",
"REVERSE",
"SLICE",
"SPLIT",
"CONCAT",
"FLATTEN",
"RANGE",
"THROW",
"ERROR",
"IS",
"ASSERT",
"WARN",
"INFO",
"EACH",
"WITH",
"TO",
"STEP",
"BEGIN",
"SYSTEM",
"PROMPT",
"SCHEDULE",
"REFRESH",
"ALLOW",
"ROLE",
"ANSWER",
"MODE",
"SYNCHRONIZE",
"TABLE",
"ON",
"EMAIL",
"REPORT",
"RESET",
"WAIT",
"FIRST",
"LAST",
"LLM",
"INSTR",
"NUMERIC",
"LEN",
"LEFT",
"RIGHT",
"MID",
"LOWER",
"UPPER",
"TRIM",
"LTRIM",
"RTRIM",
"REPLACE",
"LIKE",
"DELEGATE",
"PRIORITY",
"BOTS",
"REMOVE",
"MEMBER",
"BOOK",
"REMEMBER",
"TASK",
"SITE",
"DRAFT",
"INSTAGRAM",
"FACEBOOK",
"LINKEDIN",
"TWITTER",
"METRICS",
"HEADER",
"HEADERS",
"GRAPHQL",
"SOAP",
"HTTP",
"DESCRIPTION",
"PARAM",
"REQUIRED",
"WEBSITE",
"MODEL",
"DETECT",
"LLM",
"TALK",
];
let _identifier_re = Regex::new(r"([a-zA-Z_][a-zA-Z0-9_]*)").expect("valid regex");
for line in script.lines() {
let trimmed = line.trim();
if trimmed.starts_with("REM") || trimmed.starts_with('\'') || trimmed.starts_with("//")
{
continue;
}
// Skip lines with custom syntax that should not be lowercased
// These are registered directly with Rhai in uppercase
let trimmed_upper = trimmed.to_uppercase();
if trimmed_upper.contains("ADD_SUGGESTION_TOOL") ||
trimmed_upper.contains("ADD_SUGGESTION_TEXT") ||
trimmed_upper.starts_with("ADD_SUGGESTION_") ||
trimmed_upper.starts_with("ADD_MEMBER") {
// Keep original line as-is
result.push_str(line);
result.push('\n');
continue;
}
let mut processed_line = String::new();
let mut chars = line.chars().peekable();
let mut in_string = false;
let mut string_char = '"';
let mut current_word = String::new();
while let Some(c) = chars.next() {
if in_string {
processed_line.push(c);
if c == string_char {
in_string = false;
} else if c == '\\' {
if let Some(&next) = chars.peek() {
processed_line.push(next);
chars.next();
}
}
} else if c == '"' || c == '\'' {
if !current_word.is_empty() {
processed_line.push_str(&Self::normalize_word(&current_word, &keywords));
current_word.clear();
}
in_string = true;
string_char = c;
processed_line.push(c);
} else if c.is_alphanumeric() || c == '_' {
current_word.push(c);
} else {
if !current_word.is_empty() {
processed_line.push_str(&Self::normalize_word(&current_word, &keywords));
current_word.clear();
}
processed_line.push(c);
}
}
if !current_word.is_empty() {
processed_line.push_str(&Self::normalize_word(&current_word, &keywords));
}
result.push_str(&processed_line);
result.push('\n');
}
result
}
/// Convert ALL multi-word keywords to underscore versions (function calls) /// Convert ALL multi-word keywords to underscore versions (function calls)
/// This avoids Rhai custom syntax conflicts and makes the system more secure /// This avoids Rhai custom syntax conflicts and makes the system more secure
@ -2163,24 +1450,7 @@ impl ScriptService {
params params
} }
fn normalize_word(word: &str, keywords: &[&str]) -> String { pub(crate) fn preprocess_llm_keyword(script: &str) -> String {
let upper = word.to_uppercase();
if keywords.contains(&upper.as_str()) {
upper
} else if word
.chars()
.next()
.map(|c| c.is_ascii_digit())
.unwrap_or(false)
{
word.to_string()
} else {
word.to_lowercase()
}
}
fn preprocess_llm_keyword(script: &str) -> String {
// Transform LLM "prompt" to LLM "prompt" WITH OPTIMIZE FOR "speed" // Transform LLM "prompt" to LLM "prompt" WITH OPTIMIZE FOR "speed"
// Handle cases like: // Handle cases like:
// LLM "text" // LLM "text"

10
src/core/automation/mod.rs
View file

@ -145,14 +145,10 @@ impl AutomationService {
script_service.load_bot_config_params(&self.state, automation.bot_id); script_service.load_bot_config_params(&self.state, automation.bot_id);
match script_service.compile(&script_content) { match script_service.run(&script_content) {
Ok(ast) => { Ok(_) => {}
if let Err(e) = script_service.run(&ast) {
error!("Script execution failed: {}", e);
}
}
Err(e) => { Err(e) => {
error!("Script compilation failed: {}", e); error!("Script execution failed: {}", e);
} }
} }
Ok(()) Ok(())

177
src/core/bot/mod.rs
View file

@ -634,78 +634,73 @@ impl BotOrchestrator {
trace!("Executing start.bas for session {} at: {}", actual_session_id, start_script_path); trace!("Executing start.bas for session {} at: {}", actual_session_id, start_script_path);
// Use pre-compiled .ast if available (avoids preprocessing) // Load pre-compiled .ast only (compilation happens in Drive Monitor)
let ast_path = start_script_path.replace(".bas", ".ast"); let ast_path = start_script_path.replace(".bas", ".ast");
let (script_content, is_preprocessed) = if std::path::Path::new(&ast_path).exists() { let ast_content = match tokio::fs::read_to_string(&ast_path).await {
(tokio::fs::read_to_string(&ast_path).await.unwrap_or_default(), true) Ok(content) if !content.is_empty() => content,
} else { _ => {
(tokio::fs::read_to_string(&start_script_path).await.unwrap_or_default(), false) let content = tokio::fs::read_to_string(&start_script_path).await.unwrap_or_default();
if content.is_empty() {
trace!("No start.bas/start.ast found for bot {}", bot_name_for_context);
return Ok(());
}
content
}
}; };
if !script_content.is_empty() { let state_clone = self.state.clone();
let state_clone = self.state.clone(); let actual_session_id_for_task = session.id;
let actual_session_id_for_task = session.id; let bot_id_clone = session.bot_id;
let bot_id_clone = session.bot_id;
// Execute start.bas synchronously (blocking) // Execute start.bas synchronously (blocking)
let result = tokio::task::spawn_blocking(move || { let result = tokio::task::spawn_blocking(move || {
let session_result = { let session_result = {
let mut sm = state_clone.session_manager.blocking_lock(); let mut sm = state_clone.session_manager.blocking_lock();
sm.get_session_by_id(actual_session_id_for_task) sm.get_session_by_id(actual_session_id_for_task)
}; };
let sess = match session_result { let sess = match session_result {
Ok(Some(s)) => s, Ok(Some(s)) => s,
Ok(None) => { Ok(None) => {
return Err(format!("Session {} not found during start.bas execution", actual_session_id_for_task)); return Err(format!("Session {} not found during start.bas execution", actual_session_id_for_task));
}
Err(e) => return Err(format!("Failed to get session: {}", e)),
};
let mut script_service = crate::basic::ScriptService::new(
state_clone.clone(),
sess
);
script_service.load_bot_config_params(&state_clone, bot_id_clone);
let compile_result = if is_preprocessed {
script_service.compile_preprocessed(&script_content)
} else {
script_service.compile(&script_content)
};
match compile_result {
Ok(ast) => match script_service.run(&ast) {
Ok(_) => Ok(()),
Err(e) => Err(format!("Script execution error: {}", e)),
},
Err(e) => Err(format!("Script compilation error: {}", e)),
} }
}).await; Err(e) => return Err(format!("Failed to get session: {}", e)),
};
match result { let mut script_service = crate::basic::ScriptService::new(
Ok(Ok(())) => { state_clone.clone(),
trace!("start.bas completed successfully for session {}", actual_session_id); sess
);
script_service.load_bot_config_params(&state_clone, bot_id_clone);
// Mark start.bas as executed for this session to prevent re-running match script_service.run(&ast_content) {
if let Some(cache) = &self.state.cache { Ok(_) => Ok(()),
if let Ok(mut conn) = cache.get_multiplexed_async_connection().await { Err(e) => Err(format!("Script execution error: {}", e)),
let _: Result<(), redis::RedisError> = redis::cmd("SET") }
.arg(&start_bas_key) }).await;
.arg("1")
.arg("EX") match result {
.arg("86400") // Expire after 24 hours Ok(Ok(())) => {
.query_async(&mut conn) trace!("start.bas completed successfully for session {}", actual_session_id);
.await;
} // Mark start.bas as executed for this session to prevent re-running
if let Some(cache) = &self.state.cache {
if let Ok(mut conn) = cache.get_multiplexed_async_connection().await {
let _: Result<(), redis::RedisError> = redis::cmd("SET")
.arg(&start_bas_key)
.arg("1")
.arg("EX")
.arg("86400") // Expire after 24 hours
.query_async(&mut conn)
.await;
} }
} }
Ok(Err(e)) => { }
error!("start.bas error for session {}: {}", actual_session_id, e); Ok(Err(e)) => {
} error!("start.bas error for session {}: {}", actual_session_id, e);
Err(e) => { }
error!("start.bas task error for session {}: {}", actual_session_id, e); Err(e) => {
} error!("start.bas task error for session {}: {}", actual_session_id, e);
} }
} }
} // End of if should_execute_start_bas } // End of if should_execute_start_bas
@ -1147,13 +1142,17 @@ impl BotOrchestrator {
// DEBUG: Log LLM output for troubleshooting HTML rendering issues // DEBUG: Log LLM output for troubleshooting HTML rendering issues
let has_html = full_response.contains("</") || full_response.contains("<!--"); let has_html = full_response.contains("</") || full_response.contains("<!--");
let preview = if full_response.len() > 500 { let has_div = full_response.contains("<div") || full_response.contains("</div>");
format!("{}... ({} chars total)", &full_response[..500], full_response.len()) let has_style = full_response.contains("<style");
let is_truncated = !full_response.trim_end().ends_with("</div>") && has_div;
let preview = if full_response.len() > 800 {
format!("{}... ({} chars total)", &full_response[..800], full_response.len())
} else { } else {
full_response.clone() full_response.clone()
}; };
info!("[LLM_OUTPUT] session={} has_html={} preview=\"{}\"", info!("[LLM_OUTPUT] session={} has_html={} has_div={} has_style={} is_truncated={} len={} preview=\"{}\"",
session_id, has_html, preview.replace('\n', "\\n")); session_id, has_html, has_div, has_style, is_truncated, full_response.len(),
preview.replace('\n', "\\n"));
trace!("LLM stream complete. Full response: {}", full_response); trace!("LLM stream complete. Full response: {}", full_response);
@ -1433,27 +1432,22 @@ async fn handle_websocket(
info!("Looking for start.bas at: {}", start_script_path); info!("Looking for start.bas at: {}", start_script_path);
// Check for pre-compiled .ast file first (avoids preprocessing overhead) // Load pre-compiled .ast only (compilation happens in Drive Monitor)
let ast_path = start_script_path.replace(".bas", ".ast"); let ast_path = start_script_path.replace(".bas", ".ast");
let (script_content, is_preprocessed) = if tokio::fs::metadata(&ast_path).await.is_ok() { let ast_content = match tokio::fs::read_to_string(&ast_path).await {
if let Ok(content) = tokio::fs::read_to_string(&ast_path).await { Ok(content) if !content.is_empty() => content,
info!("Using pre-compiled start.ast for {}", bot_name); _ => {
(content, true) let content = tokio::fs::read_to_string(&start_script_path).await.unwrap_or_default();
} else { if content.is_empty() {
(String::new(), false) info!("No start.bas/start.ast found for bot {}", bot_name);
String::new()
} else {
content
}
} }
} else if tokio::fs::metadata(&start_script_path).await.is_ok() {
if let Ok(content) = tokio::fs::read_to_string(&start_script_path).await {
info!("Compiling start.bas for {}", bot_name);
(content, false)
} else {
(String::new(), false)
}
} else {
(String::new(), false)
}; };
if !script_content.is_empty() { if !ast_content.is_empty() {
info!( info!(
"Executing start.bas for bot {} on session {}", "Executing start.bas for bot {} on session {}",
bot_name, session_id bot_name, session_id
@ -1464,8 +1458,6 @@ async fn handle_websocket(
let bot_id_str = bot_id.to_string(); let bot_id_str = bot_id.to_string();
let session_id_str = session_id.to_string(); let session_id_str = session_id.to_string();
let mut send_ready_rx = send_ready_rx; let mut send_ready_rx = send_ready_rx;
let script_content_owned = script_content.clone();
let is_preprocessed_owned = is_preprocessed;
tokio::spawn(async move { tokio::spawn(async move {
let _ = send_ready_rx.recv().await; let _ = send_ready_rx.recv().await;
@ -1502,18 +1494,9 @@ async fn handle_websocket(
); );
script_service.load_bot_config_params(&state_for_start, bot_id); script_service.load_bot_config_params(&state_for_start, bot_id);
let compile_result = if is_preprocessed_owned { match script_service.run(&ast_content) {
script_service.compile_preprocessed(&script_content_owned) Ok(_) => Ok(()),
} else { Err(e) => Err(format!("Script execution error: {}", e)),
script_service.compile(&script_content_owned)
};
match compile_result {
Ok(ast) => match script_service.run(&ast) {
Ok(_) => Ok(()),
Err(e) => Err(format!("Script execution error: {}", e)),
},
Err(e) => Err(format!("Script compilation error: {}", e)),
} }
}).await; }).await;

90
src/core/bot/tool_executor.rs
View file

@ -157,46 +157,25 @@ impl ToolExecutor {
} }
}; };
// Load the .bas tool file (prefer pre-compiled .ast if available) // Load the pre-compiled .ast file (compilation happens only in Drive Monitor)
let bas_path = Self::get_tool_bas_path(bot_name, &tool_call.tool_name); let ast_path = Self::get_tool_ast_path(bot_name, &tool_call.tool_name);
let ast_path = bas_path.with_extension("ast");
// Check for .ast first (pre-compiled), fallback to .bas let ast_content = match tokio::fs::read_to_string(&ast_path).await {
let (script_content, is_preprocessed) = if ast_path.exists() { Ok(content) => content,
match tokio::fs::read_to_string(&ast_path).await { Err(e) => {
Ok(script) => { let error_msg = format!("Failed to read tool .ast file {}: {}", ast_path.display(), e);
trace!("Using pre-compiled .ast for tool: {}", tool_call.tool_name); Self::log_tool_error(bot_name, &tool_call.tool_name, &error_msg);
(script, true) return ToolExecutionResult {
} tool_call_id: tool_call.id.clone(),
Err(_) => (String::new(), false) success: false,
result: String::new(),
error: Some(Self::format_user_friendly_error(&tool_call.tool_name, &error_msg)),
};
} }
} else if bas_path.exists() {
match tokio::fs::read_to_string(&bas_path).await {
Ok(script) => (script, false),
Err(e) => {
let error_msg = format!("Failed to read tool file: {}", e);
Self::log_tool_error(bot_name, &tool_call.tool_name, &error_msg);
return ToolExecutionResult {
tool_call_id: tool_call.id.clone(),
success: false,
result: String::new(),
error: Some(Self::format_user_friendly_error(&tool_call.tool_name, &error_msg)),
};
}
}
} else {
let error_msg = format!("Tool file not found: {:?}", bas_path);
Self::log_tool_error(bot_name, &tool_call.tool_name, &error_msg);
return ToolExecutionResult {
tool_call_id: tool_call.id.clone(),
success: false,
result: String::new(),
error: Some(Self::format_user_friendly_error(&tool_call.tool_name, &error_msg)),
};
}; };
if script_content.is_empty() { if ast_content.is_empty() {
let error_msg = "Tool script is empty".to_string(); let error_msg = "Tool .ast file is empty".to_string();
Self::log_tool_error(bot_name, &tool_call.tool_name, &error_msg); Self::log_tool_error(bot_name, &tool_call.tool_name, &error_msg);
return ToolExecutionResult { return ToolExecutionResult {
tool_call_id: tool_call.id.clone(), tool_call_id: tool_call.id.clone(),
@ -245,10 +224,9 @@ impl ToolExecutor {
&bot_name_clone, &bot_name_clone,
bot_id_clone, bot_id_clone,
&session, &session,
&script_content, &ast_content,
&tool_name_clone, &tool_name_clone,
&arguments_clone, &arguments_clone,
is_preprocessed,
) )
}) })
.await; .await;
@ -274,10 +252,9 @@ impl ToolExecutor {
bot_name: &str, bot_name: &str,
bot_id: Uuid, bot_id: Uuid,
session: &crate::core::shared::models::UserSession, session: &crate::core::shared::models::UserSession,
script_content: &str, ast_content: &str,
tool_name: &str, tool_name: &str,
arguments: &Value, arguments: &Value,
is_preprocessed: bool,
) -> ToolExecutionResult { ) -> ToolExecutionResult {
let tool_call_id = format!("tool_{}", uuid::Uuid::new_v4()); let tool_call_id = format!("tool_{}", uuid::Uuid::new_v4());
@ -304,30 +281,8 @@ impl ToolExecutor {
} }
} }
// Compile: use compile_preprocessed for .ast files, compile_tool_script for .bas // Run the pre-compiled .ast content (compilation happens only in Drive Monitor)
let ast = if is_preprocessed { match script_service.run(ast_content) {
script_service.compile_preprocessed(script_content)
} else {
script_service.compile_tool_script(script_content)
};
let ast = match ast {
Ok(ast) => ast,
Err(e) => {
let error_msg = format!("Compilation error: {}", e);
Self::log_tool_error(bot_name, tool_name, &error_msg);
let user_message = Self::format_user_friendly_error(tool_name, &error_msg);
return ToolExecutionResult {
tool_call_id,
success: false,
result: String::new(),
error: Some(user_message),
};
}
};
// Run the script
match script_service.run(&ast) {
Ok(result) => { Ok(result) => {
trace!("Tool '{}' executed successfully", tool_name); trace!("Tool '{}' executed successfully", tool_name);
@ -365,13 +320,12 @@ impl ToolExecutor {
.ok() .ok()
} }
/// Get the path to a tool's .bas file /// Get the path to a tool's pre-compiled .ast file
fn get_tool_bas_path(bot_name: &str, tool_name: &str) -> std::path::PathBuf { fn get_tool_ast_path(bot_name: &str, tool_name: &str) -> std::path::PathBuf {
// Use work directory for compiled .bas files
let work_path = std::path::PathBuf::from(crate::core::shared::utils::get_work_path()) let work_path = std::path::PathBuf::from(crate::core::shared::utils::get_work_path())
.join(format!("{}.gbai", bot_name)) .join(format!("{}.gbai", bot_name))
.join(format!("{}.gbdialog", bot_name)) .join(format!("{}.gbdialog", bot_name))
.join(format!("{}.bas", tool_name)); .join(format!("{}.ast", tool_name));
work_path work_path
} }

12
src/directory/mod.rs
View file

@ -213,13 +213,11 @@ pub async fn auth_handler(
let mut script_service = let mut script_service =
crate::basic::ScriptService::new(state_clone.clone(), session_clone); crate::basic::ScriptService::new(state_clone.clone(), session_clone);
script_service.load_bot_config_params(&state_clone, bot_id); script_service.load_bot_config_params(&state_clone, bot_id);
match script_service.compile(&auth_script) {
Ok(ast) => match script_service.run(&ast) { match script_service.run(&auth_script) {
Ok(_) => Ok(()), Ok(_) => Ok(()),
Err(e) => Err(format!("Script execution error: {}", e)), Err(e) => Err(format!("Script execution error: {}", e)),
},
Err(e) => Err(format!("Script compilation error: {}", e)),
} }
}) })
.await .await

15
src/llm/mod.rs
View file

@ -1,6 +1,6 @@
use async_trait::async_trait; use async_trait::async_trait;
use futures::StreamExt; use futures::StreamExt;
use log::{error, info}; use log::{error, info, trace};
use serde_json::Value; use serde_json::Value;
use std::sync::Arc; use std::sync::Arc;
use tokio::sync::{mpsc, RwLock}; use tokio::sync::{mpsc, RwLock};
@ -413,7 +413,8 @@ impl LLMProvider for OpenAIClient {
let mut request_body = serde_json::json!({ let mut request_body = serde_json::json!({
"model": model, "model": model,
"messages": messages, "messages": messages,
"stream": true "stream": true,
"max_tokens": 16384
}); });
// Add tools to the request if provided // Add tools to the request if provided
@ -477,7 +478,17 @@ impl LLMProvider for OpenAIClient {
for line in chunk_str.lines() { for line in chunk_str.lines() {
if line.starts_with("data: ") && !line.contains("[DONE]") { if line.starts_with("data: ") && !line.contains("[DONE]") {
if let Ok(data) = serde_json::from_str::<Value>(&line[6..]) { if let Ok(data) = serde_json::from_str::<Value>(&line[6..]) {
// Kimi K2.5 and other reasoning models send thinking in "reasoning" field
// Only process "content" (actual response), ignore "reasoning" (thinking)
let content = data["choices"][0]["delta"]["content"].as_str(); let content = data["choices"][0]["delta"]["content"].as_str();
let reasoning = data["choices"][0]["delta"]["reasoning"].as_str();
// Log first chunk to help debug reasoning models
if reasoning.is_some() && content.is_none() {
trace!("[LLM] Kimi reasoning chunk (no content yet): {} chars",
reasoning.unwrap_or("").len());
}
if let Some(content) = content { if let Some(content) = content {
let processed = handler.process_content(content); let processed = handler.process_content(content);
if !processed.is_empty() { if !processed.is_empty() {