GeneralBots/generalbots
1
0
Fork 0
Code Activity Actions 5

revert: restore llm/mod.rs to stable April 9 version
All checks were successful
BotServer CI/CD / build (push) Successful in 3m26s
Details

Browse source 
Co-authored-by: Qwen-Coder <qwen-coder@alibabacloud.com>
This commit is contained in:
Rodrigo Rodriguez (Pragmatismo) 2026-04-13 15:07:19 -03:00
parent 765bd624f4
commit c5d30adebe
1 changed files with 49 additions and 91 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

60
src/llm/mod.rs
View file

@ -1,6 +1,6 @@
use async_trait::async_trait; use async_trait::async_trait;
use futures::StreamExt; use futures::StreamExt;
use log::{error, info, trace}; use log::{error, info};
use serde_json::Value; use serde_json::Value;
use std::sync::Arc; use std::sync::Arc;
use tokio::sync::{mpsc, RwLock}; use tokio::sync::{mpsc, RwLock};
@ -11,7 +11,6 @@ pub mod episodic_memory;
pub mod glm; pub mod glm;
pub mod hallucination_detector; pub mod hallucination_detector;
pub mod llm_models; pub mod llm_models;
#[cfg(feature = "llm")]
pub mod local; pub mod local;
pub mod observability; pub mod observability;
pub mod rate_limiter; pub mod rate_limiter;
@ -290,7 +289,7 @@ impl LLMProvider for OpenAIClient {
128000 // Cerebras gpt-oss models and GPT-4 variants 128000 // Cerebras gpt-oss models and GPT-4 variants
} else if model.contains("gpt-3.5") { } else if model.contains("gpt-3.5") {
16385 16385
} else if model == "local" || model.is_empty() { } else if model.starts_with("http://localhost:808") || model == "local" {
768 // Local llama.cpp server context limit 768 // Local llama.cpp server context limit
} else { } else {
32768 // Default conservative limit for modern models 32768 // Default conservative limit for modern models
@ -379,7 +378,7 @@ impl LLMProvider for OpenAIClient {
128000 // Cerebras gpt-oss models and GPT-4 variants 128000 // Cerebras gpt-oss models and GPT-4 variants
} else if model.contains("gpt-3.5") { } else if model.contains("gpt-3.5") {
16385 16385
} else if model == "local" || model.is_empty() { } else if model.starts_with("http://localhost:808") || model == "local" {
768 // Local llama.cpp server context limit 768 // Local llama.cpp server context limit
} else { } else {
32768 // Default conservative limit for modern models 32768 // Default conservative limit for modern models
@ -413,8 +412,7 @@ impl LLMProvider for OpenAIClient {
let mut request_body = serde_json::json!({ let mut request_body = serde_json::json!({
"model": model, "model": model,
"messages": messages, "messages": messages,
"stream": true, "stream": true
"max_tokens": 16384
}); });
// Add tools to the request if provided // Add tools to the request if provided
@ -453,43 +451,13 @@ impl LLMProvider for OpenAIClient {
// Accumulate tool calls here because OpenAI streams them in fragments // Accumulate tool calls here because OpenAI streams them in fragments
let mut active_tool_calls: Vec<serde_json::Value> = Vec::new(); let mut active_tool_calls: Vec<serde_json::Value> = Vec::new();
// Add timeout to stream reads - if Kimi/Nvidia stops responding, fail gracefully while let Some(chunk_result) = stream.next().await {
const STREAM_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60); let chunk = chunk_result?;
loop {
let chunk_opt = match tokio::time::timeout(
STREAM_TIMEOUT,
stream.next(),
).await {
Ok(opt) => opt,
Err(_) => {
// Timeout - LLM stopped sending data
log::warn!("[LLM] Stream timed out after {}s for model {}",
STREAM_TIMEOUT.as_secs(), model);
let _ = tx.send(format!("[ERROR] LLM response timed out after {} seconds.",
STREAM_TIMEOUT.as_secs())).await;
break;
}
};
match chunk_opt {
Some(Ok(chunk)) => {
let chunk_str = String::from_utf8_lossy(&chunk); let chunk_str = String::from_utf8_lossy(&chunk);
for line in chunk_str.lines() { for line in chunk_str.lines() {
if line.starts_with("data: ") && !line.contains("[DONE]") { if line.starts_with("data: ") && !line.contains("[DONE]") {
if let Ok(data) = serde_json::from_str::<Value>(&line[6..]) { if let Ok(data) = serde_json::from_str::<Value>(&line[6..]) {
// Kimi K2.5 and other reasoning models send thinking in "reasoning" field if let Some(content) = data["choices"][0]["delta"]["content"].as_str() {
// Only process "content" (actual response), ignore "reasoning" (thinking)
let content = data["choices"][0]["delta"]["content"].as_str();
let reasoning = data["choices"][0]["delta"]["reasoning"].as_str();
// Log first chunk to help debug reasoning models
if reasoning.is_some() && content.is_none() {
trace!("[LLM] Kimi reasoning chunk (no content yet): {} chars",
reasoning.unwrap_or("").len());
}
if let Some(content) = content {
let processed = handler.process_content(content); let processed = handler.process_content(content);
if !processed.is_empty() { if !processed.is_empty() {
let _ = tx.send(processed).await; let _ = tx.send(processed).await;
@ -537,16 +505,6 @@ impl LLMProvider for OpenAIClient {
} }
} }
} }
Some(Err(e)) => {
log::error!("[LLM] Stream error: {}", e);
break;
}
None => {
// Stream ended
break;
}
}
}
// Send accumulated tool calls when stream finishes // Send accumulated tool calls when stream finishes
for tool_call in active_tool_calls { for tool_call in active_tool_calls {
@ -927,10 +885,10 @@ mod tests {
fn test_openai_client_new_custom_url() { fn test_openai_client_new_custom_url() {
let client = OpenAIClient::new( let client = OpenAIClient::new(
"test_key".to_string(), "test_key".to_string(),
Some("".to_string()), Some("http://localhost:9000".to_string()),
None, None,
); );
assert_eq!(client.base_url, ""); assert_eq!(client.base_url, "http://localhost:9000");
} }
#[test] #[test]