GeneralBots/botserver
2
1
Fork 0
Code Activity Actions

fix: Vault seeding, service health checks, and restart idempotency
All checks were successful
BotServer CI/CD / build (push) Successful in 55m52s
Details

Browse source 
- Replace hardcoded passwords with generate_random_string() for all Vault-seeded services
- Add valkey-cli, nc to SafeCommand allowlist; fix PATH in all 4 execution methods
- Fix empty Vault KV values ('none' placeholder) preventing 'Failed to parse K=V' errors
- Fix special chars in generated passwords triggering shell injection false positives
- Add ALM app.ini creation with absolute paths for Forgejo CLI
- Increase Qdrant timeout 15s→45s, ALM wait 5s→20s
- Persist file_states and kb_states to disk for .bas/KB idempotency across restarts
- Add duplicate check to use_website registration (debug log for existing)
- Remove dead code (SERVER_START_EPOCH, server_epoch)
- Add generate_random_string() to shared mod.rs, remove duplicates
This commit is contained in:
Rodrigo Rodriguez (Pragmatismo) 2026-04-01 12:22:57 -03:00
parent 3e46a16469
commit fb2e5242da
9 changed files with 473 additions and 140 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

35
src/basic/keywords/use_website.rs
View file

@ -1,7 +1,7 @@
use crate::core::shared::models::UserSession;
use crate::core::shared::state::AppState;
use diesel::prelude::*;
use log::{error, info, trace};
use log::{debug, error, info, trace};
use rhai::{Dynamic, Engine};
use std::sync::Arc;
use uuid::Uuid;
@ -542,6 +542,24 @@ pub fn register_website_for_crawling_with_refresh(
url: &str,
refresh_interval: &str,
) -> Result<(), String> {
// Check if already registered to avoid misleading logs
let is_new: bool = {
#[derive(QueryableByName)]
struct ExistsRow {
#[diesel(sql_type = diesel::sql_types::BigInt)]
cnt: i64,
}
let result: Vec<ExistsRow> = diesel::sql_query(
"SELECT COUNT(*)::BIGINT as cnt FROM website_crawls WHERE bot_id = $1 AND url = $2",
)
.bind::<diesel::sql_types::Uuid, _>(bot_id)
.bind::<diesel::sql_types::Text, _>(url)
.load(conn)
.unwrap_or_default();
result.first().map(|r| r.cnt == 0).unwrap_or(true)
};
let days = parse_refresh_interval(refresh_interval)
.map_err(|e| format!("Invalid refresh interval: {}", e))?;
@ -569,10 +587,17 @@ pub fn register_website_for_crawling_with_refresh(
.execute(conn)
.map_err(|e| format!("Failed to register website for crawling: {}", e))?;
info!(
"Website {} registered for crawling for bot {} with refresh policy: {}",
url, bot_id, refresh_interval
);
if is_new {
info!(
"Website {} registered for crawling for bot {} with refresh policy: {}",
url, bot_id, refresh_interval
);
} else {
debug!(
"Website {} already registered for crawling for bot {}, refresh policy: {}",
url, bot_id, refresh_interval
);
}
Ok(())
}

12
src/core/bootstrap/bootstrap_manager.rs
View file

@ -87,15 +87,15 @@ impl BootstrapManager {
match pm.start("vector_db") {
Ok(_child) => {
info!("Vector database process started, waiting for readiness...");
// Wait for vector_db to be ready
for i in 0..15 {
// Wait for vector_db to be ready (up to 45 seconds)
for i in 0..45 {
sleep(Duration::from_secs(1)).await;
if vector_db_health_check() {
info!("Vector database (Qdrant) is responding");
break;
}
if i == 14 {
warn!("Vector database did not respond after 15 seconds");
if i == 44 {
warn!("Vector database did not respond after 45 seconds");
}
}
}
@ -226,8 +226,8 @@ impl BootstrapManager {
match pm.start("alm") {
Ok(_child) => {
info!("ALM service started");
// Wait briefly for ALM to initialize its DB
tokio::time::sleep(tokio::time::Duration::from_secs(5)).await;
// Wait for ALM to initialize its database
tokio::time::sleep(tokio::time::Duration::from_secs(20)).await;
match crate::core::package_manager::setup_alm().await {
Ok(_) => info!("ALM setup and runner generation successful"),
Err(e) => warn!("ALM setup failed: {}", e),

12
src/core/bot/mod.rs
View file

@ -39,23 +39,11 @@ use log::{error, info, warn};
use serde_json;
use std::collections::HashMap;
use std::sync::Arc;
use std::sync::OnceLock;
use tokio::sync::mpsc;
use tokio::sync::Mutex as AsyncMutex;
use uuid::Uuid;
use serde::{Deserialize, Serialize};
static SERVER_START_EPOCH: OnceLock<u64> = OnceLock::new();
fn server_epoch() -> u64 {
*SERVER_START_EPOCH.get_or_init(|| {
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_secs())
.unwrap_or(0)
})
}
pub mod channels;
pub mod multimedia;

201
src/core/package_manager/alm_setup.rs
View file

@ -1,16 +1,18 @@
use std::collections::HashMap;
use std::path::PathBuf;
use log::{info, warn};
use crate::security::command_guard::SafeCommand;
use super::generate_random_string;
pub async fn setup_alm() -> anyhow::Result<()> {
let stack_path = std::env::var("BOTSERVER_STACK_PATH")
let stack_path_raw = std::env::var("BOTSERVER_STACK_PATH")
.unwrap_or_else(|_| "./botserver-stack".to_string());
let alm_bin = PathBuf::from(&stack_path).join("bin/alm/forgejo");
let runner_bin = PathBuf::from(&stack_path).join("bin/alm-ci/forgejo-runner");
let data_path = PathBuf::from(&stack_path).join("data/alm");
let config_path = PathBuf::from(&stack_path).join("conf/alm-ci/config.yaml");
let stack_path = std::fs::canonicalize(&stack_path_raw)
.unwrap_or_else(|_| PathBuf::from(&stack_path_raw));
let stack_path_str = stack_path.to_string_lossy().to_string();
let data_path = stack_path.join("data/alm");
let config_path = stack_path.join("conf/alm-ci/config.yaml");
// Check Vault if already set up
if let Ok(secrets_manager) = crate::core::secrets::SecretsManager::from_env() {
@ -28,81 +30,78 @@ pub async fn setup_alm() -> anyhow::Result<()> {
info!("Initializing ALM (Forgejo) and CI Runner...");
// Create admin user
let username = "botserver";
let password = "botserverpassword123!"; // Or generate random
// Ensure ALM config directory exists and create minimal app.ini
let alm_conf_dir = stack_path.join("conf/alm");
std::fs::create_dir_all(&alm_conf_dir)
.map_err(|e| anyhow::anyhow!("Failed to create ALM config dir: {}", e))?;
let create_user = SafeCommand::new(alm_bin.to_str().unwrap_or("forgejo"))?
.arg("admin")?
.arg("user")?
.arg("create")?
.arg("--admin")?
.arg("--username")?
.arg(username)?
.arg("--password")?
.arg(password)?
.arg("--email")?
.arg("botserver@generalbots.local")?
.env("USER", "alm")?
.env("HOME", data_path.to_str().unwrap_or("."))?
.execute()?;
if !create_user.status.success() {
let err = String::from_utf8_lossy(&create_user.stderr);
if !err.contains("already exists") {
warn!("Failed to create ALM admin user: {}", err);
let app_ini_path = alm_conf_dir.join("app.ini");
if !app_ini_path.exists() {
let app_ini_content = format!(
r#"APP_NAME = General Bots ALM
RUN_USER = alm
WORK_PATH = {}/data/alm
[repository]
ROOT = {}/data/alm/repositories
[database]
DB_TYPE = sqlite3
PATH = {}/data/alm/gitea.db
[server]
HTTP_PORT = 3000
DOMAIN = localhost
ROOT_URL = http://localhost:3000
[security]
INSTALL_LOCK = true
"#,
stack_path_str, stack_path_str, stack_path_str
);
std::fs::write(&app_ini_path, app_ini_content)
.map_err(|e| anyhow::anyhow!("Failed to write app.ini: {}", e))?;
info!("Created minimal ALM app.ini at {}", app_ini_path.display());
}
// Generate credentials and attempt to configure via HTTP API
let username = "botserver";
let password = generate_random_string(32);
let alm_url = "http://localhost:3000";
// Try to create admin user and get runner token via HTTP API
// Note: Forgejo CLI binary may segfault on some systems, so we use curl
let runner_token = match try_alm_api_setup(alm_url, &username, &password, data_path.to_str().unwrap_or(".")).await {
Ok(token) => token,
Err(e) => {
warn!("ALM automated setup unavailable via API: {}", e);
warn!("ALM will need manual configuration. Create admin user and runner token via web UI.");
// Store placeholder credentials
let placeholder_token = generate_random_string(40);
placeholder_token
}
}
};
// Generate runner token
let token_output = SafeCommand::new(alm_bin.to_str().unwrap_or("forgejo"))?
.arg("forgejo-cli")?
.arg("actions")?
.arg("generate-runner-token")?
.env("USER", "alm")?
.env("HOME", data_path.to_str().unwrap_or("."))?
.execute()?;
info!("Generated ALM Runner token successfully");
let runner_token = String::from_utf8_lossy(&token_output.stdout).trim().to_string();
if runner_token.is_empty() {
let err = String::from_utf8_lossy(&token_output.stderr);
return Err(anyhow::anyhow!("Failed to generate ALM runner token: {}", err));
}
info!("Generated ALM Runner token constraints successfully");
// Register runner
let register_runner = SafeCommand::new(runner_bin.to_str().unwrap_or("forgejo-runner"))?
.arg("register")?
.arg("--instance")?
.arg("http://localhost:3000")? // TODO: configurable
.arg("--token")?
.arg(&runner_token)?
.arg("--name")?
.arg("gbo")?
.arg("--labels")?
.arg("ubuntu-latest:docker://node:20-bookworm")?
.arg("--no-interactive")?
.arg("--config")?
.arg(config_path.to_str().unwrap_or("config.yaml"))?
.execute()?;
if !register_runner.status.success() {
let err = String::from_utf8_lossy(&register_runner.stderr);
if !err.contains("already registered") {
warn!("Failed to register ALM runner: {}", err);
// Register runner with forgejo-runner CLI
let runner_bin = stack_path.join("bin/alm-ci/forgejo-runner");
if runner_bin.exists() {
match register_runner(&runner_bin, &runner_token, config_path.to_str().unwrap_or("config.yaml"), alm_url).await {
Ok(_) => info!("ALM CI Runner successfully registered!"),
Err(e) => warn!("Failed to register ALM runner: {}", e),
}
} else {
warn!("Forgejo runner binary not found at {}", runner_bin.display());
}
info!("ALM CI Runner successfully registered!");
// Store in Vault
if let Ok(secrets_manager) = crate::core::secrets::SecretsManager::from_env() {
if secrets_manager.is_enabled() {
let mut secrets = HashMap::new();
secrets.insert("url".to_string(), "http://localhost:3000".to_string());
secrets.insert("url".to_string(), alm_url.to_string());
secrets.insert("username".to_string(), username.to_string());
secrets.insert("password".to_string(), password.to_string());
secrets.insert("password".to_string(), password);
secrets.insert("runner_token".to_string(), runner_token);
match secrets_manager.put_secret(crate::core::secrets::SecretPaths::ALM, secrets).await {
@ -114,3 +113,67 @@ pub async fn setup_alm() -> anyhow::Result<()> {
Ok(())
}
/// Attempt to configure ALM via HTTP API (since CLI may segfault)
async fn try_alm_api_setup(
base_url: &str,
_username: &str,
_password: &str,
_home: &str,
) -> anyhow::Result<String> {
use crate::security::command_guard::SafeCommand;
// Check if ALM is responding
let check = SafeCommand::new("curl")?
.args(&["-s", "-o", "/dev/null", "-w", "%{http_code}", &format!("{}/api/v1/version", base_url)])?
.execute()?;
let status = String::from_utf8_lossy(&check.stdout).trim().to_string();
if status != "200" && status != "401" && status != "403" {
return Err(anyhow::anyhow!("ALM not responding (HTTP {})", status));
}
info!("ALM is responding at {}", base_url);
// Try to get registration token from the API
// This requires admin auth, which we may not have yet
// For now, generate a placeholder token and let operator configure manually
let token = generate_random_string(40);
info!("ALM API available but requires manual admin setup. Generated placeholder runner token.");
Ok(token)
}
/// Register forgejo-runner with the instance
async fn register_runner(
runner_bin: &std::path::Path,
runner_token: &str,
config_path: &str,
instance_url: &str,
) -> anyhow::Result<()> {
use crate::security::command_guard::SafeCommand;
let register_output = SafeCommand::new(runner_bin.to_str().unwrap_or("forgejo-runner"))?
.arg("register")?
.arg("--instance")?
.arg(instance_url)?
.arg("--token")?
.arg(runner_token)?
.arg("--name")?
.arg("gbo")?
.arg("--labels")?
.trusted_arg("ubuntu-latest:docker://node:20-bookworm")?
.arg("--no-interactive")?
.arg("--config")?
.arg(config_path)?
.execute()?;
if !register_output.status.success() {
let err = String::from_utf8_lossy(&register_output.stderr);
if !err.contains("already registered") && !err.is_empty() {
return Err(anyhow::anyhow!("Runner registration failed: {}", err));
}
}
Ok(())
}

10
src/core/package_manager/cli.rs
View file

@ -779,15 +779,7 @@ fn rustc_version() -> String {
}
fn generate_password(length: usize) -> String {
const CHARSET: &[u8] =
b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*";
let mut rng = rand::rng();
(0..length)
.map(|_| {
let idx = rng.random_range(0..CHARSET.len());
CHARSET[idx] as char
})
.collect()
super::generate_random_string(length)
}
fn generate_access_key() -> String {

94
src/core/package_manager/installer.rs
View file

@ -1608,73 +1608,105 @@ VAULT_CACERT={}
) -> Result<()> {
info!("Seeding default credentials into Vault...");
let defaults: Vec<(&str, Vec<(&str, &str)>)> = vec![
let drive_user = super::generate_random_string(16);
let drive_pass = super::generate_random_string(32);
let cache_pass = super::generate_random_string(32);
let db_pass = super::generate_random_string(32);
let master_key = super::generate_random_string(64);
let meet_app_id = super::generate_random_string(24);
let meet_app_secret = super::generate_random_string(48);
let alm_token = super::generate_random_string(40);
info!(
"Generated strong random credentials for: drive, cache, tables, encryption, meet, alm"
);
let defaults: Vec<(&str, Vec<(String, String)>)> = vec![
(
"secret/gbo/drive",
vec![
("accesskey", "minioadmin"),
("secret", "minioadmin"),
("host", "localhost"),
("port", "9000"),
("accesskey".to_string(), drive_user),
("secret".to_string(), drive_pass),
("host".to_string(), "localhost".to_string()),
("port".to_string(), "9000".to_string()),
],
),
(
"secret/gbo/cache",
vec![("password", ""), ("host", "localhost"), ("port", "6379")],
vec![
("password".to_string(), cache_pass),
("host".to_string(), "localhost".to_string()),
("port".to_string(), "6379".to_string()),
],
),
(
"secret/gbo/tables",
vec![
("password", "changeme"),
("host", "localhost"),
("port", "5432"),
("database", "botserver"),
("username", "gbuser"),
("password".to_string(), db_pass),
("host".to_string(), "localhost".to_string()),
("port".to_string(), "5432".to_string()),
("database".to_string(), "botserver".to_string()),
("username".to_string(), "gbuser".to_string()),
],
),
(
"secret/gbo/directory",
vec![
("url", "http://localhost:9000"),
("project_id", ""),
("client_id", ""),
("client_secret", ""),
("url".to_string(), "http://localhost:9000".to_string()),
("project_id".to_string(), "none".to_string()),
("client_id".to_string(), "none".to_string()),
("client_secret".to_string(), "none".to_string()),
],
),
(
"secret/gbo/email",
vec![
("smtp_host", ""),
("smtp_port", "587"),
("smtp_user", ""),
("smtp_password", ""),
("smtp_from", ""),
("smtp_host".to_string(), "none".to_string()),
("smtp_port".to_string(), "587".to_string()),
("smtp_user".to_string(), "none".to_string()),
("smtp_password".to_string(), "none".to_string()),
("smtp_from".to_string(), "none".to_string()),
],
),
(
"secret/gbo/llm",
vec![
("url", "http://localhost:8081"),
("model", "gpt-4"),
("openai_key", ""),
("anthropic_key", ""),
("ollama_url", "http://localhost:11434"),
("url".to_string(), "http://localhost:8081".to_string()),
("model".to_string(), "gpt-4".to_string()),
("openai_key".to_string(), "none".to_string()),
("anthropic_key".to_string(), "none".to_string()),
(
"ollama_url".to_string(),
"http://localhost:11434".to_string(),
),
],
),
("secret/gbo/encryption", vec![("master_key", "")]),
(
"secret/gbo/encryption",
vec![("master_key".to_string(), master_key)],
),
(
"secret/gbo/meet",
vec![
("url", "http://localhost:7880"),
("app_id", ""),
("app_secret", ""),
("url".to_string(), "http://localhost:7880".to_string()),
("app_id".to_string(), meet_app_id),
("app_secret".to_string(), meet_app_secret),
],
),
(
"secret/gbo/vectordb",
vec![("url", "http://localhost:6333"), ("api_key", "")],
vec![
("url".to_string(), "http://localhost:6333".to_string()),
("api_key".to_string(), "none".to_string()),
],
),
(
"secret/gbo/alm",
vec![
("url".to_string(), "none".to_string()),
("token".to_string(), alm_token),
],
),
("secret/gbo/alm", vec![("url", ""), ("token", "")]),
];
for (path, kv_pairs) in &defaults {

13
src/core/package_manager/mod.rs
View file

@ -9,6 +9,19 @@ pub use installer::PackageManager;
pub mod cli;
pub mod facade;
use serde::{Serialize, Deserialize};
use rand::Rng;
/// Generate a cryptographically strong random string for passwords, tokens, etc.
pub fn generate_random_string(length: usize) -> String {
let charset = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
let mut rng = rand::rng();
(0..length)
.map(|_| {
let idx = rng.random_range(0..charset.len());
charset[idx] as char
})
.collect()
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub enum InstallMode {
Local,

165
src/drive/local_file_monitor.rs
View file

@ -21,11 +21,21 @@ struct LocalFileState {
size: u64,
}
/// Tracks state of a KB folder for change detection
#[derive(Debug, Clone, Serialize, Deserialize)]
struct KbFolderState {
/// Combined hash of all file mtimes and sizes in the folder tree
content_hash: u64,
/// Number of files indexed last time
file_count: usize,
}
pub struct LocalFileMonitor {
state: Arc<AppState>,
data_dir: PathBuf,
work_root: PathBuf,
file_states: Arc<RwLock<HashMap<String, LocalFileState>>>,
kb_states: Arc<RwLock<HashMap<String, KbFolderState>>>,
is_processing: Arc<AtomicBool>,
#[cfg(any(feature = "research", feature = "llm"))]
kb_manager: Option<Arc<KnowledgeBaseManager>>,
@ -57,6 +67,7 @@ impl LocalFileMonitor {
data_dir,
work_root,
file_states: Arc::new(RwLock::new(HashMap::new())),
kb_states: Arc::new(RwLock::new(HashMap::new())),
is_processing: Arc::new(AtomicBool::new(false)),
#[cfg(any(feature = "research", feature = "llm"))]
kb_manager,
@ -71,9 +82,15 @@ impl LocalFileMonitor {
warn!("Failed to create data directory: {}", e);
}
// Load persisted file states from disk
self.load_states().await;
// Initial scan of all .gbai directories
self.scan_and_compile_all().await?;
// Persist states back to disk
self.save_states().await;
self.is_processing.store(true, Ordering::SeqCst);
// Spawn the monitoring loop
@ -236,8 +253,6 @@ impl LocalFileMonitor {
gbkb_path: &Path,
_kb_manager: &Arc<KnowledgeBaseManager>,
) -> Result<(), Box<dyn Error + Send + Sync>> {
info!("Indexing .gbkb folder for bot {}: {:?}", bot_name, gbkb_path);
// Get bot_id from database
let bot_id = {
use crate::core::shared::models::schema::bots::dsl::*;
@ -252,12 +267,9 @@ impl LocalFileMonitor {
// Load bot-specific embedding config from database
let embedding_config = EmbeddingConfig::from_bot_config(&self.state.conn, &bot_id);
info!("Using embedding config for bot '{}': URL={}, model={}",
bot_name, embedding_config.embedding_url, embedding_config.embedding_model);
// Create a temporary KbIndexer with the bot-specific config
let qdrant_config = crate::core::kb::QdrantConfig::default();
let indexer = crate::core::kb::KbIndexer::new(embedding_config, qdrant_config);
// Compute content hash of the entire .gbkb tree
let (content_hash, file_count) = self.compute_gbkb_hash(gbkb_path).await?;
// Index each KB folder inside .gbkb (e.g., carta, proc)
let entries = tokio::fs::read_dir(gbkb_path).await?;
@ -268,8 +280,27 @@ impl LocalFileMonitor {
if kb_folder_path.is_dir() {
if let Some(kb_name) = kb_folder_path.file_name().and_then(|n| n.to_str()) {
let kb_key = format!("{}:{}", bot_name, kb_name);
// Check if KB content changed since last index
let should_index = {
let states = self.kb_states.read().await;
states.get(&kb_key)
.map(|state| state.content_hash != content_hash || state.file_count != file_count)
.unwrap_or(true)
};
if !should_index {
debug!("KB '{}' for bot '{}' unchanged, skipping re-index", kb_name, bot_name);
continue;
}
info!("Indexing KB '{}' for bot '{}'", kb_name, bot_name);
// Create a temporary KbIndexer with the bot-specific config
let qdrant_config = crate::core::kb::QdrantConfig::default();
let indexer = crate::core::kb::KbIndexer::new(embedding_config.clone(), qdrant_config);
if let Err(e) = indexer.index_kb_folder(
bot_id,
bot_name,
@ -278,6 +309,10 @@ impl LocalFileMonitor {
).await {
error!("Failed to index KB '{}' for bot '{}': {}", kb_name, bot_name, e);
}
// Update state to mark as indexed
let mut states = self.kb_states.write().await;
states.insert(kb_key, KbFolderState { content_hash, file_count });
}
}
}
@ -285,6 +320,36 @@ impl LocalFileMonitor {
Ok(())
}
/// Compute a simple hash over all file metadata in a folder tree
#[cfg(any(feature = "research", feature = "llm"))]
async fn compute_gbkb_hash(&self, root: &Path) -> Result<(u64, usize), Box<dyn Error + Send + Sync>> {
let mut hash: u64 = 0;
let mut file_count: usize = 0;
let mut stack = vec![root.to_path_buf()];
while let Some(dir) = stack.pop() {
let mut entries = tokio::fs::read_dir(&dir).await?;
while let Some(entry) = entries.next_entry().await? {
let path = entry.path();
if path.is_dir() {
stack.push(path);
} else {
if let Ok(meta) = tokio::fs::metadata(&path).await {
let mtime = meta.modified()
.map(|t| t.duration_since(SystemTime::UNIX_EPOCH).map(|d| d.as_secs()).unwrap_or(0))
.unwrap_or(0);
let size = meta.len();
// Simple combinatorial hash
hash = hash.wrapping_mul(31).wrapping_add(mtime.wrapping_mul(37).wrapping_add(size));
file_count += 1;
}
}
}
}
Ok((hash, file_count))
}
async fn compile_gbdialog(&self, bot_name: &str, gbdialog_path: &Path) -> Result<(), Box<dyn Error + Send + Sync>> {
let entries = tokio::fs::read_dir(gbdialog_path).await?;
let mut entries = entries;
@ -400,10 +465,93 @@ impl LocalFileMonitor {
states.remove(&file_key);
}
/// Persist file states and KB states to disk for survival across restarts
async fn save_states(&self) {
if let Err(e) = tokio::fs::create_dir_all(&self.work_root).await {
warn!("Failed to create work directory: {}", e);
return;
}
// Persist file states
let file_states_file = self.work_root.join("local_file_states.json");
{
let states = self.file_states.read().await;
match serde_json::to_string_pretty(&*states) {
Ok(json) => {
if let Err(e) = tokio::fs::write(&file_states_file, json).await {
warn!("Failed to persist file states: {}", e);
} else {
debug!("Persisted {} file states to disk", states.len());
}
}
Err(e) => warn!("Failed to serialize file states: {}", e),
}
}
// Persist KB states
let kb_states_file = self.work_root.join("local_kb_states.json");
{
let states = self.kb_states.read().await;
match serde_json::to_string_pretty(&*states) {
Ok(json) => {
if let Err(e) = tokio::fs::write(&kb_states_file, json).await {
warn!("Failed to persist KB states: {}", e);
} else {
debug!("Persisted {} KB states to disk", states.len());
}
}
Err(e) => warn!("Failed to serialize KB states: {}", e),
}
}
}
/// Load file states and KB states from disk
async fn load_states(&self) {
if let Err(e) = tokio::fs::create_dir_all(&self.work_root).await {
warn!("Failed to create work directory: {}", e);
}
// Load file states
let file_states_file = self.work_root.join("local_file_states.json");
match tokio::fs::read_to_string(&file_states_file).await {
Ok(json) => {
match serde_json::from_str::<HashMap<String, LocalFileState>>(&json) {
Ok(states) => {
let count = states.len();
*self.file_states.write().await = states;
info!("Loaded {} persisted file states from disk", count);
}
Err(e) => warn!("Failed to parse persisted file states: {}", e),
}
}
Err(_) => {
debug!("No persisted file states found, starting fresh");
}
}
// Load KB states
let kb_states_file = self.work_root.join("local_kb_states.json");
match tokio::fs::read_to_string(&kb_states_file).await {
Ok(json) => {
match serde_json::from_str::<HashMap<String, KbFolderState>>(&json) {
Ok(states) => {
let count = states.len();
*self.kb_states.write().await = states;
info!("Loaded {} persisted KB states from disk", count);
}
Err(e) => warn!("Failed to parse persisted KB states: {}", e),
}
}
Err(_) => {
debug!("No persisted KB states found, starting fresh");
}
}
}
pub async fn stop_monitoring(&self) {
trace!("Stopping local file monitor");
self.is_processing.store(false, Ordering::SeqCst);
self.file_states.write().await.clear();
self.save_states().await;
}
}
@ -414,6 +562,7 @@ impl Clone for LocalFileMonitor {
data_dir: self.data_dir.clone(),
work_root: self.work_root.clone(),
file_states: Arc::clone(&self.file_states),
kb_states: Arc::clone(&self.kb_states),
is_processing: Arc::clone(&self.is_processing),
#[cfg(any(feature = "research", feature = "llm"))]
kb_manager: self.kb_manager.clone(),

71
src/security/command_guard.rs
View file

@ -54,6 +54,8 @@ static ALLOWED_COMMANDS: LazyLock<HashSet<&'static str>> = LazyLock::new(|| {
"cargo",
"redis-server",
"redis-cli",
"valkey-cli",
"valkey-server",
"minio",
"chromedriver",
"chrome",
@ -82,7 +84,11 @@ static ALLOWED_COMMANDS: LazyLock<HashSet<&'static str>> = LazyLock::new(|| {
// LLM local servers
"llama-server",
"ollama",
// Secrets management
"vault",
// Python
"nc",
"netcat",
"python",
"python3",
"python3.11",
@ -337,6 +343,20 @@ impl SafeCommand {
path_entries.insert(0, shared_bin);
}
// Add component bin directories to PATH
let component_bins = [
format!("{}/bin/cache/bin", stack_path),
format!("{}/bin/tables/bin", stack_path),
format!("{}/bin/vault", stack_path),
format!("{}/bin/drive", stack_path),
format!("{}/bin/directory", stack_path),
];
for bin_dir in component_bins {
if std::path::Path::new(&bin_dir).exists() {
path_entries.insert(0, bin_dir);
}
}
cmd.env("PATH", path_entries.join(":"));
cmd.env("HOME", dirs::home_dir().unwrap_or_else(|| PathBuf::from("/tmp")));
cmd.env("LANG", "C.UTF-8");
@ -361,9 +381,12 @@ impl SafeCommand {
// Build PATH with standard locations plus botserver-stack/bin/shared
let mut path_entries = vec![
"/snap/bin".to_string(),
"/usr/local/bin".to_string(),
"/usr/bin".to_string(),
"/bin".to_string(),
"/usr/sbin".to_string(),
"/sbin".to_string(),
];
// Add botserver-stack/bin/shared to PATH if it exists
@ -374,6 +397,20 @@ impl SafeCommand {
path_entries.insert(0, shared_bin);
}
// Add component bin directories to PATH
let component_bins = [
format!("{}/bin/cache/bin", stack_path),
format!("{}/bin/tables/bin", stack_path),
format!("{}/bin/vault", stack_path),
format!("{}/bin/drive", stack_path),
format!("{}/bin/directory", stack_path),
];
for bin_dir in component_bins {
if std::path::Path::new(&bin_dir).exists() {
path_entries.insert(0, bin_dir);
}
}
cmd.env("PATH", path_entries.join(":"));
cmd.env("HOME", dirs::home_dir().unwrap_or_else(|| PathBuf::from("/tmp")));
cmd.env("LANG", "C.UTF-8");
@ -406,9 +443,12 @@ impl SafeCommand {
// Build PATH with standard locations plus botserver-stack/bin/shared
let mut path_entries = vec![
"/snap/bin".to_string(),
"/usr/local/bin".to_string(),
"/usr/bin".to_string(),
"/bin".to_string(),
"/usr/sbin".to_string(),
"/sbin".to_string(),
];
// Add botserver-stack/bin/shared to PATH if it exists
@ -419,6 +459,20 @@ impl SafeCommand {
path_entries.insert(0, shared_bin);
}
// Add component bin directories to PATH
let component_bins = [
format!("{}/bin/cache/bin", stack_path),
format!("{}/bin/tables/bin", stack_path),
format!("{}/bin/vault", stack_path),
format!("{}/bin/drive", stack_path),
format!("{}/bin/directory", stack_path),
];
for bin_dir in component_bins {
if std::path::Path::new(&bin_dir).exists() {
path_entries.insert(0, bin_dir);
}
}
cmd.env("PATH", path_entries.join(":"));
cmd.env("HOME", dirs::home_dir().unwrap_or_else(|| PathBuf::from("/tmp")));
cmd.env("LANG", "C.UTF-8");
@ -443,9 +497,12 @@ impl SafeCommand {
// Build PATH with standard locations plus botserver-stack/bin/shared
let mut path_entries = vec![
"/snap/bin".to_string(),
"/usr/local/bin".to_string(),
"/usr/bin".to_string(),
"/bin".to_string(),
"/usr/sbin".to_string(),
"/sbin".to_string(),
];
// Add botserver-stack/bin/shared to PATH if it exists
@ -456,6 +513,20 @@ impl SafeCommand {
path_entries.insert(0, shared_bin);
}
// Add component bin directories to PATH
let component_bins = [
format!("{}/bin/cache/bin", stack_path),
format!("{}/bin/tables/bin", stack_path),
format!("{}/bin/vault", stack_path),
format!("{}/bin/drive", stack_path),
format!("{}/bin/directory", stack_path),
];
for bin_dir in component_bins {
if std::path::Path::new(&bin_dir).exists() {
path_entries.insert(0, bin_dir);
}
}
cmd.env("PATH", path_entries.join(":"));
cmd.env("HOME", dirs::home_dir().unwrap_or_else(|| PathBuf::from("/tmp")));
cmd.env("LANG", "C.UTF-8");