use crate::core::package_manager::component::ComponentConfig; use crate::core::package_manager::os::detect_os; use crate::core::package_manager::{InstallMode, OsType}; use crate::core::shared::utils::get_stack_path; use crate::security::command_guard::SafeCommand; use anyhow::{Context, Result}; use log::{error, info, trace, warn}; use serde::Deserialize; use std::collections::HashMap; use std::path::PathBuf; #[cfg(unix)] use std::os::unix::fs::PermissionsExt; #[derive(Deserialize, Debug)] struct ComponentEntry { url: String, } #[derive(Deserialize, Debug)] struct ThirdPartyConfig { components: HashMap, } static THIRDPARTY_CONFIG: std::sync::OnceLock = std::sync::OnceLock::new(); fn get_thirdparty_config() -> &'static ThirdPartyConfig { THIRDPARTY_CONFIG.get_or_init(|| { let toml_str = include_str!("../../../3rdparty.toml"); match toml::from_str::(toml_str) { Ok(config) => config, Err(e) => { error!("CRITICAL: Failed to parse embedded 3rdparty.toml: {e}"); ThirdPartyConfig { components: HashMap::new(), } } } }) } fn get_component_url(name: &str) -> Option { get_thirdparty_config() .components .get(name) .map(|c| c.url.clone()) } #[cfg(target_os = "windows")] fn safe_nvcc_version() -> Option { SafeCommand::new("nvcc") .and_then(|c| c.arg("--version")) .ok() .and_then(|cmd| cmd.execute().ok()) } fn safe_sh_command(script: &str) -> Option { SafeCommand::new("sh") .and_then(|c| c.arg("-c")) .and_then(|c| c.trusted_shell_script_arg(script)) .ok() .and_then(|cmd| cmd.execute().ok()) } fn safe_pgrep(args: &[&str]) -> Option { SafeCommand::new("pgrep") .and_then(|c| c.args(args)) .ok() .and_then(|cmd| cmd.execute().ok()) } const LLAMA_CPP_VERSION: &str = "b7345"; fn get_llama_cpp_url() -> Option { #[cfg(target_os = "linux")] { #[cfg(target_arch = "x86_64")] { if std::path::Path::new("/usr/local/cuda").exists() || std::path::Path::new("/opt/cuda").exists() || std::env::var("CUDA_HOME").is_ok() { // CUDA versions not currently in 3rdparty.toml for Linux, falling back to Vulkan or CPU if not added // Or if we had them: get_component_url("llm_linux_cuda12") } if std::path::Path::new("/usr/share/vulkan").exists() || std::env::var("VULKAN_SDK").is_ok() { info!("Detected Vulkan - using Vulkan build"); return get_component_url("llm_linux_vulkan"); } info!("Using standard Ubuntu x64 build (CPU)"); get_component_url("llm") } #[cfg(target_arch = "s390x")] { info!("Detected s390x architecture"); return get_component_url("llm_linux_s390x"); } #[cfg(target_arch = "aarch64")] { info!("Detected ARM64 architecture on Linux"); warn!("No pre-built llama.cpp for Linux ARM64 - LLM will not be available"); return None; } } #[cfg(target_os = "macos")] { #[cfg(target_arch = "aarch64")] { info!("Detected macOS ARM64 (Apple Silicon)"); return get_component_url("llm_macos_arm64"); } #[cfg(target_arch = "x86_64")] { info!("Detected macOS x64 (Intel)"); return get_component_url("llm_macos_x64"); } } #[cfg(target_os = "windows")] { #[cfg(target_arch = "x86_64")] { if std::env::var("CUDA_PATH").is_ok() { if let Some(output) = safe_nvcc_version() { let version_str = String::from_utf8_lossy(&output.stdout); if version_str.contains("13.") { info!("Detected CUDA 13.x on Windows"); return get_component_url("llm_win_cuda13"); } else if version_str.contains("12.") { info!("Detected CUDA 12.x on Windows"); return get_component_url("llm_win_cuda12"); } } } if std::env::var("VULKAN_SDK").is_ok() { info!("Detected Vulkan SDK on Windows"); return get_component_url("llm_win_vulkan"); } info!("Using standard Windows x64 CPU build"); return get_component_url("llm_win_cpu_x64"); } #[cfg(target_arch = "aarch64")] { info!("Detected Windows ARM64"); return get_component_url("llm_win_cpu_arm64"); } } } #[derive(Debug)] pub struct PackageManager { pub mode: InstallMode, pub os_type: OsType, pub base_path: PathBuf, pub tenant: String, pub components: HashMap, } impl PackageManager { pub fn new(mode: InstallMode, tenant: Option) -> Result { let os_type = detect_os(); let base_path = if mode == InstallMode::Container { PathBuf::from("/opt/gbo") } else if let Ok(custom_path) = std::env::var("BOTSERVER_STACK_PATH") { PathBuf::from(custom_path) } else { std::env::current_dir()?.join("botserver-stack") }; let tenant = tenant.unwrap_or_else(|| "default".to_string()); let mut pm = Self { mode, os_type, base_path, tenant, components: HashMap::new(), }; pm.register_components(); Ok(pm) } pub fn with_base_path( mode: InstallMode, tenant: Option, base_path: PathBuf, ) -> Result { let os_type = detect_os(); let tenant = tenant.unwrap_or_else(|| "default".to_string()); let mut pm = Self { mode, os_type, base_path, tenant, components: HashMap::new(), }; pm.register_components(); Ok(pm) } fn register_components(&mut self) { self.register_vault(); self.register_tables(); self.register_cache(); self.register_drive(); self.register_llm(); self.register_email(); self.register_proxy(); self.register_dns(); self.register_directory(); self.register_alm(); self.register_alm_ci(); self.register_meeting(); self.register_remote_terminal(); self.register_devtools(); self.register_vector_db(); self.register_timeseries_db(); self.register_observability(); self.register_host(); self.register_webmail(); self.register_table_editor(); self.register_doc_editor(); } fn register_drive(&mut self) { self.components.insert( "drive".to_string(), ComponentConfig { name: "drive".to_string(), ports: vec![9100, 9101], dependencies: vec![], linux_packages: vec![], macos_packages: vec![], windows_packages: vec![], download_url: get_component_url("drive"), binary_name: Some("minio".to_string()), pre_install_cmds_linux: vec![], post_install_cmds_linux: vec![], pre_install_cmds_macos: vec![], post_install_cmds_macos: vec![], pre_install_cmds_windows: vec![], post_install_cmds_windows: vec![], env_vars: HashMap::from([ ("MINIO_ROOT_USER".to_string(), "$DRIVE_ACCESSKEY".to_string()), ("MINIO_ROOT_PASSWORD".to_string(), "$DRIVE_SECRET".to_string()), ]), data_download_list: Vec::new(), exec_cmd: "nohup {{BIN_PATH}}/minio server {{DATA_PATH}} --address 127.0.0.1:9100 --console-address 127.0.0.1:9101 --certs-dir {{CONF_PATH}}/drive/certs > {{LOGS_PATH}}/minio.log 2>&1 &".to_string(), check_cmd: "curl -sf --cacert {{CONF_PATH}}/drive/certs/CAs/ca.crt https://127.0.0.1:9100/minio/health/live >/dev/null 2>&1".to_string(), }, ); } fn register_tables(&mut self) { self.components.insert( "tables".to_string(), ComponentConfig { name: "tables".to_string(), ports: vec![5432], dependencies: vec![], linux_packages: vec![], macos_packages: vec![], windows_packages: vec![], download_url: get_component_url("tables"), binary_name: Some("postgres".to_string()), pre_install_cmds_linux: vec![], post_install_cmds_linux: vec![ "chmod +x ./bin/*".to_string(), "if [ ! -d \"{{DATA_PATH}}/pgdata\" ]; then PG_PASSWORD='{{DB_PASSWORD}}' ./bin/initdb -D {{DATA_PATH}}/pgdata -U gbuser --pwfile=<(echo \"$PG_PASSWORD\"); fi".to_string(), "echo \"data_directory = '{{DATA_PATH}}/pgdata'\" > {{CONF_PATH}}/postgresql.conf".to_string(), "echo \"ident_file = '{{CONF_PATH}}/pg_ident.conf'\" >> {{CONF_PATH}}/postgresql.conf".to_string(), "echo \"port = 5432\" >> {{CONF_PATH}}/postgresql.conf".to_string(), "echo \"listen_addresses = '*'\" >> {{CONF_PATH}}/postgresql.conf".to_string(), "echo \"ssl = on\" >> {{CONF_PATH}}/postgresql.conf".to_string(), "echo \"ssl_cert_file = '{{CONF_PATH}}/system/certificates/tables/server.crt'\" >> {{CONF_PATH}}/postgresql.conf".to_string(), "echo \"ssl_key_file = '{{CONF_PATH}}/system/certificates/tables/server.key'\" >> {{CONF_PATH}}/postgresql.conf".to_string(), "echo \"ssl_ca_file = '{{CONF_PATH}}/system/certificates/ca/ca.crt'\" >> {{CONF_PATH}}/postgresql.conf".to_string(), "echo \"log_directory = '{{LOGS_PATH}}'\" >> {{CONF_PATH}}/postgresql.conf".to_string(), "echo \"logging_collector = on\" >> {{CONF_PATH}}/postgresql.conf".to_string(), "echo \"hostssl all all all md5\" > {{CONF_PATH}}/pg_hba.conf".to_string(), "touch {{CONF_PATH}}/pg_ident.conf".to_string(), "./bin/pg_ctl -D {{DATA_PATH}}/pgdata -l {{LOGS_PATH}}/postgres.log start -w -t 30".to_string(), "sleep 5".to_string(), "for i in $(seq 1 30); do ./bin/pg_isready -h localhost -p 5432 -d postgres >/dev/null 2>&1 && echo 'PostgreSQL is ready' && break || echo \"Waiting for PostgreSQL... attempt $i/30\" >&2; sleep 2; done".to_string(), "./bin/pg_isready -h localhost -p 5432 -d postgres || { echo 'ERROR: PostgreSQL failed to start properly' >&2; cat {{LOGS_PATH}}/postgres.log >&2; exit 1; }".to_string(), "PGPASSWORD='{{DB_PASSWORD}}' ./bin/psql -h localhost -p 5432 -U gbuser -d postgres -c \"CREATE DATABASE botserver WITH OWNER gbuser\" 2>&1 | grep -v 'already exists' || true".to_string(), ], pre_install_cmds_macos: vec![], post_install_cmds_macos: vec![ "chmod +x ./bin/*".to_string(), "if [ ! -d \"{{DATA_PATH}}/pgdata\" ]; then ./bin/initdb -A -D {{DATA_PATH}}/pgdata -U postgres; fi".to_string(), ], pre_install_cmds_windows: vec![], post_install_cmds_windows: vec![], env_vars: HashMap::new(), data_download_list: Vec::new(), exec_cmd: "./bin/pg_ctl -D {{DATA_PATH}}/pgdata -l {{LOGS_PATH}}/postgres.log start -w -t 30 > {{LOGS_PATH}}/stdout.log 2>&1 &".to_string(), check_cmd: "{{BIN_PATH}}/bin/pg_isready -h localhost -p 5432 -d postgres >/dev/null 2>&1".to_string(), }, ); } fn register_cache(&mut self) { self.components.insert( "cache".to_string(), ComponentConfig { name: "cache".to_string(), ports: vec![6379], dependencies: vec![], linux_packages: vec![], macos_packages: vec![], windows_packages: vec![], download_url: get_component_url("cache"), binary_name: Some("valkey-server".to_string()), pre_install_cmds_linux: vec![], post_install_cmds_linux: vec![ "mkdir -p {{BIN_PATH}}/bin && cd {{BIN_PATH}}/bin && tar -xzf {{CACHE_FILE}} --strip-components=1 -C {{BIN_PATH}}/bin 2>/dev/null || true".to_string(), "chmod +x {{BIN_PATH}}/bin/valkey-server 2>/dev/null || true".to_string(), "chmod +x {{BIN_PATH}}/bin/valkey-cli 2>/dev/null || true".to_string(), "chmod +x {{BIN_PATH}}/bin/valkey-benchmark 2>/dev/null || true".to_string(), "chmod +x {{BIN_PATH}}/bin/valkey-check-aof 2>/dev/null || true".to_string(), "chmod +x {{BIN_PATH}}/bin/valkey-check-rdb 2>/dev/null || true".to_string(), "chmod +x {{BIN_PATH}}/bin/valkey-sentinel 2>/dev/null || true".to_string(), ], pre_install_cmds_macos: vec![], post_install_cmds_macos: vec![], pre_install_cmds_windows: vec![], post_install_cmds_windows: vec![], env_vars: HashMap::new(), data_download_list: Vec::new(), exec_cmd: "nohup {{BIN_PATH}}/bin/valkey-server --port 6379 --bind 127.0.0.1 --dir {{DATA_PATH}} --logfile {{LOGS_PATH}}/valkey.log --daemonize yes > {{LOGS_PATH}}/valkey-startup.log 2>&1".to_string(), check_cmd: "pgrep -x valkey-server >/dev/null 2>&1".to_string(), }, ); } fn register_llm(&mut self) { let download_url = get_llama_cpp_url(); if download_url.is_none() { warn!("No llama.cpp binary available for this platform"); warn!("Local LLM will not be available - use external API instead"); } info!( "LLM component using llama.cpp {} for this platform", LLAMA_CPP_VERSION ); self.components.insert( "llm".to_string(), ComponentConfig { name: "llm".to_string(), ports: vec![8081, 8082], dependencies: vec![], linux_packages: vec![], macos_packages: vec![], windows_packages: vec![], download_url, binary_name: Some("llama-server".to_string()), pre_install_cmds_linux: vec![], post_install_cmds_linux: vec![], pre_install_cmds_macos: vec![], post_install_cmds_macos: vec![], pre_install_cmds_windows: vec![], post_install_cmds_windows: vec![], env_vars: HashMap::new(), data_download_list: vec![ "https://huggingface.co/bartowski/DeepSeek-R1-Distill-Qwen-1.5B-GGUF/resolve/main/DeepSeek-R1-Distill-Qwen-1.5B-Q3_K_M.gguf".to_string(), "https://huggingface.co/CompendiumLabs/bge-small-en-v1.5-gguf/resolve/main/bge-small-en-v1.5-f32.gguf".to_string(), ], exec_cmd: "nohup {{BIN_PATH}}/build/bin/llama-server --port 8081 --ssl-key-file {{CONF_PATH}}/system/certificates/llm/server.key --ssl-cert-file {{CONF_PATH}}/system/certificates/llm/server.crt -m {{DATA_PATH}}/DeepSeek-R1-Distill-Qwen-1.5B-Q3_K_M.gguf --ubatch-size 512 > {{LOGS_PATH}}/llm.log 2>&1 & nohup {{BIN_PATH}}/build/bin/llama-server --port 8082 --ssl-key-file {{CONF_PATH}}/system/certificates/embedding/server.key --ssl-cert-file {{CONF_PATH}}/system/certificates/embedding/server.crt -m {{DATA_PATH}}/bge-small-en-v1.5-f32.gguf --embedding --ubatch-size 512 > {{LOGS_PATH}}/embedding.log 2>&1 &".to_string(), check_cmd: "curl -f -k --connect-timeout 2 -m 5 https://localhost:8081/health >/dev/null 2>&1 && curl -f -k --connect-timeout 2 -m 5 https://localhost:8082/health >/dev/null 2>&1".to_string(), }, ); } fn register_email(&mut self) { self.components.insert( "email".to_string(), ComponentConfig { name: "email".to_string(), ports: vec![25, 143, 465, 993, 8025], dependencies: vec![], linux_packages: vec![], macos_packages: vec![], windows_packages: vec![], download_url: get_component_url("email"), binary_name: Some("stalwart-mail".to_string()), pre_install_cmds_linux: vec![], post_install_cmds_linux: vec![], pre_install_cmds_macos: vec![], post_install_cmds_macos: vec![], pre_install_cmds_windows: vec![], post_install_cmds_windows: vec![], env_vars: HashMap::from([ ("STALWART_TLS_ENABLE".to_string(), "true".to_string()), ("STALWART_TLS_CERT".to_string(), "{{CONF_PATH}}/system/certificates/email/server.crt".to_string()), ("STALWART_TLS_KEY".to_string(), "{{CONF_PATH}}/system/certificates/email/server.key".to_string()), ]), data_download_list: Vec::new(), exec_cmd: "{{BIN_PATH}}/stalwart-mail --config {{CONF_PATH}}/email/config.toml".to_string(), check_cmd: "curl -f -k --connect-timeout 2 -m 5 https://localhost:8025/health >/dev/null 2>&1".to_string(), }, ); } fn register_proxy(&mut self) { self.components.insert( "proxy".to_string(), ComponentConfig { name: "proxy".to_string(), ports: vec![80, 443], dependencies: vec![], linux_packages: vec![], macos_packages: vec![], windows_packages: vec![], download_url: get_component_url("proxy"), binary_name: Some("caddy".to_string()), pre_install_cmds_linux: vec![], post_install_cmds_linux: vec![ "setcap 'cap_net_bind_service=+ep' {{BIN_PATH}}/caddy".to_string(), ], pre_install_cmds_macos: vec![], post_install_cmds_macos: vec![], pre_install_cmds_windows: vec![], post_install_cmds_windows: vec![], env_vars: HashMap::from([( "XDG_DATA_HOME".to_string(), "{{DATA_PATH}}".to_string(), )]), data_download_list: Vec::new(), exec_cmd: "{{BIN_PATH}}/caddy run --config {{CONF_PATH}}/Caddyfile".to_string(), check_cmd: "curl -f --connect-timeout 2 -m 5 http://localhost >/dev/null 2>&1" .to_string(), }, ); } fn register_directory(&mut self) { self.components.insert( "directory".to_string(), ComponentConfig { name: "directory".to_string(), ports: vec![9000], dependencies: vec!["tables".to_string()], linux_packages: vec![], macos_packages: vec![], windows_packages: vec![], download_url: get_component_url("directory"), binary_name: Some("zitadel".to_string()), pre_install_cmds_linux: vec![ "mkdir -p {{CONF_PATH}}/directory".to_string(), "mkdir -p {{LOGS_PATH}}".to_string(), // Create Zitadel steps YAML: configures a machine user (service account) // with IAM_OWNER role and writes a PAT file for API bootstrap concat!( "cat > {{CONF_PATH}}/directory/zitadel-init-steps.yaml << 'STEPSEOF'\n", "FirstInstance:\n", " Org:\n", " Machine:\n", " Machine:\n", " Username: gb-service-account\n", " Name: General Bots Service Account\n", " MachineKey:\n", " Type: 1\n", " Pat:\n", " ExpirationDate: '2099-01-01T00:00:00Z'\n", " PatPath: {{CONF_PATH}}/directory/admin-pat.txt\n", " MachineKeyPath: {{CONF_PATH}}/directory/machine-key.json\n", "STEPSEOF", ).to_string(), ], post_install_cmds_linux: vec![ // Create zitadel DB user before start-from-init "PGPASSWORD='{{DB_PASSWORD}}' {{STACK_PATH}}/bin/tables/bin/psql -h localhost -p 5432 -U gbuser -d postgres -c \"CREATE ROLE zitadel WITH LOGIN PASSWORD 'zitadel'\" 2>&1 | grep -v 'already exists' || true".to_string(), "PGPASSWORD='{{DB_PASSWORD}}' {{STACK_PATH}}/bin/tables/bin/psql -h localhost -p 5432 -U gbuser -d postgres -c \"CREATE DATABASE zitadel WITH OWNER zitadel\" 2>&1 | grep -v 'already exists' || true".to_string(), "PGPASSWORD='{{DB_PASSWORD}}' {{STACK_PATH}}/bin/tables/bin/psql -h localhost -p 5432 -U gbuser -d postgres -c \"GRANT ALL PRIVILEGES ON DATABASE zitadel TO zitadel\" 2>&1 || true".to_string(), // Start Zitadel with --steps pointing to our init file (creates machine user + PAT) concat!( "ZITADEL_PORT=8300 ", "ZITADEL_DATABASE_POSTGRES_HOST=localhost ", "ZITADEL_DATABASE_POSTGRES_PORT=5432 ", "ZITADEL_DATABASE_POSTGRES_DATABASE=zitadel ", "ZITADEL_DATABASE_POSTGRES_USER_USERNAME=zitadel ", "ZITADEL_DATABASE_POSTGRES_USER_PASSWORD=zitadel ", "ZITADEL_DATABASE_POSTGRES_USER_SSL_MODE=disable ", "ZITADEL_DATABASE_POSTGRES_ADMIN_USERNAME=gbuser ", "ZITADEL_DATABASE_POSTGRES_ADMIN_PASSWORD={{DB_PASSWORD}} ", "ZITADEL_DATABASE_POSTGRES_ADMIN_SSL_MODE=disable ", "ZITADEL_EXTERNALSECURE=false ", "ZITADEL_EXTERNALDOMAIN=localhost ", "ZITADEL_EXTERNALPORT=8300 ", "ZITADEL_TLS_ENABLED=false ", "nohup {{BIN_PATH}}/zitadel start-from-init ", "--masterkey MasterkeyNeedsToHave32Characters ", "--tlsMode disabled ", "--steps {{CONF_PATH}}/directory/zitadel-init-steps.yaml ", "> {{LOGS_PATH}}/zitadel.log 2>&1 &", ).to_string(), // Wait for Zitadel to be ready "for i in $(seq 1 120); do curl -sf http://localhost:8300/debug/healthz && echo 'Zitadel is ready!' && break || sleep 2; done".to_string(), // Wait for PAT token to be written to logs with retry loop // Zitadel may take several seconds to write the PAT after health check passes "echo 'Waiting for PAT token in logs...'; for i in $(seq 1 30); do sync; if grep -q -E '^[A-Za-z0-9_-]{40,}$' {{LOGS_PATH}}/zitadel.log 2>/dev/null; then echo \"PAT token found in logs after $((i*2)) seconds\"; break; fi; sleep 2; done".to_string(), // Extract PAT token from logs if Zitadel printed it to stdout instead of file // The PAT appears as a standalone line (alphanumeric with hyphens/underscores) after machine key JSON "if [ ! -f '{{CONF_PATH}}/directory/admin-pat.txt' ]; then grep -E '^[A-Za-z0-9_-]{40,}$' {{LOGS_PATH}}/zitadel.log 2>/dev/null | head -1 > {{CONF_PATH}}/directory/admin-pat.txt && echo 'PAT extracted from logs' || echo 'Could not extract PAT from logs'; fi".to_string(), // Verify PAT file was created and is not empty "sync; sleep 1; if [ -f '{{CONF_PATH}}/directory/admin-pat.txt' ] && [ -s '{{CONF_PATH}}/directory/admin-pat.txt' ]; then echo 'PAT token created successfully'; cat {{CONF_PATH}}/directory/admin-pat.txt; else echo 'WARNING: PAT file not found or empty'; fi".to_string(), ], pre_install_cmds_macos: vec![ "mkdir -p {{CONF_PATH}}/directory".to_string(), ], post_install_cmds_macos: vec![], pre_install_cmds_windows: vec![], post_install_cmds_windows: vec![], env_vars: HashMap::from([ ("ZITADEL_PORT".to_string(), "8300".to_string()), ("ZITADEL_EXTERNALSECURE".to_string(), "false".to_string()), ("ZITADEL_EXTERNALDOMAIN".to_string(), "localhost".to_string()), ("ZITADEL_EXTERNALPORT".to_string(), "8300".to_string()), ("ZITADEL_TLS_ENABLED".to_string(), "false".to_string()), ]), data_download_list: Vec::new(), exec_cmd: concat!( "ZITADEL_PORT=8300 ", "ZITADEL_DATABASE_POSTGRES_HOST=localhost ", "ZITADEL_DATABASE_POSTGRES_PORT=5432 ", "ZITADEL_DATABASE_POSTGRES_DATABASE=zitadel ", "ZITADEL_DATABASE_POSTGRES_USER_USERNAME=zitadel ", "ZITADEL_DATABASE_POSTGRES_USER_PASSWORD=zitadel ", "ZITADEL_DATABASE_POSTGRES_USER_SSL_MODE=disable ", "ZITADEL_DATABASE_POSTGRES_ADMIN_USERNAME=gbuser ", "ZITADEL_DATABASE_POSTGRES_ADMIN_PASSWORD={{DB_PASSWORD}} ", "ZITADEL_DATABASE_POSTGRES_ADMIN_SSL_MODE=disable ", "ZITADEL_EXTERNALSECURE=false ", "ZITADEL_EXTERNALDOMAIN=localhost ", "ZITADEL_EXTERNALPORT=8300 ", "ZITADEL_TLS_ENABLED=false ", "nohup {{BIN_PATH}}/zitadel start ", "--masterkey MasterkeyNeedsToHave32Characters ", "--tlsMode disabled ", "> {{LOGS_PATH}}/zitadel.log 2>&1 &", ).to_string(), check_cmd: "curl -f --connect-timeout 2 -m 5 http://localhost:8300/debug/healthz >/dev/null 2>&1".to_string(), }, ); } fn register_alm(&mut self) { self.components.insert( "alm".to_string(), ComponentConfig { name: "alm".to_string(), ports: vec![3000], dependencies: vec![], linux_packages: vec![], macos_packages: vec![], windows_packages: vec![], download_url: get_component_url("alm"), binary_name: Some("forgejo".to_string()), pre_install_cmds_linux: vec![], post_install_cmds_linux: vec![], pre_install_cmds_macos: vec![], post_install_cmds_macos: vec![], pre_install_cmds_windows: vec![], post_install_cmds_windows: vec![], env_vars: HashMap::from([ ("USER".to_string(), "alm".to_string()), ("HOME".to_string(), "{{DATA_PATH}}".to_string()), ]), data_download_list: Vec::new(), exec_cmd: "nohup {{BIN_PATH}}/forgejo web --work-path {{DATA_PATH}} --port 3000 --cert {{CONF_PATH}}/system/certificates/alm/server.crt --key {{CONF_PATH}}/system/certificates/alm/server.key > {{LOGS_PATH}}/forgejo.log 2>&1 &".to_string(), check_cmd: "curl -f -k --connect-timeout 2 -m 5 https://localhost:3000 >/dev/null 2>&1".to_string(), }, ); } fn register_alm_ci(&mut self) { self.components.insert( "alm-ci".to_string(), ComponentConfig { name: "alm-ci".to_string(), ports: vec![], dependencies: vec!["alm".to_string()], linux_packages: vec![], macos_packages: vec!["git".to_string(), "node".to_string()], windows_packages: vec![], download_url: get_component_url("alm_ci"), binary_name: Some("forgejo-runner".to_string()), pre_install_cmds_linux: vec![ "mkdir -p {{CONF_PATH}}/alm-ci".to_string(), ], post_install_cmds_linux: vec![ "echo 'To register the runner, run:'".to_string(), "echo '{{BIN_PATH}}/forgejo-runner register --instance $ALM_URL --token $ALM_RUNNER_TOKEN --name gbo --labels ubuntu-latest:docker://node:20-bookworm'".to_string(), "echo 'Then start with: {{BIN_PATH}}/forgejo-runner daemon --config {{CONF_PATH}}/alm-ci/config.yaml'".to_string(), ], pre_install_cmds_macos: vec![], post_install_cmds_macos: vec![], pre_install_cmds_windows: vec![], post_install_cmds_windows: vec![], env_vars: { let mut env = HashMap::new(); env.insert("ALM_URL".to_string(), "$ALM_URL".to_string()); env.insert("ALM_RUNNER_TOKEN".to_string(), "$ALM_RUNNER_TOKEN".to_string()); env }, data_download_list: Vec::new(), exec_cmd: "nohup {{BIN_PATH}}/forgejo-runner daemon --config {{CONF_PATH}}/alm-ci/config.yaml > {{LOGS_PATH}}/forgejo-runner.log 2>&1 &".to_string(), check_cmd: "ps -ef | grep forgejo-runner | grep -v grep | grep {{BIN_PATH}} >/dev/null 2>&1".to_string(), }, ); } fn register_dns(&mut self) { self.components.insert( "dns".to_string(), ComponentConfig { name: "dns".to_string(), ports: vec![53], dependencies: vec![], linux_packages: vec![], macos_packages: vec![], windows_packages: vec![], download_url: get_component_url("dns"), binary_name: Some("coredns".to_string()), pre_install_cmds_linux: vec![], post_install_cmds_linux: vec![], pre_install_cmds_macos: vec![], post_install_cmds_macos: vec![], pre_install_cmds_windows: vec![], post_install_cmds_windows: vec![], env_vars: HashMap::new(), data_download_list: Vec::new(), exec_cmd: "{{BIN_PATH}}/coredns -conf {{CONF_PATH}}/dns/Corefile".to_string(), check_cmd: "dig @localhost botserver.local >/dev/null 2>&1".to_string(), }, ); } fn register_webmail(&mut self) { self.components.insert( "webmail".to_string(), ComponentConfig { name: "webmail".to_string(), ports: vec![8300], dependencies: vec!["email".to_string()], linux_packages: vec![ "ca-certificates".to_string(), "apt-transport-https".to_string(), "php8.1".to_string(), "php8.1-fpm".to_string(), ], macos_packages: vec!["php".to_string()], windows_packages: vec![], download_url: get_component_url("webmail"), binary_name: None, pre_install_cmds_linux: vec![], post_install_cmds_linux: vec![], pre_install_cmds_macos: vec![], post_install_cmds_macos: vec![], pre_install_cmds_windows: vec![], post_install_cmds_windows: vec![], env_vars: HashMap::new(), data_download_list: Vec::new(), exec_cmd: "php -S 0.0.0.0:9000 -t {{DATA_PATH}}/roundcubemail".to_string(), check_cmd: "curl -f -k --connect-timeout 2 -m 5 https://localhost:8300 >/dev/null 2>&1" .to_string(), }, ); } fn register_meeting(&mut self) { self.components.insert( "meet".to_string(), ComponentConfig { name: "meet".to_string(), ports: vec![7880], dependencies: vec![], linux_packages: vec![], macos_packages: vec![], windows_packages: vec![], download_url: get_component_url("meet"), binary_name: Some("livekit-server".to_string()), pre_install_cmds_linux: vec![], post_install_cmds_linux: vec![], pre_install_cmds_macos: vec![], post_install_cmds_macos: vec![], pre_install_cmds_windows: vec![], post_install_cmds_windows: vec![], env_vars: HashMap::new(), data_download_list: Vec::new(), exec_cmd: "{{BIN_PATH}}/livekit-server --config {{CONF_PATH}}/meet/config.yaml --key-file {{CONF_PATH}}/system/certificates/meet/server.key --cert-file {{CONF_PATH}}/system/certificates/meet/server.crt".to_string(), check_cmd: "curl -f -k --connect-timeout 2 -m 5 https://localhost:7880 >/dev/null 2>&1".to_string(), }, ); } fn register_table_editor(&mut self) { self.components.insert( "table_editor".to_string(), ComponentConfig { name: "table_editor".to_string(), ports: vec![5757], dependencies: vec!["tables".to_string()], linux_packages: vec![], macos_packages: vec![], windows_packages: vec![], download_url: get_component_url("table_editor"), binary_name: Some("nocodb".to_string()), pre_install_cmds_linux: vec![], post_install_cmds_linux: vec![], pre_install_cmds_macos: vec![], post_install_cmds_macos: vec![], pre_install_cmds_windows: vec![], post_install_cmds_windows: vec![], env_vars: HashMap::new(), data_download_list: Vec::new(), exec_cmd: "{{BIN_PATH}}/nocodb".to_string(), check_cmd: "curl -f -k --connect-timeout 2 -m 5 https://localhost:5757 >/dev/null 2>&1" .to_string(), }, ); } fn register_doc_editor(&mut self) { self.components.insert( "doc_editor".to_string(), ComponentConfig { name: "doc_editor".to_string(), ports: vec![9980], dependencies: vec![], linux_packages: vec![], macos_packages: vec![], windows_packages: vec![], download_url: None, binary_name: Some("coolwsd".to_string()), pre_install_cmds_linux: vec![], post_install_cmds_linux: vec![], pre_install_cmds_macos: vec![], post_install_cmds_macos: vec![], pre_install_cmds_windows: vec![], post_install_cmds_windows: vec![], env_vars: HashMap::new(), data_download_list: Vec::new(), exec_cmd: "coolwsd --config-file={{CONF_PATH}}/coolwsd.xml".to_string(), check_cmd: "curl -f -k --connect-timeout 2 -m 5 https://localhost:9980 >/dev/null 2>&1" .to_string(), }, ); } fn register_remote_terminal(&mut self) { self.components.insert( "remote_terminal".to_string(), ComponentConfig { name: "remote_terminal".to_string(), ports: vec![3389], dependencies: vec![], linux_packages: vec!["xvfb".to_string(), "xrdp".to_string(), "xfce4".to_string()], macos_packages: vec![], windows_packages: vec![], download_url: None, binary_name: None, pre_install_cmds_linux: vec![], post_install_cmds_linux: vec![], pre_install_cmds_macos: vec![], post_install_cmds_macos: vec![], pre_install_cmds_windows: vec![], post_install_cmds_windows: vec![], env_vars: HashMap::new(), data_download_list: Vec::new(), exec_cmd: "xrdp --nodaemon".to_string(), check_cmd: "netstat -tln | grep :3389 >/dev/null 2>&1".to_string(), }, ); } fn register_devtools(&mut self) { self.components.insert( "devtools".to_string(), ComponentConfig { name: "devtools".to_string(), ports: vec![], dependencies: vec![], linux_packages: vec![], macos_packages: vec![], windows_packages: vec![], download_url: None, binary_name: None, pre_install_cmds_linux: vec![], post_install_cmds_linux: vec![], pre_install_cmds_macos: vec![], post_install_cmds_macos: vec![], pre_install_cmds_windows: vec![], post_install_cmds_windows: vec![], env_vars: HashMap::new(), data_download_list: Vec::new(), exec_cmd: "".to_string(), check_cmd: "".to_string(), }, ); } fn _register_botserver(&mut self) { self.components.insert( "system".to_string(), ComponentConfig { name: "system".to_string(), ports: vec![8000], dependencies: vec![], linux_packages: vec!["curl".to_string(), "unzip".to_string(), "git".to_string()], macos_packages: vec![], windows_packages: vec![], download_url: None, binary_name: None, pre_install_cmds_linux: vec![], post_install_cmds_linux: vec![], pre_install_cmds_macos: vec![], post_install_cmds_macos: vec![], pre_install_cmds_windows: vec![], post_install_cmds_windows: vec![], env_vars: HashMap::new(), data_download_list: Vec::new(), exec_cmd: "".to_string(), check_cmd: "".to_string(), }, ); } fn register_vector_db(&mut self) { self.components.insert( "vector_db".to_string(), ComponentConfig { name: "vector_db".to_string(), ports: vec![6334], dependencies: vec![], linux_packages: vec![], macos_packages: vec![], windows_packages: vec![], download_url: get_component_url("vector_db"), binary_name: Some("qdrant".to_string()), pre_install_cmds_linux: vec![], post_install_cmds_linux: vec![], pre_install_cmds_macos: vec![], post_install_cmds_macos: vec![], pre_install_cmds_windows: vec![], post_install_cmds_windows: vec![], env_vars: HashMap::new(), data_download_list: Vec::new(), exec_cmd: "nohup {{BIN_PATH}}/qdrant --config-path {{CONF_PATH}}/vector_db/config.yaml > {{LOGS_PATH}}/qdrant.log 2>&1 &".to_string(), check_cmd: "pgrep -x qdrant >/dev/null 2>&1".to_string(), }, ); } fn register_timeseries_db(&mut self) { self.components.insert( "timeseries_db".to_string(), ComponentConfig { name: "timeseries_db".to_string(), ports: vec![8086, 8083], dependencies: vec![], linux_packages: vec![], macos_packages: vec![], windows_packages: vec![], download_url: get_component_url("timeseries_db"), binary_name: Some("influxd".to_string()), pre_install_cmds_linux: vec![ "mkdir -p {{DATA_PATH}}/influxdb".to_string(), "mkdir -p {{CONF_PATH}}/influxdb".to_string(), ], post_install_cmds_linux: vec![ "{{BIN_PATH}}/influx setup --org system --bucket metrics --username admin --password {{GENERATED_PASSWORD}} --force".to_string(), ], pre_install_cmds_macos: vec![ "mkdir -p {{DATA_PATH}}/influxdb".to_string(), ], post_install_cmds_macos: vec![], pre_install_cmds_windows: vec![], post_install_cmds_windows: vec![], env_vars: { let mut env = HashMap::new(); env.insert("INFLUXD_ENGINE_PATH".to_string(), "{{DATA_PATH}}/influxdb/engine".to_string()); env.insert("INFLUXD_BOLT_PATH".to_string(), "{{DATA_PATH}}/influxdb/influxd.bolt".to_string()); env.insert("INFLUXD_HTTP_BIND_ADDRESS".to_string(), ":8086".to_string()); env.insert("INFLUXD_REPORTING_DISABLED".to_string(), "true".to_string()); env }, data_download_list: Vec::new(), exec_cmd: "{{BIN_PATH}}/influxd --bolt-path={{DATA_PATH}}/influxdb/influxd.bolt --engine-path={{DATA_PATH}}/influxdb/engine --http-bind-address=:8086".to_string(), check_cmd: "curl -f --connect-timeout 2 -m 5 http://localhost:8086/health >/dev/null 2>&1".to_string(), }, ); } fn register_vault(&mut self) { self.components.insert( "vault".to_string(), ComponentConfig { name: "vault".to_string(), ports: vec![8200], dependencies: vec![], linux_packages: vec![], macos_packages: vec![], windows_packages: vec![], download_url: get_component_url("vault"), binary_name: Some("vault".to_string()), pre_install_cmds_linux: vec![ "mkdir -p {{DATA_PATH}}/vault".to_string(), "mkdir -p {{CONF_PATH}}/vault".to_string(), "mkdir -p {{LOGS_PATH}}".to_string(), r#"cat > {{CONF_PATH}}/vault/config.hcl << 'EOF' storage "file" { path = "{{DATA_PATH}}/vault" } listener "tcp" { address = "0.0.0.0:8200" tls_disable = false tls_cert_file = "{{CONF_PATH}}/system/certificates/vault/server.crt" tls_key_file = "{{CONF_PATH}}/system/certificates/vault/server.key" tls_client_ca_file = "{{CONF_PATH}}/system/certificates/ca/ca.crt" } api_addr = "https://localhost:8200" cluster_addr = "https://localhost:8201" ui = true disable_mlock = true EOF"#.to_string(), ], post_install_cmds_linux: vec![ "mkdir -p {{CONF_PATH}}/system/certificates/ca".to_string(), "mkdir -p {{CONF_PATH}}/system/certificates/vault".to_string(), "mkdir -p {{CONF_PATH}}/system/certificates/botserver".to_string(), "mkdir -p {{CONF_PATH}}/system/certificates/tables".to_string(), "openssl genrsa -out {{CONF_PATH}}/system/certificates/ca/ca.key 4096 2>/dev/null".to_string(), "openssl req -new -x509 -days 3650 -key {{CONF_PATH}}/system/certificates/ca/ca.key -out {{CONF_PATH}}/system/certificates/ca/ca.crt -subj '/C=BR/ST=SP/L=São Paulo/O=BotServer Internal CA/CN=BotServer CA' 2>/dev/null".to_string(), "openssl genrsa -out {{CONF_PATH}}/system/certificates/vault/server.key 4096 2>/dev/null".to_string(), "openssl req -new -key {{CONF_PATH}}/system/certificates/vault/server.key -out {{CONF_PATH}}/system/certificates/vault/server.csr -subj '/C=BR/ST=SP/L=São Paulo/O=BotServer/CN=localhost' 2>/dev/null".to_string(), "echo -e 'subjectAltName = DNS:localhost,IP:127.0.0.1\\nkeyUsage = digitalSignature,keyEncipherment\\nextendedKeyUsage = serverAuth' > {{CONF_PATH}}/system/certificates/vault/server.ext".to_string(), "openssl x509 -req -days 3650 -in {{CONF_PATH}}/system/certificates/vault/server.csr -CA {{CONF_PATH}}/system/certificates/ca/ca.crt -CAkey {{CONF_PATH}}/system/certificates/ca/ca.key -CAcreateserial -out {{CONF_PATH}}/system/certificates/vault/server.crt -extfile {{CONF_PATH}}/system/certificates/vault/server.ext 2>/dev/null".to_string(), "openssl genrsa -out {{CONF_PATH}}/system/certificates/botserver/client.key 4096 2>/dev/null".to_string(), "openssl req -new -key {{CONF_PATH}}/system/certificates/botserver/client.key -out {{CONF_PATH}}/system/certificates/botserver/client.csr -subj '/C=BR/ST=SP/L=São Paulo/O=BotServer/CN=botserver' 2>/dev/null".to_string(), "openssl x509 -req -days 3650 -in {{CONF_PATH}}/system/certificates/botserver/client.csr -CA {{CONF_PATH}}/system/certificates/ca/ca.crt -CAkey {{CONF_PATH}}/system/certificates/ca/ca.key -CAcreateserial -out {{CONF_PATH}}/system/certificates/botserver/client.crt 2>/dev/null".to_string(), "openssl genrsa -out {{CONF_PATH}}/system/certificates/tables/server.key 4096 2>/dev/null".to_string(), "openssl req -new -key {{CONF_PATH}}/system/certificates/tables/server.key -out {{CONF_PATH}}/system/certificates/tables/server.csr -subj '/C=BR/ST=SP/L=São Paulo/O=BotServer/CN=localhost' 2>/dev/null".to_string(), "echo -e 'subjectAltName = DNS:localhost,IP:127.0.0.1\\nkeyUsage = digitalSignature,keyEncipherment\\nextendedKeyUsage = serverAuth' > {{CONF_PATH}}/system/certificates/tables/server.ext".to_string(), "openssl x509 -req -days 3650 -in {{CONF_PATH}}/system/certificates/tables/server.csr -CA {{CONF_PATH}}/system/certificates/ca/ca.crt -CAkey {{CONF_PATH}}/system/certificates/ca/ca.key -CAcreateserial -out {{CONF_PATH}}/system/certificates/tables/server.crt -extfile {{CONF_PATH}}/system/certificates/tables/server.ext 2>/dev/null".to_string(), "echo 'Certificates generated successfully'".to_string(), ], pre_install_cmds_macos: vec![ "mkdir -p {{DATA_PATH}}/vault".to_string(), "mkdir -p {{CONF_PATH}}/vault".to_string(), "mkdir -p {{LOGS_PATH}}".to_string(), r#"cat > {{CONF_PATH}}/vault/config.hcl << 'EOF' storage "file" { path = "{{DATA_PATH}}/vault" } listener "tcp" { address = "0.0.0.0:8200" tls_disable = false tls_cert_file = "{{CONF_PATH}}/system/certificates/vault/server.crt" tls_key_file = "{{CONF_PATH}}/system/certificates/vault/server.key" tls_client_ca_file = "{{CONF_PATH}}/system/certificates/ca/ca.crt" } api_addr = "https://localhost:8200" cluster_addr = "https://localhost:8201" ui = true disable_mlock = true EOF"#.to_string(), ], post_install_cmds_macos: vec![], pre_install_cmds_windows: vec![], post_install_cmds_windows: vec![], env_vars: { let mut env = HashMap::new(); env.insert( "VAULT_ADDR".to_string(), "https://localhost:8200".to_string(), ); env.insert( "VAULT_CACERT".to_string(), format!("{}/conf/system/certificates/ca/ca.crt", get_stack_path()), ); env }, data_download_list: Vec::new(), exec_cmd: "nohup {{BIN_PATH}}/vault server -config={{CONF_PATH}}/vault/config.hcl > {{LOGS_PATH}}/vault.log 2>&1 &" .to_string(), check_cmd: "if [ -f {{CONF_PATH}}/system/certificates/botserver/client.crt ]; then curl -f -sk --connect-timeout 2 -m 5 --cert {{CONF_PATH}}/system/certificates/botserver/client.crt --key {{CONF_PATH}}/system/certificates/botserver/client.key 'https://localhost:8200/v1/sys/health?standbyok=true&uninitcode=200&sealedcode=200' >/dev/null 2>&1; else curl -f -sk --connect-timeout 2 -m 5 'https://localhost:8200/v1/sys/health?standbyok=true&uninitcode=200&sealedcode=200' >/dev/null 2>&1; fi" .to_string(), }, ); } fn register_observability(&mut self) { self.components.insert( "observability".to_string(), ComponentConfig { name: "observability".to_string(), ports: vec![8686], dependencies: vec!["timeseries_db".to_string()], linux_packages: vec![], macos_packages: vec![], windows_packages: vec![], download_url: get_component_url("observability"), binary_name: Some("vector".to_string()), pre_install_cmds_linux: vec![ "mkdir -p {{CONF_PATH}}/monitoring".to_string(), "mkdir -p {{DATA_PATH}}/vector".to_string(), ], post_install_cmds_linux: vec![], pre_install_cmds_macos: vec![ "mkdir -p {{CONF_PATH}}/monitoring".to_string(), "mkdir -p {{DATA_PATH}}/vector".to_string(), ], post_install_cmds_macos: vec![], pre_install_cmds_windows: vec![], post_install_cmds_windows: vec![], env_vars: HashMap::new(), data_download_list: Vec::new(), exec_cmd: "{{BIN_PATH}}/vector --config {{CONF_PATH}}/monitoring/vector.toml" .to_string(), check_cmd: "curl -f --connect-timeout 2 -m 5 http://localhost:8686/health >/dev/null 2>&1" .to_string(), }, ); } fn register_host(&mut self) { self.components.insert( "host".to_string(), ComponentConfig { name: "host".to_string(), ports: vec![], dependencies: vec![], linux_packages: vec![], macos_packages: vec![], windows_packages: vec![], download_url: None, binary_name: None, pre_install_cmds_linux: vec![ "echo 'net.ipv4.ip_forward=1' | tee -a /etc/sysctl.conf".to_string(), "sysctl -p".to_string(), ], post_install_cmds_linux: vec![ "lxd init --dump >/dev/null 2>&1 || lxd init --auto".to_string(), "lxc storage show default >/dev/null 2>&1 || lxc storage create default dir".to_string(), "lxc profile device include default root >/dev/null 2>&1 || lxc profile device add default root disk path=/ pool=default".to_string(), "lxc profile device show default | grep lxd-sock >/dev/null 2>&1 || lxc profile device add default lxd-sock proxy connect=unix:/var/lib/lxd/unix.socket listen=unix:/tmp/lxd.sock bind=container uid=0 gid=0 mode=0660".to_string(), ], pre_install_cmds_macos: vec![], post_install_cmds_macos: vec![], pre_install_cmds_windows: vec![], post_install_cmds_windows: vec![], env_vars: HashMap::new(), data_download_list: Vec::new(), exec_cmd: "".to_string(), check_cmd: "".to_string(), }, ); } pub fn start(&self, component: &str) -> Result { if let Some(component) = self.components.get(component) { let bin_path = self.base_path.join("bin").join(&component.name); let data_path = self.base_path.join("data").join(&component.name); let conf_path = self.base_path.join("conf"); let logs_path = self.base_path.join("logs").join(&component.name); let check_cmd = component .check_cmd .replace("{{BIN_PATH}}", &bin_path.to_string_lossy()) .replace("{{DATA_PATH}}", &data_path.to_string_lossy()) .replace("{{CONF_PATH}}", &conf_path.to_string_lossy()) .replace("{{LOGS_PATH}}", &logs_path.to_string_lossy()); let check_output = safe_sh_command(&check_cmd) .map(|o| o.status.success()) .unwrap_or(false); if check_output { info!( "Component {} is already running, skipping start", component.name ); return SafeCommand::noop_child() .map_err(|e| anyhow::anyhow!("Failed to create noop process: {}", e)); } // Generate qdrant config.yaml if missing if component.name == "vector_db" { let qdrant_conf = conf_path.join("vector_db/config.yaml"); if !qdrant_conf.exists() { let storage = data_path.join("storage"); let snapshots = data_path.join("snapshots"); let _ = std::fs::create_dir_all(&storage); let _ = std::fs::create_dir_all(&snapshots); let yaml = format!( "storage:\n storage_path: {}\n snapshots_path: {}\n\nservice:\n host: 0.0.0.0\n http_port: 6333\n grpc_port: 6334\n enable_tls: false\n\nlog_level: INFO\n", storage.display(), snapshots.display() ); if let Err(e) = std::fs::write(&qdrant_conf, yaml) { warn!("Failed to write qdrant config: {}", e); } else { info!("Generated qdrant config at {}", qdrant_conf.display()); } } } let rendered_cmd = component .exec_cmd .replace("{{BIN_PATH}}", &bin_path.to_string_lossy()) .replace("{{DATA_PATH}}", &data_path.to_string_lossy()) .replace("{{CONF_PATH}}", &conf_path.to_string_lossy()) .replace("{{LOGS_PATH}}", &logs_path.to_string_lossy()); trace!( "Starting component {} with command: {}", component.name, rendered_cmd ); trace!( "Working directory: {}, logs_path: {}", bin_path.display(), logs_path.display() ); let vault_credentials = Self::fetch_vault_credentials(); let mut evaluated_envs = HashMap::new(); for (k, v) in &component.env_vars { if let Some(var_name) = v.strip_prefix('$') { let value = vault_credentials .get(var_name) .cloned() .or_else(|| std::env::var(var_name).ok()) .unwrap_or_default(); evaluated_envs.insert(k.clone(), value); } else { evaluated_envs.insert(k.clone(), v.clone()); } } trace!( "About to spawn shell command for {}: {}", component.name, rendered_cmd ); trace!("Working dir: {}", bin_path.display()); let child = SafeCommand::new("sh") .and_then(|c| c.arg("-c")) .and_then(|c| c.trusted_shell_script_arg(&rendered_cmd)) .and_then(|c| c.working_dir(&bin_path)) .and_then(|cmd| cmd.spawn_with_envs(&evaluated_envs)) .map_err(|e| anyhow::anyhow!("Failed to spawn process: {}", e)); trace!("Spawn result for {}: {:?}", component.name, child.is_ok()); std::thread::sleep(std::time::Duration::from_secs(2)); trace!( "Checking if {} process exists after 2s sleep...", component.name ); let check_proc = safe_pgrep(&["-f", &component.name]); if let Some(output) = check_proc { let pids = String::from_utf8_lossy(&output.stdout); trace!("pgrep '{}' result: '{}'", component.name, pids.trim()); } match child { Ok(c) => { trace!("Component {} started successfully", component.name); // Initialize Vault after successful start (local mode only) if component.name == "vault" && self.mode == InstallMode::Local { if let Err(e) = self.initialize_vault_local() { warn!("Failed to initialize Vault: {}", e); warn!("Vault started but may need manual initialization"); } } Ok(c) } Err(e) => { error!("Spawn failed for {}: {}", component.name, e); let err_msg = e.to_string(); if err_msg.contains("already running") || err_msg.contains("be running") || component.name == "tables" { trace!( "Component {} may already be running, continuing anyway", component.name ); // Even if vault was already running, ensure .env exists if component.name == "vault" && self.mode == InstallMode::Local { let _ = self.ensure_env_file_exists(); } SafeCommand::noop_child() .map_err(|e| anyhow::anyhow!("Failed to create noop process: {}", e)) } else { Err(e) } } } } else { Err(anyhow::anyhow!("Component not found: {}", component)) } } fn fetch_vault_credentials() -> HashMap { let mut credentials = HashMap::new(); dotenvy::dotenv().ok(); let base_path = std::env::var("BOTSERVER_STACK_PATH") .map(std::path::PathBuf::from) .unwrap_or_else(|_| { std::env::current_dir() .unwrap_or_else(|_| std::path::PathBuf::from(".")) .join("botserver-stack") }); let vault_addr = std::env::var("VAULT_ADDR").unwrap_or_else(|_| "https://localhost:8200".to_string()); let vault_token = std::env::var("VAULT_TOKEN").unwrap_or_default(); if vault_token.is_empty() { warn!("VAULT_TOKEN not set, cannot fetch credentials from Vault"); return credentials; } let client_cert = base_path.join("conf/system/certificates/botserver/client.crt"); let client_key = base_path.join("conf/system/certificates/botserver/client.key"); let vault_check = SafeCommand::new("curl") .and_then(|c| { c.args(&[ "-sfk", "--cert", &client_cert.to_string_lossy(), "--key", &client_key.to_string_lossy(), &format!("{}/v1/sys/health", vault_addr), ]) }) .and_then(|c| c.execute()) .map(|o| o.status.success()) .unwrap_or(false); if !vault_check { trace!( "Vault not reachable at {}, skipping credential fetch", vault_addr ); return credentials; } let vault_bin = base_path.join("bin/vault/vault"); let ca_cert_path = std::env::var("VAULT_CACERT").unwrap_or_else(|_| { base_path .join("conf/system/certificates/ca/ca.crt") .to_string_lossy() .to_string() }); let services = [ ("drive", "secret/gbo/drive"), ("cache", "secret/gbo/cache"), ("tables", "secret/gbo/tables"), ("vectordb", "secret/gbo/vectordb"), ("directory", "secret/gbo/directory"), ("llm", "secret/gbo/llm"), ("meet", "secret/gbo/meet"), ("alm", "secret/gbo/alm"), ("encryption", "secret/gbo/encryption"), ]; for (service_name, vault_path) in &services { let result = SafeCommand::new(vault_bin.to_str().unwrap_or("vault")) .and_then(|c| { c.env("VAULT_ADDR", &vault_addr) .and_then(|c| c.env("VAULT_TOKEN", &vault_token)) .and_then(|c| c.env("VAULT_CACERT", &ca_cert_path)) }) .and_then(|c| { c.args(&["kv", "get", "-format=json", "-tls-skip-verify", vault_path]) }) .and_then(|c| c.execute()); if let Ok(output) = result { if output.status.success() { let json_str = String::from_utf8_lossy(&output.stdout); if let Ok(json) = serde_json::from_str::(&json_str) { if let Some(data) = json.get("data").and_then(|d| d.get("data")) { if let Some(obj) = data.as_object() { let prefix = service_name.to_uppercase(); for (key, value) in obj { if let Some(v) = value.as_str() { let env_key = format!("{}_{}", prefix, key.to_uppercase()); credentials.insert(env_key, v.to_string()); } } } } } } } } trace!("Fetched {} credentials from Vault", credentials.len()); credentials } /// Initialize Vault locally (non-LXC mode) and create .env file /// /// This function: /// 1. Checks if Vault is already initialized (via health endpoint or data dir) /// 2. If initialized but sealed, unseals with existing keys from vault-unseal-keys /// 3. If not initialized, runs `vault operator init` to get root token and unseal keys /// 4. Creates .env file with VAULT_ADDR and VAULT_TOKEN /// 5. Creates vault-unseal-keys file with proper permissions /// 6. Unseals Vault with 3 keys fn initialize_vault_local(&self) -> Result<()> { use std::io::Write; info!("Initializing Vault locally (non-LXC mode)..."); let bin_path = self.base_path.join("bin/vault"); let conf_path = self.base_path.join("conf"); let vault_bin = bin_path.join("vault"); let vault_data = self.base_path.join("data/vault"); // Check if Vault data directory exists (real indicator of initialized state) let vault_data_exists = vault_data.exists(); if !vault_data_exists { info!("Vault data directory not found, will initialize fresh"); } else { info!("Vault data directory found, checking health..."); } // Wait for Vault to be ready info!("Waiting for Vault to start..."); std::thread::sleep(std::time::Duration::from_secs(3)); let vault_addr = std::env::var("VAULT_ADDR").unwrap_or_else(|_| "https://localhost:8200".to_string()); let ca_cert = conf_path.join("system/certificates/ca/ca.crt"); // Only attempt recovery if data directory exists if vault_data_exists { // Check if Vault is already initialized via health endpoint let health_cmd = format!( "curl -f -s --connect-timeout 2 -k {}/v1/sys/health", vault_addr ); let health_output = safe_sh_command(&health_cmd); let already_initialized = if let Some(ref output) = health_output { if output.status.success() { if let Ok(json) = serde_json::from_str::( &String::from_utf8_lossy(&output.stdout), ) { json.get("initialized") .and_then(|v| v.as_bool()) .unwrap_or(false) } else { false } } else { // Health endpoint returns 503 when sealed but initialized let stderr = String::from_utf8_lossy(&output.stderr); let stdout = String::from_utf8_lossy(&output.stdout); stdout.contains("\"initialized\":true") || stderr.contains("\"initialized\":true") } } else { false }; if already_initialized { info!("Vault already initialized (detected via health/data), skipping init"); return self.recover_existing_vault(); } } // Initialize Vault let init_cmd = format!( "{} operator init -tls-skip-verify -key-shares=5 -key-threshold=3 -format=json -address={}", vault_bin.display(), vault_addr ); info!("Running vault operator init..."); let output = safe_sh_command(&init_cmd) .ok_or_else(|| anyhow::anyhow!("Failed to execute vault init command"))?; if !output.status.success() { let stderr = String::from_utf8_lossy(&output.stderr); if stderr.contains("already initialized") { warn!("Vault already initialized, recovering existing data"); return self.recover_existing_vault(); } return Err(anyhow::anyhow!("Failed to initialize Vault: {}", stderr)); } let init_output = String::from_utf8_lossy(&output.stdout); let init_json_val: serde_json::Value = serde_json::from_str(&init_output).context("Failed to parse Vault init output")?; let unseal_keys = init_json_val["unseal_keys_b64"] .as_array() .context("No unseal keys in output")?; let root_token = init_json_val["root_token"] .as_str() .context("No root token in output")?; // Save init.json let init_json = self.base_path.join("conf/vault/init.json"); std::fs::create_dir_all(init_json.parent().unwrap())?; std::fs::write(&init_json, serde_json::to_string_pretty(&init_json_val)?)?; info!("Created {}", init_json.display()); // Create .env file with Vault credentials let env_file = std::path::PathBuf::from(".env"); let env_content = format!( r#" # Vault Configuration (auto-generated) VAULT_ADDR={} VAULT_TOKEN={} VAULT_CACERT={} "#, vault_addr, root_token, ca_cert.display() ); if env_file.exists() { let existing = std::fs::read_to_string(&env_file)?; if existing.contains("VAULT_ADDR=") { warn!(".env already contains VAULT_ADDR, not overwriting"); } else { let mut file = std::fs::OpenOptions::new().append(true).open(&env_file)?; file.write_all(env_content.as_bytes())?; info!("Appended Vault config to .env"); } } else { std::fs::write(&env_file, env_content.trim_start())?; info!("Created .env with Vault config"); } // Create vault-unseal-keys file in botserver directory (next to .env) let unseal_keys_file = self.base_path.join("vault-unseal-keys"); let keys_content: String = unseal_keys .iter() .enumerate() .map(|(i, key): (usize, &serde_json::Value)| { format!( "VAULT_UNSEAL_KEY_{}={}\n", i + 1, key.as_str().unwrap_or("") ) }) .collect(); std::fs::write(&unseal_keys_file, keys_content)?; #[cfg(unix)] { std::fs::set_permissions(&unseal_keys_file, std::fs::Permissions::from_mode(0o600))?; } #[cfg(not(unix))] { let _ = &unseal_keys_file; // suppress unused warning } info!("Created {} (chmod 600)", unseal_keys_file.display()); // Unseal Vault (need 3 keys) self.unseal_vault(&vault_bin, &vault_addr)?; info!("Vault initialized and unsealed successfully"); info!("✓ Created .env with VAULT_ADDR, VAULT_TOKEN"); info!("✓ Created /opt/gbo/secrets/vault-unseal-keys (chmod 600)"); // Enable KV2 secrets engine at 'secret/' path info!("Enabling KV2 secrets engine at 'secret/'..."); let enable_kv2_cmd = format!( "VAULT_ADDR={} VAULT_TOKEN={} VAULT_CACERT={} {} secrets enable -path=secret kv-v2", vault_addr, root_token, ca_cert.display(), vault_bin.display() ); match safe_sh_command(&enable_kv2_cmd) { Some(output) => { if output.status.success() { info!("KV2 secrets engine enabled at 'secret/'"); } else { let stderr = String::from_utf8_lossy(&output.stderr); if stderr.contains("path is already in use") { info!("KV2 secrets engine already enabled"); } else { warn!("Failed to enable KV2 secrets engine: {}", stderr); } } } None => { warn!("Failed to execute KV2 enable command"); } } // Write default credentials to Vault for all components self.seed_vault_defaults(&vault_addr, root_token, &ca_cert, &vault_bin)?; Ok(()) } /// Seed default credentials into Vault KV2 after initialization fn seed_vault_defaults( &self, vault_addr: &str, root_token: &str, ca_cert: &std::path::Path, vault_bin: &std::path::Path, ) -> Result<()> { info!("Seeding default credentials into Vault..."); let drive_user = super::generate_random_string(16); let drive_pass = super::generate_random_string(32); let cache_pass = super::generate_random_string(32); let db_pass = super::generate_random_string(32); let master_key = super::generate_random_string(64); let meet_app_id = super::generate_random_string(24); let meet_app_secret = super::generate_random_string(48); let alm_token = super::generate_random_string(40); info!( "Generated strong random credentials for: drive, cache, tables, encryption, meet, alm" ); let defaults: Vec<(&str, Vec<(String, String)>)> = vec![ ( "secret/gbo/drive", vec![ ("accesskey".to_string(), drive_user), ("secret".to_string(), drive_pass), ("host".to_string(), "localhost".to_string()), ("port".to_string(), "9000".to_string()), ("url".to_string(), "http://localhost:9000".to_string()), ], ), ( "secret/gbo/cache", vec![ ("password".to_string(), cache_pass), ("host".to_string(), "localhost".to_string()), ("port".to_string(), "6379".to_string()), ("url".to_string(), "redis://localhost:6379".to_string()), ], ), ( "secret/gbo/tables", vec![ ("password".to_string(), db_pass), ("host".to_string(), "localhost".to_string()), ("port".to_string(), "5432".to_string()), ("database".to_string(), "botserver".to_string()), ("username".to_string(), "gbuser".to_string()), ("url".to_string(), "postgres://localhost:5432".to_string()), ], ), ( "secret/gbo/directory", vec![ ("url".to_string(), "http://localhost:9000".to_string()), ("host".to_string(), "localhost".to_string()), ("port".to_string(), "9000".to_string()), ("project_id".to_string(), "none".to_string()), ("client_id".to_string(), "none".to_string()), ("client_secret".to_string(), "none".to_string()), ], ), ( "secret/gbo/email", vec![ ("smtp_host".to_string(), "none".to_string()), ("smtp_port".to_string(), "587".to_string()), ("smtp_user".to_string(), "none".to_string()), ("smtp_password".to_string(), "none".to_string()), ("smtp_from".to_string(), "none".to_string()), ], ), ( "secret/gbo/llm", vec![ ("url".to_string(), "http://localhost:8081".to_string()), ("host".to_string(), "localhost".to_string()), ("port".to_string(), "8081".to_string()), ("model".to_string(), "gpt-4".to_string()), ("openai_key".to_string(), "none".to_string()), ("anthropic_key".to_string(), "none".to_string()), ( "ollama_url".to_string(), "http://localhost:11434".to_string(), ), ], ), ( "secret/gbo/encryption", vec![("master_key".to_string(), master_key)], ), ( "secret/gbo/meet", vec![ ("url".to_string(), "http://localhost:7880".to_string()), ("host".to_string(), "localhost".to_string()), ("port".to_string(), "7880".to_string()), ("app_id".to_string(), meet_app_id), ("app_secret".to_string(), meet_app_secret), ], ), ( "secret/gbo/vectordb", vec![ ("url".to_string(), "http://localhost:6333".to_string()), ("host".to_string(), "localhost".to_string()), ("port".to_string(), "6333".to_string()), ("grpc_port".to_string(), "6334".to_string()), ("api_key".to_string(), "none".to_string()), ], ), ( "secret/gbo/alm", vec![ ("url".to_string(), "http://localhost:9000".to_string()), ("host".to_string(), "localhost".to_string()), ("port".to_string(), "9000".to_string()), ("token".to_string(), alm_token), ("default_org".to_string(), "none".to_string()), ], ), ]; for (path, kv_pairs) in &defaults { let mut args = vec![ "kv".to_string(), "put".to_string(), "-tls-skip-verify".to_string(), format!("-address={}", vault_addr), path.to_string(), ]; for (k, v) in kv_pairs.iter() { args.push(format!("{}={}", k, v)); } let result = SafeCommand::new(vault_bin.to_str().unwrap_or("vault")) .and_then(|c| { let mut cmd = c; for arg in &args { cmd = cmd.trusted_arg(arg)?; } Ok(cmd) }) .and_then(|c| { c.env("VAULT_ADDR", vault_addr) .and_then(|c| c.env("VAULT_TOKEN", root_token)) .and_then(|c| c.env("VAULT_CACERT", ca_cert.to_str().unwrap_or(""))) }) .and_then(|c| c.execute()); match result { Ok(output) => { if output.status.success() { info!("Seeded Vault defaults at {}", path); } else { let stderr = String::from_utf8_lossy(&output.stderr); warn!("Failed to seed {} in Vault: {}", path, stderr); } } Err(e) => { warn!("Failed to execute vault put for {}: {}", path, e); } } } info!("Vault defaults seeded successfully"); Ok(()) } /// Recover existing Vault installation (already initialized but may be sealed) fn recover_existing_vault(&self) -> Result<()> { use std::io::Write; info!("Recovering existing Vault installation..."); let vault_addr = std::env::var("VAULT_ADDR").unwrap_or_else(|_| "https://localhost:8200".to_string()); let ca_cert = self.base_path.join("conf/system/certificates/ca/ca.crt"); let vault_bin = self.base_path.join("bin/vault/vault"); // Try to read existing unseal keys let unseal_keys_file = self.base_path.join("vault-unseal-keys"); let unseal_keys = if unseal_keys_file.exists() { info!("Found existing vault-unseal-keys file"); let content = std::fs::read_to_string(&unseal_keys_file)?; content .lines() .filter_map(|line| { line.strip_prefix("VAULT_UNSEAL_KEY_") .and_then(|rest| rest.split_once('=')) .map(|(_, key)| key.to_string()) }) .collect::>() } else { Vec::new() }; // Try to read existing init.json for root token let init_json = self.base_path.join("conf/vault/init.json"); let root_token = if init_json.exists() { let content = std::fs::read_to_string(&init_json)?; if let Ok(json) = serde_json::from_str::(&content) { json.get("root_token") .and_then(|v| v.as_str()) .map(String::from) } else { None } } else { None }; // Unseal if we have keys if !unseal_keys.is_empty() { info!("Unsealing Vault with existing keys..."); for (i, key) in unseal_keys.iter().take(3).enumerate() { let unseal_cmd = format!( "{} operator unseal -tls-skip-verify -address={} {}", vault_bin.display(), vault_addr, key ); let unseal_output = safe_sh_command(&unseal_cmd); if let Some(ref output) = unseal_output { if !output.status.success() { warn!("Unseal step {} may have failed", i + 1); } } } } // Create .env if we have root token if let Some(ref token) = root_token { let env_file = std::path::PathBuf::from(".env"); let env_content = format!( r#" # Vault Configuration (auto-generated) VAULT_ADDR={} VAULT_TOKEN={} VAULT_CACERT={} "#, vault_addr, token, ca_cert.display() ); if env_file.exists() { let existing = std::fs::read_to_string(&env_file)?; if !existing.contains("VAULT_ADDR=") { let mut file = std::fs::OpenOptions::new().append(true).open(&env_file)?; file.write_all(env_content.as_bytes())?; info!("Appended Vault config to .env"); } } else { std::fs::write(&env_file, env_content.trim_start())?; info!("Created .env with Vault config"); } } else { warn!("No root token found - Vault may need manual recovery"); } // Seed defaults if we have a token (ensure credentials exist even during recovery) if let Some(ref token) = root_token { let _ = self.seed_vault_defaults(&vault_addr, token, &ca_cert, &vault_bin); } info!("Vault recovery complete"); Ok(()) } /// Unseal Vault with 3 keys fn unseal_vault(&self, vault_bin: &std::path::Path, vault_addr: &str) -> Result<()> { info!("Unsealing Vault..."); let unseal_keys_file = self.base_path.join("vault-unseal-keys"); if unseal_keys_file.exists() { let content = std::fs::read_to_string(&unseal_keys_file)?; let keys: Vec = content .lines() .filter_map(|line| { line.strip_prefix("VAULT_UNSEAL_KEY_") .and_then(|rest| rest.split_once('=')) .map(|(_, key)| key.to_string()) }) .collect(); for (i, key) in keys.iter().take(3).enumerate() { let unseal_cmd = format!( "{} operator unseal -tls-skip-verify -address={} {}", vault_bin.display(), vault_addr, key ); let unseal_output = safe_sh_command(&unseal_cmd); if let Some(ref output) = unseal_output { if !output.status.success() { warn!("Unseal step {} may have failed", i + 1); } } } } Ok(()) } /// Ensure .env file exists with Vault credentials fn ensure_env_file_exists(&self) -> Result<()> { let init_json = self.base_path.join("conf/vault/init.json"); let env_file = std::path::PathBuf::from(".env"); if !init_json.exists() { return Ok(()); // No init, no .env needed yet } let init_content = std::fs::read_to_string(&init_json)?; let init_json_val: serde_json::Value = serde_json::from_str(&init_content)?; let root_token = init_json_val["root_token"] .as_str() .context("No root_token in init.json")?; let conf_path = self.base_path.join("conf"); let ca_cert = conf_path.join("system/certificates/ca/ca.crt"); let vault_addr = std::env::var("VAULT_ADDR").unwrap_or_else(|_| "https://localhost:8200".to_string()); let env_content = format!( r#" # Vault Configuration (auto-generated) VAULT_ADDR={} VAULT_TOKEN={} VAULT_CACERT={} "#, vault_addr, root_token, ca_cert.display() ); if env_file.exists() { let existing = std::fs::read_to_string(&env_file)?; if existing.contains("VAULT_ADDR=") { return Ok(()); } let mut file = std::fs::OpenOptions::new().append(true).open(&env_file)?; use std::io::Write; file.write_all(env_content.as_bytes())?; } else { std::fs::write(&env_file, env_content.trim_start())?; } info!("Created .env with Vault credentials"); Ok(()) } }