Compare commits
2 commits
| Author | SHA1 | Date | |
|---|---|---|---|
| aaba2976b4 | |||
| 116f11cb42 |
36 changed files with 844 additions and 15514 deletions
|
|
@ -1,5 +1,5 @@
|
||||||
[build]
|
[build]
|
||||||
# rustc-wrapper = "sccache"
|
rustc-wrapper = "sccache"
|
||||||
|
|
||||||
[target.x86_64-unknown-linux-gnu]
|
[target.x86_64-unknown-linux-gnu]
|
||||||
linker = "clang"
|
linker = "clang"
|
||||||
|
|
|
||||||
|
|
@ -1,8 +0,0 @@
|
||||||
# General Bots Environment Configuration
|
|
||||||
# Copy this file to .env and fill in values
|
|
||||||
# NEVER commit .env to version control
|
|
||||||
|
|
||||||
# Vault connection
|
|
||||||
VAULT_ADDR=https://127.0.0.1:8200
|
|
||||||
VAULT_TOKEN=<your-vault-token-here>
|
|
||||||
VAULT_CACERT=./botserver-stack/vault/certs/ca.crt
|
|
||||||
52
.gitignore
vendored
52
.gitignore
vendored
|
|
@ -2,17 +2,11 @@
|
||||||
target/
|
target/
|
||||||
*.out
|
*.out
|
||||||
bin/
|
bin/
|
||||||
*.png
|
|
||||||
*.jpg
|
|
||||||
# Logs
|
# Logs
|
||||||
*.log
|
*.log
|
||||||
*logfile*
|
*logfile*
|
||||||
*-log*
|
*-log*
|
||||||
.vscode
|
|
||||||
.zed
|
|
||||||
.gemini
|
|
||||||
.claude
|
|
||||||
|
|
||||||
|
|
||||||
# Temporary files
|
# Temporary files
|
||||||
.tmp*
|
.tmp*
|
||||||
|
|
@ -36,47 +30,7 @@ botserver-installers/*
|
||||||
!botserver-installers/.gitkeep
|
!botserver-installers/.gitkeep
|
||||||
botserver-stack
|
botserver-stack
|
||||||
TODO*
|
TODO*
|
||||||
work
|
|
||||||
.swp
|
|
||||||
# Lock file
|
|
||||||
# Cargo.lock (should be tracked)
|
|
||||||
.kiro
|
|
||||||
config
|
|
||||||
|
|
||||||
# Data directory (contains bot configs and API keys)
|
|
||||||
data/
|
|
||||||
|
|
||||||
# Playwright
|
# Lock file (regenerated from Cargo.toml)
|
||||||
node_modules/
|
Cargo.lock
|
||||||
/test-results/
|
|
||||||
/playwright-report/
|
|
||||||
/blob-report/
|
|
||||||
/playwright/.cache/
|
|
||||||
/playwright/.auth/
|
|
||||||
.playwright*
|
|
||||||
.ruff_cache
|
|
||||||
.opencode
|
|
||||||
config/directory_config.json
|
|
||||||
# CI cache bust: Fri Feb 13 22:33:51 UTC 2026
|
|
||||||
|
|
||||||
# Secrets - NEVER commit these files
|
|
||||||
vault-unseal-keys
|
|
||||||
start-and-unseal.sh
|
|
||||||
vault-token-*
|
|
||||||
init.json
|
|
||||||
*.pem
|
|
||||||
*.key
|
|
||||||
*.crt
|
|
||||||
*.cert
|
|
||||||
$null
|
|
||||||
AppData/
|
|
||||||
build_errors*.txt
|
|
||||||
build_errors_utf8.txt
|
|
||||||
check.json
|
|
||||||
clippy*.txt
|
|
||||||
errors.txt
|
|
||||||
errors_utf8.txt
|
|
||||||
|
|
||||||
vault-unseal-keysdefault-vault.tar
|
|
||||||
prompts/sec-bots.md
|
|
||||||
AGENTS-PROD.md
|
|
||||||
|
|
|
||||||
23
.gitmodules
vendored
23
.gitmodules
vendored
|
|
@ -1,43 +1,42 @@
|
||||||
[submodule "botapp"]
|
[submodule "botapp"]
|
||||||
path = botapp
|
path = botapp
|
||||||
url = ../botapp.git
|
url = https://github.com/GeneralBots/botapp.git
|
||||||
|
|
||||||
[submodule "botserver"]
|
[submodule "botserver"]
|
||||||
path = botserver
|
path = botserver
|
||||||
url = ../BotServer.git
|
url = https://github.com/GeneralBots/botserver.git
|
||||||
|
|
||||||
[submodule "botlib"]
|
[submodule "botlib"]
|
||||||
path = botlib
|
path = botlib
|
||||||
url = ../botlib.git
|
url = https://github.com/GeneralBots/botlib.git
|
||||||
|
|
||||||
[submodule "botui"]
|
[submodule "botui"]
|
||||||
path = botui
|
path = botui
|
||||||
url = ../botui.git
|
url = https://github.com/GeneralBots/botui.git
|
||||||
|
|
||||||
[submodule "botbook"]
|
[submodule "botbook"]
|
||||||
path = botbook
|
path = botbook
|
||||||
url = ../botbook.git
|
url = https://github.com/GeneralBots/botbook.git
|
||||||
|
|
||||||
[submodule "bottest"]
|
[submodule "bottest"]
|
||||||
path = bottest
|
path = bottest
|
||||||
url = ../bottest.git
|
url = https://github.com/GeneralBots/bottest.git
|
||||||
|
|
||||||
[submodule "botdevice"]
|
[submodule "botdevice"]
|
||||||
path = botdevice
|
path = botdevice
|
||||||
url = ../botdevice.git
|
url = https://github.com/GeneralBots/botdevice.git
|
||||||
|
|
||||||
[submodule "botmodels"]
|
[submodule "botmodels"]
|
||||||
path = botmodels
|
path = botmodels
|
||||||
url = ../botmodels.git
|
url = https://github.com/GeneralBots/botmodels.git
|
||||||
|
|
||||||
[submodule "botplugin"]
|
[submodule "botplugin"]
|
||||||
path = botplugin
|
path = botplugin
|
||||||
url = ../botplugin.git
|
url = https://github.com/GeneralBots/botplugin.git
|
||||||
|
|
||||||
[submodule "bottemplates"]
|
[submodule "bottemplates"]
|
||||||
path = bottemplates
|
path = bottemplates
|
||||||
url = ../bottemplates.git
|
url = https://github.com/GeneralBots/bottemplates.git
|
||||||
|
|
||||||
[submodule ".github"]
|
[submodule ".github"]
|
||||||
path = .github
|
path = .github
|
||||||
url = ../.github.git
|
url = https://github.com/GeneralBots/.github.git
|
||||||
|
|
|
||||||
24
.vscode/launch.json
vendored
Normal file
24
.vscode/launch.json
vendored
Normal file
|
|
@ -0,0 +1,24 @@
|
||||||
|
{
|
||||||
|
"version": "0.2.0",
|
||||||
|
"configurations": [
|
||||||
|
|
||||||
|
{
|
||||||
|
"type": "lldb",
|
||||||
|
"request": "launch",
|
||||||
|
"name": "Debug executable 'botserver'",
|
||||||
|
"cargo": {
|
||||||
|
"args": ["run", "--bin=botserver", "--package=botserver", "--manifest-path=${workspaceFolder}/botserver/Cargo.toml"],
|
||||||
|
"filter": {
|
||||||
|
"name": "botserver",
|
||||||
|
"kind": "bin"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"args": [],
|
||||||
|
"env": {
|
||||||
|
"RUST_LOG": "trace,aws_sigv4=off,aws_smithy_checksums=off,mio=off,reqwest=off,aws_runtime=off,aws_smithy_http_client=off,rustls=off,hyper_util=off,aws_smithy_runtime=off,aws_smithy_runtime_api=off,tracing=off,aws_sdk_s3=off"
|
||||||
|
|
||||||
|
},
|
||||||
|
"cwd": "${workspaceFolder}/botserver"
|
||||||
|
},
|
||||||
|
]
|
||||||
|
}
|
||||||
5
.vscode/settings.json
vendored
Normal file
5
.vscode/settings.json
vendored
Normal file
|
|
@ -0,0 +1,5 @@
|
||||||
|
{
|
||||||
|
"git.ignoreLimitWarning": true,
|
||||||
|
"Codegeex.SidebarUI.LanguagePreference": "English",
|
||||||
|
"Codegeex.RepoIndex": true
|
||||||
|
}
|
||||||
710
AGENTS.md
710
AGENTS.md
|
|
@ -1,710 +0,0 @@
|
||||||
# General Bots AI Agent Guidelines
|
|
||||||
|
|
||||||
- Use apenas a língua culta ao falar.
|
|
||||||
- Never save files on root! Use `/tmp` for temp files.
|
|
||||||
- Never push to ALM without asking first — it is production!
|
|
||||||
- If in trouble with a tool, go to the official website for install instructions.
|
|
||||||
- See `botserver/src/drive/local_file_monitor.rs` to load bots from `/opt/gbo/data`.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 📁 Workspace Structure
|
|
||||||
|
|
||||||
| Crate | Purpose | Port | Tech Stack |
|
|
||||||
|-------|---------|------|------------|
|
|
||||||
| **botserver** | Main API server, business logic | 8080 | Axum, Diesel, Rhai BASIC |
|
|
||||||
| **botui** | Web UI server (dev) + proxy | 3000 | Axum, HTML/HTMX/CSS |
|
|
||||||
| **botapp** | Desktop app wrapper | - | Tauri 2 |
|
|
||||||
| **botlib** | Shared library | - | Core types, errors |
|
|
||||||
| **botbook** | Documentation | - | mdBook |
|
|
||||||
| **bottest** | Integration tests | - | tokio-test |
|
|
||||||
| **botdevice** | IoT/Device support | - | Rust |
|
|
||||||
| **botplugin** | Browser extension | - | JS |
|
|
||||||
|
|
||||||
### Key Paths
|
|
||||||
- **Binary:** `target/debug/botserver`
|
|
||||||
- **Run from:** `botserver/` directory
|
|
||||||
- **Env file:** `botserver/.env`
|
|
||||||
- **UI Files:** `botui/ui/suite/`
|
|
||||||
- **Bot data:** `/opt/gbo/data` (primary)
|
|
||||||
- **Test web:** `http://localhost:3000` — Login: `http://localhost:3000/suite/auth/login.html`
|
|
||||||
|
|
||||||
### 📦 Data Directory Structure
|
|
||||||
|
|
||||||
```
|
|
||||||
# DEV LOCAL (quando botserver-stack existe)
|
|
||||||
├── ./botserver-stack/data/system/work/{bot}.gbai/{bot}.gbdialog/
|
|
||||||
│ ├── *.bas # Scripts compilados (gerado automático)
|
|
||||||
│ └── *.ast # Cache compilado (deletar para forçar recompilação)
|
|
||||||
|
|
||||||
# PRODUCTION (com container Incus)
|
|
||||||
├── /opt/gbo/data/ # FONTE dos bots
|
|
||||||
└── (compilação fica em memória ou /opt/gbo/work/ se existir)
|
|
||||||
```
|
|
||||||
|
|
||||||
**IMPORTANTE:**
|
|
||||||
- **FONTE**: `/opt/gbo/data/{bot}.gbai/{bot}.gbdialog/{tool}.bas`
|
|
||||||
- **DEV LOCAL**: `./botserver-stack/data/system/work/{bot}.gbai/{bot}.gbdialog/`
|
|
||||||
- O botserver compila `.bas` → `.ast` automaticamente
|
|
||||||
- Se cache, deletar `.ast` para forçar recompilação
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 🧪 Debugging & Testing Tools
|
|
||||||
|
|
||||||
### 🔍 Ver Erros de Execução
|
|
||||||
```bash
|
|
||||||
tail -f botserver.log | grep -i "error\|tool"
|
|
||||||
```
|
|
||||||
|
|
||||||
### 🧪 Testar Ferramenta Específica
|
|
||||||
|
|
||||||
1. **Identificar o erro no log:**
|
|
||||||
```bash
|
|
||||||
grep -A5 "Tool error" botserver.log
|
|
||||||
```
|
|
||||||
|
|
||||||
2. **Corrigir o arquivo `.bas` na fonte:**
|
|
||||||
- **Dev local:** `./botserver-stack/data/system/work/{bot}.gbai/{bot}.gbdialog/{tool}.bas`
|
|
||||||
- **Production:** `/opt/gbo/data/{bot}.gbai/{bot}.gbdialog/{tool}.bas`
|
|
||||||
|
|
||||||
3. **Forçar recompilação (se necessário):**
|
|
||||||
```bash
|
|
||||||
rm ./botserver-stack/data/system/work/{bot}.gbai/{bot}.gbdialog/{tool}.ast
|
|
||||||
```
|
|
||||||
- Em dev local o AST fica em `./botserver-stack/...`
|
|
||||||
- Em production pode ficar em `/opt/gbo/work/...` se existir
|
|
||||||
|
|
||||||
4. **Testar novamente no browser:**
|
|
||||||
```
|
|
||||||
http://localhost:3000/{botname}
|
|
||||||
```
|
|
||||||
|
|
||||||
### ⚠️ Erros Comuns em Scripts BASIC
|
|
||||||
|
|
||||||
| Erro | Causa | Solução |
|
|
||||||
|------|-------|---------|
|
|
||||||
| `=== is not a valid operator` | BASIC usa `==`, não `===` | Substituir `===` por `--` em strings |
|
|
||||||
| `Syntax error` | Erro de sintaxe BASIC | Verificar parênteses, vírgulas |
|
|
||||||
| `Tool execution failed` | Erro no script | Ver logs para stack trace |
|
|
||||||
|
|
||||||
### 📝 Exemplo: Corrigir Operador Inválido
|
|
||||||
```bas
|
|
||||||
# ERRADO (JavaScript syntax):
|
|
||||||
PRINT "=== RESULTADO ==="
|
|
||||||
|
|
||||||
# CORRETO (BASIC syntax):
|
|
||||||
PRINT "-- RESULTADO --"
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 🧭 LLM Navigation Guide
|
|
||||||
|
|
||||||
1. Start with **[Component Dependency Graph](../README.md#-component-dependency-graph)**
|
|
||||||
2. Review **[Module Responsibility Matrix](../README.md#-module-responsibility-matrix)**
|
|
||||||
3. Study **[Data Flow Patterns](../README.md#-data-flow-patterns)**
|
|
||||||
4. Reference **[Common Architectural Patterns](../README.md#-common-architectural-patterns)**
|
|
||||||
5. Check [Security Rules](#-security-directives---mandatory) — violations are blocking
|
|
||||||
6. Follow [Code Patterns](#-mandatory-code-patterns) — consistency is mandatory
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## ❌ Absolute Prohibitions
|
|
||||||
|
|
||||||
### Build & Deploy
|
|
||||||
- ❌ **NEVER** search `/target` folder
|
|
||||||
- ❌ **NEVER** build in release mode or use `--release`
|
|
||||||
- ❌ **NEVER** run `cargo build` — use `cargo check` for verification
|
|
||||||
- ❌ **NEVER** run `cargo clean` — causes 30min rebuilds; use `./reset.sh` for DB issues
|
|
||||||
- ❌ **NEVER** deploy manually — ALWAYS use CI/CD pipeline (push → ALM → alm-ci builds → deploys)
|
|
||||||
- ❌ **NEVER** use `scp`, direct SSH binary copy, or manual deployment
|
|
||||||
- ❌ **NEVER** run the binary directly — use `systemctl` or `./restart.sh`
|
|
||||||
|
|
||||||
### Code Quality
|
|
||||||
- ❌ **NEVER** use `panic!()`, `todo!()`, `unimplemented!()`, `unwrap()`, `expect()`
|
|
||||||
- ❌ **NEVER** use `Command::new()` directly — use `SafeCommand`
|
|
||||||
- ❌ **NEVER** return raw error strings to HTTP clients — use `ErrorSanitizer`
|
|
||||||
- ❌ **NEVER** use `#[allow()]` or lint exceptions in `Cargo.toml` — FIX the code
|
|
||||||
- ❌ **NEVER** use `_` prefix for unused vars — DELETE or USE them
|
|
||||||
- ❌ **NEVER** leave unused imports, dead code, or commented-out code
|
|
||||||
- ❌ **NEVER** use CDN links — all assets must be local
|
|
||||||
- ❌ **NEVER** create `.md` docs without checking `botbook/` first
|
|
||||||
- ❌ **NEVER** hardcode credentials — use `generate_random_string()` or env vars
|
|
||||||
|
|
||||||
### Build Pattern (MANDATORY) - Fix Fast Loop
|
|
||||||
When building botserver, use this pattern to fix errors ASAP:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Run cargo in background, kill at 20 lines, fix errors, loop
|
|
||||||
# IMPORTANT: Never use --all-features (pulls docs/slides dependencies)
|
|
||||||
cd /home/rodriguez/src/gb
|
|
||||||
cargo check -p botserver > /tmp/check.log 2>&1 &
|
|
||||||
CARGO_PID=$!
|
|
||||||
while kill -0 $CARGO_PID 2>/dev/null; do
|
|
||||||
LINES=$(wc -l < /tmp/check.log 2>/dev/null || echo 0)
|
|
||||||
if [ "$LINES" -gt 20 ]; then
|
|
||||||
kill $CARGO_PID 2>/dev/null
|
|
||||||
echo "=== Got $LINES lines, killing cargo ==="
|
|
||||||
break
|
|
||||||
fi
|
|
||||||
sleep 1
|
|
||||||
done
|
|
||||||
# Check for errors - use strings to handle binary output
|
|
||||||
if strings /tmp/check.log | grep -q "^error"; then
|
|
||||||
echo "❌ Errors found:"
|
|
||||||
strings /tmp/check.log | grep "^error" | head -20
|
|
||||||
# Fix errors, then re-run this pattern
|
|
||||||
else
|
|
||||||
echo "✅ No errors - build clean!"
|
|
||||||
fi
|
|
||||||
```
|
|
||||||
|
|
||||||
**Key Rule:** Kill cargo at 20 lines, fix errors immediately, loop until clean.
|
|
||||||
|
|
||||||
**Why:** Compiling takes 2-3+ minutes. Getting errors in 20s saves 10+ minutes per error.
|
|
||||||
|
|
||||||
### Security
|
|
||||||
- ❌ **NEVER** include sensitive data (IPs, tokens, keys) in docs or code
|
|
||||||
- ❌ **NEVER** write internal IPs to logs — mask them (e.g., "10.x.x.x")
|
|
||||||
- ❌ **NEVER** create files with secrets in repo root
|
|
||||||
|
|
||||||
> **Secret files MUST be placed in `/tmp/` only** (ephemeral, not tracked by git).
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 🔐 Security Directives — MANDATORY
|
|
||||||
|
|
||||||
### 1. Error Handling — No Panics
|
|
||||||
```rust
|
|
||||||
// ❌ FORBIDDEN: unwrap(), expect(), panic!(), todo!()
|
|
||||||
// ✅ REQUIRED:
|
|
||||||
value?
|
|
||||||
value.ok_or_else(|| Error::NotFound)?
|
|
||||||
value.unwrap_or_default()
|
|
||||||
if let Some(v) = value { ... }
|
|
||||||
```
|
|
||||||
|
|
||||||
### 2. Command Execution — SafeCommand
|
|
||||||
```rust
|
|
||||||
// ❌ FORBIDDEN: Command::new("cmd").arg(user_input).output()
|
|
||||||
// ✅ REQUIRED:
|
|
||||||
use crate::security::command_guard::SafeCommand;
|
|
||||||
SafeCommand::new("allowed_command")?.arg("safe_arg")?.execute()
|
|
||||||
```
|
|
||||||
|
|
||||||
### 3. Error Responses — ErrorSanitizer
|
|
||||||
```rust
|
|
||||||
// ❌ FORBIDDEN: Json(json!({ "error": e.to_string() }))
|
|
||||||
// ✅ REQUIRED:
|
|
||||||
use crate::security::error_sanitizer::log_and_sanitize;
|
|
||||||
let sanitized = log_and_sanitize(&e, "context", None);
|
|
||||||
(StatusCode::INTERNAL_SERVER_ERROR, sanitized)
|
|
||||||
```
|
|
||||||
|
|
||||||
### 4. SQL — sql_guard
|
|
||||||
```rust
|
|
||||||
// ❌ FORBIDDEN: format!("SELECT * FROM {}", user_table)
|
|
||||||
// ✅ REQUIRED:
|
|
||||||
use crate::security::sql_guard::{sanitize_identifier, validate_table_name};
|
|
||||||
let safe_table = sanitize_identifier(&user_table);
|
|
||||||
validate_table_name(&safe_table)?;
|
|
||||||
```
|
|
||||||
|
|
||||||
### 5. Rate Limiting
|
|
||||||
- General: 100 req/s, Auth: 10 req/s, API: 50 req/s per token, WebSocket: 10 msgs/s
|
|
||||||
- Use `governor` crate with per-IP and per-User tracking
|
|
||||||
|
|
||||||
### 6. CSRF Protection
|
|
||||||
- ALL state-changing endpoints (POST/PUT/DELETE/PATCH) MUST require CSRF token
|
|
||||||
- Use `tower_csrf`, bound to user session. Exempt: Bearer Token endpoints
|
|
||||||
|
|
||||||
### 7. Security Headers (ALL responses)
|
|
||||||
`Content-Security-Policy`, `Strict-Transport-Security`, `X-Frame-Options: DENY`, `X-Content-Type-Options: nosniff`, `Referrer-Policy: strict-origin-when-cross-origin`, `Permissions-Policy: geolocation=(), microphone=(), camera=()`
|
|
||||||
|
|
||||||
### 8. Dependency Management
|
|
||||||
- App crates track `Cargo.lock`; lib crates don't
|
|
||||||
- Critical deps: exact versions (`=1.0.1`); regular: caret (`1.0`)
|
|
||||||
- Run `cargo audit` weekly; update only via PR with testing
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## ✅ Mandatory Code Patterns
|
|
||||||
|
|
||||||
```rust
|
|
||||||
impl MyStruct { fn new() -> Self { Self { } } } // Use Self, not type name
|
|
||||||
#[derive(PartialEq, Eq)] // Always derive both
|
|
||||||
format!("Hello {name}") // Inline format args
|
|
||||||
match x { A | B => do_thing(), C => other() } // Combine identical arms
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 📏 File Size Limits
|
|
||||||
|
|
||||||
- **Max 450 lines per file** — split proactively at 350 lines
|
|
||||||
- Split by: `types.rs`, `handlers.rs`, `operations.rs`, `utils.rs`, `mod.rs`
|
|
||||||
- Re-export all public items in `mod.rs`
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 🔥 Error Fixing Workflow
|
|
||||||
|
|
||||||
### Preferred: Offline Batch Fix
|
|
||||||
1. Read ENTIRE error list first
|
|
||||||
2. Group errors by file
|
|
||||||
3. For each file: view → fix ALL errors → write once
|
|
||||||
4. Verify with build/diagnostics only AFTER all fixes
|
|
||||||
|
|
||||||
### ⚡ Streaming Build Rule
|
|
||||||
Don't wait for `cargo` to finish — cancel at first errors, fix, re-run.
|
|
||||||
|
|
||||||
### 🧠 Memory Issues (process "Killed")
|
|
||||||
```bash
|
|
||||||
pkill -9 cargo; pkill -9 rustc; pkill -9 botserver
|
|
||||||
CARGO_BUILD_JOBS=1 cargo check -p botserver 2>&1 | tail -200
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 🔄 Modos de Execução
|
|
||||||
|
|
||||||
O botserver suporta **dois modos** de execução:
|
|
||||||
|
|
||||||
### Modo 1: Local Standalone (sem Docker/Incus)
|
|
||||||
|
|
||||||
O botserver sobe **tudo localmente** (PostgreSQL, Valkey, MinIO, Vault, LLM).
|
|
||||||
|
|
||||||
```bash
|
|
||||||
cd /home/rodriguez/src/gb/botserver
|
|
||||||
cargo run -- --install # Instala dependências (PostgreSQL, Valkey, MinIO, etc.)
|
|
||||||
cargo run # Sobe tudo e inicia o servidor
|
|
||||||
```
|
|
||||||
|
|
||||||
**O que acontece:**
|
|
||||||
- `PackageManager` baixa e extrai binários para `botserver-stack/bin/`
|
|
||||||
- Cria `botserver-stack/data/pgdata/` com PostgreSQL
|
|
||||||
- Inicia PostgreSQL na porta 5432
|
|
||||||
- Inicia Valkey na porta 6379
|
|
||||||
- Inicia MinIO na porta 9100
|
|
||||||
- Configura Vault para secrets
|
|
||||||
- Baixa modelo LLM (llama.cpp) para detecção de anomalias
|
|
||||||
- Ao final: `http://localhost:8080`
|
|
||||||
|
|
||||||
**Verificar se está rodando:**
|
|
||||||
```bash
|
|
||||||
curl http://localhost:8080/health
|
|
||||||
curl http://localhost:5432 # PostgreSQL
|
|
||||||
curl http://localhost:6379 # Valkey
|
|
||||||
```
|
|
||||||
|
|
||||||
**Testar com Playwright:**
|
|
||||||
```bash
|
|
||||||
# Navegar para bot de teste
|
|
||||||
npx playwright open http://localhost:3000/salesianos
|
|
||||||
# Ou diretamente
|
|
||||||
npx playwright open http://localhost:3000/detecta
|
|
||||||
```
|
|
||||||
|
|
||||||
### Modo 2: Container (Incus) — Produção
|
|
||||||
|
|
||||||
Os serviços rodam em containers Incus separados.
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Subir todos os containers
|
|
||||||
sudo incus start system tables vault directory drive cache llm vector_db
|
|
||||||
|
|
||||||
# Verificar status
|
|
||||||
sudo incus list
|
|
||||||
|
|
||||||
# Acessar container system (onde roda botserver)
|
|
||||||
sudo incus exec system -- bash
|
|
||||||
|
|
||||||
# Ver logs do botserver
|
|
||||||
sudo incus exec system -- journalctl -u botserver -f
|
|
||||||
```
|
|
||||||
|
|
||||||
**Arquitetura de Containers:**
|
|
||||||
|
|
||||||
| Container | Services | Portas |
|
|
||||||
|-----------|----------|--------|
|
|
||||||
| system | BotServer, Valkey | 8080, 6379 |
|
|
||||||
| tables | PostgreSQL | 5432 |
|
|
||||||
| vault | Vault | 8200 |
|
|
||||||
| directory | Zitadel | 9000 |
|
|
||||||
| drive | MinIO | 9100 |
|
|
||||||
| cache | Valkey (backup) | 6379 |
|
|
||||||
| llm | llama.cpp | 8081 |
|
|
||||||
| vector_db | Qdrant | 6333 |
|
|
||||||
|
|
||||||
### reset.sh (Ambiente Local)
|
|
||||||
```bash
|
|
||||||
./reset.sh # Limpa e reinicia tudo localmente
|
|
||||||
```
|
|
||||||
|
|
||||||
### Service Commands
|
|
||||||
```bash
|
|
||||||
ps aux | grep -E "(botserver|botui)" | grep -v grep
|
|
||||||
curl http://localhost:8080/health
|
|
||||||
./restart.sh # Restart services
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 🎭 Playwright Browser Testing
|
|
||||||
|
|
||||||
### Browser Setup
|
|
||||||
If browser fails: `pkill -9 -f brave; pkill -9 -f chrome; pkill -9 -f chromium` → wait 3s → navigate again.
|
|
||||||
|
|
||||||
### Bot Testing Flow
|
|
||||||
1. Navigate to `http://localhost:3000/<botname>`
|
|
||||||
2. Snapshot → verify welcome message + suggestion buttons + Portuguese accents
|
|
||||||
3. Click suggestion → wait 3-5s → snapshot → fill data → submit
|
|
||||||
4. Verify DB records and backend logs
|
|
||||||
|
|
||||||
### Desktop UI Note
|
|
||||||
Chat window may cover other apps — click **middle button** (restore) to minimize, or navigate directly via URL.
|
|
||||||
|
|
||||||
### WhatsApp Testing
|
|
||||||
- Webhook is **global** — bot routing by typing bot name as first message
|
|
||||||
- Single WhatsApp number serves ALL bots; routing via `whatsapp-id` in `config.csv`
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## ➕ Adding New Features
|
|
||||||
|
|
||||||
### Checklist
|
|
||||||
- [ ] Which module owns this? (Check Module Responsibility Matrix)
|
|
||||||
- [ ] Database migrations needed?
|
|
||||||
- [ ] New API endpoints?
|
|
||||||
- [ ] Security: input validation, auth, rate limiting, error sanitization?
|
|
||||||
- [ ] Screens in botui?
|
|
||||||
- [ ] No `unwrap()`/`expect()`?
|
|
||||||
|
|
||||||
### Pattern: types → schema → Diesel model → business logic → API endpoint → BASIC keyword (if applicable) → tests → docs in `botbook/`
|
|
||||||
|
|
||||||
### Commit & Deploy
|
|
||||||
```bash
|
|
||||||
cd botserver && git push alm main && git push origin main
|
|
||||||
cd .. && git add botserver && git commit -m "Update botserver: <desc>" && git push alm main && git push origin main
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 🎨 Frontend Standards
|
|
||||||
|
|
||||||
- **HTMX-first** — server returns HTML fragments, not JSON
|
|
||||||
- **Local assets only** — NO CDN links
|
|
||||||
- Use `hx-get`, `hx-post`, `hx-target`, `hx-swap`; WebSocket via htmx-ws
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 🚀 Performance & Quality
|
|
||||||
|
|
||||||
- `cargo clippy --workspace` must pass with **0 warnings**
|
|
||||||
- `cargo tree --duplicates` / `cargo machete` / `cargo audit` weekly
|
|
||||||
- Release profile: `opt-level = "z"`, `lto = true`, `codegen-units = 1`, `strip = true`, `panic = "abort"`
|
|
||||||
- Use `default-features = false` and opt-in to needed features
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 🧪 Testing
|
|
||||||
|
|
||||||
- **Unit:** per-crate `tests/` or `#[cfg(test)]` modules — `cargo test -p <crate>`
|
|
||||||
- **Integration:** `bottest/` crate — `cargo test -p bottest`
|
|
||||||
- **Coverage:** 80%+ on critical paths; ALL error paths and security guards tested
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 🚢 Deploy Workflow (CI/CD Only)
|
|
||||||
|
|
||||||
1. Push to ALM (triggers CI automatically)
|
|
||||||
2. CI builds on alm-ci → deploys to system container via SSH
|
|
||||||
3. Service auto-restarts on binary update
|
|
||||||
4. Verify: check service status + logs after ~10 min
|
|
||||||
|
|
||||||
### Container Architecture
|
|
||||||
|
|
||||||
| Container | Service | Port |
|
|
||||||
|-----------|---------|------|
|
|
||||||
| system | BotServer + Valkey | 8080/6379 |
|
|
||||||
| tables | PostgreSQL | 5432 |
|
|
||||||
| vault | Vault | 8200 |
|
|
||||||
---
|
|
||||||
|
|
||||||
## 🔑 Core Directives Summary
|
|
||||||
|
|
||||||
- **OFFLINE FIRST** — fix all errors from list before compiling
|
|
||||||
- **BATCH BY FILE** — fix ALL errors in a file at once, write once
|
|
||||||
- **VERIFY LAST** — only compile after ALL fixes applied
|
|
||||||
- **DELETE DEAD CODE** — never keep unused code
|
|
||||||
- **GIT WORKFLOW** — always push to ALL repositories
|
|
||||||
- **0 warnings, 0 errors** — loop until clean
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 🔧 Bot Scripts Architecture
|
|
||||||
|
|
||||||
### File Types
|
|
||||||
| File | Purpose |
|
|
||||||
|------|---------|
|
|
||||||
| `start.bas` | Entry point, executed on session start |
|
|
||||||
| `{tool}.bas` | Tool implementation (e.g., `detecta.bas`) |
|
|
||||||
| `tables.bas` | **SPECIAL** - Defines database tables, auto-creates on compile |
|
|
||||||
| `init_folha.bas` | Initialization script for specific features |
|
|
||||||
|
|
||||||
### tables.bas — SPECIAL FILE
|
|
||||||
- **DO NOT call via CALL keyword** - it's processed automatically
|
|
||||||
- Parsed at compile time by `process_table_definitions()`
|
|
||||||
- Tables are created/updated in database via `sync_bot_tables()`
|
|
||||||
- Location: `/opt/gbo/data/{bot}.gbai/{bot}.gbdialog/tables.bas`
|
|
||||||
|
|
||||||
### Tool Button Execution (TOOL_EXEC)
|
|
||||||
- Frontend sends `message_type: 6` via WebSocket
|
|
||||||
- Backend handles in `stream_response()` when `message_type == MessageType::TOOL_EXEC`
|
|
||||||
- Tool executes directly, skips KB injection and LLM
|
|
||||||
- Result appears in chat (tool output), no "/tool" text shown
|
|
||||||
|
|
||||||
### CALL Keyword
|
|
||||||
- Can call in-memory procedures OR .bas scripts
|
|
||||||
- Syntax: `CALL "script_name"` or `CALL "procedure_name"`
|
|
||||||
- If not in memory, looks for `{name}.bas` in bot's gbdialog folder
|
|
||||||
|
|
||||||
### DETECT Keyword
|
|
||||||
- Analyzes database table for anomalies
|
|
||||||
- Requires table to exist (defined in tables.bas)
|
|
||||||
- Example: `result = DETECT "folha_salarios"`
|
|
||||||
- Calls BotModels API at `/api/anomaly/detect`
|
|
||||||
|
|
||||||
### start.bas Execution
|
|
||||||
- Executed on WebSocket connect (for web clients)
|
|
||||||
- Also on first user message (blocking, once per session)
|
|
||||||
- Loads suggestions via `ADD_SUGGESTION_TOOL`
|
|
||||||
- Marks session with Redis key to prevent re-run
|
|
||||||
|
|
||||||
### MessageType Enum (botlib/src/message_types.rs)
|
|
||||||
| ID | Name | Purpose |
|
|
||||||
|----|------|---------|
|
|
||||||
| 0 | EXTERNAL | External message |
|
|
||||||
| 1 | USER | User message |
|
|
||||||
| 2 | BOT_RESPONSE | Bot response |
|
|
||||||
| 3 | CONTINUE | Continue processing |
|
|
||||||
| 4 | SUGGESTION | Suggestion button |
|
|
||||||
| 5 | CONTEXT_CHANGE | Context change |
|
|
||||||
| 6 | TOOL_EXEC | Direct tool execution (skips KB/LLM) |
|
|
||||||
|
|
||||||
**Usage:** When frontend sends `message_type: 6`, backend executes tool directly without going through LLM.
|
|
||||||
|
|
||||||
### 🚨 FUNDAMENTAL: Submodule Push Rule (MANDATORY)
|
|
||||||
|
|
||||||
**Every time you push the main repo, you MUST also push ALL submodules!**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# After ANY main repo push, ALWAYS run:
|
|
||||||
cd botserver && git push origin main && git push alm main
|
|
||||||
cd ../botui && git push origin main && git push alm main
|
|
||||||
cd ../botlib && git push origin main && git push alm main
|
|
||||||
# ... repeat for ALL submodules
|
|
||||||
```
|
|
||||||
|
|
||||||
**Why:** CI builds based on submodule commits. If submodule isn't pushed, CI deploys old code.
|
|
||||||
|
|
||||||
**Checklist before pushing:**
|
|
||||||
- [ ] botserver pushed?
|
|
||||||
- [ ] botui pushed?
|
|
||||||
- [ ] botlib pushed?
|
|
||||||
- [ ] All other submodules pushed?
|
|
||||||
- [ ] Main repo points to new submodule commits?
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 🔐 Zitadel Setup (Directory Service)
|
|
||||||
|
|
||||||
### Container Architecture
|
|
||||||
- **directory container**: Zitadel running on port **8080** internally
|
|
||||||
- **tables container**: PostgreSQL database on port 5432
|
|
||||||
- Use database **PROD-DIRECTORY** for Zitadel data
|
|
||||||
|
|
||||||
### Network Access (Container Mode)
|
|
||||||
- **Internal API**: `http://<directory-ip>:8080`
|
|
||||||
- **External port 9000** redirected via iptables NAT to directory:8080
|
|
||||||
- **Health check**: `curl -sf http://localhost:8080/debug/healthz`
|
|
||||||
|
|
||||||
### Zitadel Installation Steps
|
|
||||||
|
|
||||||
1. **Reset database** (on tables container):
|
|
||||||
```bash
|
|
||||||
psql -h localhost -U postgres -d postgres -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = 'PROD-DIRECTORY' AND pid <> pg_backend_pid();"
|
|
||||||
psql -h localhost -U postgres -d postgres -c "DROP DATABASE IF EXISTS \"PROD-DIRECTORY\";"
|
|
||||||
psql -h localhost -U postgres -d postgres -c "CREATE DATABASE \"PROD-DIRECTORY\";"
|
|
||||||
```
|
|
||||||
|
|
||||||
2. **Create init config** (on directory container):
|
|
||||||
```bash
|
|
||||||
cat > /opt/gbo/conf/directory/zitadel-init-steps.yaml << "EOF"
|
|
||||||
FirstInstance:
|
|
||||||
InstanceName: "BotServer"
|
|
||||||
DefaultLanguage: "en"
|
|
||||||
PatPath: "/opt/gbo/conf/directory/admin-pat.txt"
|
|
||||||
Org:
|
|
||||||
Name: "BotServer"
|
|
||||||
Machine:
|
|
||||||
Machine:
|
|
||||||
Username: "admin-sa"
|
|
||||||
Name: "Admin Service Account"
|
|
||||||
Pat:
|
|
||||||
ExpirationDate: "2099-01-01T00:00:00Z"
|
|
||||||
Human:
|
|
||||||
UserName: "admin"
|
|
||||||
FirstName: "Admin"
|
|
||||||
LastName: "User"
|
|
||||||
Email:
|
|
||||||
Address: "admin@localhost"
|
|
||||||
Verified: true
|
|
||||||
Password: "Admin123!"
|
|
||||||
PasswordChangeRequired: false
|
|
||||||
EOF
|
|
||||||
```
|
|
||||||
|
|
||||||
3. **Start Zitadel** (on directory container):
|
|
||||||
```bash
|
|
||||||
pkill -9 zitadel || true
|
|
||||||
nohup env \
|
|
||||||
ZITADEL_DATABASE_POSTGRES_HOST=<tables-ip> \
|
|
||||||
ZITADEL_DATABASE_POSTGRES_PORT=5432 \
|
|
||||||
ZITADEL_DATABASE_POSTGRES_DATABASE=PROD-DIRECTORY \
|
|
||||||
ZITADEL_DATABASE_POSTGRES_USER_USERNAME=postgres \
|
|
||||||
ZITADEL_DATABASE_POSTGRES_USER_PASSWORD=postgres \
|
|
||||||
ZITADEL_DATABASE_POSTGRES_USER_SSL_MODE=disable \
|
|
||||||
ZITADEL_DATABASE_POSTGRES_ADMIN_USERNAME=postgres \
|
|
||||||
ZITADEL_DATABASE_POSTGRES_ADMIN_PASSWORD=postgres \
|
|
||||||
ZITADEL_DATABASE_POSTGRES_ADMIN_SSL_MODE=disable \
|
|
||||||
ZITADEL_EXTERNALSECURE=false \
|
|
||||||
ZITADEL_EXTERNALDOMAIN=<directory-ip> \
|
|
||||||
ZITADEL_EXTERNALPORT=9000 \
|
|
||||||
ZITADEL_TLS_ENABLED=false \
|
|
||||||
/opt/gbo/bin/zitadel start-from-init \
|
|
||||||
--masterkey MasterkeyNeedsToHave32Characters \
|
|
||||||
--tlsMode disabled \
|
|
||||||
--externalDomain <directory-ip> \
|
|
||||||
--externalPort 9000 \
|
|
||||||
--steps /opt/gbo/conf/directory/zitadel-init-steps.yaml \
|
|
||||||
> /opt/gbo/logs/zitadel.log 2>&1 &
|
|
||||||
```
|
|
||||||
|
|
||||||
4. **Wait for bootstrap** (~90 seconds), then verify:
|
|
||||||
```bash
|
|
||||||
curl -sf http://localhost:8080/debug/healthz
|
|
||||||
cat /opt/gbo/conf/directory/admin-pat.txt
|
|
||||||
```
|
|
||||||
|
|
||||||
5. **Configure iptables** (on system container):
|
|
||||||
```bash
|
|
||||||
iptables -t nat -A PREROUTING -p tcp --dport 9000 -j DNAT --to-destination <directory-ip>:8080
|
|
||||||
iptables -t nat -A OUTPUT -p tcp -d <external-ip> --dport 9000 -j DNAT --to-destination <directory-ip>:8080
|
|
||||||
```
|
|
||||||
|
|
||||||
### Zitadel API Usage
|
|
||||||
|
|
||||||
**PAT file location**: `/opt/gbo/conf/directory/admin-pat.txt` (on directory container)
|
|
||||||
|
|
||||||
#### Get IAM Info (internal)
|
|
||||||
```bash
|
|
||||||
curl -s -H "Authorization: Bearer $PAT" http://<directory-ip>:8080/management/v1/iam
|
|
||||||
```
|
|
||||||
|
|
||||||
#### Get IAM Info (external via port 9000)
|
|
||||||
```bash
|
|
||||||
curl -s -H "Authorization: Bearer $PAT" -H "Host: <directory-ip>" http://<external-ip>:9000/management/v1/iam
|
|
||||||
```
|
|
||||||
|
|
||||||
#### Create Human User
|
|
||||||
```bash
|
|
||||||
curl -s -X POST \
|
|
||||||
-H "Authorization: Bearer $PAT" \
|
|
||||||
-H "Host: <directory-ip>" \
|
|
||||||
-H "Content-Type: application/json" \
|
|
||||||
http://<external-ip>:9000/management/v1/users/human \
|
|
||||||
-d '{
|
|
||||||
"userName": "janedoe",
|
|
||||||
"name": "Jane Doe",
|
|
||||||
"profile": {"firstName": "Jane", "lastName": "Doe"},
|
|
||||||
"email": {"email": "jane@example.com"}
|
|
||||||
}'
|
|
||||||
```
|
|
||||||
|
|
||||||
### Zitadel API Endpoints Reference
|
|
||||||
|
|
||||||
| Endpoint | Method | Description |
|
|
||||||
|----------|--------|-------------|
|
|
||||||
| `/management/v1/iam` | GET | Get IAM info |
|
|
||||||
| `/management/v1/orgs/me` | GET | Get current org |
|
|
||||||
| `/management/v1/users/human` | POST | Create human user |
|
|
||||||
| `/management/v1/users/machine` | POST | Create machine user |
|
|
||||||
| `/oauth/v2/token` | POST | Get access token |
|
|
||||||
| `/debug/healthz` | GET | Health check |
|
|
||||||
|
|
||||||
### Important Notes
|
|
||||||
|
|
||||||
- **Zitadel listens on port 8080 internally**
|
|
||||||
- **External port 9000** is forwarded via iptables NAT
|
|
||||||
- **Use Host header** with directory IP for external API calls
|
|
||||||
- **PAT file**: `/opt/gbo/conf/directory/admin-pat.txt`
|
|
||||||
- **Admin credentials**: `admin` / `Admin123!` (human user)
|
|
||||||
- **Database**: `PROD-DIRECTORY` on tables container
|
|
||||||
- **Zitadel v4.13.1** is the current version
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 📊 SEPLAGSE Bot Configuration
|
|
||||||
|
|
||||||
### Bot Location
|
|
||||||
- **Source**: `/opt/gbo/data/seplagse.gbai/seplagse.gbdialog/`
|
|
||||||
- **Work**: `./botserver-stack/data/system/work/seplagse.gbai/seplagse.gbdialog/`
|
|
||||||
|
|
||||||
### Key Files
|
|
||||||
| File | Purpose |
|
|
||||||
|------|---------|
|
|
||||||
| `start.bas` | Entry point with suggestion buttons |
|
|
||||||
| `detecta.bas` | Tool for detecting anomalies in folha_salarios |
|
|
||||||
| `init_folha.bas` | Tool to initialize test data (INSERT keyword has issues) |
|
|
||||||
| `tables.bas` | Table definitions - auto-processed on compile |
|
|
||||||
|
|
||||||
### Tool Button Configuration (start.bas)
|
|
||||||
```bas
|
|
||||||
ADD_SUGGESTION_TOOL "detecta" AS "🔍 Detectar Desvios na Folha"
|
|
||||||
ADD_SUGGESTION_TOOL "init_folha" AS "⚙️ Inicializar Dados de Teste"
|
|
||||||
```
|
|
||||||
|
|
||||||
### Detection Flow
|
|
||||||
1. User clicks "Detectar Desvios na Folha" button
|
|
||||||
2. Frontend sends `message_type: 6` (TOOL_EXEC) via WebSocket
|
|
||||||
3. Backend executes `detecta.bas` directly (skips KB/LLM)
|
|
||||||
4. `detecta.bas` calls `DETECT "folha_salarios"` keyword
|
|
||||||
5. Keyword queries bot-specific database for table data
|
|
||||||
6. Data sent to BotModels API at `/api/anomaly/detect`
|
|
||||||
7. Results displayed in chat
|
|
||||||
|
|
||||||
### Fixes Applied
|
|
||||||
1. **TOOL_EXEC message type**: Added `MessageType::TOOL_EXEC` (id=6)
|
|
||||||
2. **Frontend WebSocket**: Sends `message_type: 6` for tool buttons
|
|
||||||
3. **Backend handler**: `stream_response()` handles TOOL_EXEC directly
|
|
||||||
4. **DETECT keyword**: Fixed to use bot-specific database (`bot_database_manager`)
|
|
||||||
5. **Bot execution**: Tool buttons work - no "/tool" text shown
|
|
||||||
|
|
||||||
### Known Issues
|
|
||||||
- **INSERT keyword**: Has parsing issues in multi-line scripts
|
|
||||||
- **Test data**: `init_folha.bas` cannot insert data due to INSERT issues
|
|
||||||
- **Workaround**: Insert data manually or via external SQL tool
|
|
||||||
|
|
||||||
### Testing
|
|
||||||
```bash
|
|
||||||
# Restart services
|
|
||||||
./restart.sh
|
|
||||||
|
|
||||||
# Test in browser
|
|
||||||
http://localhost:3000/seplagse
|
|
||||||
|
|
||||||
# Check logs
|
|
||||||
tail -f botserver.log | grep -i "detecta\|error"
|
|
||||||
```
|
|
||||||
11677
Cargo.lock
generated
11677
Cargo.lock
generated
File diff suppressed because it is too large
Load diff
|
|
@ -9,7 +9,6 @@ members = [
|
||||||
"bottest",
|
"bottest",
|
||||||
"botui",
|
"botui",
|
||||||
]
|
]
|
||||||
exclude = ["backup-to-s3"]
|
|
||||||
|
|
||||||
[workspace.lints.rust]
|
[workspace.lints.rust]
|
||||||
|
|
||||||
|
|
@ -110,7 +109,6 @@ url = "2.5"
|
||||||
dirs = "5.0"
|
dirs = "5.0"
|
||||||
tempfile = "3"
|
tempfile = "3"
|
||||||
walkdir = "2.5.0"
|
walkdir = "2.5.0"
|
||||||
notify = "8.0"
|
|
||||||
|
|
||||||
# ─── COMPRESSION / ARCHIVES ───
|
# ─── COMPRESSION / ARCHIVES ───
|
||||||
flate2 = "1.0"
|
flate2 = "1.0"
|
||||||
|
|
@ -176,7 +174,7 @@ indicatif = "0.18.0"
|
||||||
|
|
||||||
# ─── MEMORY ALLOCATOR ───
|
# ─── MEMORY ALLOCATOR ───
|
||||||
tikv-jemallocator = "0.6"
|
tikv-jemallocator = "0.6"
|
||||||
tikv-jemalloc-ctl = { version = "0.6", default-features = false, features = ["stats"] }
|
tikv-jemalloc-ctl = { version = "0.6", default-features = false }
|
||||||
|
|
||||||
# ─── SECRETS / VAULT ───
|
# ─── SECRETS / VAULT ───
|
||||||
vaultrs = "0.7"
|
vaultrs = "0.7"
|
||||||
|
|
@ -198,7 +196,7 @@ csv = "1.3"
|
||||||
tonic = { version = "0.14.2", default-features = false }
|
tonic = { version = "0.14.2", default-features = false }
|
||||||
|
|
||||||
# ─── STATIC FILES ───
|
# ─── STATIC FILES ───
|
||||||
rust-embed = { version = "8.5", features = ["interpolate-folder-path"] }
|
rust-embed = "8.5"
|
||||||
mime_guess = "2.0"
|
mime_guess = "2.0"
|
||||||
|
|
||||||
# ─── TAURI (Desktop/Mobile) ───
|
# ─── TAURI (Desktop/Mobile) ───
|
||||||
|
|
|
||||||
204
DEPENDENCIES-DEV.sh
Normal file
204
DEPENDENCIES-DEV.sh
Normal file
|
|
@ -0,0 +1,204 @@
|
||||||
|
#!/bin/bash
|
||||||
|
#
|
||||||
|
# DEPENDENCIES-DEV.sh - Development Dependencies for General Bots
|
||||||
|
#
|
||||||
|
# This script installs additional packages needed for BUILDING botserver from source.
|
||||||
|
# Only install these if you plan to compile the code yourself.
|
||||||
|
#
|
||||||
|
# Usage: sudo ./DEPENDENCIES-DEV.sh
|
||||||
|
#
|
||||||
|
|
||||||
|
set -e
|
||||||
|
|
||||||
|
# Colors
|
||||||
|
RED='\033[0;31m'
|
||||||
|
GREEN='\033[0;32m'
|
||||||
|
YELLOW='\033[1;33m'
|
||||||
|
NC='\033[0m'
|
||||||
|
|
||||||
|
echo -e "${GREEN}========================================${NC}"
|
||||||
|
echo -e "${GREEN} General Bots Development Dependencies${NC}"
|
||||||
|
echo -e "${GREEN}========================================${NC}"
|
||||||
|
|
||||||
|
# Check root
|
||||||
|
if [ "$EUID" -ne 0 ]; then
|
||||||
|
echo -e "${RED}Error: Run as root (use sudo)${NC}"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Detect OS
|
||||||
|
if [ -f /etc/os-release ]; then
|
||||||
|
. /etc/os-release
|
||||||
|
OS=$ID
|
||||||
|
else
|
||||||
|
echo -e "${RED}Error: Cannot detect OS${NC}"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo -e "${YELLOW}OS: $OS${NC}"
|
||||||
|
|
||||||
|
install_debian_ubuntu() {
|
||||||
|
apt-get update
|
||||||
|
apt-get install -y \
|
||||||
|
build-essential \
|
||||||
|
gcc \
|
||||||
|
g++ \
|
||||||
|
clang \
|
||||||
|
llvm-dev \
|
||||||
|
libclang-dev \
|
||||||
|
cmake \
|
||||||
|
make \
|
||||||
|
git \
|
||||||
|
pkg-config \
|
||||||
|
libssl-dev \
|
||||||
|
libpq-dev \
|
||||||
|
liblzma-dev \
|
||||||
|
zlib1g-dev \
|
||||||
|
libabseil-dev \
|
||||||
|
protobuf-compiler \
|
||||||
|
libprotobuf-dev \
|
||||||
|
automake \
|
||||||
|
bison \
|
||||||
|
flex \
|
||||||
|
gperf \
|
||||||
|
libtool \
|
||||||
|
m4 \
|
||||||
|
nasm \
|
||||||
|
python3 \
|
||||||
|
python3-pip \
|
||||||
|
nodejs \
|
||||||
|
npm
|
||||||
|
|
||||||
|
# Cross-compilation toolchains
|
||||||
|
apt-get install -y \
|
||||||
|
gcc-aarch64-linux-gnu \
|
||||||
|
gcc-arm-linux-gnueabihf \
|
||||||
|
gcc-x86-64-linux-gnu || true
|
||||||
|
}
|
||||||
|
|
||||||
|
install_fedora_rhel() {
|
||||||
|
dnf groupinstall -y "Development Tools"
|
||||||
|
dnf install -y \
|
||||||
|
gcc \
|
||||||
|
gcc-c++ \
|
||||||
|
clang \
|
||||||
|
llvm-devel \
|
||||||
|
clang-devel \
|
||||||
|
cmake \
|
||||||
|
make \
|
||||||
|
git \
|
||||||
|
pkgconf-devel \
|
||||||
|
openssl-devel \
|
||||||
|
libpq-devel \
|
||||||
|
xz-devel \
|
||||||
|
zlib-devel \
|
||||||
|
abseil-cpp-devel \
|
||||||
|
protobuf-compiler \
|
||||||
|
protobuf-devel \
|
||||||
|
automake \
|
||||||
|
bison \
|
||||||
|
flex \
|
||||||
|
gperf \
|
||||||
|
libtool \
|
||||||
|
m4 \
|
||||||
|
nasm \
|
||||||
|
python3 \
|
||||||
|
python3-pip \
|
||||||
|
nodejs \
|
||||||
|
npm
|
||||||
|
}
|
||||||
|
|
||||||
|
install_arch() {
|
||||||
|
pacman -Sy --noconfirm \
|
||||||
|
base-devel \
|
||||||
|
gcc \
|
||||||
|
clang \
|
||||||
|
llvm \
|
||||||
|
cmake \
|
||||||
|
make \
|
||||||
|
git \
|
||||||
|
pkgconf \
|
||||||
|
openssl \
|
||||||
|
postgresql-libs \
|
||||||
|
xz \
|
||||||
|
zlib \
|
||||||
|
abseil-cpp \
|
||||||
|
protobuf \
|
||||||
|
automake \
|
||||||
|
bison \
|
||||||
|
flex \
|
||||||
|
gperf \
|
||||||
|
libtool \
|
||||||
|
m4 \
|
||||||
|
nasm \
|
||||||
|
python \
|
||||||
|
python-pip \
|
||||||
|
nodejs \
|
||||||
|
npm
|
||||||
|
}
|
||||||
|
|
||||||
|
install_alpine() {
|
||||||
|
apk add --no-cache \
|
||||||
|
build-base \
|
||||||
|
gcc \
|
||||||
|
g++ \
|
||||||
|
clang \
|
||||||
|
llvm-dev \
|
||||||
|
clang-dev \
|
||||||
|
cmake \
|
||||||
|
make \
|
||||||
|
git \
|
||||||
|
pkgconf-dev \
|
||||||
|
openssl-dev \
|
||||||
|
postgresql-dev \
|
||||||
|
xz-dev \
|
||||||
|
zlib-dev \
|
||||||
|
abseil-cpp-dev \
|
||||||
|
protobuf-dev \
|
||||||
|
protoc \
|
||||||
|
automake \
|
||||||
|
bison \
|
||||||
|
flex \
|
||||||
|
gperf \
|
||||||
|
libtool \
|
||||||
|
m4 \
|
||||||
|
nasm \
|
||||||
|
python3 \
|
||||||
|
py3-pip \
|
||||||
|
nodejs \
|
||||||
|
npm
|
||||||
|
}
|
||||||
|
|
||||||
|
case $OS in
|
||||||
|
ubuntu|debian|linuxmint|pop)
|
||||||
|
install_debian_ubuntu
|
||||||
|
;;
|
||||||
|
fedora|rhel|centos|rocky|almalinux)
|
||||||
|
install_fedora_rhel
|
||||||
|
;;
|
||||||
|
arch|manjaro)
|
||||||
|
install_arch
|
||||||
|
;;
|
||||||
|
alpine)
|
||||||
|
install_alpine
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
echo -e "${RED}Unsupported OS: $OS${NC}"
|
||||||
|
echo "Required development packages:"
|
||||||
|
echo " - build-essential/base-devel"
|
||||||
|
echo " - gcc, g++, clang"
|
||||||
|
echo " - cmake, make, git"
|
||||||
|
echo " - Development headers for:"
|
||||||
|
echo " - OpenSSL, PostgreSQL, XZ, zlib"
|
||||||
|
echo " - Abseil, Protobuf, LLVM"
|
||||||
|
exit 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
echo -e "${GREEN}Development dependencies installed!${NC}"
|
||||||
|
echo ""
|
||||||
|
echo "Install Rust if not already installed:"
|
||||||
|
echo " curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh"
|
||||||
|
echo ""
|
||||||
|
echo "Then build with:"
|
||||||
|
echo " cargo build --release"
|
||||||
|
|
@ -1,77 +0,0 @@
|
||||||
<#
|
|
||||||
.SYNOPSIS
|
|
||||||
Installs runtime dependencies for General Bots on Windows.
|
|
||||||
|
|
||||||
.DESCRIPTION
|
|
||||||
This script downloads and configures the system libraries required to build
|
|
||||||
and run BotServer on Windows. It downloads PostgreSQL binaries (for libpq)
|
|
||||||
and sets the PQ_LIB_DIR environment variable permanently.
|
|
||||||
|
|
||||||
.EXAMPLE
|
|
||||||
PS> .\DEPENDENCIES.ps1
|
|
||||||
#>
|
|
||||||
|
|
||||||
$ErrorActionPreference = 'Stop'
|
|
||||||
|
|
||||||
# ─── COLORS ───
|
|
||||||
function Write-Step { param($msg) Write-Host " * $msg" -ForegroundColor Green }
|
|
||||||
function Write-Warn { param($msg) Write-Host " ! $msg" -ForegroundColor Yellow }
|
|
||||||
function Write-Err { param($msg) Write-Host " x $msg" -ForegroundColor Red }
|
|
||||||
|
|
||||||
Write-Host "========================================" -ForegroundColor Green
|
|
||||||
Write-Host " General Bots Runtime Dependencies" -ForegroundColor Green
|
|
||||||
Write-Host " (Windows)" -ForegroundColor Green
|
|
||||||
Write-Host "========================================" -ForegroundColor Green
|
|
||||||
Write-Host ""
|
|
||||||
|
|
||||||
# ─── PostgreSQL binaries (libpq.lib for Diesel ORM) ───
|
|
||||||
$PgsqlDir = "C:\pgsql\pgsql"
|
|
||||||
$PgsqlLib = "$PgsqlDir\lib\libpq.lib"
|
|
||||||
$PgsqlZipUrl = "https://get.enterprisedb.com/postgresql/postgresql-17.4-1-windows-x64-binaries.zip"
|
|
||||||
$PgsqlZip = "$env:TEMP\pgsql.zip"
|
|
||||||
|
|
||||||
if (Test-Path $PgsqlLib) {
|
|
||||||
Write-Step "PostgreSQL binaries already present at $PgsqlDir"
|
|
||||||
} else {
|
|
||||||
Write-Host "`nDownloading PostgreSQL binaries..." -ForegroundColor Cyan
|
|
||||||
Write-Host " URL: $PgsqlZipUrl"
|
|
||||||
Write-Host " This may take a few minutes (~300MB)...`n"
|
|
||||||
|
|
||||||
Invoke-WebRequest -Uri $PgsqlZipUrl -OutFile $PgsqlZip -UseBasicParsing
|
|
||||||
|
|
||||||
Write-Host "Extracting to C:\pgsql ..."
|
|
||||||
if (Test-Path "C:\pgsql") { Remove-Item "C:\pgsql" -Recurse -Force }
|
|
||||||
Expand-Archive -Path $PgsqlZip -DestinationPath "C:\pgsql" -Force
|
|
||||||
Remove-Item $PgsqlZip -Force -ErrorAction SilentlyContinue
|
|
||||||
|
|
||||||
if (Test-Path $PgsqlLib) {
|
|
||||||
Write-Step "PostgreSQL binaries installed successfully."
|
|
||||||
} else {
|
|
||||||
Write-Err "Failed to find libpq.lib after extraction!"
|
|
||||||
exit 1
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
# Set PQ_LIB_DIR permanently for the current user
|
|
||||||
$CurrentPqDir = [System.Environment]::GetEnvironmentVariable("PQ_LIB_DIR", "User")
|
|
||||||
if ($CurrentPqDir -ne "$PgsqlDir\lib") {
|
|
||||||
[System.Environment]::SetEnvironmentVariable("PQ_LIB_DIR", "$PgsqlDir\lib", "User")
|
|
||||||
$env:PQ_LIB_DIR = "$PgsqlDir\lib"
|
|
||||||
Write-Step "PQ_LIB_DIR set to '$PgsqlDir\lib' (User environment variable)"
|
|
||||||
} else {
|
|
||||||
Write-Step "PQ_LIB_DIR already configured."
|
|
||||||
}
|
|
||||||
|
|
||||||
# ─── Summary ───
|
|
||||||
Write-Host ""
|
|
||||||
Write-Host "========================================" -ForegroundColor Green
|
|
||||||
Write-Host " Dependencies installed!" -ForegroundColor Green
|
|
||||||
Write-Host "========================================" -ForegroundColor Green
|
|
||||||
Write-Host ""
|
|
||||||
Write-Host "You can now build and run:" -ForegroundColor Cyan
|
|
||||||
Write-Host " cargo build -p botserver"
|
|
||||||
Write-Host " cargo build -p botui"
|
|
||||||
Write-Host " .\restart.ps1"
|
|
||||||
Write-Host ""
|
|
||||||
Write-Host "NOTE: If this is the first time, restart your terminal" -ForegroundColor Yellow
|
|
||||||
Write-Host " so PQ_LIB_DIR takes effect." -ForegroundColor Yellow
|
|
||||||
249
DEPENDENCIES.sh
249
DEPENDENCIES.sh
|
|
@ -1,133 +1,140 @@
|
||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
#
|
#
|
||||||
# DEPENDENCIES.sh - Runtime Dependencies for General Bots
|
# DEPENDENCIES.sh - Runtime Dependencies for General Bots
|
||||||
#
|
#
|
||||||
# This script installs all system packages required to RUN botserver binary.
|
# This script installs all system packages required to RUN botserver binary.
|
||||||
# These are the minimal dependencies needed for production deployment.
|
# These are the minimal dependencies needed for production deployment.
|
||||||
#
|
#
|
||||||
# Usage: sudo ./DEPENDENCIES.sh
|
# Usage: sudo ./DEPENDENCIES.sh
|
||||||
#
|
#
|
||||||
|
|
||||||
set -e
|
set -e
|
||||||
|
|
||||||
# Colors
|
# Colors
|
||||||
RED='\033[0;31m'
|
RED='\033[0;31m'
|
||||||
GREEN='\033[0;32m'
|
GREEN='\033[0;32m'
|
||||||
YELLOW='\033[1;33m'
|
YELLOW='\033[1;33m'
|
||||||
NC='\033[0m'
|
NC='\033[0m'
|
||||||
|
|
||||||
echo -e "${GREEN}========================================${NC}"
|
echo -e "${GREEN}========================================${NC}"
|
||||||
echo -e "${GREEN} General Bots Runtime Dependencies${NC}"
|
echo -e "${GREEN} General Bots Runtime Dependencies${NC}"
|
||||||
echo -e "${GREEN}========================================${NC}"
|
echo -e "${GREEN}========================================${NC}"
|
||||||
|
|
||||||
# Check root
|
# Check root
|
||||||
if [ "$EUID" -ne 0 ]; then
|
if [ "$EUID" -ne 0 ]; then
|
||||||
echo -e "${RED}Error: Run as root (use sudo)${NC}"
|
echo -e "${RED}Error: Run as root (use sudo)${NC}"
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Detect OS
|
# Detect OS
|
||||||
if [ -f /etc/os-release ]; then
|
if [ -f /etc/os-release ]; then
|
||||||
. /etc/os-release
|
. /etc/os-release
|
||||||
OS=$ID
|
OS=$ID
|
||||||
else
|
else
|
||||||
echo -e "${RED}Error: Cannot detect OS${NC}"
|
echo -e "${RED}Error: Cannot detect OS${NC}"
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
echo -e "${YELLOW}OS: $OS${NC}"
|
echo -e "${YELLOW}OS: $OS${NC}"
|
||||||
|
|
||||||
install_debian_ubuntu() {
|
install_debian_ubuntu() {
|
||||||
|
apt-get update
|
||||||
apt-get install -y \
|
apt-get install -y \
|
||||||
libpq5 \
|
libpq5 \
|
||||||
libssl3 \
|
libssl3 \
|
||||||
liblzma5 \
|
liblzma5 \
|
||||||
zlib1g \
|
zlib1g \
|
||||||
ca-certificates \
|
ca-certificates \
|
||||||
curl \
|
curl \
|
||||||
wget \
|
wget \
|
||||||
libclang1 \
|
libabseil20210324 \
|
||||||
pkg-config \
|
libclang1 \
|
||||||
snapd
|
pkg-config \
|
||||||
|
snapd
|
||||||
|
|
||||||
|
# LXC for containers
|
||||||
|
snap install lxd || apt-get install -y lxd || true
|
||||||
|
|
||||||
|
# Initialize LXD
|
||||||
|
if command -v lxd &> /dev/null && ! lxc list &> /dev/null 2>&1; then
|
||||||
|
lxd init --auto || true
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
install_fedora_rhel() {
|
||||||
|
dnf install -y \
|
||||||
|
libpq \
|
||||||
|
openssl-libs \
|
||||||
|
xz-libs \
|
||||||
|
zlib \
|
||||||
|
ca-certificates \
|
||||||
|
curl \
|
||||||
|
wget \
|
||||||
|
abseil-cpp \
|
||||||
|
clang-libs \
|
||||||
|
pkgconf-pkg-config \
|
||||||
|
lxc \
|
||||||
|
lxc-templates
|
||||||
|
}
|
||||||
|
|
||||||
}
|
install_arch() {
|
||||||
|
pacman -Sy --noconfirm \
|
||||||
|
postgresql-libs \
|
||||||
|
openssl \
|
||||||
|
xz \
|
||||||
|
zlib \
|
||||||
|
ca-certificates \
|
||||||
|
curl \
|
||||||
|
wget \
|
||||||
|
abseil-cpp \
|
||||||
|
clang \
|
||||||
|
pkgconf \
|
||||||
|
lxc
|
||||||
|
}
|
||||||
|
|
||||||
install_fedora_rhel() {
|
install_alpine() {
|
||||||
dnf install -y \
|
apk add --no-cache \
|
||||||
libpq \
|
libpq \
|
||||||
openssl-libs \
|
openssl \
|
||||||
xz-libs \
|
xz-libs \
|
||||||
zlib \
|
zlib \
|
||||||
ca-certificates \
|
ca-certificates \
|
||||||
curl \
|
curl \
|
||||||
wget \
|
wget \
|
||||||
abseil-cpp \
|
abseil-cpp \
|
||||||
clang-libs \
|
clang \
|
||||||
pkgconf-pkg-config \
|
pkgconf \
|
||||||
lxc \
|
lxc
|
||||||
lxc-templates
|
}
|
||||||
}
|
|
||||||
|
|
||||||
install_arch() {
|
case $OS in
|
||||||
pacman -Sy --noconfirm \
|
ubuntu|debian|linuxmint|pop)
|
||||||
postgresql-libs \
|
install_debian_ubuntu
|
||||||
openssl \
|
;;
|
||||||
xz \
|
fedora|rhel|centos|rocky|almalinux)
|
||||||
zlib \
|
install_fedora_rhel
|
||||||
ca-certificates \
|
;;
|
||||||
curl \
|
arch|manjaro)
|
||||||
wget \
|
install_arch
|
||||||
abseil-cpp \
|
;;
|
||||||
clang \
|
alpine)
|
||||||
pkgconf \
|
install_alpine
|
||||||
lxc
|
;;
|
||||||
}
|
*)
|
||||||
|
echo -e "${RED}Unsupported OS: $OS${NC}"
|
||||||
|
echo "Required libraries:"
|
||||||
|
echo " - libpq (PostgreSQL client)"
|
||||||
|
echo " - libssl (OpenSSL)"
|
||||||
|
echo " - liblzma (XZ compression)"
|
||||||
|
echo " - zlib (compression)"
|
||||||
|
echo " - abseil-cpp (Google Abseil)"
|
||||||
|
echo " - clang (LLVM runtime)"
|
||||||
|
echo " - LXC (containers)"
|
||||||
|
exit 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
|
||||||
install_alpine() {
|
echo -e "${GREEN}Runtime dependencies installed!${NC}"
|
||||||
apk add --no-cache \
|
echo ""
|
||||||
libpq \
|
echo "You can now run:"
|
||||||
openssl \
|
echo " ./botserver"
|
||||||
xz-libs \
|
|
||||||
zlib \
|
|
||||||
ca-certificates \
|
|
||||||
curl \
|
|
||||||
wget \
|
|
||||||
abseil-cpp \
|
|
||||||
clang \
|
|
||||||
pkgconf \
|
|
||||||
lxc
|
|
||||||
}
|
|
||||||
|
|
||||||
case $OS in
|
|
||||||
ubuntu|debian|linuxmint|pop)
|
|
||||||
install_debian_ubuntu
|
|
||||||
;;
|
|
||||||
fedora|rhel|centos|rocky|almalinux)
|
|
||||||
install_fedora_rhel
|
|
||||||
;;
|
|
||||||
arch|manjaro)
|
|
||||||
install_arch
|
|
||||||
;;
|
|
||||||
alpine)
|
|
||||||
install_alpine
|
|
||||||
;;
|
|
||||||
*)
|
|
||||||
echo -e "${RED}Unsupported OS: $OS${NC}"
|
|
||||||
echo "Required libraries:"
|
|
||||||
echo " - libpq (PostgreSQL client)"
|
|
||||||
echo " - libssl (OpenSSL)"
|
|
||||||
echo " - liblzma (XZ compression)"
|
|
||||||
echo " - zlib (compression)"
|
|
||||||
echo " - abseil-cpp (Google Abseil)"
|
|
||||||
echo " - clang (LLVM runtime)"
|
|
||||||
echo " - LXC (containers)"
|
|
||||||
exit 1
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
|
|
||||||
echo -e "${GREEN}Runtime dependencies installed!${NC}"
|
|
||||||
echo ""
|
|
||||||
echo "You can now run:"
|
|
||||||
echo " ./botserver"
|
|
||||||
|
|
|
||||||
383
PROMPT.md
Normal file
383
PROMPT.md
Normal file
|
|
@ -0,0 +1,383 @@
|
||||||
|
# General Bots Workspace - Master Development Guide
|
||||||
|
|
||||||
|
**Version:** 6.2.0 - DO NOT CHANGE
|
||||||
|
**Project:** General Bots Workspace (Rust Monorepo)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 📁 WORKSPACE STRUCTURE
|
||||||
|
|
||||||
|
| Crate | Purpose | Port | Tech Stack |
|
||||||
|
|-------|---------|------|------------|
|
||||||
|
| **botserver** | Main API server, business logic | 8088 | Axum, Diesel, Rhai BASIC |
|
||||||
|
| **botui** | Web UI server (dev) + proxy | 3000 | Axum, HTML/HTMX/CSS |
|
||||||
|
| **botapp** | Desktop app wrapper | - | Tauri 2 |
|
||||||
|
| **botlib** | Shared library | - | Core types, errors |
|
||||||
|
| **botbook** | Documentation | - | mdBook |
|
||||||
|
| **bottest** | Integration tests | - | tokio-test |
|
||||||
|
| **botdevice** | IoT/Device support | - | Rust |
|
||||||
|
| **botmodels** | Data models visualization | - | - |
|
||||||
|
| **botplugin** | Browser extension | - | JS |
|
||||||
|
|
||||||
|
### Key Paths
|
||||||
|
- **Binary:** `target/debug/botserver`
|
||||||
|
- **Run from:** `botserver/` directory
|
||||||
|
- **Env file:** `botserver/.env`
|
||||||
|
- **Stack:** `botserver/botserver-stack/`
|
||||||
|
- **UI Files:** `botui/ui/suite/`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 🔥 ERROR FIXING WORKFLOW
|
||||||
|
|
||||||
|
### Mode 1: OFFLINE Batch Fix (PREFERRED)
|
||||||
|
|
||||||
|
When given error output:
|
||||||
|
|
||||||
|
```
|
||||||
|
1. Read ENTIRE error list first
|
||||||
|
2. Group errors by file
|
||||||
|
3. For EACH file with errors:
|
||||||
|
a. View file → understand context
|
||||||
|
b. Fix ALL errors in that file
|
||||||
|
c. Write once with all fixes
|
||||||
|
4. Move to next file
|
||||||
|
5. REPEAT until ALL errors addressed
|
||||||
|
6. ONLY THEN → verify with build/diagnostics
|
||||||
|
```
|
||||||
|
|
||||||
|
**NEVER run cargo build/check/clippy DURING fixing**
|
||||||
|
**Fix ALL errors OFFLINE first, verify ONCE at the end**
|
||||||
|
|
||||||
|
### Mode 2: Interactive Loop
|
||||||
|
|
||||||
|
```
|
||||||
|
LOOP UNTIL (0 warnings AND 0 errors):
|
||||||
|
1. Run diagnostics → pick file with issues
|
||||||
|
2. Read entire file
|
||||||
|
3. Fix ALL issues in that file
|
||||||
|
4. Write file once with all fixes
|
||||||
|
5. Verify with diagnostics
|
||||||
|
6. CONTINUE LOOP
|
||||||
|
END LOOP
|
||||||
|
```
|
||||||
|
|
||||||
|
### Common Error Patterns
|
||||||
|
|
||||||
|
| Error | Fix |
|
||||||
|
|-------|-----|
|
||||||
|
| `expected i64, found u64` | `value as i64` |
|
||||||
|
| `expected Option<T>, found T` | `Some(value)` |
|
||||||
|
| `expected T, found Option<T>` | `value.unwrap_or(default)` |
|
||||||
|
| `cannot multiply f32 by f64` | `f64::from(f32_val) * f64_val` |
|
||||||
|
| `no field X on type Y` | Check struct definition |
|
||||||
|
| `no variant X found` | Check enum definition |
|
||||||
|
| `function takes N arguments` | Match function signature |
|
||||||
|
| `cannot find function` | Add missing function or fix import |
|
||||||
|
| `unused variable` | Delete or use with `..` in patterns |
|
||||||
|
| `unused import` | Delete the import line |
|
||||||
|
| `cannot move out of X because borrowed` | Use scoping `{ }` to limit borrow |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 🧠 MEMORY MANAGEMENT
|
||||||
|
|
||||||
|
When compilation fails due to memory issues (process "Killed"):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
pkill -9 cargo; pkill -9 rustc; pkill -9 botserver
|
||||||
|
CARGO_BUILD_JOBS=1 cargo check -p botserver 2>&1 | tail -200
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 📏 FILE SIZE LIMITS - MANDATORY
|
||||||
|
|
||||||
|
### Maximum 1000 Lines Per File
|
||||||
|
|
||||||
|
When a file grows beyond this limit:
|
||||||
|
|
||||||
|
1. **Identify logical groups** - Find related functions
|
||||||
|
2. **Create subdirectory module** - e.g., `handlers/`
|
||||||
|
3. **Split by responsibility:**
|
||||||
|
- `crud.rs` - Create, Read, Update, Delete
|
||||||
|
- `ai.rs` - AI/ML handlers
|
||||||
|
- `export.rs` - Export/import
|
||||||
|
- `validation.rs` - Validation
|
||||||
|
- `mod.rs` - Re-exports
|
||||||
|
4. **Keep files focused** - Single responsibility
|
||||||
|
5. **Update mod.rs** - Re-export all public items
|
||||||
|
|
||||||
|
**NEVER let a single file exceed 1000 lines - split proactively at 800 lines**
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 🚀 PERFORMANCE & SIZE STANDARDS
|
||||||
|
|
||||||
|
### Binary Size Optimization
|
||||||
|
- **Release Profile**: Always maintain `opt-level = "z"`, `lto = true`, `codegen-units = 1`, `strip = true`, `panic = "abort"`.
|
||||||
|
- **Dependencies**:
|
||||||
|
- Run `cargo tree --duplicates` weekly to find and resolve duplicate versions.
|
||||||
|
- Run `cargo machete` to remove unused dependencies.
|
||||||
|
- Use `default-features = false` and explicitly opt-in to needed features.
|
||||||
|
|
||||||
|
### Memory Optimization
|
||||||
|
- **Strings**: Prefer `&str` over `String` where possible. Use `Cow<str>` for conditional ownership.
|
||||||
|
- **Collections**: Use `Vec::with_capacity` when size is known. Consider `SmallVec` for hot paths.
|
||||||
|
- **Allocations**: Minimize heap allocations in hot paths.
|
||||||
|
|
||||||
|
### Linting & Code Quality
|
||||||
|
- **Clippy**: Code MUST pass `cargo clippy --all-targets --all-features` with **0 warnings**.
|
||||||
|
- **No Allow**: Do not use `#[allow(clippy::...)]` unless absolutely necessary and documented. Fix the underlying issue.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 🔐 SECURITY DIRECTIVES - MANDATORY
|
||||||
|
|
||||||
|
### Error Handling - NO PANICS IN PRODUCTION
|
||||||
|
|
||||||
|
```rust
|
||||||
|
// ❌ FORBIDDEN
|
||||||
|
value.unwrap()
|
||||||
|
value.expect("message")
|
||||||
|
panic!("error")
|
||||||
|
todo!()
|
||||||
|
unimplemented!()
|
||||||
|
|
||||||
|
// ✅ REQUIRED
|
||||||
|
value?
|
||||||
|
value.ok_or_else(|| Error::NotFound)?
|
||||||
|
value.unwrap_or_default()
|
||||||
|
value.unwrap_or_else(|e| { log::error!("{}", e); default })
|
||||||
|
if let Some(v) = value { ... }
|
||||||
|
match value { Ok(v) => v, Err(e) => return Err(e.into()) }
|
||||||
|
```
|
||||||
|
|
||||||
|
### Command Execution - USE SafeCommand
|
||||||
|
|
||||||
|
```rust
|
||||||
|
// ❌ FORBIDDEN
|
||||||
|
Command::new("some_command").arg(user_input).output()
|
||||||
|
|
||||||
|
// ✅ REQUIRED
|
||||||
|
use crate::security::command_guard::SafeCommand;
|
||||||
|
SafeCommand::new("allowed_command")?
|
||||||
|
.arg("safe_arg")?
|
||||||
|
.execute()
|
||||||
|
```
|
||||||
|
|
||||||
|
### Error Responses - USE ErrorSanitizer
|
||||||
|
|
||||||
|
```rust
|
||||||
|
// ❌ FORBIDDEN
|
||||||
|
Json(json!({ "error": e.to_string() }))
|
||||||
|
format!("Database error: {}", e)
|
||||||
|
|
||||||
|
// ✅ REQUIRED
|
||||||
|
use crate::security::error_sanitizer::log_and_sanitize;
|
||||||
|
let sanitized = log_and_sanitize(&e, "context", None);
|
||||||
|
(StatusCode::INTERNAL_SERVER_ERROR, sanitized)
|
||||||
|
```
|
||||||
|
|
||||||
|
### SQL - USE sql_guard
|
||||||
|
|
||||||
|
```rust
|
||||||
|
// ❌ FORBIDDEN
|
||||||
|
format!("SELECT * FROM {}", user_table)
|
||||||
|
|
||||||
|
// ✅ REQUIRED
|
||||||
|
use crate::security::sql_guard::{sanitize_identifier, validate_table_name};
|
||||||
|
let safe_table = sanitize_identifier(&user_table);
|
||||||
|
validate_table_name(&safe_table)?;
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## ❌ ABSOLUTE PROHIBITIONS
|
||||||
|
|
||||||
|
```
|
||||||
|
❌ NEVER use .unwrap() or .expect() in production code (tests OK)
|
||||||
|
❌ NEVER use panic!(), todo!(), unimplemented!()
|
||||||
|
❌ NEVER use Command::new() directly - use SafeCommand
|
||||||
|
❌ NEVER return raw error strings to HTTP clients
|
||||||
|
❌ NEVER use #[allow()] in source code - FIX the code instead
|
||||||
|
❌ NEVER add lint exceptions to Cargo.toml - FIX the code instead
|
||||||
|
❌ NEVER use _ prefix for unused variables - DELETE or USE them
|
||||||
|
❌ NEVER leave unused imports or dead code
|
||||||
|
❌ NEVER add comments - code must be self-documenting
|
||||||
|
❌ NEVER modify Cargo.toml lints section!
|
||||||
|
❌ NEVER use CDN links - all assets must be local
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## ✅ MANDATORY CODE PATTERNS
|
||||||
|
|
||||||
|
### Use Self in Impl Blocks
|
||||||
|
```rust
|
||||||
|
impl MyStruct {
|
||||||
|
fn new() -> Self { Self { } } // ✅ Not MyStruct
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Derive Eq with PartialEq
|
||||||
|
```rust
|
||||||
|
#[derive(PartialEq, Eq)] // ✅ Always both
|
||||||
|
struct MyStruct { }
|
||||||
|
```
|
||||||
|
|
||||||
|
### Inline Format Args
|
||||||
|
```rust
|
||||||
|
format!("Hello {name}") // ✅ Not format!("{}", name)
|
||||||
|
```
|
||||||
|
|
||||||
|
### Combine Match Arms
|
||||||
|
```rust
|
||||||
|
match x {
|
||||||
|
A | B => do_thing(), // ✅ Combine identical arms
|
||||||
|
C => other(),
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 🖥️ UI Architecture (botui + botserver)
|
||||||
|
|
||||||
|
### Two Servers During Development
|
||||||
|
|
||||||
|
| Server | Port | Purpose |
|
||||||
|
|--------|------|---------|
|
||||||
|
| **botui** | 3000 | Serves UI files + proxies API to botserver |
|
||||||
|
| **botserver** | 8088 | Backend API + embedded UI fallback |
|
||||||
|
|
||||||
|
### How It Works
|
||||||
|
|
||||||
|
```
|
||||||
|
Browser → localhost:3000 → botui (serves HTML/CSS/JS)
|
||||||
|
→ /api/* proxied to botserver:8088
|
||||||
|
→ /suite/* served from botui/ui/suite/
|
||||||
|
```
|
||||||
|
|
||||||
|
### Adding New Suite Apps
|
||||||
|
|
||||||
|
1. Create folder: `botui/ui/suite/<appname>/`
|
||||||
|
2. Add to `SUITE_DIRS` in `botui/src/ui_server/mod.rs`
|
||||||
|
3. Rebuild botui: `cargo build -p botui`
|
||||||
|
4. Add menu entry in `botui/ui/suite/index.html`
|
||||||
|
|
||||||
|
### Hot Reload
|
||||||
|
|
||||||
|
- **UI files (HTML/CSS/JS)**: Edit & refresh browser (no restart)
|
||||||
|
- **botui Rust code**: Rebuild + restart botui
|
||||||
|
- **botserver Rust code**: Rebuild + restart botserver
|
||||||
|
|
||||||
|
### Production (Single Binary)
|
||||||
|
|
||||||
|
When `botui/ui/suite/` folder not found, botserver uses **embedded UI** compiled into binary via `rust-embed`.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 🎨 FRONTEND STANDARDS
|
||||||
|
|
||||||
|
### HTMX-First Approach
|
||||||
|
- Use HTMX to minimize JavaScript
|
||||||
|
- Server returns HTML fragments, not JSON
|
||||||
|
- Use `hx-get`, `hx-post`, `hx-target`, `hx-swap`
|
||||||
|
- WebSocket via htmx-ws extension
|
||||||
|
|
||||||
|
### Local Assets Only - NO CDN
|
||||||
|
```html
|
||||||
|
<!-- ✅ CORRECT -->
|
||||||
|
<script src="js/vendor/htmx.min.js"></script>
|
||||||
|
|
||||||
|
<!-- ❌ WRONG -->
|
||||||
|
<script src="https://unpkg.com/htmx.org@1.9.10"></script>
|
||||||
|
```
|
||||||
|
|
||||||
|
### Vendor Libraries Location
|
||||||
|
```
|
||||||
|
ui/suite/js/vendor/
|
||||||
|
├── htmx.min.js
|
||||||
|
├── htmx-ws.js
|
||||||
|
├── marked.min.js
|
||||||
|
└── gsap.min.js
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 📋 PROJECT-SPECIFIC PROMPTS
|
||||||
|
|
||||||
|
Each crate has its own PROMPT.md with specific guidelines:
|
||||||
|
|
||||||
|
| Crate | PROMPT.md Location | Focus |
|
||||||
|
|-------|-------------------|-------|
|
||||||
|
| botserver | `botserver/PROMPT.md` | API, security, Rhai BASIC |
|
||||||
|
| botui | `botui/PROMPT.md` | UI, HTMX, CSS design system |
|
||||||
|
| botapp | `botapp/PROMPT.md` | Tauri, desktop features |
|
||||||
|
| botlib | `botlib/PROMPT.md` | Shared types, errors |
|
||||||
|
| botbook | `botbook/PROMPT.md` | Documentation, mdBook |
|
||||||
|
| bottest | `bottest/PROMPT.md` | Test infrastructure |
|
||||||
|
|
||||||
|
### Special Prompts
|
||||||
|
| File | Purpose |
|
||||||
|
|------|---------|
|
||||||
|
| `botserver/src/tasks/PROMPT.md` | AutoTask LLM executor |
|
||||||
|
| `botserver/src/auto_task/APP_GENERATOR_PROMPT.md` | App generation |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 🚀 STARTING DEVELOPMENT
|
||||||
|
|
||||||
|
### Start Both Servers
|
||||||
|
```bash
|
||||||
|
# Terminal 1: botserver
|
||||||
|
cd botserver && cargo run -- --noconsole
|
||||||
|
|
||||||
|
# Terminal 2: botui
|
||||||
|
cd botui && BOTSERVER_URL="http://localhost:8088" cargo run
|
||||||
|
```
|
||||||
|
|
||||||
|
### Build Commands
|
||||||
|
```bash
|
||||||
|
# Check single crate
|
||||||
|
cargo check -p botserver
|
||||||
|
|
||||||
|
# Build workspace
|
||||||
|
cargo build
|
||||||
|
|
||||||
|
# Run tests
|
||||||
|
cargo test -p bottest
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 📋 CONTINUATION PROMPT
|
||||||
|
|
||||||
|
When starting a new session or continuing work:
|
||||||
|
|
||||||
|
```
|
||||||
|
Continue on gb/ workspace. Follow PROMPT.md strictly:
|
||||||
|
|
||||||
|
1. Check current state with build/diagnostics
|
||||||
|
2. Fix ALL warnings and errors - NO #[allow()] attributes
|
||||||
|
3. Delete unused code, don't suppress warnings
|
||||||
|
4. Remove unused parameters, don't prefix with _
|
||||||
|
5. Verify after each fix batch
|
||||||
|
6. Loop until 0 warnings, 0 errors
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 🔑 REMEMBER
|
||||||
|
|
||||||
|
- **OFFLINE FIRST** - Fix all errors from list before compiling
|
||||||
|
- **ZERO WARNINGS, ZERO ERRORS** - The only acceptable state
|
||||||
|
- **FIX, DON'T SUPPRESS** - No #[allow()], no Cargo.toml lint exceptions
|
||||||
|
- **SECURITY FIRST** - No unwrap, no raw errors, no direct commands
|
||||||
|
- **READ BEFORE FIX** - Always understand context first
|
||||||
|
- **BATCH BY FILE** - Fix ALL errors in a file at once
|
||||||
|
- **WRITE ONCE** - Single edit per file with all fixes
|
||||||
|
- **VERIFY LAST** - Only compile/diagnostics after ALL fixes
|
||||||
|
- **DELETE DEAD CODE** - Don't keep unused code around
|
||||||
|
- **Version 6.2.0** - Do not change without approval
|
||||||
|
- **GIT WORKFLOW** - ALWAYS push to ALL repositories (github, pragmatismo)
|
||||||
719
README.md
719
README.md
|
|
@ -1,644 +1,85 @@
|
||||||
# General Bots Workspace
|
# General Bots Workspace
|
||||||
|
|
||||||
## ⚠️ CRITICAL SECURITY WARNING
|
**Version:** 6.1.0
|
||||||
|
|
||||||
**NEVER CREATE FILES WITH SECRETS IN THE REPOSITORY ROOT**
|
|
||||||
|
|
||||||
Secret files MUST be placed in `/tmp/` only:
|
|
||||||
- ✅ `/tmp/vault-token-gb` - Vault root token
|
|
||||||
- ✅ `/tmp/vault-unseal-key-gb` - Vault unseal key
|
|
||||||
- ❌ `vault-unseal-keys` - FORBIDDEN (tracked by git)
|
|
||||||
- ❌ `start-and-unseal.sh` - FORBIDDEN (contains secrets)
|
|
||||||
|
|
||||||
**Files added to .gitignore:** `vault-unseal-keys`, `start-and-unseal.sh`, `vault-token-*`
|
|
||||||
|
|
||||||
**Why `/tmp/`?**
|
|
||||||
- Cleared on reboot (ephemeral)
|
|
||||||
- Not tracked by git
|
|
||||||
- Standard Unix security practice
|
|
||||||
- Prevents accidental commits
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
|
|
||||||
**Version:** 6.3.0
|
|
||||||
**Type:** Rust Workspace (Monorepo with Independent Subproject Repos)
|
**Type:** Rust Workspace (Monorepo with Independent Subproject Repos)
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## Overview
|
## Structure
|
||||||
|
|
||||||
General Bots is a comprehensive automation platform built with Rust, providing a unified workspace for building AI-powered bots, web interfaces, desktop applications, and integration tools. The workspace follows a modular architecture with independent subprojects that can be developed and deployed separately while sharing common libraries and standards.
|
This workspace contains multiple General Bots projects:
|
||||||
|
|
||||||
For comprehensive documentation, see **[docs.pragmatismo.com.br](https://docs.pragmatismo.com.br)** or the **[BotBook](./botbook)** for detailed guides, API references, and tutorials.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 📁 Workspace Structure
|
|
||||||
|
|
||||||
| Crate | Purpose | Port | Tech Stack |
|
|
||||||
|-------|---------|------|------------|
|
|
||||||
| **botserver** | Main API server, business logic | 9000 | Axum, Diesel, Rhai BASIC |
|
|
||||||
| **botui** | Web UI server (dev) + proxy | 3000 | Axum, HTML/HTMX/CSS |
|
|
||||||
| **botapp** | Desktop app wrapper | - | Tauri 2 |
|
|
||||||
| **botlib** | Shared library | - | Core types, errors |
|
|
||||||
| **botbook** | Documentation | - | mdBook |
|
|
||||||
| **bottest** | Integration tests | - | tokio-test |
|
|
||||||
| **botdevice** | IoT/Device support | - | Rust |
|
|
||||||
| **botmodels** | Data models visualization | - | - |
|
|
||||||
| **botplugin** | Browser extension | - | JS |
|
|
||||||
|
|
||||||
### Key Paths
|
|
||||||
- **Binary:** `target/debug/botserver`
|
|
||||||
- **Run from:** `botserver/` directory
|
|
||||||
- **Env file:** `botserver/.env`
|
|
||||||
- **Stack:** `botserver-stack/`
|
|
||||||
- **UI Files:** `botui/ui/suite/`
|
|
||||||
- **Local Bot Data:** `/opt/gbo/data/` (place `.gbai` packages here)
|
|
||||||
|
|
||||||
### Local Bot Data Directory
|
|
||||||
|
|
||||||
Place local bot packages in `/opt/gbo/data/` for automatic loading and monitoring:
|
|
||||||
|
|
||||||
**Directory Structure:**
|
|
||||||
```
|
```
|
||||||
/opt/gbo/data/
|
gb/
|
||||||
└── mybot.gbai/
|
├── PROMPT.md ← Workspace-level development guide (READ THIS FIRST)
|
||||||
├── mybot.gbdialog/
|
├── Cargo.toml ← Workspace configuration
|
||||||
│ ├── start.bas
|
├── README.md ← This file
|
||||||
│ └── main.bas
|
│
|
||||||
└── mybot.gbot/
|
├── botapp/ ← Desktop application (Tauri)
|
||||||
└── config.csv
|
├── botserver/ ← Main server (API + business logic)
|
||||||
```
|
├── botlib/ ← Shared library (types, utilities)
|
||||||
|
├── botui/ ← Web UI (HTML/CSS/JS)
|
||||||
**Features:**
|
├── botbook/ ← Documentation
|
||||||
- **Auto-loading:** Bots automatically mounted on server startup
|
├── bottest/ ← Integration tests
|
||||||
- **Auto-compilation:** `.bas` files compiled to `.ast` on change
|
├── botdevice/ ← Device integration
|
||||||
- **Auto-creation:** New bots automatically added to database
|
├── botmodels/ ← AI models
|
||||||
- **Hot-reload:** Changes trigger immediate recompilation
|
├── botplugin/ ← Plugin system
|
||||||
- **Monitored by:** LocalFileMonitor and ConfigWatcher services
|
├── bottemplates/ ← Templates
|
||||||
|
└── target/ ← Build artifacts
|
||||||
**Usage:**
|
|
||||||
1. Create bot directory structure in `/opt/gbo/data/`
|
|
||||||
2. Add `.bas` files to `<bot_name>.gbai/<bot_name>.gbdialog/`
|
|
||||||
3. Server automatically detects and loads the bot
|
|
||||||
4. Optional: Add `config.csv` for bot configuration
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 🏗️ BotServer Component Architecture
|
|
||||||
|
|
||||||
### 🔧 Infrastructure Components (Auto-Managed)
|
|
||||||
|
|
||||||
BotServer automatically installs, configures, and manages all infrastructure components on first run. **DO NOT manually start these services** - BotServer handles everything.
|
|
||||||
|
|
||||||
**Automatic Service Lifecycle:**
|
|
||||||
1. **Start**: When botserver starts, it automatically launches all infrastructure components (PostgreSQL, Vault, MinIO, Valkey, Qdrant, etc.)
|
|
||||||
2. **Credentials**: BotServer retrieves all service credentials (passwords, tokens, API keys) from Vault
|
|
||||||
3. **Connection**: BotServer uses these credentials to establish secure connections to each service
|
|
||||||
4. **Query**: All database queries, cache operations, and storage requests are authenticated using Vault-managed credentials
|
|
||||||
|
|
||||||
**Credential Flow:**
|
|
||||||
```
|
|
||||||
botserver starts
|
|
||||||
↓
|
|
||||||
Launch PostgreSQL, MinIO, Valkey, Qdrant
|
|
||||||
↓
|
|
||||||
Connect to Vault
|
|
||||||
↓
|
|
||||||
Retrieve service credentials (from database)
|
|
||||||
↓
|
|
||||||
Authenticate with each service using retrieved credentials
|
|
||||||
↓
|
|
||||||
Ready to handle requests
|
|
||||||
```
|
|
||||||
|
|
||||||
| Component | Purpose | Port | Binary Location | Credentials From |
|
|
||||||
|-----------|---------|------|-----------------|------------------|
|
|
||||||
| **Vault** | Secrets management | 8200 | `botserver-stack/bin/vault/vault` | Auto-unsealed |
|
|
||||||
| **PostgreSQL** | Primary database | 5432 | `botserver-stack/bin/tables/bin/postgres` | Vault → database |
|
|
||||||
| **MinIO** | Object storage (S3-compatible) | 9000/9001 | `botserver-stack/bin/drive/minio` | Vault → database |
|
|
||||||
| **Zitadel** | Identity/Authentication | 8300 | `botserver-stack/bin/directory/zitadel` | Vault → database |
|
|
||||||
| **Qdrant** | Vector database (embeddings) | 6333 | `botserver-stack/bin/vector_db/qdrant` | Vault → database |
|
|
||||||
| **Valkey** | Cache/Queue (Redis-compatible) | 6379 | `botserver-stack/bin/cache/valkey-server` | Vault → database |
|
|
||||||
| **Llama.cpp** | Local LLM server | 8081 | `botserver-stack/bin/llm/build/bin/llama-server` | Vault → database |
|
|
||||||
|
|
||||||
### 📦 Component Installation System
|
|
||||||
|
|
||||||
Components are defined in `botserver/3rdparty.toml` and managed by the `PackageManager` (`botserver/src/core/package_manager/`):
|
|
||||||
|
|
||||||
```toml
|
|
||||||
[components.cache]
|
|
||||||
name = "Valkey Cache (Redis-compatible)"
|
|
||||||
url = "https://github.com/valkey-io/valkey/archive/refs/tags/8.0.2.tar.gz"
|
|
||||||
filename = "valkey-8.0.2.tar.gz"
|
|
||||||
|
|
||||||
[components.llm]
|
|
||||||
name = "Llama.cpp Server"
|
|
||||||
url = "https://github.com/ggml-org/llama.cpp/releases/download/b7345/llama-b7345-bin-ubuntu-x64.zip"
|
|
||||||
filename = "llama-b7345-bin-ubuntu-x64.zip"
|
|
||||||
```
|
|
||||||
|
|
||||||
**Installation Flow:**
|
|
||||||
1. **Download:** Components downloaded to `botserver-installers/` (cached)
|
|
||||||
2. **Extract/Build:** Binaries placed in `botserver-stack/bin/<component>/`
|
|
||||||
3. **Configure:** Config files generated in `botserver-stack/conf/<component>/`
|
|
||||||
4. **Start:** Components started with proper TLS certificates
|
|
||||||
5. **Monitor:** Components monitored and auto-restarted if needed
|
|
||||||
|
|
||||||
**Bootstrap Process:**
|
|
||||||
- First run: Full bootstrap (downloads, installs, configures all components)
|
|
||||||
- Subsequent runs: Only starts existing components (uses cached binaries)
|
|
||||||
- Config stored in: `botserver-stack/conf/system/bootstrap.json`
|
|
||||||
|
|
||||||
### 🚀 PROPER STARTUP PROCEDURES
|
|
||||||
|
|
||||||
**❌ FORBIDDEN:**
|
|
||||||
- NEVER manually start infrastructure components (Vault, PostgreSQL, MinIO, etc.)
|
|
||||||
- NEVER run `cargo run` or `cargo build` for botserver directly without ./restart.sh
|
|
||||||
- NEVER modify botserver-stack/ files manually (use botserver API)
|
|
||||||
|
|
||||||
**✅ REQUIRED:**
|
|
||||||
|
|
||||||
**Option 1: Development (Recommended)**
|
|
||||||
```bash
|
|
||||||
./restart.sh
|
|
||||||
```
|
|
||||||
This script:
|
|
||||||
1. Kills existing processes cleanly
|
|
||||||
2. Builds botserver and botui sequentially (no race conditions)
|
|
||||||
3. Starts botserver in background with logging to `botserver.log`
|
|
||||||
4. Starts botui in background with logging to `botui.log`
|
|
||||||
5. Shows process IDs and access URLs
|
|
||||||
|
|
||||||
**Option 2: Production/Release**
|
|
||||||
```bash
|
|
||||||
# Build release binary first
|
|
||||||
cargo build --release -p botserver
|
|
||||||
|
|
||||||
# Start with release binary
|
|
||||||
RUST_LOG=info ./target/release/botserver --noconsole 2>&1 | tee botserver.log &
|
|
||||||
```
|
|
||||||
|
|
||||||
**Option 3: Using Exec (Systemd/Supervisord)**
|
|
||||||
```bash
|
|
||||||
# In systemd service or similar
|
|
||||||
ExecStart=/home/rodriguez/src/gb/target/release/botserver --noconsole
|
|
||||||
```
|
|
||||||
|
|
||||||
### 🔒 Component Communication
|
|
||||||
|
|
||||||
All components communicate through internal networks with mTLS:
|
|
||||||
- **Vault**: mTLS for secrets access
|
|
||||||
- **PostgreSQL**: TLS encrypted connections
|
|
||||||
- **MinIO**: TLS with client certificates
|
|
||||||
- **Zitadel**: mTLS for user authentication
|
|
||||||
|
|
||||||
Certificates auto-generated in: `botserver-stack/conf/system/certificates/`
|
|
||||||
|
|
||||||
### 📊 Component Status
|
|
||||||
|
|
||||||
Check component status anytime:
|
|
||||||
```bash
|
|
||||||
# Check if all components are running
|
|
||||||
ps aux | grep -E "vault|postgres|minio|zitadel|qdrant|valkey" | grep -v grep
|
|
||||||
|
|
||||||
# View component logs
|
|
||||||
tail -f botserver-stack/logs/vault/vault.log
|
|
||||||
tail -f botserver-stack/logs/tables/postgres.log
|
|
||||||
tail -f botserver-stack/logs/drive/minio.log
|
|
||||||
|
|
||||||
# Test component connectivity
|
|
||||||
cd botserver-stack/bin/vault && ./vault status
|
|
||||||
cd botserver-stack/bin/cache && ./valkey-cli ping
|
|
||||||
```
|
```
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 🏗️ Component Dependency Graph
|
## CRITICAL: PROMPT.md Files
|
||||||
|
|
||||||
```
|
**Each project has a PROMPT.md that defines its development rules.**
|
||||||
┌─────────────────────────────────────────────────────────────────┐
|
|
||||||
│ Client Layer │
|
|
||||||
├─────────────────────────────────────────────────────────────────┤
|
|
||||||
│ botui (Web UI) │ botapp (Desktop) │ botplugin (Ext) │
|
|
||||||
│ HTMX + Axum │ Tauri 2 Wrapper │ Browser Extension │
|
|
||||||
└─────────┬───────────────────┬──────────────────┬─────────────────┘
|
|
||||||
│ │ │
|
|
||||||
└───────────────────┼──────────────────┘
|
|
||||||
│
|
|
||||||
┌─────────▼─────────┐
|
|
||||||
│ botlib │
|
|
||||||
│ (Shared Types) │
|
|
||||||
└─────────┬─────────┘
|
|
||||||
│
|
|
||||||
┌───────────────────┼───────────────────┐
|
|
||||||
│ │ │
|
|
||||||
┌─────▼─────┐ ┌─────▼─────┐ ┌─────▼─────┐
|
|
||||||
│ botserver │ │ bottest │ │ botdevice │
|
|
||||||
│ API Core │ │ Tests │ │ IoT/Device │
|
|
||||||
└───────────┘ └───────────┘ └───────────┘
|
|
||||||
```
|
|
||||||
|
|
||||||
### Dependency Rules
|
The diagnostics tool reads and respects these PROMPT.md files.
|
||||||
|
|
||||||
| Crate | Depends On | Why |
|
### Hierarchy
|
||||||
|-------|-----------|-----|
|
|
||||||
| **botserver** | botlib | Shared types, error handling, models |
|
|
||||||
| **botui** | botlib | Common data structures, API client |
|
|
||||||
| **botapp** | botlib | Shared types, desktop-specific utilities |
|
|
||||||
| **bottest** | botserver, botlib | Integration testing with real components |
|
|
||||||
| **botdevice** | botlib | Device types, communication protocols |
|
|
||||||
| **botplugin** | - | Standalone browser extension (JS) |
|
|
||||||
|
|
||||||
**Key Principle:** `botlib` contains ONLY shared types and utilities. No business logic. All business logic lives in botserver or specialized crates.
|
1. **`PROMPT.md`** (this directory) - Workspace-wide rules
|
||||||
|
2. **`botapp/PROMPT.md`** - Desktop app specifics
|
||||||
|
3. **`botserver/PROMPT.md`** - Server specifics
|
||||||
|
4. **`botlib/PROMPT.md`** - Library specifics
|
||||||
|
5. **`botui/PROMPT.md`** - UI specifics
|
||||||
|
6. **`botbook/PROMPT.md`** - Documentation specifics
|
||||||
|
7. **`bottest/PROMPT.md`** - Test specifics
|
||||||
|
|
||||||
## 📦 Module Responsibility Matrix
|
**ALWAYS read the relevant PROMPT.md before working on a project.**
|
||||||
|
|
||||||
### botserver/src/ Module Structure
|
---
|
||||||
|
|
||||||
| Module | Responsibility | Key Types | Dependencies |
|
## Main Directive
|
||||||
|--------|---------------|-----------|--------------|
|
|
||||||
| **core/bot/** | WebSocket handling, bot orchestration | BotOrchestrator, UserMessage | basic, shared |
|
|
||||||
| **core/session/** | Session management, conversation history | SessionManager, UserSession | shared, database |
|
|
||||||
| **basic/** | Rhai BASIC scripting engine | ScriptService, Engine | rhai, keywords |
|
|
||||||
| **basic/keywords/** | BASIC keyword implementations (TALK, HEAR, etc.) | Keyword functions | basic, state |
|
|
||||||
| **llm/** | Multi-vendor LLM API integration | LLMClient, ModelConfig | reqwest, shared |
|
|
||||||
| **drive/** | S3 file storage and monitoring | DriveMonitor, compile_tool | s3, basic |
|
|
||||||
| **security/** | Security guards (command, SQL, error) | SafeCommand, ErrorSanitizer | state |
|
|
||||||
| **shared/** | Database models, schema definitions | Bot, Session, Message | diesel |
|
|
||||||
| **tasks/** | AutoTask execution system | TaskRunner, TaskScheduler | core/basic |
|
|
||||||
| **auto_task/** | LLM-powered app generation | AppGenerator, template engine | llm, tasks |
|
|
||||||
| **learn/** | Knowledge base management | KBManager, vector storage | database, drive |
|
|
||||||
| **attendance/** | LLM-assisted customer service | AttendantManager, queue | core/bot |
|
|
||||||
|
|
||||||
### Data Flow Patterns
|
**LOOP AND COMPACT UNTIL 0 WARNINGS - MAXIMUM YOLO**
|
||||||
|
|
||||||
```
|
- 0 warnings
|
||||||
1. User Request Flow:
|
- 0 errors
|
||||||
Client → WebSocket → botserver/src/core/bot/mod.rs
|
- Trust project diagnostics
|
||||||
↓
|
- Respect all rules
|
||||||
BotOrchestrator::stream_response()
|
- No `#[allow()]` in source code
|
||||||
↓
|
- Real code fixes only
|
||||||
┌───────────┴───────────┐
|
|
||||||
│ │
|
|
||||||
LLM API Call Script Execution
|
|
||||||
(llm/mod.rs) (basic/mod.rs)
|
|
||||||
│ │
|
|
||||||
└───────────┬───────────┘
|
|
||||||
↓
|
|
||||||
Response → WebSocket → Client
|
|
||||||
|
|
||||||
2. File Sync Flow:
|
|
||||||
S3 Drive → drive_monitor/src/drive_monitor/mod.rs
|
|
||||||
↓
|
|
||||||
Download .bas files
|
|
||||||
↓
|
|
||||||
compile_file() → Generate .ast
|
|
||||||
↓
|
|
||||||
Store in ./work/{bot_name}.gbai/
|
|
||||||
|
|
||||||
3. Script Execution Flow:
|
|
||||||
.bas file → ScriptService::compile()
|
|
||||||
↓
|
|
||||||
preprocess_basic_script()
|
|
||||||
↓
|
|
||||||
engine.compile() → AST
|
|
||||||
↓
|
|
||||||
ScriptService::run() → Execute
|
|
||||||
↓
|
|
||||||
TALK commands → WebSocket messages
|
|
||||||
```
|
|
||||||
|
|
||||||
### Common Architectural Patterns
|
|
||||||
|
|
||||||
| Pattern | Where Used | Purpose |
|
|
||||||
|---------|-----------|---------|
|
|
||||||
| **State via Arc<AppState>** | All handlers | Shared async state (DB, cache, config) |
|
|
||||||
| **Extension(state) extractor** | Axum handlers | Inject Arc<AppState> into route handlers |
|
|
||||||
| **tokio::spawn_blocking** | CPU-intensive tasks | Offload blocking work from async runtime |
|
|
||||||
| **WebSocket with split()** | Real-time comms | Separate sender/receiver for WS streams |
|
|
||||||
| **ErrorSanitizer for responses** | All HTTP errors | Prevent leaking sensitive info in errors |
|
|
||||||
| **SafeCommand for execution** | Command running | Whitelist-based command validation |
|
|
||||||
| **Rhai for scripting** | BASIC interpreter | Embeddable scripting language |
|
|
||||||
| **Diesel ORM** | Database access | Type-safe SQL queries |
|
|
||||||
| **Redis for cache** | Session data | Fast key-value storage |
|
|
||||||
| **S3 for storage** | File system | Scalable object storage |
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## Quick Start
|
## Quick Start
|
||||||
|
|
||||||
### 🚀 Simple Startup (ALWAYS USE restart.sh)
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
./restart.sh
|
|
||||||
```
|
|
||||||
|
|
||||||
**⚠️ CRITICAL: ALWAYS use restart.sh - NEVER start servers individually!**
|
|
||||||
|
|
||||||
The script handles BOTH servers properly:
|
|
||||||
1. Stop existing processes cleanly
|
|
||||||
2. Build botserver and botui sequentially (no race conditions)
|
|
||||||
3. Start botserver in background → **automatically starts all infrastructure services (PostgreSQL, Vault, MinIO, Valkey, Qdrant)**
|
|
||||||
4. BotServer retrieves credentials from Vault and authenticates with all services
|
|
||||||
5. Start botui in background → proxy to botserver
|
|
||||||
6. Show process IDs and monitoring commands
|
|
||||||
|
|
||||||
**Infrastructure services are fully automated - no manual configuration required!**
|
|
||||||
|
|
||||||
**Monitor startup:**
|
|
||||||
```bash
|
|
||||||
tail -f botserver.log botui.log
|
|
||||||
```
|
|
||||||
|
|
||||||
**Access:**
|
|
||||||
- Web UI: http://localhost:3000
|
|
||||||
- API: http://localhost:9000
|
|
||||||
|
|
||||||
### 📊 Monitor & Debug
|
|
||||||
|
|
||||||
```bash
|
|
||||||
tail -f botserver.log botui.log
|
|
||||||
```
|
|
||||||
|
|
||||||
**Quick status check:**
|
|
||||||
```bash
|
|
||||||
ps aux | grep -E "botserver|botui" | grep -v grep
|
|
||||||
```
|
|
||||||
|
|
||||||
**Quick error scan:**
|
|
||||||
```bash
|
|
||||||
grep -E " E |W |CLIENT:" botserver.log | tail -20
|
|
||||||
```
|
|
||||||
|
|
||||||
### 🔧 Manual Startup (If needed)
|
|
||||||
|
|
||||||
**⚠️ WARNING: Only use if restart.sh fails. Always prefer restart.sh!**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
cd botserver && cargo run -- --noconsole > ../botserver.log 2>&1 &
|
|
||||||
cd botui && BOTSERVER_URL="http://localhost:9000" cargo run > ../botui.log 2>&1 &
|
|
||||||
```
|
|
||||||
|
|
||||||
### 🛑 Stop Servers
|
|
||||||
|
|
||||||
```bash
|
|
||||||
pkill -f botserver; pkill -f botui
|
|
||||||
```
|
|
||||||
|
|
||||||
### ⚠️ Common Issues
|
|
||||||
|
|
||||||
**Vault init error?** Delete stale state:
|
|
||||||
```bash
|
|
||||||
rm -rf botserver-stack/data/vault botserver-stack/conf/vault/init.json && ./restart.sh
|
|
||||||
```
|
|
||||||
|
|
||||||
**Port in use?** Find and kill:
|
|
||||||
```bash
|
|
||||||
lsof -ti:9000 | xargs kill -9
|
|
||||||
lsof -ti:3000 | xargs kill -9
|
|
||||||
```
|
|
||||||
|
|
||||||
**⚠️ IMPORTANT: Stack Services Management**
|
|
||||||
All infrastructure services (PostgreSQL, Vault, Redis, Qdrant, MinIO, etc.) are **automatically started by botserver** and managed through `botserver-stack/` directory, NOT global system installations. The system uses:
|
|
||||||
|
|
||||||
- **Local binaries:** `botserver-stack/bin/` (PostgreSQL, Vault, Redis, etc.)
|
|
||||||
- **Configurations:** `botserver-stack/conf/`
|
|
||||||
- **Data storage:** `botserver-stack/data/`
|
|
||||||
- **Service logs:** `botserver-stack/logs/` (check here for troubleshooting)
|
|
||||||
- **Credentials:** Stored in Vault, retrieved by botserver at startup
|
|
||||||
|
|
||||||
**Do NOT install or reference global PostgreSQL, Redis, or other services.** When botserver starts, it automatically:
|
|
||||||
1. Launches all required stack services
|
|
||||||
2. Connects to Vault
|
|
||||||
3. Retrieves credentials from the `bot_configuration` database table
|
|
||||||
4. Authenticates with each service using retrieved credentials
|
|
||||||
5. Begins handling requests with authenticated connections
|
|
||||||
|
|
||||||
If you encounter service errors, check the individual service logs in `./botserver-stack/logs/[service]/` directories.
|
|
||||||
|
|
||||||
### UI File Deployment - Production Options
|
|
||||||
|
|
||||||
**Option 1: Embedded UI (Recommended for Production)**
|
|
||||||
|
|
||||||
The `embed-ui` feature compiles UI files directly into the botui binary, eliminating the need for separate file deployment:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Build with embedded UI files
|
|
||||||
cargo build --release -p botui --features embed-ui
|
|
||||||
|
|
||||||
# The binary now contains all UI files - no additional deployment needed!
|
|
||||||
# The botui binary is self-contained and production-ready
|
|
||||||
```
|
|
||||||
|
|
||||||
**Benefits of embed-ui:**
|
|
||||||
- ✅ Single binary deployment (no separate UI files)
|
|
||||||
- ✅ Faster startup (no filesystem access)
|
|
||||||
- ✅ Smaller attack surface
|
|
||||||
- ✅ Simpler deployment process
|
|
||||||
|
|
||||||
**Option 2: Filesystem Deployment (Development Only)**
|
|
||||||
|
|
||||||
For development, UI files are served from the filesystem:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# UI files must exist at botui/ui/suite/
|
|
||||||
# This is automatically available in development builds
|
|
||||||
```
|
|
||||||
|
|
||||||
**Option 3: Manual File Deployment (Legacy)**
|
|
||||||
|
|
||||||
If you need to deploy UI files separately (not recommended):
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Deploy UI files to production location
|
|
||||||
./botserver/deploy/deploy-ui.sh /opt/gbo
|
|
||||||
|
|
||||||
# Verify deployment
|
|
||||||
ls -la /opt/gbo/bin/ui/suite/index.html
|
|
||||||
```
|
|
||||||
|
|
||||||
See `botserver/deploy/README.md` for deployment scripts.
|
|
||||||
|
|
||||||
### Start Both Servers (Automated)
|
|
||||||
```bash
|
|
||||||
# Use restart script (RECOMMENDED)
|
|
||||||
./restart.sh
|
|
||||||
```
|
|
||||||
|
|
||||||
### Start Both Servers (Manual)
|
|
||||||
```bash
|
|
||||||
# Terminal 1: botserver
|
|
||||||
cd botserver && cargo run -- --noconsole
|
|
||||||
|
|
||||||
# Terminal 2: botui
|
|
||||||
cd botui && BOTSERVER_URL="http://localhost:9000" cargo run
|
|
||||||
```
|
|
||||||
|
|
||||||
### Build Commands
|
|
||||||
```bash
|
|
||||||
# Check single crate
|
|
||||||
cargo check -p botserver
|
|
||||||
|
|
||||||
# Build workspace
|
|
||||||
cargo build
|
cargo build
|
||||||
|
cargo test
|
||||||
# Run tests
|
|
||||||
cargo test -p bottest
|
|
||||||
```
|
```
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 🤖 AI Agent Guidelines
|
## Development Workflow
|
||||||
|
|
||||||
> **For LLM instructions, coding rules, security directives, testing workflows, and error handling patterns, see [AGENTS.md](./AGENTS.md).**
|
1. Read `PROMPT.md` (workspace-level rules)
|
||||||
|
2. Read `<project>/PROMPT.md` (project-specific rules)
|
||||||
---
|
3. Use diagnostics tool to check warnings
|
||||||
|
4. Fix all warnings with full file rewrites
|
||||||
## 📖 Glossary
|
5. Verify with diagnostics after each file
|
||||||
|
6. Never suppress warnings with `#[allow()]`
|
||||||
| Term | Definition | Usage |
|
|
||||||
|------|-----------|-------|
|
|
||||||
| **Bot** | AI agent with configuration, scripts, and knowledge bases | Primary entity in system |
|
|
||||||
| **Session** | Single conversation instance between user and bot | Stored in `sessions` table |
|
|
||||||
| **Dialog** | Collection of BASIC scripts (.bas files) for bot logic | Stored in `{bot_name}.gbdialog/` |
|
|
||||||
| **Tool** | Reusable function callable by LLM | Defined in .bas files, compiled to .ast |
|
|
||||||
| **Knowledge Base (KB)** | Vector database of documents for semantic search | Managed in `learn/` module |
|
|
||||||
| **Scheduler** | Time-triggered task execution | Cron-like scheduling in BASIC scripts |
|
|
||||||
| **Drive** | S3-compatible storage for files | Abstracted in `drive/` module |
|
|
||||||
| **Rhai** | Embedded scripting language for BASIC dialect | Rhai engine in `basic/` module |
|
|
||||||
| **WebSocket Adapter** | Component that sends messages to connected clients | `web_adapter` in state |
|
|
||||||
| **AutoTask** | LLM-generated task automation system | In `auto_task/` and `tasks/` modules |
|
|
||||||
| **Orchestrator** | Coordinates LLM, tools, KBs, and user input | `BotOrchestrator` in `core/bot/` |
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
## 🖥️ UI Architecture (botui + botserver)
|
|
||||||
|
|
||||||
### Two Servers During Development
|
|
||||||
|
|
||||||
| Server | Port | Purpose |
|
|
||||||
|--------|------|---------|
|
|
||||||
| **botui** | 3000 | Serves UI files + proxies API to botserver |
|
|
||||||
| **botserver** | 9000 | Backend API + embedded UI fallback |
|
|
||||||
|
|
||||||
### How It Works
|
|
||||||
|
|
||||||
```
|
|
||||||
Browser → localhost:3000 → botui (serves HTML/CSS/JS)
|
|
||||||
→ /api/* proxied to botserver:9000
|
|
||||||
→ /suite/* served from botui/ui/suite/
|
|
||||||
```
|
|
||||||
|
|
||||||
### Adding New Suite Apps
|
|
||||||
|
|
||||||
1. Create folder: `botui/ui/suite/<appname>/`
|
|
||||||
2. Add to `SUITE_DIRS` in `botui/src/ui_server/mod.rs`
|
|
||||||
3. Rebuild botui: `cargo build -p botui`
|
|
||||||
4. Add menu entry in `botui/ui/suite/index.html`
|
|
||||||
|
|
||||||
### Hot Reload
|
|
||||||
|
|
||||||
- **UI files (HTML/CSS/JS)**: Edit & refresh browser (no restart)
|
|
||||||
- **botui Rust code**: Rebuild + restart botui
|
|
||||||
- **botserver Rust code**: Rebuild + restart botserver
|
|
||||||
|
|
||||||
### Production (Single Binary)
|
|
||||||
|
|
||||||
When `botui/ui/suite/` folder not found, botserver uses **embedded UI** compiled into binary via `rust-embed`.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 🎨 Frontend Standards
|
|
||||||
|
|
||||||
### HTMX-First Approach
|
|
||||||
- Use HTMX to minimize JavaScript
|
|
||||||
- Server returns HTML fragments, not JSON
|
|
||||||
- Use `hx-get`, `hx-post`, `hx-target`, `hx-swap`
|
|
||||||
- WebSocket via htmx-ws extension
|
|
||||||
|
|
||||||
### Local Assets Only - NO CDN
|
|
||||||
```html
|
|
||||||
<!-- ✅ CORRECT -->
|
|
||||||
<script src="js/vendor/htmx.min.js"></script>
|
|
||||||
|
|
||||||
<!-- ❌ WRONG -->
|
|
||||||
<script src="https://unpkg.com/htmx.org@1.9.10"></script>
|
|
||||||
```
|
|
||||||
|
|
||||||
### Vendor Libraries Location
|
|
||||||
```
|
|
||||||
ui/suite/js/vendor/
|
|
||||||
├── htmx.min.js
|
|
||||||
├── htmx-ws.js
|
|
||||||
├── marked.min.js
|
|
||||||
└── gsap.min.js
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 📋 Project-Specific Guidelines
|
|
||||||
|
|
||||||
Each crate has its own README.md with specific guidelines:
|
|
||||||
|
|
||||||
| Crate | README.md Location | Focus |
|
|
||||||
|-------|-------------------|-------|
|
|
||||||
| botserver | `botserver/README.md` | API, security, Rhai BASIC |
|
|
||||||
| botui | `botui/README.md` | UI, HTMX, CSS design system |
|
|
||||||
| botapp | `botapp/README.md` | Tauri, desktop features |
|
|
||||||
| botlib | `botlib/README.md` | Shared types, errors |
|
|
||||||
| botbook | `botbook/README.md` | Documentation, mdBook |
|
|
||||||
| bottest | `bottest/README.md` | Test infrastructure |
|
|
||||||
|
|
||||||
### Special Prompts
|
|
||||||
| File | Purpose |
|
|
||||||
|------|---------|
|
|
||||||
| `botserver/src/tasks/README.md` | AutoTask LLM executor |
|
|
||||||
| `botserver/src/auto_task/APP_GENERATOR_PROMPT.md` | App generation |
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 📚 Documentation
|
|
||||||
|
|
||||||
For complete documentation, guides, and API references:
|
|
||||||
|
|
||||||
- **[docs.pragmatismo.com.br](https://docs.pragmatismo.com.br)** - Full online documentation
|
|
||||||
- **[BotBook](./botbook)** - Local comprehensive guide with tutorials and examples
|
|
||||||
- **[General Bots Repository](https://github.com/GeneralBots/BotServer)** - Main project repository
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 🔧 Immediate Technical Debt
|
|
||||||
|
|
||||||
### Critical Issues to Address
|
|
||||||
|
|
||||||
1. **Error Handling Debt**: 955 instances of `unwrap()`/`expect()` in production code
|
|
||||||
2. **Performance Debt**: 12,973 excessive `clone()`/`to_string()` calls
|
|
||||||
3. **File Size Debt**: 7 files exceed 450 lines (largest: 3220 lines)
|
|
||||||
4. **Test Coverage**: Missing integration tests for critical paths
|
|
||||||
5. **Documentation**: Missing inline documentation for complex algorithms
|
|
||||||
|
|
||||||
### Weekly Maintenance Tasks
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Check for duplicate dependencies
|
|
||||||
cargo tree --duplicates
|
|
||||||
|
|
||||||
# Remove unused dependencies
|
|
||||||
cargo machete
|
|
||||||
|
|
||||||
# Check binary size
|
|
||||||
cargo build --release && ls -lh target/release/botserver
|
|
||||||
|
|
||||||
# Performance profiling
|
|
||||||
cargo bench
|
|
||||||
|
|
||||||
# Security audit
|
|
||||||
cargo audit
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
|
@ -646,57 +87,43 @@ cargo audit
|
||||||
|
|
||||||
**Note:** Each subproject has its own git repository. This root repository only tracks workspace-level files:
|
**Note:** Each subproject has its own git repository. This root repository only tracks workspace-level files:
|
||||||
|
|
||||||
|
- `PROMPT.md` - Development guide
|
||||||
- `Cargo.toml` - Workspace configuration
|
- `Cargo.toml` - Workspace configuration
|
||||||
- `README.md` - This file
|
- `README.md` - This file
|
||||||
- `.gitignore` - Ignore patterns
|
- `.gitignore` - Ignore patterns
|
||||||
- `ADDITIONAL-SUGGESTIONS.md` - Enhancement ideas
|
|
||||||
- `TODO-*.md` - Task tracking files
|
|
||||||
|
|
||||||
Subprojects (botapp, botserver, botui, etc.) are **independent repositories referenced as git submodules**.
|
Subprojects (botapp, botserver, etc.) are **not** git submodules - they are independent repositories.
|
||||||
|
|
||||||
### ⚠️ CRITICAL: Submodule Push Workflow
|
|
||||||
|
|
||||||
When making changes to any submodule (botserver, botui, botlib, etc.):
|
|
||||||
|
|
||||||
1. **Commit and push changes within the submodule directory:**
|
|
||||||
```bash
|
|
||||||
cd botserver
|
|
||||||
git add .
|
|
||||||
git commit -m "Your changes"
|
|
||||||
git push pragmatismo main
|
|
||||||
git push github main
|
|
||||||
```
|
|
||||||
|
|
||||||
2. **Update the global gb repository submodule reference:**
|
|
||||||
```bash
|
|
||||||
cd .. # Back to gb root
|
|
||||||
git add botserver
|
|
||||||
git commit -m "Update botserver submodule to latest commit"
|
|
||||||
git push pragmatismo main
|
|
||||||
git push github main
|
|
||||||
```
|
|
||||||
|
|
||||||
**Failure to push the global gb repository will cause submodule changes to not trigger CI/CD pipelines.**
|
|
||||||
|
|
||||||
Both repositories must be pushed for changes to take effect in production.
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## Development Workflow
|
## Rules Summary
|
||||||
|
|
||||||
1. Read this README.md (workspace structure)
|
```
|
||||||
2. Read **[AGENTS.md](./AGENTS.md)** (coding rules & workflows)
|
✅ FULL FILE REWRITES ONLY
|
||||||
3. **BEFORE creating any .md file, search botbook/ for existing documentation**
|
✅ BATCH ALL FIXES BEFORE WRITING
|
||||||
4. Read `<project>/README.md` (project-specific rules)
|
✅ VERIFY WITH DIAGNOSTICS AFTER EACH FILE
|
||||||
5. Use diagnostics tool to check warnings
|
✅ TRUST PROJECT DIAGNOSTICS
|
||||||
6. Fix all warnings with full file rewrites
|
✅ RESPECT ALL RULES
|
||||||
7. Verify with diagnostics after each file
|
|
||||||
8. Never suppress warnings with `#[allow()]`
|
❌ NEVER use #[allow()] in source code
|
||||||
|
❌ NEVER use partial edits
|
||||||
|
❌ NEVER run cargo check/clippy manually
|
||||||
|
❌ NEVER leave unused code
|
||||||
|
❌ NEVER use .unwrap()/.expect()
|
||||||
|
❌ NEVER use panic!/todo!/unimplemented!()
|
||||||
|
❌ NEVER add comments
|
||||||
|
```
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
## Links
|
||||||
|
|
||||||
|
- Main Server: http://localhost:8081
|
||||||
|
- Desktop App: Uses Tauri to wrap botui
|
||||||
|
- Documentation: See botbook/
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
## License
|
## License
|
||||||
|
|
||||||
See individual project repositories for license information.
|
See individual project repositories for license information.
|
||||||
2
botapp
2
botapp
|
|
@ -1 +1 @@
|
||||||
Subproject commit 0b556948f970832e8606f886853793e2bc8dc35c
|
Subproject commit 1a1e17fa1012e4db10a0f716c9b63a03b4863c9f
|
||||||
2
botbook
2
botbook
|
|
@ -1 +1 @@
|
||||||
Subproject commit 7b2b7ab3c53c65a68930a8cb2e7ca359d8e22bcf
|
Subproject commit 827e011ac05084396aaf2c3098409bf5e02b5cf9
|
||||||
|
|
@ -1 +1 @@
|
||||||
Subproject commit 35411f4f9e64e54b1039360ab654d537cd2958c9
|
Subproject commit 97778e06dd804be55ff761c7fe2788af0ef50626
|
||||||
2
botlib
2
botlib
|
|
@ -1 +1 @@
|
||||||
Subproject commit e926818f35879c1db4086efaa9634caac45f5743
|
Subproject commit bfaa68dc35e96ced2915d43ffe6fca8267a9a598
|
||||||
|
|
@ -1 +1 @@
|
||||||
Subproject commit e088a8e69eb8fe064bf1510a720d42abe159ab00
|
Subproject commit 462a6dfa51b12f22e87712e613a559f66f9013cb
|
||||||
|
|
@ -1 +1 @@
|
||||||
Subproject commit 1727e48307fdb7b54c726af8cd6b12669764e908
|
Subproject commit 17a3caebabddbe843c2b7fd93f624b0ccd9c44fb
|
||||||
|
|
@ -1 +1 @@
|
||||||
Subproject commit 73002b36cc3def17546085574cbafe0f42c7b04f
|
Subproject commit 84458b2a6905af7db72b15f5e833bb7942ccdaa9
|
||||||
|
|
@ -1 +1 @@
|
||||||
Subproject commit 3110dd587290047f283300d674ad325f4f9b3046
|
Subproject commit dd3d8c74dd58a1cc6d6b18d22108819519aaf9c3
|
||||||
2
bottest
2
bottest
|
|
@ -1 +1 @@
|
||||||
Subproject commit 346120cb0b916f72abd2fdad577ae1c606aba1a2
|
Subproject commit 706391b272e0fb7c5b2646cc4cc72180195e07f4
|
||||||
2
botui
2
botui
|
|
@ -1 +1 @@
|
||||||
Subproject commit ff6dfcf126dc20ff8a46d3ac44ef04045cfaf268
|
Subproject commit 661edc09fa1063673e84b63d2dcb5cfbe0f91232
|
||||||
1113
prompts/container.md
1113
prompts/container.md
File diff suppressed because it is too large
Load diff
146
prompts/crmex.md
146
prompts/crmex.md
|
|
@ -1,146 +0,0 @@
|
||||||
# Email Campaigns — Feature Plan
|
|
||||||
|
|
||||||
## Existing Foundation (botserver/src/marketing/)
|
|
||||||
- `campaigns.rs` — CrmCampaign model, CRUD handlers
|
|
||||||
- `metrics.rs` — CampaignMetrics, ChannelBreakdown, open/click/conversion rates
|
|
||||||
- `lists.rs` — recipient lists
|
|
||||||
- `templates.rs` — content templates
|
|
||||||
- `triggers.rs` — event-based sending
|
|
||||||
- `email/tracking.rs` — open/click tracking pixels
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Features to Build
|
|
||||||
|
|
||||||
### 1. Insights Dashboard
|
|
||||||
**What:** Time series views of delivery + engagement metrics per campaign.
|
|
||||||
|
|
||||||
**Data points per time bucket (hourly/daily):**
|
|
||||||
- Sent, delivered, bounced, failed
|
|
||||||
- Opens (unique + total), clicks, replies, unsubscribes
|
|
||||||
- Delivery rate, open rate, click-to-open rate (CTOR)
|
|
||||||
|
|
||||||
**Filters/pivots:**
|
|
||||||
- By mailbox provider (Gmail, Outlook, Yahoo, etc. — parsed from MX/SMTP response)
|
|
||||||
- By sender identity (from address / domain)
|
|
||||||
- By campaign or list
|
|
||||||
- Message search → show exact SMTP response from provider
|
|
||||||
|
|
||||||
**Implementation:**
|
|
||||||
- Add `email_delivery_events` table: `(id, campaign_id, recipient_id, event_type, provider, smtp_response, ts)`
|
|
||||||
- API: `GET /api/campaigns/:id/insights?from=&to=&group_by=provider|identity|day`
|
|
||||||
- UI: HTMX + chart.js time series (local vendor)
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### 2. Advisor Recommendations
|
|
||||||
**What:** Analyze sending config + results and surface actionable fixes.
|
|
||||||
|
|
||||||
**Checks to run:**
|
|
||||||
| Check | Signal | Recommendation |
|
|
||||||
|---|---|---|
|
|
||||||
| SPF/DKIM/DMARC | DNS lookup | "Add missing record" |
|
|
||||||
| Bounce rate > 5% | delivery_events | "Clean list — remove hard bounces" |
|
|
||||||
| Open rate < 15% | metrics | "Improve subject line / send time" |
|
|
||||||
| Spam complaints > 0.1% | FBL data | "Remove complainers immediately" |
|
|
||||||
| Sending from new IP | warmup_schedule | "Follow warmup plan" |
|
|
||||||
| List age > 6 months | list.last_sent | "Re-engagement campaign before bulk send" |
|
|
||||||
|
|
||||||
**Implementation:**
|
|
||||||
- `marketing/advisor.rs` — `AdvisorEngine::analyze(campaign_id) -> Vec<Recommendation>`
|
|
||||||
- API: `GET /api/campaigns/:id/advisor`
|
|
||||||
- Runs automatically after each campaign completes
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### 3. IP Warmup (like OneSignal / Mailchimp)
|
|
||||||
**What:** Gradually increase daily send volume over 4–6 weeks to build sender reputation.
|
|
||||||
|
|
||||||
**Warmup schedule (standard):**
|
|
||||||
| Day | Max emails/day |
|
|
||||||
|---|---|
|
|
||||||
| 1–2 | 50 |
|
|
||||||
| 3–4 | 100 |
|
|
||||||
| 5–7 | 500 |
|
|
||||||
| 8–10 | 1,000 |
|
|
||||||
| 11–14 | 5,000 |
|
|
||||||
| 15–21 | 10,000 |
|
|
||||||
| 22–28 | 50,000 |
|
|
||||||
| 29+ | unlimited |
|
|
||||||
|
|
||||||
**Rules:**
|
|
||||||
- Only send to most engaged subscribers first (opened in last 90 days)
|
|
||||||
- Stop warmup if bounce rate > 3% or complaint rate > 0.1%
|
|
||||||
- Resume next day at same volume if paused
|
|
||||||
|
|
||||||
**Implementation:**
|
|
||||||
- `marketing/warmup.rs` — `WarmupSchedule`, `WarmupEngine::get_daily_limit(ip, day) -> u32`
|
|
||||||
- `warmup_schedules` table: `(id, ip, started_at, current_day, status, paused_reason)`
|
|
||||||
- Scheduler checks warmup limit before each send batch
|
|
||||||
- API: `GET /api/warmup/status`, `POST /api/warmup/start`
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### 4. Optimized Shared Delivery
|
|
||||||
**What:** Auto-select best sending IP based on real-time reputation signals.
|
|
||||||
|
|
||||||
**Logic:**
|
|
||||||
- Track per-IP: bounce rate, complaint rate, delivery rate (last 24h)
|
|
||||||
- Score each IP: `score = delivery_rate - (bounce_rate * 10) - (complaint_rate * 100)`
|
|
||||||
- Route each send to highest-scored IP for that destination provider
|
|
||||||
- Rotate IPs to spread load and preserve reputation
|
|
||||||
|
|
||||||
**Implementation:**
|
|
||||||
- `marketing/ip_router.rs` — `IpRouter::select(destination_domain) -> IpAddr`
|
|
||||||
- `ip_reputation` table: `(ip, provider, bounces, complaints, delivered, window_start)`
|
|
||||||
- Plugs into Stalwart send path via botserver API
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### 5. Modern Email Marketing Features
|
|
||||||
|
|
||||||
| Feature | Description |
|
|
||||||
|---|---|
|
|
||||||
| **Send time optimization** | ML-based per-contact best send time (based on past open history) |
|
|
||||||
| **A/B testing** | Split subject/content, auto-pick winner after N hours |
|
|
||||||
| **Suppression list** | Global unsubscribe/bounce/complaint list, auto-applied to all sends |
|
|
||||||
| **Re-engagement flows** | Auto-trigger "we miss you" to contacts inactive > 90 days |
|
|
||||||
| **Transactional + marketing separation** | Separate IPs/domains for transactional vs bulk |
|
|
||||||
| **One-click unsubscribe** | RFC 8058 `List-Unsubscribe-Post` header on all bulk sends |
|
|
||||||
| **Preview & spam score** | Pre-send SpamAssassin score check |
|
|
||||||
| **Link tracking** | Redirect all links through tracker, record clicks per contact |
|
|
||||||
| **Webhook events** | Push delivery events to external URLs (Stalwart webhook → botserver) |
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## DB Tables to Add
|
|
||||||
|
|
||||||
```sql
|
|
||||||
email_delivery_events (id, campaign_id, recipient_id, event_type, provider, smtp_code, smtp_response, ts)
|
|
||||||
warmup_schedules (id, ip, started_at, current_day, daily_limit, status, paused_reason)
|
|
||||||
ip_reputation (id, ip, provider, delivered, bounced, complained, window_start)
|
|
||||||
advisor_recommendations (id, campaign_id, check_name, severity, message, created_at, dismissed)
|
|
||||||
ab_tests (id, campaign_id, variant_a, variant_b, split_pct, winner, decided_at)
|
|
||||||
suppression_list (id, org_id, email, reason, added_at)
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Files to Create
|
|
||||||
```
|
|
||||||
botserver/src/marketing/
|
|
||||||
├── warmup.rs — IP warmup engine + schedule
|
|
||||||
├── advisor.rs — recommendation engine
|
|
||||||
├── ip_router.rs — optimized IP selection
|
|
||||||
├── ab_test.rs — A/B test logic
|
|
||||||
├── suppression.rs — global suppression list
|
|
||||||
└── send_time.rs — send time optimization
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Existing Code to Extend
|
|
||||||
- `marketing/metrics.rs` → add time-series queries + provider breakdown
|
|
||||||
- `marketing/campaigns.rs` → add warmup_enabled, ab_test_id fields
|
|
||||||
- `email/tracking.rs` → already has open/click tracking, extend with provider parsing
|
|
||||||
- `core/shared/schema/` → add new tables above
|
|
||||||
|
|
@ -1,59 +0,0 @@
|
||||||
# SEPLAGSE - Detecção de Desvios na Folha
|
|
||||||
|
|
||||||
## Objetivo
|
|
||||||
- Bot seplagse deve usar start.bas para inserir dados via init_folha.bas
|
|
||||||
- detecta.bas deve detectar anomalias nos dados inseridos
|
|
||||||
|
|
||||||
## ✅ Status Atual
|
|
||||||
|
|
||||||
### Correção REM em mod.rs (FEITA)
|
|
||||||
**Arquivo:** `botserver/src/basic/mod.rs` linha ~588-594
|
|
||||||
|
|
||||||
Filtro adicionado para `REM ` e `REM\t` no `compile_tool_script`:
|
|
||||||
```rust
|
|
||||||
!(trimmed.starts_with("PARAM ") ||
|
|
||||||
trimmed.starts_with("PARAM\t") ||
|
|
||||||
trimmed.starts_with("DESCRIPTION ") ||
|
|
||||||
trimmed.starts_with("DESCRIPTION\t") ||
|
|
||||||
trimmed.starts_with("REM ") || // <-- ADICIONADO
|
|
||||||
trimmed.starts_with("REM\t") || // <-- ADICIONADO
|
|
||||||
trimmed.starts_with('\'') ||
|
|
||||||
trimmed.starts_with('#') ||
|
|
||||||
trimmed.is_empty())
|
|
||||||
```
|
|
||||||
|
|
||||||
### Arquivos Envolvidos (VERIFICADOS)
|
|
||||||
- `/opt/gbo/data/seplagse.gbai/seplagse.gbdialog/start.bas` ✅ OK
|
|
||||||
- Contém botões de sugestão: detecta e init_folha
|
|
||||||
- `/opt/gbo/data/seplagse.gbai/seplagse.gbdialog/init_folha.bas` ✅ OK
|
|
||||||
- 4 INSERT statements para dados de exemplo
|
|
||||||
- `/opt/gbo/data/seplagse.gbai/seplagse.gbdialog/detecta.bas` ✅ OK
|
|
||||||
- Usa DETECT keyword
|
|
||||||
- `/opt/gbo/data/seplagse.gbai/seplagse.gbdialog/tables.bas` ✅ OK
|
|
||||||
- TABLE folha_salarios definida
|
|
||||||
|
|
||||||
### Botserver (RODANDO)
|
|
||||||
- ✅ Botserver compilado com sucesso
|
|
||||||
- ✅ Botserver rodando em http://localhost:8080
|
|
||||||
- ✅ Health check OK
|
|
||||||
|
|
||||||
## Próximos Passos (Pendentes)
|
|
||||||
|
|
||||||
1. **Testar via navegador** - Necessário instalar Playwright browsers
|
|
||||||
- Navegar para http://localhost:3000/seplagse
|
|
||||||
- Clicar em "⚙️ Inicializar Dados de Teste"
|
|
||||||
- Verificar se INSERT funciona
|
|
||||||
- Clicar em "🔍 Detectar Desvios na Folha"
|
|
||||||
- Verificar se DETECT funciona
|
|
||||||
|
|
||||||
2. **Verificar se há warnings relevantes**
|
|
||||||
- Alguns warnings de código podem precisar ser corrigidos
|
|
||||||
|
|
||||||
## Cache
|
|
||||||
- AST limpo: `rm ./botserver-stack/data/system/work/seplagse.gbai/seplagse.gbdialog/*.ast`
|
|
||||||
- Reiniciado: `./restart.sh`
|
|
||||||
- Botserver: ✅ Rodando
|
|
||||||
|
|
||||||
## Arquivos de Trabalho
|
|
||||||
- Work directory: `./botserver-stack/data/system/work/seplagse.gbai/seplagse.gbdialog/`
|
|
||||||
- Todos os arquivos BASIC estão presentes e parecem válidos
|
|
||||||
|
|
@ -1,272 +0,0 @@
|
||||||
# Integrated Suite — Conversational Interface Plan
|
|
||||||
|
|
||||||
> **Pattern:** Every suite app exposes its own `PROMPT.md` + internal tools.
|
|
||||||
> The shared chat bar activates app-specific context when the user is inside that app.
|
|
||||||
> WhatsApp campaigns is the first full example.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Architecture
|
|
||||||
|
|
||||||
```
|
|
||||||
User (WhatsApp / Suite chat bar)
|
|
||||||
↓
|
|
||||||
BotOrchestrator (core/bot/mod.rs)
|
|
||||||
↓
|
|
||||||
detect active app context
|
|
||||||
↓
|
|
||||||
load app PROMPT.md + app InternalTools
|
|
||||||
↓
|
|
||||||
LLM with tools → tool_executor.rs
|
|
||||||
↓
|
|
||||||
app data / actions
|
|
||||||
```
|
|
||||||
|
|
||||||
### Key existing pieces
|
|
||||||
| File | Role |
|
|
||||||
|---|---|
|
|
||||||
| `core/bot/mod.rs` | `get_session_tools()` + `ToolExecutor::execute_tool_call()` |
|
|
||||||
| `tasks/PROMPT.md` | Pattern for app-level LLM prompt |
|
|
||||||
| `marketing/whatsapp.rs` | WhatsApp campaign send/metrics |
|
|
||||||
| `marketing/campaigns.rs` | Campaign CRUD |
|
|
||||||
| `marketing/lists.rs` | Recipient lists |
|
|
||||||
| `botui/ui/suite/campaigns/` | Campaigns UI |
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Standard: Every Suite App
|
|
||||||
|
|
||||||
### 1. `PROMPT.md` per app folder
|
|
||||||
Location: `botserver/src/<app>/PROMPT.md`
|
|
||||||
|
|
||||||
```markdown
|
|
||||||
# <App> — Internal Tools Guide
|
|
||||||
|
|
||||||
You are the <App> assistant. When the user is in <App>, you have access to:
|
|
||||||
- tool: list_<entities>
|
|
||||||
- tool: create_<entity>
|
|
||||||
- tool: search_<entity>
|
|
||||||
- tool: <app_specific_action>
|
|
||||||
|
|
||||||
Rules:
|
|
||||||
- Always confirm destructive actions before executing
|
|
||||||
- Show results as structured summaries, not raw JSON
|
|
||||||
- If user uploads a file, parse it and confirm before acting
|
|
||||||
```
|
|
||||||
|
|
||||||
### 2. `tools.rs` per app
|
|
||||||
Location: `botserver/src/<app>/tools.rs`
|
|
||||||
|
|
||||||
Registers `Vec<Tool>` (LLM function-calling schema) + handler mapping.
|
|
||||||
Loaded by `get_session_tools()` when session's active app = this app.
|
|
||||||
|
|
||||||
### 3. App context detection
|
|
||||||
`core/bot/mod.rs` reads `session.active_app` (set by UI via `POST /api/session/context`).
|
|
||||||
Loads `<app>/PROMPT.md` as system prompt prefix + `<app>/tools.rs` tools.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## WhatsApp Campaigns — Full Conversational Flow
|
|
||||||
|
|
||||||
### Meta Rules (enforced in tools)
|
|
||||||
- Only approved Message Templates for marketing (non-session-initiated)
|
|
||||||
- 24h session window for free-form after user replies
|
|
||||||
- Media: image/video/document via Media Upload API before send
|
|
||||||
- Opt-out: always honor STOP, add to suppression list immediately
|
|
||||||
- Rate: respect per-phone-number rate limits (1000 msg/s business tier)
|
|
||||||
- Template category: MARKETING requires explicit opt-in from recipient
|
|
||||||
|
|
||||||
### Conversation Flow (WhatsApp → campaign creation)
|
|
||||||
|
|
||||||
```
|
|
||||||
User sends to bot number:
|
|
||||||
"I want to send a campaign"
|
|
||||||
↓
|
|
||||||
Bot: "Great! Send me:
|
|
||||||
1. Your contact list (.xlsx or .csv)
|
|
||||||
2. The message text
|
|
||||||
3. An image (optional)
|
|
||||||
4. When to send (or 'now')"
|
|
||||||
↓
|
|
||||||
User uploads contacts.xlsx
|
|
||||||
↓
|
|
||||||
[tool: parse_contact_file]
|
|
||||||
→ extract phone numbers, names
|
|
||||||
→ validate E.164 format
|
|
||||||
→ show preview: "Found 342 contacts. First 3: +55..."
|
|
||||||
↓
|
|
||||||
User sends message text
|
|
||||||
↓
|
|
||||||
[tool: check_template_compliance]
|
|
||||||
→ check if free-form or needs approved template
|
|
||||||
→ if template needed: list available approved templates
|
|
||||||
→ suggest closest match
|
|
||||||
↓
|
|
||||||
User sends image (optional)
|
|
||||||
↓
|
|
||||||
[tool: upload_media]
|
|
||||||
→ upload to Meta Media API
|
|
||||||
→ return media_id
|
|
||||||
↓
|
|
||||||
Bot: "Ready to send to 342 contacts at 14:00 today.
|
|
||||||
Preview: [image] Hello {name}, ...
|
|
||||||
Estimated cost: $X
|
|
||||||
Confirm? (yes/no)"
|
|
||||||
↓
|
|
||||||
User: "yes"
|
|
||||||
↓
|
|
||||||
[tool: create_and_schedule_campaign]
|
|
||||||
→ create campaign record
|
|
||||||
→ apply warmup limit if IP warming
|
|
||||||
→ schedule via TaskScheduler
|
|
||||||
```
|
|
||||||
|
|
||||||
### WhatsApp Campaign Tools (`marketing/whatsapp_tools.rs`)
|
|
||||||
|
|
||||||
```rust
|
|
||||||
// Tool definitions for LLM function calling
|
|
||||||
pub fn whatsapp_campaign_tools() -> Vec<Tool> {
|
|
||||||
vec![
|
|
||||||
Tool::new("parse_contact_file", "Parse uploaded xlsx/csv into contact list"),
|
|
||||||
Tool::new("list_templates", "List approved WhatsApp message templates"),
|
|
||||||
Tool::new("check_template_compliance", "Check if message needs approved template"),
|
|
||||||
Tool::new("upload_media", "Upload image/video to Meta Media API"),
|
|
||||||
Tool::new("preview_campaign", "Show campaign preview with cost estimate"),
|
|
||||||
Tool::new("create_and_schedule_campaign", "Create campaign and schedule send"),
|
|
||||||
Tool::new("get_campaign_status", "Get delivery/read metrics for a campaign"),
|
|
||||||
Tool::new("pause_campaign", "Pause an in-progress campaign"),
|
|
||||||
Tool::new("list_campaigns", "List recent campaigns with metrics"),
|
|
||||||
Tool::new("add_to_suppression", "Add number to opt-out list"),
|
|
||||||
]
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### WhatsApp PROMPT.md (`marketing/WHATSAPP_PROMPT.md`)
|
|
||||||
|
|
||||||
```markdown
|
|
||||||
# WhatsApp Campaign Assistant
|
|
||||||
|
|
||||||
You help users create and manage WhatsApp marketing campaigns.
|
|
||||||
|
|
||||||
## Meta Platform Rules (MANDATORY)
|
|
||||||
- Marketing messages MUST use pre-approved templates outside 24h session window
|
|
||||||
- Always check opt-in status before adding to campaign
|
|
||||||
- Honor STOP/unsubscribe immediately via add_to_suppression tool
|
|
||||||
- Never send more than warmup daily limit if IP is warming up
|
|
||||||
- Image must be uploaded via upload_media before referencing in campaign
|
|
||||||
|
|
||||||
## Conversation Style
|
|
||||||
- Guide step by step: contacts → message → media → schedule → confirm
|
|
||||||
- Show cost estimate before confirming
|
|
||||||
- After send: proactively share open/read rates when available
|
|
||||||
|
|
||||||
## File Handling
|
|
||||||
- .xlsx/.csv → use parse_contact_file tool
|
|
||||||
- Images → use upload_media tool
|
|
||||||
- Always confirm parsed data before proceeding
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Integrated Suite Chat Bar — Standard
|
|
||||||
|
|
||||||
### How it works
|
|
||||||
1. User opens any suite app (CRM, Campaigns, Drive, etc.)
|
|
||||||
2. Chat bar at bottom activates with app context
|
|
||||||
3. `POST /api/session/context { app: "campaigns" }` sets `session.active_app`
|
|
||||||
4. BotOrchestrator loads `campaigns/PROMPT.md` + `campaigns/tools.rs`
|
|
||||||
5. User can ask natural language questions or trigger actions
|
|
||||||
|
|
||||||
### Examples per app
|
|
||||||
|
|
||||||
| App | Example query | Tool activated |
|
|
||||||
|---|---|---|
|
|
||||||
| **Campaigns** | "How did last week's campaign perform?" | `get_campaign_metrics` |
|
|
||||||
| **CRM** | "Show deals closing this month" | `list_deals` with filter |
|
|
||||||
| **Drive** | "Find the Q1 report" | `search_files` |
|
|
||||||
| **Tasks** | "Create a task to follow up with Acme" | `create_task` |
|
|
||||||
| **People** | "Who hasn't been contacted in 30 days?" | `list_contacts` with filter |
|
|
||||||
| **Mail** | "Summarize unread emails from clients" | `list_emails` + LLM summary |
|
|
||||||
| **Sheet** | "What's the total revenue in column D?" | `query_sheet` |
|
|
||||||
| **Learn** | "What does our refund policy say?" | `search_kb` |
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Implementation Plan
|
|
||||||
|
|
||||||
### Phase 1 — Infrastructure (1 sprint)
|
|
||||||
- [ ] `core/bot/mod.rs` — read `session.active_app`, load app PROMPT + tools
|
|
||||||
- [ ] `core/tool_context.rs` — app tool registry: `register_app_tools(app_name) -> Vec<Tool>`
|
|
||||||
- [ ] `POST /api/session/context` — set active app from UI
|
|
||||||
- [ ] Suite chat bar UI component (`botui/ui/suite/partials/chatbar.html`)
|
|
||||||
|
|
||||||
### Phase 2 — WhatsApp Campaigns (1 sprint)
|
|
||||||
- [ ] `marketing/whatsapp_tools.rs` — 10 tools above
|
|
||||||
- [ ] `marketing/WHATSAPP_PROMPT.md`
|
|
||||||
- [ ] `marketing/file_parser.rs` — xlsx/csv → contact list
|
|
||||||
- [ ] Meta warmup enforcement in send path
|
|
||||||
- [ ] Conversational campaign creation flow (state machine in session)
|
|
||||||
|
|
||||||
### Phase 3 — App-by-app rollout (1 app/sprint)
|
|
||||||
Priority order based on value:
|
|
||||||
1. CRM (deals, contacts, pipeline queries)
|
|
||||||
2. Campaigns (email + WhatsApp)
|
|
||||||
3. Tasks (create, assign, status)
|
|
||||||
4. Drive (search, summarize docs)
|
|
||||||
5. Mail (summarize, draft reply)
|
|
||||||
6. People (segment, find contacts)
|
|
||||||
7. Sheet (query, calculate)
|
|
||||||
8. Learn (KB search)
|
|
||||||
|
|
||||||
### Phase 4 — Cross-app intelligence
|
|
||||||
- [ ] Global search across all apps via single query
|
|
||||||
- [ ] "What happened today?" — aggregates activity across CRM + Mail + Tasks
|
|
||||||
- [ ] Proactive suggestions: "You have 3 deals closing this week and no follow-up tasks"
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## File Structure to Create
|
|
||||||
|
|
||||||
```
|
|
||||||
botserver/src/
|
|
||||||
├── marketing/
|
|
||||||
│ ├── whatsapp_tools.rs ← NEW: LLM tool definitions + handlers
|
|
||||||
│ ├── WHATSAPP_PROMPT.md ← NEW: WhatsApp assistant system prompt
|
|
||||||
│ ├── file_parser.rs ← NEW: xlsx/csv → contacts
|
|
||||||
│ └── warmup.rs ← NEW: (from campaigns.md plan)
|
|
||||||
├── core/
|
|
||||||
│ ├── tool_registry.rs ← NEW: app → tools mapping
|
|
||||||
│ └── bot/
|
|
||||||
│ └── app_context.rs ← NEW: load app prompt + tools per session
|
|
||||||
├── crm/
|
|
||||||
│ ├── tools.rs ← NEW
|
|
||||||
│ └── PROMPT.md ← NEW
|
|
||||||
├── tasks/
|
|
||||||
│ └── tools.rs ← NEW (PROMPT.md exists)
|
|
||||||
└── <each app>/
|
|
||||||
├── tools.rs ← NEW per app
|
|
||||||
└── PROMPT.md ← NEW per app
|
|
||||||
|
|
||||||
botui/ui/suite/
|
|
||||||
└── partials/
|
|
||||||
└── chatbar.html ← NEW: shared chat bar component
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Chat Bar UI (`partials/chatbar.html`)
|
|
||||||
|
|
||||||
```html
|
|
||||||
<div id="suite-chatbar" class="chatbar">
|
|
||||||
<div id="chatbar-messages" hx-ext="ws" ws-connect="/ws/suite-chat"></div>
|
|
||||||
<form ws-send>
|
|
||||||
<input type="hidden" name="app_context" value="{{ active_app }}">
|
|
||||||
<input type="file" id="chatbar-file" name="file" accept=".xlsx,.csv,.png,.jpg,.pdf" style="display:none">
|
|
||||||
<button type="button" onclick="document.getElementById('chatbar-file').click()">📎</button>
|
|
||||||
<input type="text" name="message" placeholder="Ask anything about {{ active_app }}...">
|
|
||||||
<button type="submit">→</button>
|
|
||||||
</form>
|
|
||||||
</div>
|
|
||||||
```
|
|
||||||
|
|
||||||
File uploads go to `POST /api/suite/upload` → stored in Drive → media_id passed to tool.
|
|
||||||
|
|
@ -1,46 +0,0 @@
|
||||||
# Progress: Removendo aws-sdk-s3 do default bundle
|
|
||||||
|
|
||||||
## Goal
|
|
||||||
Remover `aws-sdk-s3` (~120MB) do bundle default `["chat", "automation", "cache", "llm"]` e fazer compilar com:
|
|
||||||
```bash
|
|
||||||
cargo check -p botserver --no-default-features --features "chat,automation,cache,llm"
|
|
||||||
```
|
|
||||||
|
|
||||||
## ✅ COMPLETED
|
|
||||||
|
|
||||||
1. **Cargo.toml** - Features separadas: `drive` (S3) vs `local-files` (notify)
|
|
||||||
2. **main.rs** - `pub mod drive` com `#[cfg(any(feature = "drive", feature = "local-files"))]`
|
|
||||||
3. **state.rs** - `NoDrive` struct adicionada
|
|
||||||
4. **multimedia.rs** - `DefaultMultimediaHandler` com cfg gates (drive vs no-drive)
|
|
||||||
5. **drive/mod.rs** - Módulos condicionais:
|
|
||||||
- `#[cfg(feature = "drive")] pub mod document_processing;`
|
|
||||||
- `#[cfg(feature = "drive")] pub mod drive_monitor;`
|
|
||||||
- `#[cfg(feature = "drive")] pub mod vectordb;`
|
|
||||||
- `#[cfg(feature = "local-files")] pub mod local_file_monitor;`
|
|
||||||
- Todas ~21 funções com `#[cfg(feature = "drive")]`
|
|
||||||
6. **multimedia.rs - upload_media** - Duas implementações separadas com cfg gates:
|
|
||||||
- `#[cfg(feature = "drive")]` - Usa S3 client
|
|
||||||
- `#[cfg(not(feature = "drive"))]` - Usa armazenamento local
|
|
||||||
|
|
||||||
## ✅ VERIFIED
|
|
||||||
|
|
||||||
```bash
|
|
||||||
cargo check -p botserver --no-default-features --features "chat,automation,cache,llm"
|
|
||||||
```
|
|
||||||
|
|
||||||
**Resultado:** ✅ Build limpo (apenas warnings, 0 erros)
|
|
||||||
**Tempo de compilação:** 2m 29s
|
|
||||||
|
|
||||||
## Arquivo Não Fixado (opcional)
|
|
||||||
|
|
||||||
### auto_task/app_generator.rs
|
|
||||||
- `ensure_bucket_exists` method never used (warning, não impede compilação)
|
|
||||||
- Método já está com `#[cfg(feature = "drive")]` (correto)
|
|
||||||
|
|
||||||
## Resumo
|
|
||||||
|
|
||||||
O `aws-sdk-s3` foi removido com sucesso do bundle default. O sistema agora suporta dois modos:
|
|
||||||
- **Com feature "drive"**: Usa S3 (aws-sdk-s3 ~120MB)
|
|
||||||
- **Sem feature "drive"**: Usa armazenamento local (notify ~2MB)
|
|
||||||
|
|
||||||
O build padrão agora é leve (~120MB a menos) e funciona sem dependências de AWS.
|
|
||||||
259
prompts/prod.md
259
prompts/prod.md
|
|
@ -1,259 +0,0 @@
|
||||||
# Production Environment Guide
|
|
||||||
|
|
||||||
## Infrastructure
|
|
||||||
|
|
||||||
### Servers
|
|
||||||
|
|
||||||
| Host | IP | Purpose |
|
|
||||||
|------|-----|---------|
|
|
||||||
| `system` | `10.157.134.196` | Main botserver + botui container |
|
|
||||||
| `alm-ci` | `10.157.134.200` | CI/CD runner (Forgejo Actions) |
|
|
||||||
| `alm` | `10.157.134.34` | Forgejo git server |
|
|
||||||
| `dns` | `10.157.134.214` | DNS container |
|
|
||||||
| `drive` | `10.157.134.206` | Drive storage |
|
|
||||||
| `email` | `10.157.134.40` | Email service |
|
|
||||||
| `proxy` | `10.157.134.241` | Reverse proxy |
|
|
||||||
| `tables` | `10.157.134.174` | PostgreSQL |
|
|
||||||
| `table-editor` | `10.157.134.184` | Table editor |
|
|
||||||
| `webmail` | `10.157.134.86` | Webmail |
|
|
||||||
|
|
||||||
### Port Mapping (system container)
|
|
||||||
|
|
||||||
| Service | Internal Port | External URL |
|
|
||||||
|---------|--------------|--------------|
|
|
||||||
| botserver | `5858` | `https://system.pragmatismo.com.br` |
|
|
||||||
| botui | `5859` | `https://chat.pragmatismo.com.br` |
|
|
||||||
|
|
||||||
### Access
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# SSH to host
|
|
||||||
ssh administrator@63.141.255.9
|
|
||||||
|
|
||||||
# Execute inside system container
|
|
||||||
sudo incus exec system -- bash -c 'command'
|
|
||||||
|
|
||||||
# SSH from host to container (used by CI)
|
|
||||||
ssh -o StrictHostKeyChecking=no system "command"
|
|
||||||
```
|
|
||||||
|
|
||||||
## Services
|
|
||||||
|
|
||||||
### botserver.service
|
|
||||||
|
|
||||||
- **Binary**: `/opt/gbo/bin/botserver`
|
|
||||||
- **Port**: `5858`
|
|
||||||
- **User**: `gbuser`
|
|
||||||
- **Logs**: `/opt/gbo/logs/out.log`, `/opt/gbo/logs/err.log`
|
|
||||||
- **Config**: `/etc/systemd/system/botserver.service`
|
|
||||||
- **Env**: `PORT=5858`
|
|
||||||
|
|
||||||
### ui.service
|
|
||||||
|
|
||||||
- **Binary**: `/opt/gbo/bin/botui`
|
|
||||||
- **Port**: `5859`
|
|
||||||
- **Config**: `/etc/systemd/system/ui.service`
|
|
||||||
- **Env**: `BOTSERVER_URL=http://localhost:5858`
|
|
||||||
- ⚠️ MUST be `http://localhost:5858` — NOT `https://system.pragmatismo.com.br`
|
|
||||||
- Rust proxy runs server-side, needs direct localhost access
|
|
||||||
- JS client uses relative URLs through `chat.pragmatismo.com.br`
|
|
||||||
|
|
||||||
### Data Directory
|
|
||||||
|
|
||||||
- **Path**: `/opt/gbo/data/`
|
|
||||||
- **Structure**: `<botname>.gbai/<botname>.gbdialog/*.bas`
|
|
||||||
- **Bots**: cristo, fema, jucees, oerlabs, poupatempo, pragmatismogb, salesianos, sentient, seplagse
|
|
||||||
- **Work dir**: `/opt/gbo/work/` (compiled .ast cache)
|
|
||||||
|
|
||||||
### Stack Services (managed by botserver bootstrap)
|
|
||||||
|
|
||||||
- **Vault**: Secrets management
|
|
||||||
- **PostgreSQL**: Database (port 5432)
|
|
||||||
- **Valkey**: Cache (port 6379, password auth)
|
|
||||||
- **MinIO**: Object storage
|
|
||||||
- **Zitadel**: Identity provider
|
|
||||||
- **LLM**: llama.cpp
|
|
||||||
|
|
||||||
## CI/CD Pipeline
|
|
||||||
|
|
||||||
### Repositories
|
|
||||||
|
|
||||||
| Repo | ALM URL | GitHub URL |
|
|
||||||
|------|---------|------------|
|
|
||||||
| gb | `https://alm.pragmatismo.com.br/GeneralBots/gb.git` | `git@github.com:GeneralBots/gb.git` |
|
|
||||||
| botserver | `https://alm.pragmatismo.com.br/GeneralBots/BotServer.git` | `git@github.com:GeneralBots/botserver.git` |
|
|
||||||
| botui | `https://alm.pragmatismo.com.br/GeneralBots/BotUI.git` | `git@github.com:GeneralBots/botui.git` |
|
|
||||||
| botlib | `https://alm.pragmatismo.com.br/GeneralBots/botlib.git` | `git@github.com:GeneralBots/botlib.git` |
|
|
||||||
|
|
||||||
### Push Order
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# 1. Push submodules first
|
|
||||||
cd botserver && git push alm main && git push origin main && cd ..
|
|
||||||
cd botui && git push alm main && git push origin main && cd ..
|
|
||||||
|
|
||||||
# 2. Update root workspace references
|
|
||||||
git add botserver botui botlib
|
|
||||||
git commit -m "Update submodules: <description>"
|
|
||||||
git push alm main && git push origin main
|
|
||||||
```
|
|
||||||
|
|
||||||
### Build Environment
|
|
||||||
|
|
||||||
- **CI runner**: `alm-ci` container (Debian Trixie, glibc 2.41)
|
|
||||||
- **Target**: `system` container (Debian 12 Bookworm, glibc 2.36)
|
|
||||||
- **⚠️ GLIBC MISMATCH**: Building on CI runner produces binaries incompatible with system container
|
|
||||||
- **Solution**: CI workflow transfers source to system container and builds there via SSH
|
|
||||||
|
|
||||||
### Workflow File
|
|
||||||
|
|
||||||
- **Location**: `botserver/.forgejo/workflows/botserver.yaml`
|
|
||||||
- **Triggers**: Push to `main` branch
|
|
||||||
- **Steps**:
|
|
||||||
1. Setup workspace on CI runner (clone repos)
|
|
||||||
2. Transfer source to system container via `tar | ssh`
|
|
||||||
3. Build inside system container (matches glibc 2.36)
|
|
||||||
4. Deploy binary inside container
|
|
||||||
5. Verify botserver is running
|
|
||||||
|
|
||||||
## Common Operations
|
|
||||||
|
|
||||||
### Check Service Status
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# From host
|
|
||||||
sudo incus exec system -- systemctl status botserver --no-pager
|
|
||||||
sudo incus exec system -- systemctl status ui --no-pager
|
|
||||||
|
|
||||||
# Check if running
|
|
||||||
sudo incus exec system -- pgrep -f botserver
|
|
||||||
sudo incus exec system -- pgrep -f botui
|
|
||||||
```
|
|
||||||
|
|
||||||
### View Logs
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Systemd journal
|
|
||||||
sudo incus exec system -- journalctl -u botserver --no-pager -n 50
|
|
||||||
sudo incus exec system -- journalctl -u ui --no-pager -n 50
|
|
||||||
|
|
||||||
# Application logs
|
|
||||||
sudo incus exec system -- tail -50 /opt/gbo/logs/out.log
|
|
||||||
sudo incus exec system -- tail -50 /opt/gbo/logs/err.log
|
|
||||||
|
|
||||||
# Live tail
|
|
||||||
sudo incus exec system -- tail -f /opt/gbo/logs/out.log
|
|
||||||
```
|
|
||||||
|
|
||||||
### Restart Services
|
|
||||||
|
|
||||||
```bash
|
|
||||||
sudo incus exec system -- systemctl restart botserver
|
|
||||||
sudo incus exec system -- systemctl restart ui
|
|
||||||
```
|
|
||||||
|
|
||||||
### Manual Deploy (emergency)
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Kill old process
|
|
||||||
sudo incus exec system -- killall botserver
|
|
||||||
|
|
||||||
# Copy binary (from host CI workspace or local)
|
|
||||||
sudo incus exec system -- cp /opt/gbo/ci/botserver/target/debug/botserver /opt/gbo/bin/botserver
|
|
||||||
sudo incus exec system -- chmod +x /opt/gbo/bin/botserver
|
|
||||||
sudo incus exec system -- chown gbuser:gbuser /opt/gbo/bin/botserver
|
|
||||||
|
|
||||||
# Start service
|
|
||||||
sudo incus exec system -- systemctl start botserver
|
|
||||||
```
|
|
||||||
|
|
||||||
### Transfer Bot Files to Production
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# From local to prod host
|
|
||||||
tar czf /tmp/bots.tar.gz -C /opt/gbo/data <botname>.gbai
|
|
||||||
scp /tmp/bots.tar.gz administrator@63.141.255.9:/tmp/
|
|
||||||
|
|
||||||
# From host to container
|
|
||||||
sudo incus exec system -- bash -c 'tar xzf /tmp/bots.tar.gz -C /opt/gbo/data/'
|
|
||||||
|
|
||||||
# Clear compiled cache
|
|
||||||
sudo incus exec system -- find /opt/gbo/data -name "*.ast" -delete
|
|
||||||
sudo incus exec system -- find /opt/gbo/work -name "*.ast" -delete
|
|
||||||
```
|
|
||||||
|
|
||||||
### Snapshots
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# List snapshots
|
|
||||||
sudo incus snapshot list system
|
|
||||||
|
|
||||||
# Restore snapshot
|
|
||||||
sudo incus snapshot restore system <snapshot-name>
|
|
||||||
```
|
|
||||||
|
|
||||||
## Troubleshooting
|
|
||||||
|
|
||||||
### GLIBC Version Mismatch
|
|
||||||
|
|
||||||
**Symptom**: `GLIBC_2.39 not found` or `GLIBC_2.38 not found`
|
|
||||||
|
|
||||||
**Cause**: Binary compiled on CI runner (glibc 2.41) but runs in system container (glibc 2.36)
|
|
||||||
|
|
||||||
**Fix**: CI workflow must build inside the system container. Check `botserver.yaml` uses SSH to build in container.
|
|
||||||
|
|
||||||
### botserver Not Starting
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Check binary
|
|
||||||
sudo incus exec system -- ldd /opt/gbo/bin/botserver | grep "not found"
|
|
||||||
|
|
||||||
# Check direct execution
|
|
||||||
sudo incus exec system -- timeout 10 /opt/gbo/bin/botserver 2>&1
|
|
||||||
|
|
||||||
# Check data directory
|
|
||||||
sudo incus exec system -- ls -la /opt/gbo/data/
|
|
||||||
```
|
|
||||||
|
|
||||||
### botui Can't Reach botserver
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Check BOTSERVER_URL
|
|
||||||
sudo incus exec system -- grep BOTSERVER_URL /etc/systemd/system/ui.service
|
|
||||||
|
|
||||||
# Must be http://localhost:5858, NOT https://system.pragmatismo.com.br
|
|
||||||
# Fix:
|
|
||||||
sudo incus exec system -- sed -i 's|BOTSERVER_URL=.*|BOTSERVER_URL=http://localhost:5858|' /etc/systemd/system/ui.service
|
|
||||||
sudo incus exec system -- systemctl daemon-reload
|
|
||||||
sudo incus exec system -- systemctl restart ui
|
|
||||||
```
|
|
||||||
|
|
||||||
### Suggestions Not Showing
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Check bot files exist
|
|
||||||
sudo incus exec system -- ls -la /opt/gbo/data/<bot>.gbai/<bot>.gbdialog/
|
|
||||||
|
|
||||||
# Check for compilation errors
|
|
||||||
sudo incus exec system -- tail -50 /opt/gbo/logs/out.log | grep -i "error\|fail\|compile"
|
|
||||||
|
|
||||||
# Clear cache and restart
|
|
||||||
sudo incus exec system -- find /opt/gbo/work -name "*.ast" -delete
|
|
||||||
sudo incus exec system -- systemctl restart botserver
|
|
||||||
```
|
|
||||||
|
|
||||||
### IPv6 DNS Issues
|
|
||||||
|
|
||||||
**Symptom**: External API calls (Groq, Cloudflare) timeout
|
|
||||||
|
|
||||||
**Cause**: Container DNS returns AAAA records but no IPv6 connectivity
|
|
||||||
|
|
||||||
**Fix**: Container has `IPV6=no` in network config and `gai.conf` labels. If issues persist, check `RES_OPTIONS=inet4` in botserver.service.
|
|
||||||
|
|
||||||
## Security
|
|
||||||
|
|
||||||
- **NEVER** push secrets to git
|
|
||||||
- **NEVER** commit files to root with credentials
|
|
||||||
- **Vault** is single source of truth for secrets
|
|
||||||
- **CI/CD** is the only deployment method — never manually scp binaries
|
|
||||||
- **ALM** is production — ask before pushing
|
|
||||||
|
|
@ -1,154 +0,0 @@
|
||||||
# USE KB 2.0: Group-Based Knowledge Base Access
|
|
||||||
|
|
||||||
## Overview
|
|
||||||
Modify the USE KB keyword to respect user group permissions, ensuring that THINK KB queries only return answers from knowledge base folders that belong to groups the logged-in user is a member of.
|
|
||||||
|
|
||||||
## Current Architecture
|
|
||||||
|
|
||||||
### USE KB Flow
|
|
||||||
1. User executes `USE KB "kb_name"` in BASIC script
|
|
||||||
2. `use_kb.rs:add_kb_to_session()` checks if KB exists in `kb_collections`
|
|
||||||
3. Creates default KB entry if not found
|
|
||||||
4. Adds association to `session_kb_associations` table
|
|
||||||
5. KB becomes active for the session
|
|
||||||
|
|
||||||
### THINK KB Flow
|
|
||||||
1. User executes `THINK KB "query"`
|
|
||||||
2. `think_kb.rs:think_kb_search()` gets all active KBs from `session_kb_associations`
|
|
||||||
3. For each active KB, calls `KnowledgeBaseManager.search()` on its Qdrant collection
|
|
||||||
4. Returns combined results from all active KBs
|
|
||||||
|
|
||||||
### Group System
|
|
||||||
- Groups stored in `rbac_groups` table
|
|
||||||
- User membership in `rbac_user_groups` table
|
|
||||||
- Group permissions via `rbac_group_roles` table
|
|
||||||
|
|
||||||
## Proposed Changes
|
|
||||||
|
|
||||||
### 1. Database Schema Changes
|
|
||||||
|
|
||||||
Add new table `kb_group_associations`:
|
|
||||||
|
|
||||||
```sql
|
|
||||||
CREATE TABLE kb_group_associations (
|
|
||||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
||||||
kb_id UUID NOT NULL REFERENCES kb_collections(id) ON DELETE CASCADE,
|
|
||||||
group_id UUID NOT NULL REFERENCES rbac_groups(id) ON DELETE CASCADE,
|
|
||||||
granted_by UUID REFERENCES users(id) ON DELETE SET NULL,
|
|
||||||
granted_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
|
||||||
UNIQUE(kb_id, group_id)
|
|
||||||
);
|
|
||||||
```
|
|
||||||
|
|
||||||
Migration file: `botserver/migrations/6.2.0-01-kb-groups/up.sql`
|
|
||||||
|
|
||||||
### 2. Backend Logic Changes
|
|
||||||
|
|
||||||
#### Modify `think_kb_search()` in `think_kb.rs`
|
|
||||||
- Add user group lookup before searching
|
|
||||||
- Filter active KBs to only those accessible by user's groups
|
|
||||||
- Allow access if KB has no group associations (public KBs) OR user is in associated groups
|
|
||||||
|
|
||||||
```rust
|
|
||||||
async fn think_kb_search(
|
|
||||||
kb_manager: Arc<KnowledgeBaseManager>,
|
|
||||||
db_pool: DbPool,
|
|
||||||
session_id: Uuid,
|
|
||||||
bot_id: Uuid,
|
|
||||||
user_id: Uuid, // Add user_id parameter
|
|
||||||
query: &str,
|
|
||||||
) -> Result<serde_json::Value, String> {
|
|
||||||
// Get user's groups
|
|
||||||
let user_groups = get_user_groups(&db_pool, user_id)?;
|
|
||||||
|
|
||||||
// Get active KBs filtered by groups
|
|
||||||
let accessible_kbs = get_accessible_kbs_for_session(&db_pool, session_id, &user_groups)?;
|
|
||||||
|
|
||||||
// Search only accessible KBs
|
|
||||||
// ... rest of search logic
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
#### Add `get_accessible_kbs_for_session()` function
|
|
||||||
```rust
|
|
||||||
fn get_accessible_kbs_for_session(
|
|
||||||
conn_pool: &DbPool,
|
|
||||||
session_id: Uuid,
|
|
||||||
user_groups: &[String],
|
|
||||||
) -> Result<Vec<(String, String, String)>, String> {
|
|
||||||
// Query that joins session_kb_associations with kb_group_associations
|
|
||||||
// Returns KBs where group_id IS NULL (public) OR group_id IN user_groups
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
#### Modify `add_kb_to_session()` in `use_kb.rs`
|
|
||||||
- Add optional group access check
|
|
||||||
- Allow USE KB if user has access to the KB's groups
|
|
||||||
|
|
||||||
### 3. API Changes
|
|
||||||
|
|
||||||
Add new endpoints in `rbac.rs` for KB-group management:
|
|
||||||
|
|
||||||
```rust
|
|
||||||
// Assign KB to group
|
|
||||||
POST /api/rbac/kbs/{kb_id}/groups/{group_id}
|
|
||||||
|
|
||||||
// Remove KB from group
|
|
||||||
DELETE /api/rbac/kbs/{kb_id}/groups/{group_id}
|
|
||||||
|
|
||||||
// Get groups for KB
|
|
||||||
GET /api/rbac/kbs/{kb_id}/groups
|
|
||||||
|
|
||||||
// Get KBs accessible by user
|
|
||||||
GET /api/rbac/users/{user_id}/accessible-kbs
|
|
||||||
```
|
|
||||||
|
|
||||||
### 4. Frontend Changes
|
|
||||||
|
|
||||||
#### Update `botui/ui/suite/admin/groups.html`
|
|
||||||
- Add "Knowledge Bases" tab to group detail panel
|
|
||||||
- Show list of KBs assigned to the group
|
|
||||||
- Allow adding/removing KB assignments
|
|
||||||
|
|
||||||
#### Update `botui/ui/suite/drive/drive.html`
|
|
||||||
- Add group visibility indicators for KB folders
|
|
||||||
- Show which groups have access to each KB
|
|
||||||
|
|
||||||
### 5. Migration Strategy
|
|
||||||
|
|
||||||
1. Create new migration for `kb_group_associations` table
|
|
||||||
2. Run migration to create table
|
|
||||||
3. Assign existing KBs to default groups (e.g., "all_users" group)
|
|
||||||
4. Update application code
|
|
||||||
5. Deploy and test
|
|
||||||
|
|
||||||
### 6. Backward Compatibility
|
|
||||||
|
|
||||||
- Existing KBs without group associations remain public
|
|
||||||
- Existing USE KB calls continue to work
|
|
||||||
- THINK KB will filter results based on new permissions
|
|
||||||
|
|
||||||
## Implementation Steps
|
|
||||||
|
|
||||||
1. ✅ Database migration for kb_group_associations
|
|
||||||
2. ✅ Modify think_kb_search to accept user_id and filter by groups
|
|
||||||
3. ✅ Update THINK KB keyword registration to pass user_id
|
|
||||||
4. ✅ Add group access check to USE KB
|
|
||||||
5. ✅ Add API endpoints for KB-group management
|
|
||||||
6. ✅ Update admin UI for group-KB assignment
|
|
||||||
7. ✅ Update drive UI to show group access
|
|
||||||
8. ✅ Add tests for group-based access control
|
|
||||||
|
|
||||||
## Security Considerations
|
|
||||||
|
|
||||||
- All KB access checks must happen at the database level
|
|
||||||
- No client-side filtering of search results
|
|
||||||
- Group membership verified on each request
|
|
||||||
- Audit logging for KB access attempts
|
|
||||||
|
|
||||||
## Testing
|
|
||||||
|
|
||||||
- Unit tests for group access functions
|
|
||||||
- Integration tests for THINK KB with group filtering
|
|
||||||
- UI tests for admin group-KB management
|
|
||||||
- End-to-end tests with different user group scenarios
|
|
||||||
10
reset.sh
10
reset.sh
|
|
@ -1,10 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
set -e
|
|
||||||
|
|
||||||
echo "Cleaning up..."
|
|
||||||
rm -rf botserver-stack/ ./work/ .env
|
|
||||||
|
|
||||||
echo "Starting services..."
|
|
||||||
./restart.sh
|
|
||||||
|
|
||||||
echo "Reset complete!"
|
|
||||||
|
|
@ -1,49 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
|
|
||||||
echo "=== Fast Restart: botserver + botmodels only ==="
|
|
||||||
|
|
||||||
# Kill only the app services, keep infra running
|
|
||||||
pkill -f "botserver --noconsole" || true
|
|
||||||
pkill -f "botmodels" || true
|
|
||||||
|
|
||||||
# Clean logs
|
|
||||||
rm -f botserver.log botmodels.log
|
|
||||||
|
|
||||||
# Build only botserver (botui likely already built)
|
|
||||||
cargo build -p botserver
|
|
||||||
|
|
||||||
# Start botmodels
|
|
||||||
cd botmodels
|
|
||||||
source venv/bin/activate
|
|
||||||
uvicorn src.main:app --host 0.0.0.0 --port 8085 > ../botmodels.log 2>&1 &
|
|
||||||
echo " botmodels PID: $!"
|
|
||||||
cd ..
|
|
||||||
|
|
||||||
# Wait for botmodels
|
|
||||||
for i in $(seq 1 20); do
|
|
||||||
if curl -s http://localhost:8085/api/health > /dev/null 2>&1; then
|
|
||||||
echo " botmodels ready"
|
|
||||||
break
|
|
||||||
fi
|
|
||||||
sleep 1
|
|
||||||
done
|
|
||||||
|
|
||||||
# Start botserver (keep botui running if already up)
|
|
||||||
if ! pgrep -f "botui" > /dev/null; then
|
|
||||||
echo "Starting botui..."
|
|
||||||
cargo build -p botui
|
|
||||||
cd botui
|
|
||||||
BOTSERVER_URL="http://localhost:8080" ./target/debug/botui > ../botui.log 2>&1 &
|
|
||||||
echo " botui PID: $!"
|
|
||||||
cd ..
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Start botserver
|
|
||||||
BOTMODELS_HOST="http://localhost:8085" BOTMODELS_API_KEY="starter" RUST_LOG=info ./target/debug/botserver --noconsole > botserver.log 2>&1 &
|
|
||||||
echo " botserver PID: $!"
|
|
||||||
|
|
||||||
# Quick health check
|
|
||||||
sleep 2
|
|
||||||
curl -s http://localhost:8080/health > /dev/null 2>&1 && echo "✅ botserver ready" || echo "❌ botserver failed"
|
|
||||||
|
|
||||||
echo "Done. botserver $(pgrep -f 'botserver --noconsole') botui $(pgrep -f botui) botmodels $(pgrep -f botmodels)"
|
|
||||||
32
restart.ps1
32
restart.ps1
|
|
@ -1,32 +0,0 @@
|
||||||
$ErrorActionPreference = "Continue"
|
|
||||||
|
|
||||||
Write-Host "Stopping..."
|
|
||||||
Stop-Process -Name "botserver" -Force -ErrorAction SilentlyContinue
|
|
||||||
Stop-Process -Name "botui" -Force -ErrorAction SilentlyContinue
|
|
||||||
Stop-Process -Name "rustc" -Force -ErrorAction SilentlyContinue
|
|
||||||
|
|
||||||
Write-Host "Cleaning..."
|
|
||||||
Remove-Item -Path "botserver.log", "botui.log" -Force -ErrorAction SilentlyContinue
|
|
||||||
|
|
||||||
Write-Host "Building..."
|
|
||||||
cargo build -p botserver
|
|
||||||
if ($LASTEXITCODE -ne 0) { Write-Host "Failed to build botserver"; exit 1 }
|
|
||||||
|
|
||||||
cargo build -p botui
|
|
||||||
if ($LASTEXITCODE -ne 0) { Write-Host "Failed to build botui"; exit 1 }
|
|
||||||
|
|
||||||
Write-Host "Starting botserver..."
|
|
||||||
$env:PORT = "8080"
|
|
||||||
$env:RUST_LOG = "debug"
|
|
||||||
$env:PATH += ";C:\pgsql\pgsql\bin;C:\pgsql\pgsql\lib"
|
|
||||||
$botserverProcess = Start-Process -PassThru -NoNewWindow -FilePath ".\target\debug\botserver.exe" -ArgumentList "--noconsole" -RedirectStandardOutput "botserver.log" -RedirectStandardError "botserver.log"
|
|
||||||
Write-Host " PID: $($botserverProcess.Id)"
|
|
||||||
|
|
||||||
Write-Host "Starting botui..."
|
|
||||||
$env:BOTSERVER_URL = "http://localhost:8080"
|
|
||||||
$env:PORT = "3000"
|
|
||||||
$botuiProcess = Start-Process -PassThru -NoNewWindow -FilePath ".\target\debug\botui.exe" -RedirectStandardOutput "botui.log" -RedirectStandardError "botui.log"
|
|
||||||
Write-Host " PID: $($botuiProcess.Id)"
|
|
||||||
|
|
||||||
Write-Host "Done. Logs are being written to botserver.log and botui.log"
|
|
||||||
Write-Host "To view logs, you can use: Get-Content botserver.log -Wait"
|
|
||||||
55
restart.sh
55
restart.sh
|
|
@ -1,55 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
set -e
|
|
||||||
|
|
||||||
echo "=== Fast Restart: botserver only (keeps infra running) ==="
|
|
||||||
|
|
||||||
# Only kill the app services, keep infra (postgres, valkey, minio, vault, zitadel) running
|
|
||||||
pkill -f "botserver --noconsole" || true
|
|
||||||
pkill -f "botmodels" || true
|
|
||||||
|
|
||||||
# Clean app logs only
|
|
||||||
rm -f botserver.log botmodels.log
|
|
||||||
|
|
||||||
# Build botserver (incremental, should be fast)
|
|
||||||
cargo build -p botserver
|
|
||||||
|
|
||||||
# Start botmodels if not running
|
|
||||||
if ! pgrep -f "botmodels" > /dev/null; then
|
|
||||||
echo "Starting botmodels..."
|
|
||||||
cd botmodels
|
|
||||||
source venv/bin/activate
|
|
||||||
uvicorn src.main:app --host 0.0.0.0 --port 8085 > ../botmodels.log 2>&1 &
|
|
||||||
echo " botmodels PID: $!"
|
|
||||||
cd ..
|
|
||||||
|
|
||||||
# Wait for botmodels
|
|
||||||
for i in $(seq 1 15); do
|
|
||||||
if curl -s http://localhost:8085/api/health > /dev/null 2>&1; then
|
|
||||||
echo " botmodels ready"
|
|
||||||
break
|
|
||||||
fi
|
|
||||||
sleep 1
|
|
||||||
done
|
|
||||||
else
|
|
||||||
echo " botmodels already running"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Start botserver
|
|
||||||
echo "Starting botserver..."
|
|
||||||
BOTMODELS_HOST="http://localhost:8085" BOTMODELS_API_KEY="starter" RUST_LOG=info \
|
|
||||||
./target/debug/botserver --noconsole > botserver.log 2>&1 &
|
|
||||||
echo " botserver PID: $!"
|
|
||||||
|
|
||||||
# Wait for botserver health with timeout
|
|
||||||
echo "Waiting for botserver..."
|
|
||||||
for i in $(seq 1 10); do
|
|
||||||
if curl -sf http://localhost:8080/health > /dev/null 2>&1; then
|
|
||||||
echo "✅ botserver ready"
|
|
||||||
exit 0
|
|
||||||
fi
|
|
||||||
sleep 1
|
|
||||||
done
|
|
||||||
|
|
||||||
echo "❌ botserver failed to start - check botserver.log"
|
|
||||||
tail -20 botserver.log
|
|
||||||
exit 1
|
|
||||||
4
stop.sh
4
stop.sh
|
|
@ -1,4 +0,0 @@
|
||||||
pkill botui
|
|
||||||
pkill botserver -9
|
|
||||||
|
|
||||||
|
|
||||||
Loading…
Add table
Reference in a new issue