fix(drive-monitor): sync bas files when not yet indexed + INFRA.md MinIO docs
- Fix DriveMonitor skipping .bas files that exist but have indexed=false and same ETag (needs_reindex was computed but not used for bas sync) - Mark bas files as indexed after successful sync to work dir - Update INFRA.md: MinIO connection setup (port 9100, creds from drive.service) - Remove erroneously created bottemplates start.bas (bots live in MinIO)
This commit is contained in:
parent
2eebbe9b0d
commit
d723974410
4 changed files with 57 additions and 45 deletions
|
|
@ -17,6 +17,8 @@
|
|||
- NEVER use cargo clean — causes 30min rebuilds, use ./reset.sh for DB issues
|
||||
- Secret files go in /tmp/ only (vault-token-gb, vault-unseal-key-gb)
|
||||
- See botserver/src/drive/local_file_monitor.rs for how bots load from drive (MinIO)
|
||||
- ⚠️ **NEVER modify bottemplates/** - Bots are stored in MinIO (drive) on stage/production
|
||||
- To modify a bot: access stage via SSH → get Vault credentials → edit bot files in MinIO bucket
|
||||
|
||||
---
|
||||
|
||||
|
|
|
|||
43
INFRA.md
43
INFRA.md
|
|
@ -47,7 +47,7 @@ Always manage services with systemctl inside the system container. Never run bin
|
|||
| system | BotServer + Valkey | 8080/6379 | Main API + cache |
|
||||
| tables | PostgreSQL | 5432 | Primary database |
|
||||
| vault | Vault | 8200 | Secrets |
|
||||
| drive | MinIO | 9000/9100 | Object storage |
|
||||
| drive | MinIO | 9100 (9000 external) | Object storage, connect from inside container |
|
||||
| directory | Zitadel | 9000 | Identity provider |
|
||||
| llm | llama.cpp | 8081 | Local LLM |
|
||||
| vectordb | Qdrant | 6333 | Vector database |
|
||||
|
|
@ -282,26 +282,47 @@ All bot files live in MinIO buckets. Use mc CLI at /opt/gbo/bin/mc from drive co
|
|||
{bot}.gbai/{bot}.gbot/ — config.csv
|
||||
{bot}.gbai/{bot}.gbkb/ — knowledge base
|
||||
|
||||
### MinIO Connection Setup
|
||||
|
||||
MinIO credentials come from the drive.service Environment vars (MINIO_ROOT_USER, MINIO_ROOT_PASSWORD).
|
||||
The root user from .service env has full read/write access. Always connect from inside the drive container:
|
||||
|
||||
```bash
|
||||
# 1. Set mc alias (run once per session) — use credentials from drive.service Environment
|
||||
sudo incus exec drive -- bash -c 'export PATH=/opt/gbo/bin:$PATH && \
|
||||
mc alias set local http://localhost:9100 <MINIO_ROOT_USER> <MINIO_ROOT_PASSWORD>'
|
||||
|
||||
# 2. Verify connection
|
||||
sudo incus exec drive -- bash -c 'export PATH=/opt/gbo/bin:$PATH && mc ls local/'
|
||||
```
|
||||
|
||||
MinIO listens on port 9100 (NOT 9000). The 9000 port is for external DNAT only.
|
||||
Credentials are in drive.service: `sudo incus exec drive -- systemctl cat drive`
|
||||
|
||||
### Common mc Commands
|
||||
|
||||
```bash
|
||||
# All mc commands need PATH set
|
||||
# All mc commands need PATH set and mc alias configured first
|
||||
sudo incus exec drive -- bash -c 'export PATH=/opt/gbo/bin:$PATH && mc <command>'
|
||||
|
||||
mc ls local/ # List all buckets
|
||||
mc ls local/<bot>.gbai/ # List bot bucket
|
||||
mc cat local/<bot>.gbai/<bot>.gbdialog/start.bas # Read file
|
||||
mc cp local/<bot>.gbai/<bot>.gbdialog/file /tmp/ # Download
|
||||
mc cp /tmp/file local/<bot>.gbai/<bot>.gbot/config.csv # Upload (triggers DriveMonitor)
|
||||
mc stat local/<bot>.gbai/<bot>.gbot/config.csv # Show ETag/metadata
|
||||
mc mb local/newbot.gbai # Create bucket
|
||||
mc admin info local # Health check
|
||||
mc ls local/ # List all buckets
|
||||
mc ls local/<bot>.gbai/ # List bot bucket
|
||||
mc cat local/<bot>.gbai/<bot>.gbdialog/start.bas # Read file
|
||||
mc cp local/<bot>.gbai/<bot>.gbdialog/file /tmp/ # Download
|
||||
mc cp /tmp/file local/<bot>.gbai/<bot>.gbot/config.csv # Upload (triggers DriveMonitor)
|
||||
mc stat local/<bot>.gbai/<bot>.gbot/config.csv # Show ETag/metadata
|
||||
mc mb local/newbot.gbai # Create bucket
|
||||
mc admin info local # Health check
|
||||
|
||||
# Force re-sync (change ETag without content change)
|
||||
mc cp local/<bot>.gbai/<bot>.gbot/config.csv local/<bot>.gbai/<bot>.gbot/config.csv
|
||||
```
|
||||
|
||||
### Upload config.csv workflow: download via mc cat → edit locally → push via mc cp → wait 15s → verify in logs
|
||||
### Upload/Download workflow
|
||||
|
||||
1. Download: `mc cp local/<bot>.gbai/<bot>.gbdialog/file /tmp/` → edit locally → `mc cp /tmp/file local/<bot>.gbai/<bot>.gbdialog/file`
|
||||
2. Or read directly: `mc cat local/<bot>.gbai/<bot>.gbdialog/start.bas`
|
||||
3. Wait 15s for DriveMonitor to pick up changes → verify in logs
|
||||
|
||||
---
|
||||
|
||||
|
|
|
|||
|
|
@ -70,25 +70,25 @@ impl DriveMonitor {
|
|||
|
||||
let etag_changed = existing.as_ref().map_or(true, |prev| prev.etag.as_deref() != etag.as_deref());
|
||||
|
||||
if etag_changed || existing.is_none() {
|
||||
match self.file_repo.upsert_file(
|
||||
self.bot_id,
|
||||
&full_key,
|
||||
file_type,
|
||||
etag,
|
||||
None,
|
||||
) {
|
||||
Ok(_) => log::info!("DriveMonitor: Added/updated drive_files for: {} ({})", full_key, file_type),
|
||||
Err(e) => log::error!("Failed to upsert {}: {}", full_key, e),
|
||||
}
|
||||
|
||||
if file_type == "bas" {
|
||||
self.sync_bas_to_work(bot_name, &obj.key).await;
|
||||
}
|
||||
} else {
|
||||
log::trace!("{} unchanged, skipping upsert", full_key);
|
||||
if etag_changed || existing.is_none() || needs_reindex {
|
||||
match self.file_repo.upsert_file(
|
||||
self.bot_id,
|
||||
&full_key,
|
||||
file_type,
|
||||
etag,
|
||||
None,
|
||||
) {
|
||||
Ok(_) => log::info!("DriveMonitor: Added/updated drive_files for: {} ({})", full_key, file_type),
|
||||
Err(e) => log::error!("Failed to upsert {}: {}", full_key, e),
|
||||
}
|
||||
|
||||
if file_type == "bas" {
|
||||
self.sync_bas_to_work(bot_name, &obj.key).await;
|
||||
}
|
||||
} else {
|
||||
log::trace!("{} unchanged, skipping upsert", full_key);
|
||||
}
|
||||
|
||||
if needs_reindex && file_type == "kb" {
|
||||
#[cfg(any(feature = "research", feature = "llm"))]
|
||||
{
|
||||
|
|
@ -290,14 +290,16 @@ if let Err(e) = config_manager.set_config(&self.bot_id, key, value) {
|
|||
let file_name = s3_key.split('/').next_back().unwrap_or(s3_key);
|
||||
let work_path = work_dir.join(file_name);
|
||||
|
||||
match String::from_utf8(data) {
|
||||
Ok(content) => {
|
||||
if let Err(e) = std::fs::write(&work_path, &content) {
|
||||
match String::from_utf8(data) {
|
||||
Ok(content) => {
|
||||
if let Err(e) = std::fs::write(&work_path, &content) {
|
||||
log::error!("Failed to write {} to work dir: {}", work_path.display(), e);
|
||||
} else {
|
||||
log::trace!("Synced {} to work dir {}", s3_key, work_path.display());
|
||||
let full_key = format!("{}.gbai/{}", bot_name, s3_key);
|
||||
let _ = self.file_repo.mark_indexed(self.bot_id, &full_key);
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
log::error!("Failed to parse .bas as UTF-8: {}", e);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,13 +0,0 @@
|
|||
' start.bas - Salesianos Bot Configuration
|
||||
' This runs once per session
|
||||
|
||||
' Add switchers with colors
|
||||
ADD SWITCHER tables AS "Tabelas"
|
||||
ADD SWITCHER list AS "Lista"
|
||||
ADD SWITCHER cards AS "Cards"
|
||||
|
||||
' Add suggestions
|
||||
ADD SUGGESTION "Cartas"
|
||||
ADD SUGGESTION "Procedimentos"
|
||||
ADD SUGGESTION "Ramais"
|
||||
ADD SUGGESTION "Todos"
|
||||
Loading…
Add table
Reference in a new issue