fix(drive-monitor): sync bas files when not yet indexed + INFRA.md MinIO docs
Some checks failed
Botlib CI / build (push) Failing after 1s
BotServer CI / build (push) Failing after 1s
Bottest CI / build (push) Failing after 0s
BotUI CI / build (push) Failing after 1s

- Fix DriveMonitor skipping .bas files that exist but have indexed=false
  and same ETag (needs_reindex was computed but not used for bas sync)
- Mark bas files as indexed after successful sync to work dir
- Update INFRA.md: MinIO connection setup (port 9100, creds from drive.service)
- Remove erroneously created bottemplates start.bas (bots live in MinIO)
This commit is contained in:
Rodrigo Rodriguez (Pragmatismo) 2026-04-27 17:27:00 -03:00
parent 2eebbe9b0d
commit d723974410
4 changed files with 57 additions and 45 deletions

View file

@ -17,6 +17,8 @@
- NEVER use cargo clean — causes 30min rebuilds, use ./reset.sh for DB issues
- Secret files go in /tmp/ only (vault-token-gb, vault-unseal-key-gb)
- See botserver/src/drive/local_file_monitor.rs for how bots load from drive (MinIO)
- ⚠️ **NEVER modify bottemplates/** - Bots are stored in MinIO (drive) on stage/production
- To modify a bot: access stage via SSH → get Vault credentials → edit bot files in MinIO bucket
---

View file

@ -47,7 +47,7 @@ Always manage services with systemctl inside the system container. Never run bin
| system | BotServer + Valkey | 8080/6379 | Main API + cache |
| tables | PostgreSQL | 5432 | Primary database |
| vault | Vault | 8200 | Secrets |
| drive | MinIO | 9000/9100 | Object storage |
| drive | MinIO | 9100 (9000 external) | Object storage, connect from inside container |
| directory | Zitadel | 9000 | Identity provider |
| llm | llama.cpp | 8081 | Local LLM |
| vectordb | Qdrant | 6333 | Vector database |
@ -282,26 +282,47 @@ All bot files live in MinIO buckets. Use mc CLI at /opt/gbo/bin/mc from drive co
{bot}.gbai/{bot}.gbot/ — config.csv
{bot}.gbai/{bot}.gbkb/ — knowledge base
### MinIO Connection Setup
MinIO credentials come from the drive.service Environment vars (MINIO_ROOT_USER, MINIO_ROOT_PASSWORD).
The root user from .service env has full read/write access. Always connect from inside the drive container:
```bash
# 1. Set mc alias (run once per session) — use credentials from drive.service Environment
sudo incus exec drive -- bash -c 'export PATH=/opt/gbo/bin:$PATH && \
mc alias set local http://localhost:9100 <MINIO_ROOT_USER> <MINIO_ROOT_PASSWORD>'
# 2. Verify connection
sudo incus exec drive -- bash -c 'export PATH=/opt/gbo/bin:$PATH && mc ls local/'
```
MinIO listens on port 9100 (NOT 9000). The 9000 port is for external DNAT only.
Credentials are in drive.service: `sudo incus exec drive -- systemctl cat drive`
### Common mc Commands
```bash
# All mc commands need PATH set
# All mc commands need PATH set and mc alias configured first
sudo incus exec drive -- bash -c 'export PATH=/opt/gbo/bin:$PATH && mc <command>'
mc ls local/ # List all buckets
mc ls local/<bot>.gbai/ # List bot bucket
mc cat local/<bot>.gbai/<bot>.gbdialog/start.bas # Read file
mc cp local/<bot>.gbai/<bot>.gbdialog/file /tmp/ # Download
mc cp /tmp/file local/<bot>.gbai/<bot>.gbot/config.csv # Upload (triggers DriveMonitor)
mc stat local/<bot>.gbai/<bot>.gbot/config.csv # Show ETag/metadata
mc mb local/newbot.gbai # Create bucket
mc admin info local # Health check
mc ls local/ # List all buckets
mc ls local/<bot>.gbai/ # List bot bucket
mc cat local/<bot>.gbai/<bot>.gbdialog/start.bas # Read file
mc cp local/<bot>.gbai/<bot>.gbdialog/file /tmp/ # Download
mc cp /tmp/file local/<bot>.gbai/<bot>.gbot/config.csv # Upload (triggers DriveMonitor)
mc stat local/<bot>.gbai/<bot>.gbot/config.csv # Show ETag/metadata
mc mb local/newbot.gbai # Create bucket
mc admin info local # Health check
# Force re-sync (change ETag without content change)
mc cp local/<bot>.gbai/<bot>.gbot/config.csv local/<bot>.gbai/<bot>.gbot/config.csv
```
### Upload config.csv workflow: download via mc cat → edit locally → push via mc cp → wait 15s → verify in logs
### Upload/Download workflow
1. Download: `mc cp local/<bot>.gbai/<bot>.gbdialog/file /tmp/` → edit locally → `mc cp /tmp/file local/<bot>.gbai/<bot>.gbdialog/file`
2. Or read directly: `mc cat local/<bot>.gbai/<bot>.gbdialog/start.bas`
3. Wait 15s for DriveMonitor to pick up changes → verify in logs
---

View file

@ -70,25 +70,25 @@ impl DriveMonitor {
let etag_changed = existing.as_ref().map_or(true, |prev| prev.etag.as_deref() != etag.as_deref());
if etag_changed || existing.is_none() {
match self.file_repo.upsert_file(
self.bot_id,
&full_key,
file_type,
etag,
None,
) {
Ok(_) => log::info!("DriveMonitor: Added/updated drive_files for: {} ({})", full_key, file_type),
Err(e) => log::error!("Failed to upsert {}: {}", full_key, e),
}
if file_type == "bas" {
self.sync_bas_to_work(bot_name, &obj.key).await;
}
} else {
log::trace!("{} unchanged, skipping upsert", full_key);
if etag_changed || existing.is_none() || needs_reindex {
match self.file_repo.upsert_file(
self.bot_id,
&full_key,
file_type,
etag,
None,
) {
Ok(_) => log::info!("DriveMonitor: Added/updated drive_files for: {} ({})", full_key, file_type),
Err(e) => log::error!("Failed to upsert {}: {}", full_key, e),
}
if file_type == "bas" {
self.sync_bas_to_work(bot_name, &obj.key).await;
}
} else {
log::trace!("{} unchanged, skipping upsert", full_key);
}
if needs_reindex && file_type == "kb" {
#[cfg(any(feature = "research", feature = "llm"))]
{
@ -290,14 +290,16 @@ if let Err(e) = config_manager.set_config(&self.bot_id, key, value) {
let file_name = s3_key.split('/').next_back().unwrap_or(s3_key);
let work_path = work_dir.join(file_name);
match String::from_utf8(data) {
Ok(content) => {
if let Err(e) = std::fs::write(&work_path, &content) {
match String::from_utf8(data) {
Ok(content) => {
if let Err(e) = std::fs::write(&work_path, &content) {
log::error!("Failed to write {} to work dir: {}", work_path.display(), e);
} else {
log::trace!("Synced {} to work dir {}", s3_key, work_path.display());
let full_key = format!("{}.gbai/{}", bot_name, s3_key);
let _ = self.file_repo.mark_indexed(self.bot_id, &full_key);
}
}
}
Err(e) => {
log::error!("Failed to parse .bas as UTF-8: {}", e);
}

View file

@ -1,13 +0,0 @@
' start.bas - Salesianos Bot Configuration
' This runs once per session
' Add switchers with colors
ADD SWITCHER tables AS "Tabelas"
ADD SWITCHER list AS "Lista"
ADD SWITCHER cards AS "Cards"
' Add suggestions
ADD SUGGESTION "Cartas"
ADD SUGGESTION "Procedimentos"
ADD SUGGESTION "Ramais"
ADD SUGGESTION "Todos"