fix: track config.csv ETag to avoid unnecessary syncs
All checks were successful
BotServer CI/CD / build (push) Successful in 5m2s
All checks were successful
BotServer CI/CD / build (push) Successful in 5m2s
- Add ETag tracking for config.csv files in DriveMonitor - Only download and sync config.csv when ETag changes - Prevents unnecessary database updates on every check - Uses __config__ prefix for config.csv state keys
This commit is contained in:
parent
1977c4c0af
commit
4cd469afc3
1 changed files with 38 additions and 14 deletions
|
|
@ -772,20 +772,30 @@ impl DriveMonitor {
|
||||||
}
|
}
|
||||||
|
|
||||||
debug!("check_gbot: Found config.csv at path: {}", path);
|
debug!("check_gbot: Found config.csv at path: {}", path);
|
||||||
match client
|
let etag = obj.e_tag().unwrap_or_default().to_string();
|
||||||
.head_object()
|
let config_state_key = format!("__config__{}", path);
|
||||||
.bucket(&self.bucket_name)
|
let should_sync = {
|
||||||
.key(&path)
|
let states = self.file_states.read().await;
|
||||||
.send()
|
match states.get(&config_state_key) {
|
||||||
.await
|
Some(prev) => prev.etag != etag,
|
||||||
{
|
None => true,
|
||||||
Ok(_head_res) => {
|
}
|
||||||
let response = client
|
};
|
||||||
.get_object()
|
if should_sync {
|
||||||
.bucket(&self.bucket_name)
|
match client
|
||||||
.key(&path)
|
.head_object()
|
||||||
.send()
|
.bucket(&self.bucket_name)
|
||||||
.await?;
|
.key(&path)
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(_head_res) => {
|
||||||
|
let response = client
|
||||||
|
.get_object()
|
||||||
|
.bucket(&self.bucket_name)
|
||||||
|
.key(&path)
|
||||||
|
.send()
|
||||||
|
.await?;
|
||||||
let bytes = response.body.collect().await?.into_bytes();
|
let bytes = response.body.collect().await?.into_bytes();
|
||||||
let csv_content = String::from_utf8(bytes.to_vec())
|
let csv_content = String::from_utf8(bytes.to_vec())
|
||||||
.map_err(|e| format!("UTF-8 error in {}: {}", path, e))?;
|
.map_err(|e| format!("UTF-8 error in {}: {}", path, e))?;
|
||||||
|
|
@ -902,6 +912,19 @@ impl DriveMonitor {
|
||||||
self.broadcast_theme_change(&csv_content).await?;
|
self.broadcast_theme_change(&csv_content).await?;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Update file_states with config.csv ETag
|
||||||
|
let mut states = self.file_states.write().await;
|
||||||
|
states.insert(config_state_key, FileState { etag, indexed: false, last_failed_at: None, fail_count: 0 });
|
||||||
|
drop(states);
|
||||||
|
let file_states_clone = Arc::clone(&self.file_states);
|
||||||
|
let work_root_clone = self.work_root.clone();
|
||||||
|
let bucket_name_clone = self.bucket_name.clone();
|
||||||
|
tokio::spawn(async move {
|
||||||
|
if let Err(e) = Self::save_file_states_static(&file_states_clone, &work_root_clone, &bucket_name_clone).await {
|
||||||
|
warn!("Failed to save file states after config update: {}", e);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
// Check for system-prompt-file and download it
|
// Check for system-prompt-file and download it
|
||||||
let prompt_file_line = csv_content
|
let prompt_file_line = csv_content
|
||||||
.lines()
|
.lines()
|
||||||
|
|
@ -950,6 +973,7 @@ impl DriveMonitor {
|
||||||
log::error!("Config file {} not found or inaccessible: {}", path, e);
|
log::error!("Config file {} not found or inaccessible: {}", path, e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if !list_objects.is_truncated.unwrap_or(false) {
|
if !list_objects.is_truncated.unwrap_or(false) {
|
||||||
break;
|
break;
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue