Compare commits
2 Commits
925ec9f574
...
0b6b168043
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0b6b168043 | ||
|
|
1d003aeac2 |
File diff suppressed because one or more lines are too long
@@ -1 +1 @@
|
|||||||
bd-3pz
|
bd-1ht
|
||||||
|
|||||||
@@ -3,6 +3,8 @@
|
|||||||
use console::style;
|
use console::style;
|
||||||
use indicatif::{ProgressBar, ProgressStyle};
|
use indicatif::{ProgressBar, ProgressStyle};
|
||||||
use serde::Serialize;
|
use serde::Serialize;
|
||||||
|
use std::sync::atomic::{AtomicBool, Ordering};
|
||||||
|
use std::sync::Arc;
|
||||||
use tracing::Instrument;
|
use tracing::Instrument;
|
||||||
use tracing::{info, warn};
|
use tracing::{info, warn};
|
||||||
|
|
||||||
@@ -160,16 +162,39 @@ pub async fn run_sync(
|
|||||||
options.robot_mode,
|
options.robot_mode,
|
||||||
);
|
);
|
||||||
info!("Sync stage {current_stage}/{total_stages}: generating documents");
|
info!("Sync stage {current_stage}/{total_stages}: generating documents");
|
||||||
let docs_spinner = spinner.clone();
|
|
||||||
|
// Create a dedicated progress bar matching the ingest stage style
|
||||||
|
let docs_bar = if options.robot_mode {
|
||||||
|
ProgressBar::hidden()
|
||||||
|
} else {
|
||||||
|
let b = crate::cli::progress::multi().add(ProgressBar::new(0));
|
||||||
|
b.set_style(
|
||||||
|
ProgressStyle::default_bar()
|
||||||
|
.template(
|
||||||
|
" {spinner:.blue} Processing documents [{bar:30.cyan/dim}] {pos}/{len}",
|
||||||
|
)
|
||||||
|
.unwrap()
|
||||||
|
.progress_chars("=> "),
|
||||||
|
);
|
||||||
|
b
|
||||||
|
};
|
||||||
|
let docs_bar_clone = docs_bar.clone();
|
||||||
|
let tick_started = Arc::new(AtomicBool::new(false));
|
||||||
|
let tick_started_clone = Arc::clone(&tick_started);
|
||||||
let docs_cb: Box<dyn Fn(usize, usize)> = Box::new(move |processed, total| {
|
let docs_cb: Box<dyn Fn(usize, usize)> = Box::new(move |processed, total| {
|
||||||
if total > 0 {
|
if total > 0 {
|
||||||
docs_spinner.set_message(format!(
|
if !tick_started_clone.swap(true, Ordering::Relaxed) {
|
||||||
"Processing documents... ({processed}/{total})"
|
docs_bar_clone.enable_steady_tick(std::time::Duration::from_millis(100));
|
||||||
));
|
}
|
||||||
|
// Update length every callback — the regenerator's estimated_total
|
||||||
|
// can grow if new dirty items are queued during processing.
|
||||||
|
docs_bar_clone.set_length(total as u64);
|
||||||
|
docs_bar_clone.set_position(processed as u64);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
let docs_result = run_generate_docs(config, false, None, Some(docs_cb))?;
|
let docs_result = run_generate_docs(config, false, None, Some(docs_cb))?;
|
||||||
result.documents_regenerated = docs_result.regenerated;
|
result.documents_regenerated = docs_result.regenerated;
|
||||||
|
docs_bar.finish_and_clear();
|
||||||
spinner.finish_and_clear();
|
spinner.finish_and_clear();
|
||||||
} else {
|
} else {
|
||||||
info!("Sync: skipping document generation (--no-docs)");
|
info!("Sync: skipping document generation (--no-docs)");
|
||||||
@@ -185,19 +210,43 @@ pub async fn run_sync(
|
|||||||
options.robot_mode,
|
options.robot_mode,
|
||||||
);
|
);
|
||||||
info!("Sync stage {current_stage}/{total_stages}: embedding documents");
|
info!("Sync stage {current_stage}/{total_stages}: embedding documents");
|
||||||
let embed_spinner = spinner.clone();
|
|
||||||
|
// Create a dedicated progress bar matching the ingest stage style
|
||||||
|
let embed_bar = if options.robot_mode {
|
||||||
|
ProgressBar::hidden()
|
||||||
|
} else {
|
||||||
|
let b = crate::cli::progress::multi().add(ProgressBar::new(0));
|
||||||
|
b.set_style(
|
||||||
|
ProgressStyle::default_bar()
|
||||||
|
.template(
|
||||||
|
" {spinner:.blue} Generating embeddings [{bar:30.cyan/dim}] {pos}/{len}",
|
||||||
|
)
|
||||||
|
.unwrap()
|
||||||
|
.progress_chars("=> "),
|
||||||
|
);
|
||||||
|
b
|
||||||
|
};
|
||||||
|
let embed_bar_clone = embed_bar.clone();
|
||||||
|
let tick_started = Arc::new(AtomicBool::new(false));
|
||||||
|
let tick_started_clone = Arc::clone(&tick_started);
|
||||||
let embed_cb: Box<dyn Fn(usize, usize)> = Box::new(move |processed, total| {
|
let embed_cb: Box<dyn Fn(usize, usize)> = Box::new(move |processed, total| {
|
||||||
embed_spinner.set_message(format!(
|
if total > 0 {
|
||||||
"Embedding documents... ({processed}/{total})"
|
if !tick_started_clone.swap(true, Ordering::Relaxed) {
|
||||||
));
|
embed_bar_clone.enable_steady_tick(std::time::Duration::from_millis(100));
|
||||||
|
}
|
||||||
|
embed_bar_clone.set_length(total as u64);
|
||||||
|
embed_bar_clone.set_position(processed as u64);
|
||||||
|
}
|
||||||
});
|
});
|
||||||
match run_embed(config, options.full, false, Some(embed_cb)).await {
|
match run_embed(config, options.full, false, Some(embed_cb)).await {
|
||||||
Ok(embed_result) => {
|
Ok(embed_result) => {
|
||||||
result.documents_embedded = embed_result.embedded;
|
result.documents_embedded = embed_result.embedded;
|
||||||
|
embed_bar.finish_and_clear();
|
||||||
spinner.finish_and_clear();
|
spinner.finish_and_clear();
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
// Graceful degradation: Ollama down is a warning, not an error
|
// Graceful degradation: Ollama down is a warning, not an error
|
||||||
|
embed_bar.finish_and_clear();
|
||||||
spinner.finish_and_clear();
|
spinner.finish_and_clear();
|
||||||
if !options.robot_mode {
|
if !options.robot_mode {
|
||||||
eprintln!(" {} Embedding skipped ({})", style("warn").yellow(), e);
|
eprintln!(" {} Embedding skipped ({})", style("warn").yellow(), e);
|
||||||
|
|||||||
Reference in New Issue
Block a user