feat(cli): Wire --full flag for embed, add sync stage spinners
- Add --full / --no-full flag pair to EmbedArgs with overrides_with semantics matching the existing flag pattern. When active, atomically DELETEs all embedding_metadata and embeddings before re-embedding. - Thread the full flag through run_embed -> run_sync so that 'lore sync --full' triggers a complete re-embed alongside the full re-ingest it already performed. - Add indicatif spinners to sync stages with dynamic stage numbering that adjusts when --no-docs or --no-embed skip stages. Spinners are hidden in robot mode. - Update robot-docs manifest to advertise the new --full flag on the embed command. - Replace hardcoded schema version 9 in health check with the LATEST_SCHEMA_VERSION constant from db.rs. Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -21,6 +21,7 @@ pub struct EmbedCommandResult {
|
|||||||
/// Run the embed command.
|
/// Run the embed command.
|
||||||
pub async fn run_embed(
|
pub async fn run_embed(
|
||||||
config: &Config,
|
config: &Config,
|
||||||
|
full: bool,
|
||||||
retry_failed: bool,
|
retry_failed: bool,
|
||||||
) -> Result<EmbedCommandResult> {
|
) -> Result<EmbedCommandResult> {
|
||||||
let db_path = get_db_path(config.storage.db_path.as_deref());
|
let db_path = get_db_path(config.storage.db_path.as_deref());
|
||||||
@@ -37,8 +38,18 @@ pub async fn run_embed(
|
|||||||
// Health check — fail fast if Ollama is down or model missing
|
// Health check — fail fast if Ollama is down or model missing
|
||||||
client.health_check().await?;
|
client.health_check().await?;
|
||||||
|
|
||||||
// If retry_failed, clear errors so they become pending again
|
if full {
|
||||||
if retry_failed {
|
// Clear ALL embeddings and metadata atomically for a complete re-embed.
|
||||||
|
// Wrapped in a transaction so a crash between the two DELETEs can't
|
||||||
|
// leave orphaned data.
|
||||||
|
conn.execute_batch(
|
||||||
|
"BEGIN;
|
||||||
|
DELETE FROM embedding_metadata;
|
||||||
|
DELETE FROM embeddings;
|
||||||
|
COMMIT;",
|
||||||
|
)?;
|
||||||
|
} else if retry_failed {
|
||||||
|
// Clear errors so they become pending again
|
||||||
conn.execute(
|
conn.execute(
|
||||||
"UPDATE embedding_metadata SET last_error = NULL, attempt_count = 0
|
"UPDATE embedding_metadata SET last_error = NULL, attempt_count = 0
|
||||||
WHERE last_error IS NOT NULL",
|
WHERE last_error IS NOT NULL",
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
//! Sync command: unified orchestrator for ingest -> generate-docs -> embed.
|
//! Sync command: unified orchestrator for ingest -> generate-docs -> embed.
|
||||||
|
|
||||||
use console::style;
|
use console::style;
|
||||||
|
use indicatif::{ProgressBar, ProgressStyle};
|
||||||
use serde::Serialize;
|
use serde::Serialize;
|
||||||
use tracing::{info, warn};
|
use tracing::{info, warn};
|
||||||
|
|
||||||
@@ -31,6 +32,22 @@ pub struct SyncResult {
|
|||||||
pub documents_embedded: usize,
|
pub documents_embedded: usize,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Create a styled spinner for a sync stage.
|
||||||
|
fn stage_spinner(stage: u8, total: u8, msg: &str, robot_mode: bool) -> ProgressBar {
|
||||||
|
if robot_mode {
|
||||||
|
return ProgressBar::hidden();
|
||||||
|
}
|
||||||
|
let pb = ProgressBar::new_spinner();
|
||||||
|
pb.set_style(
|
||||||
|
ProgressStyle::default_spinner()
|
||||||
|
.template("{spinner:.blue} {msg}")
|
||||||
|
.expect("valid template"),
|
||||||
|
);
|
||||||
|
pb.enable_steady_tick(std::time::Duration::from_millis(80));
|
||||||
|
pb.set_message(format!("[{stage}/{total}] {msg}"));
|
||||||
|
pb
|
||||||
|
}
|
||||||
|
|
||||||
/// Run the full sync pipeline: ingest -> generate-docs -> embed.
|
/// Run the full sync pipeline: ingest -> generate-docs -> embed.
|
||||||
pub async fn run_sync(config: &Config, options: SyncOptions) -> Result<SyncResult> {
|
pub async fn run_sync(config: &Config, options: SyncOptions) -> Result<SyncResult> {
|
||||||
let mut result = SyncResult::default();
|
let mut result = SyncResult::default();
|
||||||
@@ -41,41 +58,70 @@ pub async fn run_sync(config: &Config, options: SyncOptions) -> Result<SyncResul
|
|||||||
IngestDisplay::progress_only()
|
IngestDisplay::progress_only()
|
||||||
};
|
};
|
||||||
|
|
||||||
|
let total_stages: u8 = if options.no_docs && options.no_embed {
|
||||||
|
2
|
||||||
|
} else if options.no_docs || options.no_embed {
|
||||||
|
3
|
||||||
|
} else {
|
||||||
|
4
|
||||||
|
};
|
||||||
|
let mut current_stage: u8 = 0;
|
||||||
|
|
||||||
// Stage 1: Ingest issues
|
// Stage 1: Ingest issues
|
||||||
info!("Sync stage 1/4: ingesting issues");
|
current_stage += 1;
|
||||||
|
let spinner = stage_spinner(current_stage, total_stages, "Fetching issues from GitLab...", options.robot_mode);
|
||||||
|
info!("Sync stage {current_stage}/{total_stages}: ingesting issues");
|
||||||
let issues_result = run_ingest(config, "issues", None, options.force, options.full, ingest_display).await?;
|
let issues_result = run_ingest(config, "issues", None, options.force, options.full, ingest_display).await?;
|
||||||
result.issues_updated = issues_result.issues_upserted;
|
result.issues_updated = issues_result.issues_upserted;
|
||||||
result.discussions_fetched += issues_result.discussions_fetched;
|
result.discussions_fetched += issues_result.discussions_fetched;
|
||||||
|
spinner.finish_and_clear();
|
||||||
|
|
||||||
// Stage 2: Ingest MRs
|
// Stage 2: Ingest MRs
|
||||||
info!("Sync stage 2/4: ingesting merge requests");
|
current_stage += 1;
|
||||||
|
let spinner = stage_spinner(current_stage, total_stages, "Fetching merge requests from GitLab...", options.robot_mode);
|
||||||
|
info!("Sync stage {current_stage}/{total_stages}: ingesting merge requests");
|
||||||
let mrs_result = run_ingest(config, "mrs", None, options.force, options.full, ingest_display).await?;
|
let mrs_result = run_ingest(config, "mrs", None, options.force, options.full, ingest_display).await?;
|
||||||
result.mrs_updated = mrs_result.mrs_upserted;
|
result.mrs_updated = mrs_result.mrs_upserted;
|
||||||
result.discussions_fetched += mrs_result.discussions_fetched;
|
result.discussions_fetched += mrs_result.discussions_fetched;
|
||||||
|
spinner.finish_and_clear();
|
||||||
|
|
||||||
// Stage 3: Generate documents (unless --no-docs)
|
// Stage 3: Generate documents (unless --no-docs)
|
||||||
if options.no_docs {
|
if !options.no_docs {
|
||||||
info!("Sync stage 3/4: skipping document generation (--no-docs)");
|
current_stage += 1;
|
||||||
} else {
|
let spinner = stage_spinner(current_stage, total_stages, "Processing documents...", options.robot_mode);
|
||||||
info!("Sync stage 3/4: generating documents");
|
info!("Sync stage {current_stage}/{total_stages}: generating documents");
|
||||||
let docs_result = run_generate_docs(config, false, None)?;
|
let docs_result = run_generate_docs(config, false, None)?;
|
||||||
result.documents_regenerated = docs_result.regenerated;
|
result.documents_regenerated = docs_result.regenerated;
|
||||||
|
spinner.finish_and_clear();
|
||||||
|
} else {
|
||||||
|
info!("Sync: skipping document generation (--no-docs)");
|
||||||
}
|
}
|
||||||
|
|
||||||
// Stage 4: Embed documents (unless --no-embed)
|
// Stage 4: Embed documents (unless --no-embed)
|
||||||
if options.no_embed {
|
if !options.no_embed {
|
||||||
info!("Sync stage 4/4: skipping embedding (--no-embed)");
|
current_stage += 1;
|
||||||
} else {
|
let spinner = stage_spinner(current_stage, total_stages, "Generating embeddings...", options.robot_mode);
|
||||||
info!("Sync stage 4/4: embedding documents");
|
info!("Sync stage {current_stage}/{total_stages}: embedding documents");
|
||||||
match run_embed(config, false).await {
|
match run_embed(config, options.full, false).await {
|
||||||
Ok(embed_result) => {
|
Ok(embed_result) => {
|
||||||
result.documents_embedded = embed_result.embedded;
|
result.documents_embedded = embed_result.embedded;
|
||||||
|
spinner.finish_and_clear();
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
// Graceful degradation: Ollama down is a warning, not an error
|
// Graceful degradation: Ollama down is a warning, not an error
|
||||||
|
spinner.finish_and_clear();
|
||||||
|
if !options.robot_mode {
|
||||||
|
eprintln!(
|
||||||
|
" {} Embedding skipped ({})",
|
||||||
|
style("warn").yellow(),
|
||||||
|
e
|
||||||
|
);
|
||||||
|
}
|
||||||
warn!(error = %e, "Embedding stage failed (Ollama may be unavailable), continuing");
|
warn!(error = %e, "Embedding stage failed (Ollama may be unavailable), continuing");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
} else {
|
||||||
|
info!("Sync: skipping embedding (--no-embed)");
|
||||||
}
|
}
|
||||||
|
|
||||||
info!(
|
info!(
|
||||||
|
|||||||
@@ -483,6 +483,13 @@ pub struct SyncArgs {
|
|||||||
/// Arguments for `lore embed`
|
/// Arguments for `lore embed`
|
||||||
#[derive(Parser)]
|
#[derive(Parser)]
|
||||||
pub struct EmbedArgs {
|
pub struct EmbedArgs {
|
||||||
|
/// Re-embed all documents (clears existing embeddings first)
|
||||||
|
#[arg(long, overrides_with = "no_full")]
|
||||||
|
pub full: bool,
|
||||||
|
|
||||||
|
#[arg(long = "no-full", hide = true, overrides_with = "full")]
|
||||||
|
pub no_full: bool,
|
||||||
|
|
||||||
/// Retry previously failed embeddings
|
/// Retry previously failed embeddings
|
||||||
#[arg(long, overrides_with = "no_retry_failed")]
|
#[arg(long, overrides_with = "no_retry_failed")]
|
||||||
pub retry_failed: bool,
|
pub retry_failed: bool,
|
||||||
|
|||||||
10
src/main.rs
10
src/main.rs
@@ -26,7 +26,7 @@ use lore::cli::{
|
|||||||
Cli, Commands, CountArgs, EmbedArgs, GenerateDocsArgs, IngestArgs, IssuesArgs, MrsArgs,
|
Cli, Commands, CountArgs, EmbedArgs, GenerateDocsArgs, IngestArgs, IssuesArgs, MrsArgs,
|
||||||
SearchArgs, StatsArgs, SyncArgs,
|
SearchArgs, StatsArgs, SyncArgs,
|
||||||
};
|
};
|
||||||
use lore::core::db::{create_connection, get_schema_version, run_migrations};
|
use lore::core::db::{create_connection, get_schema_version, run_migrations, LATEST_SCHEMA_VERSION};
|
||||||
use lore::core::error::{LoreError, RobotErrorOutput};
|
use lore::core::error::{LoreError, RobotErrorOutput};
|
||||||
use lore::core::paths::get_config_path;
|
use lore::core::paths::get_config_path;
|
||||||
use lore::core::paths::get_db_path;
|
use lore::core::paths::get_db_path;
|
||||||
@@ -1112,8 +1112,9 @@ async fn handle_embed(
|
|||||||
robot_mode: bool,
|
robot_mode: bool,
|
||||||
) -> Result<(), Box<dyn std::error::Error>> {
|
) -> Result<(), Box<dyn std::error::Error>> {
|
||||||
let config = Config::load(config_override)?;
|
let config = Config::load(config_override)?;
|
||||||
|
let full = args.full && !args.no_full;
|
||||||
let retry_failed = args.retry_failed && !args.no_retry_failed;
|
let retry_failed = args.retry_failed && !args.no_retry_failed;
|
||||||
let result = run_embed(&config, retry_failed).await?;
|
let result = run_embed(&config, full, retry_failed).await?;
|
||||||
if robot_mode {
|
if robot_mode {
|
||||||
print_embed_json(&result);
|
print_embed_json(&result);
|
||||||
} else {
|
} else {
|
||||||
@@ -1183,8 +1184,7 @@ async fn handle_health(
|
|||||||
match create_connection(&db_path) {
|
match create_connection(&db_path) {
|
||||||
Ok(conn) => {
|
Ok(conn) => {
|
||||||
let version = get_schema_version(&conn);
|
let version = get_schema_version(&conn);
|
||||||
let latest = 9; // Number of embedded migrations
|
(true, version, version >= LATEST_SCHEMA_VERSION)
|
||||||
(true, version, version >= latest)
|
|
||||||
}
|
}
|
||||||
Err(_) => (true, 0, false),
|
Err(_) => (true, 0, false),
|
||||||
}
|
}
|
||||||
@@ -1340,7 +1340,7 @@ fn handle_robot_docs(robot_mode: bool) -> Result<(), Box<dyn std::error::Error>>
|
|||||||
},
|
},
|
||||||
"embed": {
|
"embed": {
|
||||||
"description": "Generate vector embeddings for documents via Ollama",
|
"description": "Generate vector embeddings for documents via Ollama",
|
||||||
"flags": ["--retry-failed"],
|
"flags": ["--full", "--retry-failed"],
|
||||||
"example": "lore --robot embed"
|
"example": "lore --robot embed"
|
||||||
},
|
},
|
||||||
"migrate": {
|
"migrate": {
|
||||||
|
|||||||
Reference in New Issue
Block a user