theirongolddev/gitlore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(sync): per-project breakdown, status enrichment progress bars, and summary polish

Browse Source 
Add per-project detail rows beneath stage completion lines during multi-project
syncs, showing itemized counts (issues/MRs, discussions, events, statuses, diffs)
for each project. Previously, only aggregate totals were visible, making it hard
to diagnose which project contributed what during a sync.

Status enrichment gets proper progress bars replacing the old spinner-only
display: StatusEnrichmentStarted now carries a total count so the CLI can
render a determinate bar with rate and ETA. The enrichment SQL is tightened
to use IS NOT comparisons for diff-only UPDATEs (skip rows where values
haven't changed), and a follow-up touch_stmt ensures status_synced_at is
updated even for unchanged rows so staleness detection works correctly.

Other improvements:
- New ProjectSummary struct aggregates per-project metrics during ingestion
- SyncResult gains statuses_enriched + per-project summary vectors
- "Already up to date" message when sync finds zero changes
- Remove Arc<AtomicBool> tick_started pattern from docs/embed stages
  (enable_steady_tick is idempotent, the guard was unnecessary)
- Progress bar styling: dim spinner, dark_gray track, per_sec + eta display
- Tick intervals tightened from 100ms to 60ms for smoother animation
- statuses_without_widget calculation uses fetch_result.statuses.len()
  instead of subtracting enriched (more accurate when some statuses lack
  work item widgets)
- Status enrichment completion log downgraded from info to debug

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
teernisse
2026-02-14 11:25:19 -05:00
parent 64e73b1cab
commit bb6660178c
3 changed files with 257 additions and 113 deletions
Download Patch File Download Diff File Expand all files Collapse all files

88
src/cli/commands/ingest.rs
View File

@@ -46,6 +46,21 @@ pub struct IngestResult {
pub mr_diffs_failed: usize, pub mr_diffs_failed: usize,
pub status_enrichment_errors: usize, pub status_enrichment_errors: usize,
pub status_enrichment_projects: Vec<ProjectStatusEnrichment>, pub status_enrichment_projects: Vec<ProjectStatusEnrichment>,
pub project_summaries: Vec<ProjectSummary>,
}
/// Per-project summary for display in stage completion sub-rows.
#[derive(Debug, Default)]
pub struct ProjectSummary {
pub path: String,
pub items_upserted: usize,
pub discussions_synced: usize,
pub events_fetched: usize,
pub events_failed: usize,
pub statuses_enriched: usize,
pub statuses_seen: usize,
pub mr_diffs_fetched: usize,
pub mr_diffs_failed: usize,
} }
/// Per-project status enrichment result, collected during ingestion. /// Per-project status enrichment result, collected during ingestion.
@@ -388,11 +403,11 @@ async fn run_ingest_inner(
let s = multi.add(ProgressBar::new_spinner()); let s = multi.add(ProgressBar::new_spinner());
s.set_style( s.set_style(
ProgressStyle::default_spinner() ProgressStyle::default_spinner()
.template("{spinner:.blue} {msg}") .template("{spinner:.cyan} {msg}")
.unwrap(), .unwrap(),
); );
s.set_message(format!("Fetching {type_label} from {path}...")); s.set_message(format!("Fetching {type_label} from {path}..."));
s.enable_steady_tick(std::time::Duration::from_millis(100)); s.enable_steady_tick(std::time::Duration::from_millis(60));
s s
}; };
@@ -403,12 +418,13 @@ async fn run_ingest_inner(
b.set_style( b.set_style(
ProgressStyle::default_bar() ProgressStyle::default_bar()
.template( .template(
" {spinner:.blue} {prefix:.cyan} Syncing discussions [{bar:30.cyan/dim}] {pos}/{len}", " {spinner:.dim} {prefix:.cyan} Syncing discussions [{bar:30.cyan/dark_gray}] {pos}/{len} {per_sec:.dim} {eta:.dim}",
) )
.unwrap() .unwrap()
.progress_chars("=> "), .progress_chars(crate::cli::render::Icons::progress_chars()),
); );
b.set_prefix(path.clone()); b.set_prefix(path.clone());
b.enable_steady_tick(std::time::Duration::from_millis(60));
b b
}; };
@@ -445,7 +461,7 @@ async fn run_ingest_inner(
spinner_clone.finish_and_clear(); spinner_clone.finish_and_clear();
let agg_total = agg_disc_total_clone.fetch_add(total, Ordering::Relaxed) + total; let agg_total = agg_disc_total_clone.fetch_add(total, Ordering::Relaxed) + total;
disc_bar_clone.set_length(total as u64); disc_bar_clone.set_length(total as u64);
disc_bar_clone.enable_steady_tick(std::time::Duration::from_millis(100)); disc_bar_clone.enable_steady_tick(std::time::Duration::from_millis(60));
stage_bar_clone.set_message(format!( stage_bar_clone.set_message(format!(
"Syncing discussions... (0/{agg_total})" "Syncing discussions... (0/{agg_total})"
)); ));
@@ -465,7 +481,7 @@ async fn run_ingest_inner(
spinner_clone.finish_and_clear(); spinner_clone.finish_and_clear();
let agg_total = agg_disc_total_clone.fetch_add(total, Ordering::Relaxed) + total; let agg_total = agg_disc_total_clone.fetch_add(total, Ordering::Relaxed) + total;
disc_bar_clone.set_length(total as u64); disc_bar_clone.set_length(total as u64);
disc_bar_clone.enable_steady_tick(std::time::Duration::from_millis(100)); disc_bar_clone.enable_steady_tick(std::time::Duration::from_millis(60));
stage_bar_clone.set_message(format!( stage_bar_clone.set_message(format!(
"Syncing discussions... (0/{agg_total})" "Syncing discussions... (0/{agg_total})"
)); ));
@@ -486,11 +502,11 @@ async fn run_ingest_inner(
disc_bar_clone.set_length(total as u64); disc_bar_clone.set_length(total as u64);
disc_bar_clone.set_style( disc_bar_clone.set_style(
ProgressStyle::default_bar() ProgressStyle::default_bar()
.template(" {spinner:.blue} {prefix:.cyan} Fetching resource events [{bar:30.cyan/dim}] {pos}/{len}") .template(" {spinner:.dim} {prefix:.cyan} Fetching resource events [{bar:30.cyan/dark_gray}] {pos}/{len} {per_sec:.dim} {eta:.dim}")
.unwrap() .unwrap()
.progress_chars("=> "), .progress_chars(crate::cli::render::Icons::progress_chars()),
); );
disc_bar_clone.enable_steady_tick(std::time::Duration::from_millis(100)); disc_bar_clone.enable_steady_tick(std::time::Duration::from_millis(60));
agg_events_total_clone.fetch_add(total, Ordering::Relaxed); agg_events_total_clone.fetch_add(total, Ordering::Relaxed);
stage_bar_clone.set_message( stage_bar_clone.set_message(
"Fetching resource events...".to_string() "Fetching resource events...".to_string()
@@ -510,7 +526,7 @@ async fn run_ingest_inner(
ProgressEvent::ClosesIssuesFetchStarted { total } => { ProgressEvent::ClosesIssuesFetchStarted { total } => {
disc_bar_clone.reset(); disc_bar_clone.reset();
disc_bar_clone.set_length(total as u64); disc_bar_clone.set_length(total as u64);
disc_bar_clone.enable_steady_tick(std::time::Duration::from_millis(100)); disc_bar_clone.enable_steady_tick(std::time::Duration::from_millis(60));
stage_bar_clone.set_message( stage_bar_clone.set_message(
"Fetching closes-issues references...".to_string() "Fetching closes-issues references...".to_string()
); );
@@ -524,7 +540,7 @@ async fn run_ingest_inner(
ProgressEvent::MrDiffsFetchStarted { total } => { ProgressEvent::MrDiffsFetchStarted { total } => {
disc_bar_clone.reset(); disc_bar_clone.reset();
disc_bar_clone.set_length(total as u64); disc_bar_clone.set_length(total as u64);
disc_bar_clone.enable_steady_tick(std::time::Duration::from_millis(100)); disc_bar_clone.enable_steady_tick(std::time::Duration::from_millis(60));
stage_bar_clone.set_message( stage_bar_clone.set_message(
"Fetching MR file changes...".to_string() "Fetching MR file changes...".to_string()
); );
@@ -535,35 +551,37 @@ async fn run_ingest_inner(
ProgressEvent::MrDiffsFetchComplete { .. } => { ProgressEvent::MrDiffsFetchComplete { .. } => {
disc_bar_clone.finish_and_clear(); disc_bar_clone.finish_and_clear();
} }
ProgressEvent::StatusEnrichmentStarted => { ProgressEvent::StatusEnrichmentStarted { total } => {
spinner_clone.set_message(format!( spinner_clone.finish_and_clear();
"{path_for_cb}: Enriching work item statuses..." disc_bar_clone.reset();
)); disc_bar_clone.set_length(total as u64);
disc_bar_clone.set_style(
ProgressStyle::default_bar()
.template(" {spinner:.dim} {prefix:.cyan} Statuses [{bar:30.cyan/dark_gray}] {pos}/{len} {per_sec:.dim} {eta:.dim}")
.unwrap()
.progress_chars(crate::cli::render::Icons::progress_chars()),
);
disc_bar_clone.set_prefix(path_for_cb.clone());
disc_bar_clone.enable_steady_tick(std::time::Duration::from_millis(60));
stage_bar_clone.set_message( stage_bar_clone.set_message(
"Enriching work item statuses...".to_string() "Enriching work item statuses...".to_string()
); );
} }
ProgressEvent::StatusEnrichmentPageFetched { items_so_far } => { ProgressEvent::StatusEnrichmentPageFetched { items_so_far } => {
spinner_clone.set_message(format!( disc_bar_clone.set_position(items_so_far as u64);
"{path_for_cb}: Fetching statuses... ({items_so_far} work items)"
));
stage_bar_clone.set_message(format!( stage_bar_clone.set_message(format!(
"Enriching work item statuses... ({items_so_far} fetched)" "Enriching work item statuses... ({items_so_far} fetched)"
)); ));
} }
ProgressEvent::StatusEnrichmentWriting { total } => { ProgressEvent::StatusEnrichmentWriting { total } => {
spinner_clone.set_message(format!( disc_bar_clone.set_message(format!("Writing {total} statuses..."));
"{path_for_cb}: Writing {total} statuses..."
));
stage_bar_clone.set_message(format!( stage_bar_clone.set_message(format!(
"Writing {total} work item statuses..." "Writing {total} work item statuses..."
)); ));
} }
ProgressEvent::StatusEnrichmentComplete { enriched, cleared } => { ProgressEvent::StatusEnrichmentComplete { enriched, cleared } => {
disc_bar_clone.finish_and_clear();
if enriched > 0 || cleared > 0 { if enriched > 0 || cleared > 0 {
spinner_clone.set_message(format!(
"{path_for_cb}: {enriched} statuses enriched, {cleared} cleared"
));
stage_bar_clone.set_message(format!( stage_bar_clone.set_message(format!(
"Status enrichment: {enriched} enriched, {cleared} cleared" "Status enrichment: {enriched} enriched, {cleared} cleared"
)); ));
@@ -656,6 +674,17 @@ async fn run_ingest_inner(
first_partial_error: result.first_partial_error.clone(), first_partial_error: result.first_partial_error.clone(),
error: result.status_enrichment_error.clone(), error: result.status_enrichment_error.clone(),
}); });
total.project_summaries.push(ProjectSummary {
path: path.clone(),
items_upserted: result.issues_upserted,
discussions_synced: result.discussions_fetched,
events_fetched: result.resource_events_fetched,
events_failed: result.resource_events_failed,
statuses_enriched: result.statuses_enriched,
statuses_seen: result.statuses_seen,
mr_diffs_fetched: 0,
mr_diffs_failed: 0,
});
} }
Ok(ProjectIngestOutcome::Mrs { Ok(ProjectIngestOutcome::Mrs {
ref path, ref path,
@@ -679,6 +708,17 @@ async fn run_ingest_inner(
total.resource_events_failed += result.resource_events_failed; total.resource_events_failed += result.resource_events_failed;
total.mr_diffs_fetched += result.mr_diffs_fetched; total.mr_diffs_fetched += result.mr_diffs_fetched;
total.mr_diffs_failed += result.mr_diffs_failed; total.mr_diffs_failed += result.mr_diffs_failed;
total.project_summaries.push(ProjectSummary {
path: path.clone(),
items_upserted: result.mrs_upserted,
discussions_synced: result.discussions_fetched,
events_fetched: result.resource_events_fetched,
events_failed: result.resource_events_failed,
statuses_enriched: 0,
statuses_seen: 0,
mr_diffs_fetched: result.mr_diffs_fetched,
mr_diffs_failed: result.mr_diffs_failed,
});
} }
} }
} }

226
src/cli/commands/sync.rs
View File

@@ -1,10 +1,8 @@
use crate::cli::render::{self, Icons, Theme, format_number}; use crate::cli::render::{self, Icons, Theme, format_number};
use serde::Serialize; use serde::Serialize;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::Instant; use std::time::Instant;
use tracing::Instrument; use tracing::Instrument;
use tracing::{info, warn}; use tracing::{debug, warn};
use crate::Config; use crate::Config;
use crate::cli::progress::{finish_stage, nested_progress, stage_spinner_v2}; use crate::cli::progress::{finish_stage, nested_progress, stage_spinner_v2};
@@ -14,7 +12,7 @@ use crate::core::shutdown::ShutdownSignal;
use super::embed::run_embed; use super::embed::run_embed;
use super::generate_docs::run_generate_docs; use super::generate_docs::run_generate_docs;
use super::ingest::{DryRunPreview, IngestDisplay, run_ingest, run_ingest_dry_run}; use super::ingest::{DryRunPreview, IngestDisplay, ProjectSummary, run_ingest, run_ingest_dry_run};
#[derive(Debug, Default)] #[derive(Debug, Default)]
pub struct SyncOptions { pub struct SyncOptions {
@@ -41,6 +39,11 @@ pub struct SyncResult {
pub documents_regenerated: usize, pub documents_regenerated: usize,
pub documents_embedded: usize, pub documents_embedded: usize,
pub status_enrichment_errors: usize, pub status_enrichment_errors: usize,
pub statuses_enriched: usize,
#[serde(skip)]
pub issue_projects: Vec<ProjectSummary>,
#[serde(skip)]
pub mr_projects: Vec<ProjectSummary>,
} }
pub async fn run_sync( pub async fn run_sync(
@@ -79,7 +82,7 @@ pub async fn run_sync(
// ── Stage: Issues ── // ── Stage: Issues ──
let stage_start = Instant::now(); let stage_start = Instant::now();
let spinner = stage_spinner_v2(Icons::sync(), "Issues", "fetching...", options.robot_mode); let spinner = stage_spinner_v2(Icons::sync(), "Issues", "fetching...", options.robot_mode);
info!("Sync: ingesting issues"); debug!("Sync: ingesting issues");
let issues_result = run_ingest( let issues_result = run_ingest(
config, config,
"issues", "issues",
@@ -97,6 +100,10 @@ pub async fn run_sync(
result.resource_events_fetched += issues_result.resource_events_fetched; result.resource_events_fetched += issues_result.resource_events_fetched;
result.resource_events_failed += issues_result.resource_events_failed; result.resource_events_failed += issues_result.resource_events_failed;
result.status_enrichment_errors += issues_result.status_enrichment_errors; result.status_enrichment_errors += issues_result.status_enrichment_errors;
for sep in &issues_result.status_enrichment_projects {
result.statuses_enriched += sep.enriched;
}
result.issue_projects = issues_result.project_summaries;
let issues_summary = format!( let issues_summary = format!(
"{} issues from {} {}", "{} issues from {} {}",
format_number(result.issues_updated as i64), format_number(result.issues_updated as i64),
@@ -104,16 +111,19 @@ pub async fn run_sync(
if issues_result.projects_synced == 1 { "project" } else { "projects" } if issues_result.projects_synced == 1 { "project" } else { "projects" }
); );
finish_stage(&spinner, Icons::success(), "Issues", &issues_summary, stage_start.elapsed()); finish_stage(&spinner, Icons::success(), "Issues", &issues_summary, stage_start.elapsed());
if !options.robot_mode {
print_issue_sub_rows(&result.issue_projects);
}
if signal.is_cancelled() { if signal.is_cancelled() {
info!("Shutdown requested after issues stage, returning partial sync results"); debug!("Shutdown requested after issues stage, returning partial sync results");
return Ok(result); return Ok(result);
} }
// ── Stage: MRs ── // ── Stage: MRs ──
let stage_start = Instant::now(); let stage_start = Instant::now();
let spinner = stage_spinner_v2(Icons::sync(), "MRs", "fetching...", options.robot_mode); let spinner = stage_spinner_v2(Icons::sync(), "MRs", "fetching...", options.robot_mode);
info!("Sync: ingesting merge requests"); debug!("Sync: ingesting merge requests");
let mrs_result = run_ingest( let mrs_result = run_ingest(
config, config,
"mrs", "mrs",
@@ -132,6 +142,7 @@ pub async fn run_sync(
result.resource_events_failed += mrs_result.resource_events_failed; result.resource_events_failed += mrs_result.resource_events_failed;
result.mr_diffs_fetched += mrs_result.mr_diffs_fetched; result.mr_diffs_fetched += mrs_result.mr_diffs_fetched;
result.mr_diffs_failed += mrs_result.mr_diffs_failed; result.mr_diffs_failed += mrs_result.mr_diffs_failed;
result.mr_projects = mrs_result.project_summaries;
let mrs_summary = format!( let mrs_summary = format!(
"{} merge requests from {} {}", "{} merge requests from {} {}",
format_number(result.mrs_updated as i64), format_number(result.mrs_updated as i64),
@@ -139,9 +150,12 @@ pub async fn run_sync(
if mrs_result.projects_synced == 1 { "project" } else { "projects" } if mrs_result.projects_synced == 1 { "project" } else { "projects" }
); );
finish_stage(&spinner, Icons::success(), "MRs", &mrs_summary, stage_start.elapsed()); finish_stage(&spinner, Icons::success(), "MRs", &mrs_summary, stage_start.elapsed());
if !options.robot_mode {
print_mr_sub_rows(&result.mr_projects);
}
if signal.is_cancelled() { if signal.is_cancelled() {
info!("Shutdown requested after MRs stage, returning partial sync results"); debug!("Shutdown requested after MRs stage, returning partial sync results");
return Ok(result); return Ok(result);
} }
@@ -149,17 +163,12 @@ pub async fn run_sync(
if !options.no_docs { if !options.no_docs {
let stage_start = Instant::now(); let stage_start = Instant::now();
let spinner = stage_spinner_v2(Icons::sync(), "Docs", "generating...", options.robot_mode); let spinner = stage_spinner_v2(Icons::sync(), "Docs", "generating...", options.robot_mode);
info!("Sync: generating documents"); debug!("Sync: generating documents");
let docs_bar = nested_progress("Docs", 0, options.robot_mode); let docs_bar = nested_progress("Docs", 0, options.robot_mode);
let docs_bar_clone = docs_bar.clone(); let docs_bar_clone = docs_bar.clone();
let tick_started = Arc::new(AtomicBool::new(false));
let tick_started_clone = Arc::clone(&tick_started);
let docs_cb: Box<dyn Fn(usize, usize)> = Box::new(move |processed, total| { let docs_cb: Box<dyn Fn(usize, usize)> = Box::new(move |processed, total| {
if total > 0 { if total > 0 {
if !tick_started_clone.swap(true, Ordering::Relaxed) {
docs_bar_clone.enable_steady_tick(std::time::Duration::from_millis(100));
}
docs_bar_clone.set_length(total as u64); docs_bar_clone.set_length(total as u64);
docs_bar_clone.set_position(processed as u64); docs_bar_clone.set_position(processed as u64);
} }
@@ -173,24 +182,19 @@ pub async fn run_sync(
); );
finish_stage(&spinner, Icons::success(), "Docs", &docs_summary, stage_start.elapsed()); finish_stage(&spinner, Icons::success(), "Docs", &docs_summary, stage_start.elapsed());
} else { } else {
info!("Sync: skipping document generation (--no-docs)"); debug!("Sync: skipping document generation (--no-docs)");
} }
// ── Stage: Embed ── // ── Stage: Embed ──
if !options.no_embed { if !options.no_embed {
let stage_start = Instant::now(); let stage_start = Instant::now();
let spinner = stage_spinner_v2(Icons::sync(), "Embed", "preparing...", options.robot_mode); let spinner = stage_spinner_v2(Icons::sync(), "Embed", "preparing...", options.robot_mode);
info!("Sync: embedding documents"); debug!("Sync: embedding documents");
let embed_bar = nested_progress("Embed", 0, options.robot_mode); let embed_bar = nested_progress("Embed", 0, options.robot_mode);
let embed_bar_clone = embed_bar.clone(); let embed_bar_clone = embed_bar.clone();
let tick_started = Arc::new(AtomicBool::new(false));
let tick_started_clone = Arc::clone(&tick_started);
let embed_cb: Box<dyn Fn(usize, usize)> = Box::new(move |processed, total| { let embed_cb: Box<dyn Fn(usize, usize)> = Box::new(move |processed, total| {
if total > 0 { if total > 0 {
if !tick_started_clone.swap(true, Ordering::Relaxed) {
embed_bar_clone.enable_steady_tick(std::time::Duration::from_millis(100));
}
embed_bar_clone.set_length(total as u64); embed_bar_clone.set_length(total as u64);
embed_bar_clone.set_position(processed as u64); embed_bar_clone.set_position(processed as u64);
} }
@@ -213,10 +217,10 @@ pub async fn run_sync(
} }
} }
} else { } else {
info!("Sync: skipping embedding (--no-embed)"); debug!("Sync: skipping embedding (--no-embed)");
} }
info!( debug!(
issues = result.issues_updated, issues = result.issues_updated,
mrs = result.mrs_updated, mrs = result.mrs_updated,
discussions = result.discussions_fetched, discussions = result.discussions_fetched,
@@ -240,58 +244,78 @@ pub fn print_sync(
elapsed: std::time::Duration, elapsed: std::time::Duration,
metrics: Option<&MetricsLayer>, metrics: Option<&MetricsLayer>,
) { ) {
// Headline: what happened, how long let has_data = result.issues_updated > 0
println!( || result.mrs_updated > 0
"\n {} {} issues and {} MRs in {:.1}s", || result.discussions_fetched > 0
Theme::success().bold().render("Synced"), || result.resource_events_fetched > 0
Theme::bold().render(&result.issues_updated.to_string()), || result.mr_diffs_fetched > 0
Theme::bold().render(&result.mrs_updated.to_string()), || result.documents_regenerated > 0
elapsed.as_secs_f64() || result.documents_embedded > 0
); || result.statuses_enriched > 0;
// Detail: supporting counts, compact middle-dot format, zero-suppressed if !has_data {
let mut details: Vec<String> = Vec::new(); println!(
if result.discussions_fetched > 0 { "\n {} ({:.1}s)\n",
details.push(format!("{} discussions", result.discussions_fetched)); Theme::dim().render("Already up to date"),
} elapsed.as_secs_f64()
if result.resource_events_fetched > 0 { );
details.push(format!("{} events", result.resource_events_fetched)); } else {
} // Headline: what happened, how long
if result.mr_diffs_fetched > 0 { println!(
details.push(format!("{} diffs", result.mr_diffs_fetched)); "\n {} {} issues and {} MRs in {:.1}s",
} Theme::success().bold().render("Synced"),
if !details.is_empty() { Theme::bold().render(&result.issues_updated.to_string()),
println!(" {}", Theme::dim().render(&details.join(" \u{b7} "))); Theme::bold().render(&result.mrs_updated.to_string()),
} elapsed.as_secs_f64()
);
// Documents: regeneration + embedding as a second detail line // Detail: supporting counts, compact middle-dot format, zero-suppressed
let mut doc_parts: Vec<String> = Vec::new(); let mut details: Vec<String> = Vec::new();
if result.documents_regenerated > 0 { if result.discussions_fetched > 0 {
doc_parts.push(format!("{} docs regenerated", result.documents_regenerated)); details.push(format!("{} discussions", result.discussions_fetched));
} }
if result.documents_embedded > 0 { if result.resource_events_fetched > 0 {
doc_parts.push(format!("{} embedded", result.documents_embedded)); details.push(format!("{} events", result.resource_events_fetched));
} }
if !doc_parts.is_empty() { if result.mr_diffs_fetched > 0 {
println!(" {}", Theme::dim().render(&doc_parts.join(" \u{b7} "))); details.push(format!("{} diffs", result.mr_diffs_fetched));
} }
if result.statuses_enriched > 0 {
details.push(format!("{} statuses updated", result.statuses_enriched));
}
if !details.is_empty() {
println!(" {}", Theme::dim().render(&details.join(" \u{b7} ")));
}
// Errors: visually prominent, only if non-zero // Documents: regeneration + embedding as a second detail line
let mut errors: Vec<String> = Vec::new(); let mut doc_parts: Vec<String> = Vec::new();
if result.resource_events_failed > 0 { if result.documents_regenerated > 0 {
errors.push(format!("{} event failures", result.resource_events_failed)); doc_parts.push(format!("{} docs regenerated", result.documents_regenerated));
} }
if result.mr_diffs_failed > 0 { if result.documents_embedded > 0 {
errors.push(format!("{} diff failures", result.mr_diffs_failed)); doc_parts.push(format!("{} embedded", result.documents_embedded));
} }
if result.status_enrichment_errors > 0 { if !doc_parts.is_empty() {
errors.push(format!("{} status errors", result.status_enrichment_errors)); println!(" {}", Theme::dim().render(&doc_parts.join(" \u{b7} ")));
} }
if !errors.is_empty() {
println!(" {}", Theme::error().render(&errors.join(" \u{b7} ")));
}
println!(); // Errors: visually prominent, only if non-zero
let mut errors: Vec<String> = Vec::new();
if result.resource_events_failed > 0 {
errors.push(format!("{} event failures", result.resource_events_failed));
}
if result.mr_diffs_failed > 0 {
errors.push(format!("{} diff failures", result.mr_diffs_failed));
}
if result.status_enrichment_errors > 0 {
errors.push(format!("{} status errors", result.status_enrichment_errors));
}
if !errors.is_empty() {
println!(" {}", Theme::error().render(&errors.join(" \u{b7} ")));
}
println!();
}
if let Some(metrics) = metrics { if let Some(metrics) = metrics {
let stages = metrics.extract_timings(); let stages = metrics.extract_timings();
@@ -301,6 +325,66 @@ pub fn print_sync(
} }
} }
fn print_issue_sub_rows(projects: &[ProjectSummary]) {
if projects.len() <= 1 {
return;
}
for p in projects {
let mut parts: Vec<String> = Vec::new();
parts.push(format!(
"{} {}",
p.items_upserted,
if p.items_upserted == 1 {
"issue"
} else {
"issues"
}
));
if p.discussions_synced > 0 {
parts.push(format!("{} discussions", p.discussions_synced));
}
if p.statuses_enriched > 0 {
parts.push(format!("{} statuses updated", p.statuses_enriched));
}
if p.events_fetched > 0 {
parts.push(format!("{} events", p.events_fetched));
}
let detail = parts.join(" \u{b7} ");
let _ = crate::cli::progress::multi().println(format!(
" {}",
Theme::dim().render(&format!("{:<30} {}", p.path, detail))
));
}
}
fn print_mr_sub_rows(projects: &[ProjectSummary]) {
if projects.len() <= 1 {
return;
}
for p in projects {
let mut parts: Vec<String> = Vec::new();
parts.push(format!(
"{} {}",
p.items_upserted,
if p.items_upserted == 1 { "MR" } else { "MRs" }
));
if p.discussions_synced > 0 {
parts.push(format!("{} discussions", p.discussions_synced));
}
if p.mr_diffs_fetched > 0 {
parts.push(format!("{} diffs", p.mr_diffs_fetched));
}
if p.events_fetched > 0 {
parts.push(format!("{} events", p.events_fetched));
}
let detail = parts.join(" \u{b7} ");
let _ = crate::cli::progress::multi().println(format!(
" {}",
Theme::dim().render(&format!("{:<30} {}", p.path, detail))
));
}
}
fn section(title: &str) { fn section(title: &str) {
println!("{}", render::section_divider(title)); println!("{}", render::section_divider(title));
} }

56
src/ingestion/orchestrator.rs
View File

@@ -1,6 +1,6 @@
use futures::future::join_all; use futures::future::join_all;
use rusqlite::Connection; use rusqlite::Connection;
use tracing::{debug, info, instrument, warn}; use tracing::{debug, instrument, warn};
use crate::Config; use crate::Config;
use crate::core::dependent_queue::{ use crate::core::dependent_queue::{
@@ -45,7 +45,7 @@ pub enum ProgressEvent {
MrDiffsFetchStarted { total: usize }, MrDiffsFetchStarted { total: usize },
MrDiffFetched { current: usize, total: usize }, MrDiffFetched { current: usize, total: usize },
MrDiffsFetchComplete { fetched: usize, failed: usize }, MrDiffsFetchComplete { fetched: usize, failed: usize },
StatusEnrichmentStarted, StatusEnrichmentStarted { total: usize },
StatusEnrichmentPageFetched { items_so_far: usize }, StatusEnrichmentPageFetched { items_so_far: usize },
StatusEnrichmentWriting { total: usize }, StatusEnrichmentWriting { total: usize },
StatusEnrichmentComplete { enriched: usize, cleared: usize }, StatusEnrichmentComplete { enriched: usize, cleared: usize },
@@ -153,7 +153,16 @@ pub async fn ingest_project_issues_with_progress(
if config.sync.fetch_work_item_status && !signal.is_cancelled() { if config.sync.fetch_work_item_status && !signal.is_cancelled() {
use rusqlite::OptionalExtension; use rusqlite::OptionalExtension;
emit(ProgressEvent::StatusEnrichmentStarted); let issue_count: i64 = conn
.query_row(
"SELECT COUNT(*) FROM issues WHERE project_id = ?1",
[project_id],
|r| r.get(0),
)
.unwrap_or(0);
emit(ProgressEvent::StatusEnrichmentStarted {
total: issue_count as usize,
});
let project_path: Option<String> = conn let project_path: Option<String> = conn
.query_row( .query_row(
@@ -225,9 +234,10 @@ pub async fn ingest_project_issues_with_progress(
Ok((enriched, cleared)) => { Ok((enriched, cleared)) => {
result.statuses_enriched = enriched; result.statuses_enriched = enriched;
result.statuses_cleared = cleared; result.statuses_cleared = cleared;
result.statuses_without_widget = result.statuses_without_widget = result
result.statuses_seen.saturating_sub(enriched); .statuses_seen
info!( .saturating_sub(fetch_result.statuses.len());
debug!(
seen = result.statuses_seen, seen = result.statuses_seen,
enriched, enriched,
cleared, cleared,
@@ -282,7 +292,7 @@ pub async fn ingest_project_issues_with_progress(
if issues_needing_sync.is_empty() { if issues_needing_sync.is_empty() {
debug!("No issues need discussion sync"); debug!("No issues need discussion sync");
} else { } else {
info!( debug!(
count = issues_needing_sync.len(), count = issues_needing_sync.len(),
"Starting discussion sync for issues" "Starting discussion sync for issues"
); );
@@ -347,7 +357,7 @@ pub async fn ingest_project_issues_with_progress(
} }
} }
info!( debug!(
summary = crate::ingestion::nonzero_summary(&[ summary = crate::ingestion::nonzero_summary(&[
("fetched", result.issues_fetched), ("fetched", result.issues_fetched),
("upserted", result.issues_upserted), ("upserted", result.issues_upserted),
@@ -402,12 +412,14 @@ fn enrich_issue_statuses_txn(
} }
} }
// Phase 2: Apply new/updated statuses // Phase 2: Apply new/updated statuses (only write when values actually differ)
{ {
let mut update_stmt = tx.prepare_cached( let mut update_stmt = tx.prepare_cached(
"UPDATE issues SET status_name = ?1, status_category = ?2, status_color = ?3, "UPDATE issues SET status_name = ?1, status_category = ?2, status_color = ?3,
status_icon_name = ?4, status_synced_at = ?5 status_icon_name = ?4, status_synced_at = ?5
WHERE project_id = ?6 AND iid = ?7", WHERE project_id = ?6 AND iid = ?7
AND (status_name IS NOT ?1 OR status_category IS NOT ?2
OR status_color IS NOT ?3 OR status_icon_name IS NOT ?4)",
)?; )?;
for (iid, status) in statuses { for (iid, status) in statuses {
let rows = update_stmt.execute(rusqlite::params![ let rows = update_stmt.execute(rusqlite::params![
@@ -423,6 +435,14 @@ fn enrich_issue_statuses_txn(
enriched += 1; enriched += 1;
} }
} }
// Update synced_at timestamp for unchanged rows too
let mut touch_stmt = tx.prepare_cached(
"UPDATE issues SET status_synced_at = ?1
WHERE project_id = ?2 AND iid = ?3 AND status_synced_at IS NOT ?1",
)?;
for iid in statuses.keys() {
touch_stmt.execute(rusqlite::params![now_ms, project_id, iid])?;
}
} }
tx.commit()?; tx.commit()?;
@@ -558,7 +578,7 @@ pub async fn ingest_project_merge_requests_with_progress(
if mrs_needing_sync.is_empty() { if mrs_needing_sync.is_empty() {
debug!("No MRs need discussion sync"); debug!("No MRs need discussion sync");
} else { } else {
info!( debug!(
count = mrs_needing_sync.len(), count = mrs_needing_sync.len(),
"Starting discussion sync for MRs" "Starting discussion sync for MRs"
); );
@@ -705,7 +725,7 @@ pub async fn ingest_project_merge_requests_with_progress(
result.mr_diffs_failed = diffs_result.failed; result.mr_diffs_failed = diffs_result.failed;
} }
info!( debug!(
summary = crate::ingestion::nonzero_summary(&[ summary = crate::ingestion::nonzero_summary(&[
("fetched", result.mrs_fetched), ("fetched", result.mrs_fetched),
("upserted", result.mrs_upserted), ("upserted", result.mrs_upserted),
@@ -923,7 +943,7 @@ async fn drain_resource_events(
let reclaimed = reclaim_stale_locks(conn, config.sync.stale_lock_minutes)?; let reclaimed = reclaim_stale_locks(conn, config.sync.stale_lock_minutes)?;
if reclaimed > 0 { if reclaimed > 0 {
info!(reclaimed, "Reclaimed stale resource event locks"); debug!(reclaimed, "Reclaimed stale resource event locks");
} }
let claimable_counts = count_claimable_jobs(conn, project_id)?; let claimable_counts = count_claimable_jobs(conn, project_id)?;
@@ -1063,7 +1083,7 @@ async fn drain_resource_events(
}); });
if result.fetched > 0 || result.failed > 0 { if result.fetched > 0 || result.failed > 0 {
info!( debug!(
fetched = result.fetched, fetched = result.fetched,
failed = result.failed, failed = result.failed,
"Resource events drain complete" "Resource events drain complete"
@@ -1245,7 +1265,7 @@ async fn drain_mr_closes_issues(
let reclaimed = reclaim_stale_locks(conn, config.sync.stale_lock_minutes)?; let reclaimed = reclaim_stale_locks(conn, config.sync.stale_lock_minutes)?;
if reclaimed > 0 { if reclaimed > 0 {
info!(reclaimed, "Reclaimed stale mr_closes_issues locks"); debug!(reclaimed, "Reclaimed stale mr_closes_issues locks");
} }
let claimable_counts = count_claimable_jobs(conn, project_id)?; let claimable_counts = count_claimable_jobs(conn, project_id)?;
@@ -1373,7 +1393,7 @@ async fn drain_mr_closes_issues(
}); });
if result.fetched > 0 || result.failed > 0 { if result.fetched > 0 || result.failed > 0 {
info!( debug!(
fetched = result.fetched, fetched = result.fetched,
failed = result.failed, failed = result.failed,
"mr_closes_issues drain complete" "mr_closes_issues drain complete"
@@ -1505,7 +1525,7 @@ async fn drain_mr_diffs(
let reclaimed = reclaim_stale_locks(conn, config.sync.stale_lock_minutes)?; let reclaimed = reclaim_stale_locks(conn, config.sync.stale_lock_minutes)?;
if reclaimed > 0 { if reclaimed > 0 {
info!(reclaimed, "Reclaimed stale mr_diffs locks"); debug!(reclaimed, "Reclaimed stale mr_diffs locks");
} }
let claimable_counts = count_claimable_jobs(conn, project_id)?; let claimable_counts = count_claimable_jobs(conn, project_id)?;
@@ -1630,7 +1650,7 @@ async fn drain_mr_diffs(
}); });
if result.fetched > 0 || result.failed > 0 { if result.fetched > 0 || result.failed > 0 {
info!( debug!(
fetched = result.fetched, fetched = result.fetched,
failed = result.failed, failed = result.failed,
"mr_diffs drain complete" "mr_diffs drain complete"