From e9af529f6ed1dd8388bbc1e866aac55aee1bb2d4 Mon Sep 17 00:00:00 2001 From: Taylor Eernisse Date: Wed, 11 Feb 2026 10:22:20 -0500 Subject: [PATCH] feat(ingestion): add progress reporting for status enrichment pipeline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously the status enrichment phase (GraphQL work item status fetch) ran silently — users saw no feedback between "syncing issues" and the final enrichment summary. For projects with hundreds of issues and adaptive page-size retries, this felt like a hang. Changes across three layers: GraphQL (graphql.rs): - Extract fetch_issue_statuses_with_progress() accepting an optional on_page callback invoked after each paginated fetch with the running count of fetched IIDs - Original fetch_issue_statuses() preserved as a zero-cost delegation wrapper (no callback overhead) Orchestrator (orchestrator.rs): - Three new ProgressEvent variants: StatusEnrichmentStarted, StatusEnrichmentPageFetched, StatusEnrichmentWriting - Wire the page callback through to the new _with_progress fn CLI (ingest.rs): - Handle all three new events in the progress callback, updating both the per-project spinner and the stage bar with live counts Co-Authored-By: Claude Opus 4.6 --- src/cli/commands/ingest.rs | 27 +++++++++++++++++++++++++++ src/gitlab/graphql.rs | 12 ++++++++++++ src/ingestion/orchestrator.rs | 19 ++++++++++++++++++- 3 files changed, 57 insertions(+), 1 deletion(-) diff --git a/src/cli/commands/ingest.rs b/src/cli/commands/ingest.rs index 2c31e84..519bd12 100644 --- a/src/cli/commands/ingest.rs +++ b/src/cli/commands/ingest.rs @@ -532,8 +532,35 @@ async fn run_ingest_inner( ProgressEvent::MrDiffsFetchComplete { .. } => { disc_bar_clone.finish_and_clear(); } + ProgressEvent::StatusEnrichmentStarted => { + spinner_clone.set_message(format!( + "{path_for_cb}: Enriching work item statuses..." + )); + stage_bar_clone.set_message( + "Enriching work item statuses...".to_string() + ); + } + ProgressEvent::StatusEnrichmentPageFetched { items_so_far } => { + spinner_clone.set_message(format!( + "{path_for_cb}: Fetching statuses... ({items_so_far} work items)" + )); + stage_bar_clone.set_message(format!( + "Enriching work item statuses... ({items_so_far} fetched)" + )); + } + ProgressEvent::StatusEnrichmentWriting { total } => { + spinner_clone.set_message(format!( + "{path_for_cb}: Writing {total} statuses..." + )); + stage_bar_clone.set_message(format!( + "Writing {total} work item statuses..." + )); + } ProgressEvent::StatusEnrichmentComplete { enriched, cleared } => { if enriched > 0 || cleared > 0 { + spinner_clone.set_message(format!( + "{path_for_cb}: {enriched} statuses enriched, {cleared} cleared" + )); stage_bar_clone.set_message(format!( "Status enrichment: {enriched} enriched, {cleared} cleared" )); diff --git a/src/gitlab/graphql.rs b/src/gitlab/graphql.rs index 36ae731..5c1fd41 100644 --- a/src/gitlab/graphql.rs +++ b/src/gitlab/graphql.rs @@ -233,6 +233,14 @@ fn is_complexity_or_timeout_error(msg: &str) -> bool { pub async fn fetch_issue_statuses( client: &GraphqlClient, project_path: &str, +) -> crate::core::error::Result { + fetch_issue_statuses_with_progress(client, project_path, None).await +} + +pub async fn fetch_issue_statuses_with_progress( + client: &GraphqlClient, + project_path: &str, + on_page: Option<&dyn Fn(usize)>, ) -> crate::core::error::Result { let mut statuses = std::collections::HashMap::new(); let mut all_fetched_iids = std::collections::HashSet::new(); @@ -327,6 +335,10 @@ pub async fn fetch_issue_statuses( } } + if let Some(cb) = &on_page { + cb(all_fetched_iids.len()); + } + // Pagination if !connection.page_info.has_next_page { break; diff --git a/src/ingestion/orchestrator.rs b/src/ingestion/orchestrator.rs index 51dcf87..aadd92e 100644 --- a/src/ingestion/orchestrator.rs +++ b/src/ingestion/orchestrator.rs @@ -45,6 +45,9 @@ pub enum ProgressEvent { MrDiffsFetchStarted { total: usize }, MrDiffFetched { current: usize, total: usize }, MrDiffsFetchComplete { fetched: usize, failed: usize }, + StatusEnrichmentStarted, + StatusEnrichmentPageFetched { items_so_far: usize }, + StatusEnrichmentWriting { total: usize }, StatusEnrichmentComplete { enriched: usize, cleared: usize }, StatusEnrichmentSkipped, } @@ -150,6 +153,8 @@ pub async fn ingest_project_issues_with_progress( if config.sync.fetch_work_item_status && !signal.is_cancelled() { use rusqlite::OptionalExtension; + emit(ProgressEvent::StatusEnrichmentStarted); + let project_path: Option = conn .query_row( "SELECT path_with_namespace FROM projects WHERE id = ?1", @@ -170,7 +175,16 @@ pub async fn ingest_project_issues_with_progress( } Some(path) => { let graphql_client = client.graphql_client(); - match crate::gitlab::graphql::fetch_issue_statuses(&graphql_client, &path).await { + let page_cb = |items_so_far: usize| { + emit(ProgressEvent::StatusEnrichmentPageFetched { items_so_far }); + }; + match crate::gitlab::graphql::fetch_issue_statuses_with_progress( + &graphql_client, + &path, + Some(&page_cb), + ) + .await + { Ok(fetch_result) => { if let Some(ref reason) = fetch_result.unsupported_reason { result.status_enrichment_mode = "unsupported".into(); @@ -199,6 +213,9 @@ pub async fn ingest_project_issues_with_progress( cleared: 0, }); } else { + emit(ProgressEvent::StatusEnrichmentWriting { + total: fetch_result.all_fetched_iids.len(), + }); match enrich_issue_statuses_txn( conn, project_id,