From c6a5461d41b3419d8d42b4a2699d26dc9b38bb95 Mon Sep 17 00:00:00 2001 From: Taylor Eernisse Date: Fri, 13 Feb 2026 22:31:57 -0500 Subject: [PATCH] refactor(ingestion): compact log summaries and quieter shutdown messages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Migrate all ingestion completion logs to use nonzero_summary() for compact, zero-suppressed output. Before: 8-14 individual key=value structured fields per completion message. After: a single summary field like '42 fetched · 3 labels · 12 notes' that only shows non-zero counters. Also downgrade all 'Shutdown requested...' messages from info! to debug!. These are emitted on every Ctrl+C and add noise to the partial results output that immediately follows. They remain visible at -vv for debugging graceful shutdown behavior. Affected modules: - issues.rs: issue ingestion completion - merge_requests.rs: MR ingestion completion, full-sync cursor reset - mr_discussions.rs: discussion ingestion completion - orchestrator.rs: project-level issue and MR completion summaries, all shutdown-requested checkpoints across discussion sync, resource events drain, closes-issues drain, and MR diffs drain Co-Authored-By: Claude Opus 4.6 --- src/ingestion/issues.rs | 12 +++-- src/ingestion/merge_requests.rs | 16 ++++--- src/ingestion/mr_discussions.rs | 16 +++---- src/ingestion/orchestrator.rs | 82 +++++++++++++++++---------------- 4 files changed, 67 insertions(+), 59 deletions(-) diff --git a/src/ingestion/issues.rs b/src/ingestion/issues.rs index a43a7e7..7e75459 100644 --- a/src/ingestion/issues.rs +++ b/src/ingestion/issues.rs @@ -109,11 +109,13 @@ pub async fn ingest_issues( result.issues_needing_discussion_sync = get_issues_needing_discussion_sync(conn, project_id)?; info!( - fetched = result.fetched, - upserted = result.upserted, - labels_created = result.labels_created, - needing_sync = result.issues_needing_discussion_sync.len(), - "Issue ingestion complete" + summary = crate::ingestion::nonzero_summary(&[ + ("fetched", result.fetched), + ("upserted", result.upserted), + ("labels", result.labels_created), + ("needing sync", result.issues_needing_discussion_sync.len()), + ]), + "Issue ingestion" ); Ok(result) diff --git a/src/ingestion/merge_requests.rs b/src/ingestion/merge_requests.rs index 384d606..f1d9f0b 100644 --- a/src/ingestion/merge_requests.rs +++ b/src/ingestion/merge_requests.rs @@ -50,7 +50,7 @@ pub async fn ingest_merge_requests( if full_sync { reset_sync_cursor(conn, project_id)?; reset_discussion_watermarks(conn, project_id)?; - info!("Full sync: cursor and discussion watermarks reset"); + debug!("Full sync: cursor and discussion watermarks reset"); } let cursor = get_sync_cursor(conn, project_id)?; @@ -122,12 +122,14 @@ pub async fn ingest_merge_requests( } info!( - fetched = result.fetched, - upserted = result.upserted, - labels_created = result.labels_created, - assignees_linked = result.assignees_linked, - reviewers_linked = result.reviewers_linked, - "MR ingestion complete" + summary = crate::ingestion::nonzero_summary(&[ + ("fetched", result.fetched), + ("upserted", result.upserted), + ("labels", result.labels_created), + ("assignees", result.assignees_linked), + ("reviewers", result.reviewers_linked), + ]), + "MR ingestion" ); Ok(result) diff --git a/src/ingestion/mr_discussions.rs b/src/ingestion/mr_discussions.rs index 7fecd55..736f301 100644 --- a/src/ingestion/mr_discussions.rs +++ b/src/ingestion/mr_discussions.rs @@ -269,14 +269,14 @@ pub async fn ingest_mr_discussions( } info!( - mrs_processed = mrs.len(), - discussions_fetched = total_result.discussions_fetched, - discussions_upserted = total_result.discussions_upserted, - notes_upserted = total_result.notes_upserted, - notes_skipped = total_result.notes_skipped_bad_timestamp, - diffnotes = total_result.diffnotes_count, - pagination_succeeded = total_result.pagination_succeeded, - "MR discussion ingestion complete" + summary = crate::ingestion::nonzero_summary(&[ + ("MRs", mrs.len()), + ("discussions", total_result.discussions_fetched), + ("notes", total_result.notes_upserted), + ("skipped", total_result.notes_skipped_bad_timestamp), + ("diffnotes", total_result.diffnotes_count), + ]), + "MR discussion ingestion" ); Ok(total_result) diff --git a/src/ingestion/orchestrator.rs b/src/ingestion/orchestrator.rs index c1c68fb..54c2417 100644 --- a/src/ingestion/orchestrator.rs +++ b/src/ingestion/orchestrator.rs @@ -207,7 +207,7 @@ pub async fn ingest_project_issues_with_progress( } if signal.is_cancelled() { - info!("Shutdown requested after status fetch, skipping DB write"); + debug!("Shutdown requested after status fetch, skipping DB write"); emit(ProgressEvent::StatusEnrichmentComplete { enriched: 0, cleared: 0, @@ -275,12 +275,12 @@ pub async fn ingest_project_issues_with_progress( result.issues_skipped_discussion_sync = total_issues.saturating_sub(issues_needing_sync.len()); if signal.is_cancelled() { - info!("Shutdown requested, returning partial issue results"); + debug!("Shutdown requested, returning partial issue results"); return Ok(result); } if issues_needing_sync.is_empty() { - info!("No issues need discussion sync"); + debug!("No issues need discussion sync"); } else { info!( count = issues_needing_sync.len(), @@ -314,7 +314,7 @@ pub async fn ingest_project_issues_with_progress( } if signal.is_cancelled() { - info!("Shutdown requested, returning partial issue results"); + debug!("Shutdown requested, returning partial issue results"); return Ok(result); } @@ -348,16 +348,18 @@ pub async fn ingest_project_issues_with_progress( } info!( - issues_fetched = result.issues_fetched, - issues_upserted = result.issues_upserted, - labels_created = result.labels_created, - discussions_fetched = result.discussions_fetched, - notes_upserted = result.notes_upserted, - issues_synced = result.issues_synced_discussions, - issues_skipped = result.issues_skipped_discussion_sync, - resource_events_fetched = result.resource_events_fetched, - resource_events_failed = result.resource_events_failed, - "Project ingestion complete" + summary = crate::ingestion::nonzero_summary(&[ + ("fetched", result.issues_fetched), + ("upserted", result.issues_upserted), + ("labels", result.labels_created), + ("discussions", result.discussions_fetched), + ("notes", result.notes_upserted), + ("synced", result.issues_synced_discussions), + ("skipped", result.issues_skipped_discussion_sync), + ("events", result.resource_events_fetched), + ("event errors", result.resource_events_failed), + ]), + "Project complete" ); tracing::Span::current().record("items_processed", result.issues_upserted); @@ -445,7 +447,7 @@ async fn sync_discussions_sequential( for chunk in issues.chunks(batch_size) { if signal.is_cancelled() { - info!("Shutdown requested during discussion sync, returning partial results"); + debug!("Shutdown requested during discussion sync, returning partial results"); break; } for issue in chunk { @@ -549,12 +551,12 @@ pub async fn ingest_project_merge_requests_with_progress( result.mrs_skipped_discussion_sync = total_mrs.saturating_sub(mrs_needing_sync.len()); if signal.is_cancelled() { - info!("Shutdown requested, returning partial MR results"); + debug!("Shutdown requested, returning partial MR results"); return Ok(result); } if mrs_needing_sync.is_empty() { - info!("No MRs need discussion sync"); + debug!("No MRs need discussion sync"); } else { info!( count = mrs_needing_sync.len(), @@ -592,7 +594,7 @@ pub async fn ingest_project_merge_requests_with_progress( } if signal.is_cancelled() { - info!("Shutdown requested, returning partial MR results"); + debug!("Shutdown requested, returning partial MR results"); return Ok(result); } @@ -626,7 +628,7 @@ pub async fn ingest_project_merge_requests_with_progress( } if signal.is_cancelled() { - info!("Shutdown requested, returning partial MR results"); + debug!("Shutdown requested, returning partial MR results"); return Ok(result); } @@ -679,7 +681,7 @@ pub async fn ingest_project_merge_requests_with_progress( } if signal.is_cancelled() { - info!("Shutdown requested, returning partial MR results"); + debug!("Shutdown requested, returning partial MR results"); return Ok(result); } @@ -704,21 +706,23 @@ pub async fn ingest_project_merge_requests_with_progress( } info!( - mrs_fetched = result.mrs_fetched, - mrs_upserted = result.mrs_upserted, - labels_created = result.labels_created, - discussions_fetched = result.discussions_fetched, - notes_upserted = result.notes_upserted, - diffnotes = result.diffnotes_count, - mrs_synced = result.mrs_synced_discussions, - mrs_skipped = result.mrs_skipped_discussion_sync, - resource_events_fetched = result.resource_events_fetched, - resource_events_failed = result.resource_events_failed, - closes_issues_fetched = result.closes_issues_fetched, - closes_issues_failed = result.closes_issues_failed, - mr_diffs_fetched = result.mr_diffs_fetched, - mr_diffs_failed = result.mr_diffs_failed, - "MR project ingestion complete" + summary = crate::ingestion::nonzero_summary(&[ + ("fetched", result.mrs_fetched), + ("upserted", result.mrs_upserted), + ("labels", result.labels_created), + ("discussions", result.discussions_fetched), + ("notes", result.notes_upserted), + ("diffnotes", result.diffnotes_count), + ("synced", result.mrs_synced_discussions), + ("skipped", result.mrs_skipped_discussion_sync), + ("events", result.resource_events_fetched), + ("event errors", result.resource_events_failed), + ("closes", result.closes_issues_fetched), + ("close errors", result.closes_issues_failed), + ("diffs", result.mr_diffs_fetched), + ("diff errors", result.mr_diffs_failed), + ]), + "MR project complete" ); tracing::Span::current().record("items_processed", result.mrs_upserted); @@ -750,7 +754,7 @@ async fn sync_mr_discussions_sequential( for chunk in mrs.chunks(batch_size) { if signal.is_cancelled() { - info!("Shutdown requested during MR discussion sync, returning partial results"); + debug!("Shutdown requested during MR discussion sync, returning partial results"); break; } let prefetch_futures = chunk.iter().map(|mr| { @@ -947,7 +951,7 @@ async fn drain_resource_events( loop { if signal.is_cancelled() { - info!("Shutdown requested during resource events drain, returning partial results"); + debug!("Shutdown requested during resource events drain, returning partial results"); break; } @@ -1269,7 +1273,7 @@ async fn drain_mr_closes_issues( loop { if signal.is_cancelled() { - info!("Shutdown requested during closes_issues drain, returning partial results"); + debug!("Shutdown requested during closes_issues drain, returning partial results"); break; } @@ -1526,7 +1530,7 @@ async fn drain_mr_diffs( loop { if signal.is_cancelled() { - info!("Shutdown requested during mr_diffs drain, returning partial results"); + debug!("Shutdown requested during mr_diffs drain, returning partial results"); break; }