feat(sync): Instrument pipeline with tracing spans, run_id correlation, and metrics
Add end-to-end observability to the sync and ingest pipelines: Sync command: - Generate UUID-based run_id for each sync invocation, propagated through all child spans for log correlation across stages - Accept MetricsLayer reference to extract hierarchical StageTiming data after pipeline completion for robot-mode performance output - Record sync runs in DB via SyncRunRecorder (start/succeed/fail lifecycle) - Wrap entire sync execution in a root tracing span with run_id field Ingest command: - Wrap run_ingest in an instrumented root span with run_id and resource_type - Add project path prefix to discussion progress bars for multi-project clarity - Reset resource_events_synced_for_updated_at on --full re-sync Sync status: - Expand from single last_run to configurable recent runs list (default 10) - Parse and expose StageTiming metrics from stored metrics_json - Add run_id, total_items_processed, total_errors to SyncRunInfo - Add mr_count to DataSummary for complete entity coverage Orchestrator: - Add #[instrument] with structured fields to issue and MR ingestion functions - Record items_processed, items_skipped, errors on span close for MetricsLayer - Emit granular progress events (IssuesFetchStarted, IssuesFetchComplete) - Pass project_id through to drain_resource_events for scoped job claiming Document regenerator and embedding pipeline: - Add #[instrument] spans with items_processed, items_skipped, errors fields - Record final counts on span close for metrics extraction Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -5,6 +5,8 @@ use indicatif::{ProgressBar, ProgressStyle};
|
||||
use rusqlite::Connection;
|
||||
use serde::Serialize;
|
||||
|
||||
use tracing::Instrument;
|
||||
|
||||
use crate::Config;
|
||||
use crate::core::db::create_connection;
|
||||
use crate::core::error::{LoreError, Result};
|
||||
@@ -111,6 +113,24 @@ pub async fn run_ingest(
|
||||
force: bool,
|
||||
full: bool,
|
||||
display: IngestDisplay,
|
||||
) -> Result<IngestResult> {
|
||||
let run_id = uuid::Uuid::new_v4().simple().to_string();
|
||||
let run_id = &run_id[..8];
|
||||
let span = tracing::info_span!("ingest", %run_id, %resource_type);
|
||||
|
||||
run_ingest_inner(config, resource_type, project_filter, force, full, display)
|
||||
.instrument(span)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Inner implementation of run_ingest, instrumented with a root span.
|
||||
async fn run_ingest_inner(
|
||||
config: &Config,
|
||||
resource_type: &str,
|
||||
project_filter: Option<&str>,
|
||||
force: bool,
|
||||
full: bool,
|
||||
display: IngestDisplay,
|
||||
) -> Result<IngestResult> {
|
||||
// Validate resource type early
|
||||
if resource_type != "issues" && resource_type != "mrs" {
|
||||
@@ -162,15 +182,15 @@ pub async fn run_ingest(
|
||||
}
|
||||
for (local_project_id, _, path) in &projects {
|
||||
if resource_type == "issues" {
|
||||
// Reset issue discussion watermarks first so discussions get re-synced
|
||||
// Reset issue discussion and resource event watermarks so everything gets re-synced
|
||||
conn.execute(
|
||||
"UPDATE issues SET discussions_synced_for_updated_at = NULL WHERE project_id = ?",
|
||||
"UPDATE issues SET discussions_synced_for_updated_at = NULL, resource_events_synced_for_updated_at = NULL WHERE project_id = ?",
|
||||
[*local_project_id],
|
||||
)?;
|
||||
} else if resource_type == "mrs" {
|
||||
// Reset MR discussion watermarks
|
||||
// Reset MR discussion and resource event watermarks
|
||||
conn.execute(
|
||||
"UPDATE merge_requests SET discussions_synced_for_updated_at = NULL WHERE project_id = ?",
|
||||
"UPDATE merge_requests SET discussions_synced_for_updated_at = NULL, resource_events_synced_for_updated_at = NULL WHERE project_id = ?",
|
||||
[*local_project_id],
|
||||
)?;
|
||||
}
|
||||
@@ -255,11 +275,12 @@ pub async fn run_ingest(
|
||||
b.set_style(
|
||||
ProgressStyle::default_bar()
|
||||
.template(
|
||||
" {spinner:.blue} Syncing discussions [{bar:30.cyan/dim}] {pos}/{len}",
|
||||
" {spinner:.blue} {prefix:.cyan} Syncing discussions [{bar:30.cyan/dim}] {pos}/{len}",
|
||||
)
|
||||
.unwrap()
|
||||
.progress_chars("=> "),
|
||||
);
|
||||
b.set_prefix(path.clone());
|
||||
b
|
||||
};
|
||||
|
||||
@@ -296,7 +317,7 @@ pub async fn run_ingest(
|
||||
disc_bar_clone.set_length(total as u64);
|
||||
disc_bar_clone.set_style(
|
||||
ProgressStyle::default_bar()
|
||||
.template(" {spinner:.blue} Fetching resource events [{bar:30.cyan/dim}] {pos}/{len}")
|
||||
.template(" {spinner:.blue} {prefix:.cyan} Fetching resource events [{bar:30.cyan/dim}] {pos}/{len}")
|
||||
.unwrap()
|
||||
.progress_chars("=> "),
|
||||
);
|
||||
|
||||
Reference in New Issue
Block a user