feat(sync): concurrent drains, atomic watermarks, graceful Ctrl+C shutdown

Three fixes to the sync pipeline:

1. Atomic watermarks: wrap complete_job + update_watermark in a single
   SQLite transaction so crash between them can't leave partial state.

2. Concurrent drain loops: prefetch HTTP requests via join_all (batch
   size = dependent_concurrency), then write serially to DB. Reduces
   ~9K sequential requests from ~19 min to ~2.4 min.

3. Graceful shutdown: install Ctrl+C handler via ShutdownSignal
   (Arc<AtomicBool>), thread through orchestrator/CLI, release locked
   jobs on interrupt, record sync_run as "failed".

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Taylor Eernisse
2026-02-06 11:22:04 -05:00
parent 32783080f1
commit 405e5370dc
9 changed files with 536 additions and 92 deletions

View File

@@ -14,6 +14,7 @@ use crate::core::error::{LoreError, Result};
use crate::core::lock::{AppLock, LockOptions};
use crate::core::paths::get_db_path;
use crate::core::project::resolve_project;
use crate::core::shutdown::ShutdownSignal;
use crate::gitlab::GitLabClient;
use crate::ingestion::{
IngestMrProjectResult, IngestProjectResult, ProgressEvent, ingest_project_issues_with_progress,
@@ -113,6 +114,7 @@ pub async fn run_ingest(
dry_run: bool,
display: IngestDisplay,
stage_bar: Option<ProgressBar>,
signal: &ShutdownSignal,
) -> Result<IngestResult> {
let run_id = uuid::Uuid::new_v4().simple().to_string();
let run_id = &run_id[..8];
@@ -127,6 +129,7 @@ pub async fn run_ingest(
dry_run,
display,
stage_bar,
signal,
)
.instrument(span)
.await
@@ -228,6 +231,7 @@ async fn run_ingest_inner(
dry_run: bool,
display: IngestDisplay,
stage_bar: Option<ProgressBar>,
signal: &ShutdownSignal,
) -> Result<IngestResult> {
// In dry_run mode, we don't actually ingest - use run_ingest_dry_run instead
// This flag is passed through for consistency but the actual dry-run logic
@@ -350,6 +354,7 @@ async fn run_ingest_inner(
let agg_disc_total = Arc::clone(&agg_disc_total);
let agg_events = Arc::clone(&agg_events);
let agg_events_total = Arc::clone(&agg_events_total);
let signal = signal.clone();
async move {
let proj_conn = create_connection(&db_path)?;
@@ -506,6 +511,7 @@ async fn run_ingest_inner(
local_project_id,
gitlab_project_id,
Some(progress_callback),
&signal,
)
.await?;
@@ -522,6 +528,7 @@ async fn run_ingest_inner(
gitlab_project_id,
full,
Some(progress_callback),
&signal,
)
.await?;