feat(sync): concurrent drains, atomic watermarks, graceful Ctrl+C shutdown
Three fixes to the sync pipeline: 1. Atomic watermarks: wrap complete_job + update_watermark in a single SQLite transaction so crash between them can't leave partial state. 2. Concurrent drain loops: prefetch HTTP requests via join_all (batch size = dependent_concurrency), then write serially to DB. Reduces ~9K sequential requests from ~19 min to ~2.4 min. 3. Graceful shutdown: install Ctrl+C handler via ShutdownSignal (Arc<AtomicBool>), thread through orchestrator/CLI, release locked jobs on interrupt, record sync_run as "failed". Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -14,6 +14,7 @@ use crate::core::error::{LoreError, Result};
|
||||
use crate::core::lock::{AppLock, LockOptions};
|
||||
use crate::core::paths::get_db_path;
|
||||
use crate::core::project::resolve_project;
|
||||
use crate::core::shutdown::ShutdownSignal;
|
||||
use crate::gitlab::GitLabClient;
|
||||
use crate::ingestion::{
|
||||
IngestMrProjectResult, IngestProjectResult, ProgressEvent, ingest_project_issues_with_progress,
|
||||
@@ -113,6 +114,7 @@ pub async fn run_ingest(
|
||||
dry_run: bool,
|
||||
display: IngestDisplay,
|
||||
stage_bar: Option<ProgressBar>,
|
||||
signal: &ShutdownSignal,
|
||||
) -> Result<IngestResult> {
|
||||
let run_id = uuid::Uuid::new_v4().simple().to_string();
|
||||
let run_id = &run_id[..8];
|
||||
@@ -127,6 +129,7 @@ pub async fn run_ingest(
|
||||
dry_run,
|
||||
display,
|
||||
stage_bar,
|
||||
signal,
|
||||
)
|
||||
.instrument(span)
|
||||
.await
|
||||
@@ -228,6 +231,7 @@ async fn run_ingest_inner(
|
||||
dry_run: bool,
|
||||
display: IngestDisplay,
|
||||
stage_bar: Option<ProgressBar>,
|
||||
signal: &ShutdownSignal,
|
||||
) -> Result<IngestResult> {
|
||||
// In dry_run mode, we don't actually ingest - use run_ingest_dry_run instead
|
||||
// This flag is passed through for consistency but the actual dry-run logic
|
||||
@@ -350,6 +354,7 @@ async fn run_ingest_inner(
|
||||
let agg_disc_total = Arc::clone(&agg_disc_total);
|
||||
let agg_events = Arc::clone(&agg_events);
|
||||
let agg_events_total = Arc::clone(&agg_events_total);
|
||||
let signal = signal.clone();
|
||||
|
||||
async move {
|
||||
let proj_conn = create_connection(&db_path)?;
|
||||
@@ -506,6 +511,7 @@ async fn run_ingest_inner(
|
||||
local_project_id,
|
||||
gitlab_project_id,
|
||||
Some(progress_callback),
|
||||
&signal,
|
||||
)
|
||||
.await?;
|
||||
|
||||
@@ -522,6 +528,7 @@ async fn run_ingest_inner(
|
||||
gitlab_project_id,
|
||||
full,
|
||||
Some(progress_callback),
|
||||
&signal,
|
||||
)
|
||||
.await?;
|
||||
|
||||
|
||||
@@ -9,6 +9,7 @@ use tracing::{info, warn};
|
||||
use crate::Config;
|
||||
use crate::core::error::Result;
|
||||
use crate::core::metrics::{MetricsLayer, StageTiming};
|
||||
use crate::core::shutdown::ShutdownSignal;
|
||||
|
||||
use super::embed::run_embed;
|
||||
use super::generate_docs::run_generate_docs;
|
||||
@@ -58,6 +59,7 @@ pub async fn run_sync(
|
||||
config: &Config,
|
||||
options: SyncOptions,
|
||||
run_id: Option<&str>,
|
||||
signal: &ShutdownSignal,
|
||||
) -> Result<SyncResult> {
|
||||
let generated_id;
|
||||
let run_id = match run_id {
|
||||
@@ -112,6 +114,7 @@ pub async fn run_sync(
|
||||
false, // dry_run - sync has its own dry_run handling
|
||||
ingest_display,
|
||||
Some(spinner.clone()),
|
||||
signal,
|
||||
)
|
||||
.await?;
|
||||
result.issues_updated = issues_result.issues_upserted;
|
||||
@@ -120,6 +123,11 @@ pub async fn run_sync(
|
||||
result.resource_events_failed += issues_result.resource_events_failed;
|
||||
spinner.finish_and_clear();
|
||||
|
||||
if signal.is_cancelled() {
|
||||
info!("Shutdown requested after issues stage, returning partial sync results");
|
||||
return Ok(result);
|
||||
}
|
||||
|
||||
current_stage += 1;
|
||||
let spinner = stage_spinner(
|
||||
current_stage,
|
||||
@@ -137,6 +145,7 @@ pub async fn run_sync(
|
||||
false, // dry_run - sync has its own dry_run handling
|
||||
ingest_display,
|
||||
Some(spinner.clone()),
|
||||
signal,
|
||||
)
|
||||
.await?;
|
||||
result.mrs_updated = mrs_result.mrs_upserted;
|
||||
@@ -145,6 +154,11 @@ pub async fn run_sync(
|
||||
result.resource_events_failed += mrs_result.resource_events_failed;
|
||||
spinner.finish_and_clear();
|
||||
|
||||
if signal.is_cancelled() {
|
||||
info!("Shutdown requested after MRs stage, returning partial sync results");
|
||||
return Ok(result);
|
||||
}
|
||||
|
||||
if !options.no_docs {
|
||||
current_stage += 1;
|
||||
let spinner = stage_spinner(
|
||||
|
||||
Reference in New Issue
Block a user