theirongolddev/gitlore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(surgical-sync): add per-IID surgical sync pipeline with preflight validation

Browse Source 
Add the ability to sync specific issues or merge requests by IID without
running a full incremental sync. This enables fast, targeted data refresh
for individual entities — useful for agent workflows, debugging, and
real-time investigation of specific issues or MRs.

Architecture:
- New CLI flags: --issue <IID> and --mr <IID> (repeatable, up to 100 total)
  scoped to a single project via -p/--project
- Preflight phase validates all IIDs exist on GitLab before any DB writes,
  with TOCTOU-aware soft verification at ingest time
- 6-stage pipeline: preflight -> fetch -> ingest -> dependents -> docs -> embed
- Each stage is cancellation-aware via ShutdownSignal
- Dedicated SyncRunRecorder extensions track surgical-specific counters
  (issues_fetched, mrs_ingested, docs_regenerated, etc.)

New modules:
- src/ingestion/surgical.rs: Core surgical fetch/ingest/dependent logic
  with preflight_fetch(), ingest_issue_by_iid(), ingest_mr_by_iid(),
  and fetch_dependents_for_{issue,mr}()
- src/cli/commands/sync_surgical.rs: Full CLI orchestrator with progress
  spinners, human/robot output, and cancellation handling
- src/embedding/pipeline.rs: embed_documents_by_ids() for scoped embedding
- src/documents/regenerator.rs: regenerate_dirty_documents_for_sources()
  for scoped document regeneration

Database changes:
- Migration 027: Extends sync_runs with mode, phase, surgical_iids_json,
  per-entity counters, and cancelled_at column
- New indexes: idx_sync_runs_mode_started, idx_sync_runs_status_phase_started

GitLab client:
- get_issue_by_iid() and get_mr_by_iid() single-entity fetch methods

Error handling:
- New SurgicalPreflightFailed error variant with entity_type, iid, project,
  and reason fields. Shares exit code 6 with GitLabNotFound.

Includes comprehensive test coverage:
- 645 lines of surgical ingestion tests (wiremock-based)
- 184 lines of scoped embedding tests
- 85 lines of scoped regeneration tests
- 113 lines of GitLab client single-entity tests
- 236 lines of sync_run surgical column/counter tests
- Unit tests for SyncOptions, error codes, and CLI validation
This commit is contained in:
teernisse
2026-02-18 16:27:59 -05:00
parent ea6e45e43f
commit 9ec1344945
25 changed files with 3354 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/cli/commands/mod.rs
View File

@@ -15,6 +15,7 @@ pub mod show;
pub mod stats;
pub mod sync;
pub mod sync_status;
pub mod sync_surgical;
pub mod timeline;
pub mod trace;
pub mod who;
@@ -39,7 +40,7 @@ pub use ingest::{
DryRunPreview, IngestDisplay, print_dry_run_preview, print_dry_run_preview_json,
print_ingest_summary, print_ingest_summary_json, run_ingest, run_ingest_dry_run,
};
pub use init::{InitInputs, InitOptions, InitResult, run_init};
pub use init::{InitInputs, InitOptions, InitResult, run_init, run_token_set, run_token_show};
pub use list::{
ListFilters, MrListFilters, NoteListFilters, open_issue_in_browser, open_mr_in_browser,
print_list_issues, print_list_issues_json, print_list_mrs, print_list_mrs_json,
@@ -55,6 +56,7 @@ pub use show::{
pub use stats::{print_stats, print_stats_json, run_stats};
pub use sync::{SyncOptions, SyncResult, print_sync, print_sync_json, run_sync};
pub use sync_status::{print_sync_status, print_sync_status_json, run_sync_status};
pub use sync_surgical::run_sync_surgical;
pub use timeline::{TimelineParams, print_timeline, print_timeline_json_with_meta, run_timeline};
pub use trace::{parse_trace_path, print_trace, print_trace_json};
pub use who::{WhoRun, print_who_human, print_who_json, run_who};

175
src/cli/commands/sync.rs
View File

@@ -16,6 +16,7 @@ use super::ingest::{
DryRunPreview, IngestDisplay, ProjectStatusEnrichment, ProjectSummary, run_ingest,
run_ingest_dry_run,
};
use super::sync_surgical::run_sync_surgical;
#[derive(Debug, Default)]
pub struct SyncOptions {
@@ -26,6 +27,35 @@ pub struct SyncOptions {
pub no_events: bool,
pub robot_mode: bool,
pub dry_run: bool,
pub issue_iids: Vec<u64>,
pub mr_iids: Vec<u64>,
pub project: Option<String>,
pub preflight_only: bool,
}
impl SyncOptions {
pub const MAX_SURGICAL_TARGETS: usize = 100;
pub fn is_surgical(&self) -> bool {
!self.issue_iids.is_empty() || !self.mr_iids.is_empty()
}
}
#[derive(Debug, Default, Serialize)]
pub struct SurgicalIids {
pub issues: Vec<u64>,
pub merge_requests: Vec<u64>,
}
#[derive(Debug, Serialize)]
pub struct EntitySyncResult {
pub entity_type: String,
pub iid: u64,
pub outcome: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub error: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub toctou_reason: Option<String>,
}
#[derive(Debug, Default, Serialize)]
@@ -45,19 +75,23 @@ pub struct SyncResult {
pub embedding_failed: usize,
pub status_enrichment_errors: usize,
pub statuses_enriched: usize,
#[serde(skip_serializing_if = "Option::is_none")]
pub surgical_mode: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub surgical_iids: Option<SurgicalIids>,
#[serde(skip_serializing_if = "Option::is_none")]
pub entity_results: Option<Vec<EntitySyncResult>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub preflight_only: Option<bool>,
#[serde(skip)]
pub issue_projects: Vec<ProjectSummary>,
#[serde(skip)]
pub mr_projects: Vec<ProjectSummary>,
}
/// Apply semantic color to a stage-completion icon glyph.
/// Alias for [`Theme::color_icon`] to keep call sites concise.
fn color_icon(icon: &str, has_errors: bool) -> String {
if has_errors {
Theme::warning().render(icon)
} else {
Theme::success().render(icon)
}
Theme::color_icon(icon, has_errors)
}
pub async fn run_sync(
@@ -66,6 +100,11 @@ pub async fn run_sync(
run_id: Option<&str>,
signal: &ShutdownSignal,
) -> Result<SyncResult> {
// Surgical dispatch: if any IIDs specified, route to surgical pipeline
if options.is_surgical() {
return run_sync_surgical(config, options, run_id, signal).await;
}
let generated_id;
let run_id = match run_id {
Some(id) => id,
@@ -893,6 +932,22 @@ pub fn print_sync_dry_run_json(result: &SyncDryRunResult) {
mod tests {
use super::*;
fn default_options() -> SyncOptions {
SyncOptions {
full: false,
force: false,
no_embed: false,
no_docs: false,
no_events: false,
robot_mode: false,
dry_run: false,
issue_iids: vec![],
mr_iids: vec![],
project: None,
preflight_only: false,
}
}
#[test]
fn append_failures_skips_zeroes() {
let mut summary = "base".to_string();
@@ -1035,4 +1090,112 @@ mod tests {
assert!(rows[0].contains("0 statuses updated"));
assert!(rows[0].contains("skipped (disabled)"));
}
#[test]
fn is_surgical_with_issues() {
let opts = SyncOptions {
issue_iids: vec![1],
..default_options()
};
assert!(opts.is_surgical());
}
#[test]
fn is_surgical_with_mrs() {
let opts = SyncOptions {
mr_iids: vec![10],
..default_options()
};
assert!(opts.is_surgical());
}
#[test]
fn is_surgical_empty() {
let opts = default_options();
assert!(!opts.is_surgical());
}
#[test]
fn max_surgical_targets_is_100() {
assert_eq!(SyncOptions::MAX_SURGICAL_TARGETS, 100);
}
#[test]
fn sync_result_default_omits_surgical_fields() {
let result = SyncResult::default();
let json = serde_json::to_value(&result).unwrap();
assert!(json.get("surgical_mode").is_none());
assert!(json.get("surgical_iids").is_none());
assert!(json.get("entity_results").is_none());
assert!(json.get("preflight_only").is_none());
}
#[test]
fn sync_result_with_surgical_fields_serializes_correctly() {
let result = SyncResult {
surgical_mode: Some(true),
surgical_iids: Some(SurgicalIids {
issues: vec![7, 42],
merge_requests: vec![10],
}),
entity_results: Some(vec![
EntitySyncResult {
entity_type: "issue".to_string(),
iid: 7,
outcome: "synced".to_string(),
error: None,
toctou_reason: None,
},
EntitySyncResult {
entity_type: "issue".to_string(),
iid: 42,
outcome: "skipped_toctou".to_string(),
error: None,
toctou_reason: Some("updated_at changed".to_string()),
},
]),
preflight_only: Some(false),
..SyncResult::default()
};
let json = serde_json::to_value(&result).unwrap();
assert_eq!(json["surgical_mode"], true);
assert_eq!(json["surgical_iids"]["issues"], serde_json::json!([7, 42]));
assert_eq!(json["entity_results"].as_array().unwrap().len(), 2);
assert_eq!(json["entity_results"][1]["outcome"], "skipped_toctou");
assert_eq!(json["preflight_only"], false);
}
#[test]
fn entity_sync_result_omits_none_fields() {
let entity = EntitySyncResult {
entity_type: "merge_request".to_string(),
iid: 10,
outcome: "synced".to_string(),
error: None,
toctou_reason: None,
};
let json = serde_json::to_value(&entity).unwrap();
assert!(json.get("error").is_none());
assert!(json.get("toctou_reason").is_none());
assert!(json.get("entity_type").is_some());
}
#[test]
fn is_surgical_with_both_issues_and_mrs() {
let opts = SyncOptions {
issue_iids: vec![1, 2],
mr_iids: vec![10],
..default_options()
};
assert!(opts.is_surgical());
}
#[test]
fn is_not_surgical_with_only_project() {
let opts = SyncOptions {
project: Some("group/repo".to_string()),
..default_options()
};
assert!(!opts.is_surgical());
}
}

711
src/cli/commands/sync_surgical.rs Normal file
View File

@@ -0,0 +1,711 @@
use std::time::Instant;
use tracing::{Instrument, debug, info, warn};
use crate::Config;
use crate::cli::commands::sync::{EntitySyncResult, SurgicalIids, SyncOptions, SyncResult};
use crate::cli::progress::{format_stage_line, stage_spinner_v2};
use crate::cli::render::{Icons, Theme};
use crate::core::db::{LATEST_SCHEMA_VERSION, create_connection, get_schema_version};
use crate::core::error::{LoreError, Result};
use crate::core::lock::{AppLock, LockOptions};
use crate::core::paths::get_db_path;
use crate::core::project::resolve_project;
use crate::core::shutdown::ShutdownSignal;
use crate::core::sync_run::SyncRunRecorder;
use crate::documents::{SourceType, regenerate_dirty_documents_for_sources};
use crate::embedding::ollama::{OllamaClient, OllamaConfig};
use crate::embedding::pipeline::{DEFAULT_EMBED_CONCURRENCY, embed_documents_by_ids};
use crate::gitlab::GitLabClient;
use crate::ingestion::surgical::{
fetch_dependents_for_issue, fetch_dependents_for_mr, ingest_issue_by_iid, ingest_mr_by_iid,
preflight_fetch,
};
pub async fn run_sync_surgical(
config: &Config,
options: SyncOptions,
run_id: Option<&str>,
signal: &ShutdownSignal,
) -> Result<SyncResult> {
// ── Generate run_id ──
let generated_id;
let run_id = match run_id {
Some(id) => id,
None => {
generated_id = uuid::Uuid::new_v4().simple().to_string();
&generated_id[..8]
}
};
let span = tracing::info_span!("surgical_sync", %run_id);
async move {
let pipeline_start = Instant::now();
let mut result = SyncResult {
run_id: run_id.to_string(),
surgical_mode: Some(true),
surgical_iids: Some(SurgicalIids {
issues: options.issue_iids.clone(),
merge_requests: options.mr_iids.clone(),
}),
..SyncResult::default()
};
let mut entity_results: Vec<EntitySyncResult> = Vec::new();
// ── Resolve project ──
let project_str = options.project.as_deref().ok_or_else(|| {
LoreError::Other(
"Surgical sync requires --project. Specify the project path.".to_string(),
)
})?;
let db_path = get_db_path(config.storage.db_path.as_deref());
let conn = create_connection(&db_path)?;
let schema_version = get_schema_version(&conn);
if schema_version < LATEST_SCHEMA_VERSION {
return Err(LoreError::MigrationFailed {
version: schema_version,
message: format!(
"Database is at schema version {schema_version} but {LATEST_SCHEMA_VERSION} is required. \
Run 'lore sync' first to apply migrations."
),
source: None,
});
}
let project_id = resolve_project(&conn, project_str)?;
let gitlab_project_id: i64 = conn.query_row(
"SELECT gitlab_project_id FROM projects WHERE id = ?1",
[project_id],
|row| row.get(0),
)?;
debug!(
project_str,
project_id,
gitlab_project_id,
"Resolved project for surgical sync"
);
// ── Start recorder ──
let recorder_conn = create_connection(&db_path)?;
let recorder = SyncRunRecorder::start(&recorder_conn, "surgical-sync", run_id)?;
let iids_json = serde_json::to_string(&SurgicalIids {
issues: options.issue_iids.clone(),
merge_requests: options.mr_iids.clone(),
})
.unwrap_or_else(|_| "{}".to_string());
recorder.set_surgical_metadata(&recorder_conn, "surgical", "preflight", &iids_json)?;
// Wrap recorder in Option for consuming terminal methods
let mut recorder = Some(recorder);
// ── Build GitLab client ──
let token = config.gitlab.resolve_token()?;
let client = GitLabClient::new(
&config.gitlab.base_url,
&token,
Some(config.sync.requests_per_second),
);
// ── Build targets list ──
let mut targets: Vec<(String, i64)> = Vec::new();
for iid in &options.issue_iids {
targets.push(("issue".to_string(), *iid as i64));
}
for iid in &options.mr_iids {
targets.push(("merge_request".to_string(), *iid as i64));
}
// ── Stage: Preflight ──
let stage_start = Instant::now();
let spinner =
stage_spinner_v2(Icons::sync(), "Preflight", "fetching...", options.robot_mode);
info!(targets = targets.len(), "Preflight: fetching entities from GitLab");
let preflight = preflight_fetch(&client, gitlab_project_id, &targets).await;
// Record preflight failures
for failure in &preflight.failures {
let is_not_found = matches!(&failure.error, LoreError::GitLabNotFound { .. });
entity_results.push(EntitySyncResult {
entity_type: failure.entity_type.clone(),
iid: failure.iid as u64,
outcome: if is_not_found {
"not_found".to_string()
} else {
"preflight_failed".to_string()
},
error: Some(failure.error.to_string()),
toctou_reason: None,
});
if let Some(ref rec) = recorder {
let _ = rec.record_entity_result(&recorder_conn, &failure.entity_type, "warning");
}
}
let preflight_summary = format!(
"{} issues, {} MRs fetched ({} failed)",
preflight.issues.len(),
preflight.merge_requests.len(),
preflight.failures.len()
);
let preflight_icon = color_icon(
if preflight.failures.is_empty() {
Icons::success()
} else {
Icons::warning()
},
!preflight.failures.is_empty(),
);
emit_stage_line(
&spinner,
&preflight_icon,
"Preflight",
&preflight_summary,
stage_start.elapsed(),
options.robot_mode,
);
// ── Preflight-only early return ──
if options.preflight_only {
result.preflight_only = Some(true);
result.entity_results = Some(entity_results);
if let Some(rec) = recorder.take() {
rec.succeed(&recorder_conn, &[], 0, preflight.failures.len())?;
}
return Ok(result);
}
// ── Check cancellation ──
if signal.is_cancelled() {
if let Some(rec) = recorder.take() {
rec.cancel(&recorder_conn, "cancelled before ingest")?;
}
result.entity_results = Some(entity_results);
return Ok(result);
}
// ── Acquire lock ──
let lock_conn = create_connection(&db_path)?;
let mut lock = AppLock::new(
lock_conn,
LockOptions {
name: "sync".to_string(),
stale_lock_minutes: config.sync.stale_lock_minutes,
heartbeat_interval_seconds: config.sync.heartbeat_interval_seconds,
},
);
lock.acquire(options.force)?;
// Wrap the rest in a closure-like block to ensure lock release on error
let pipeline_result = run_pipeline_stages(
&conn,
&recorder_conn,
config,
&client,
&options,
&preflight,
project_id,
gitlab_project_id,
&mut entity_results,
&mut result,
recorder.as_ref(),
signal,
)
.await;
match pipeline_result {
Ok(()) => {
// ── Finalize: succeed ──
if let Some(ref rec) = recorder {
let _ = rec.update_phase(&recorder_conn, "finalize");
}
let total_items = result.issues_updated
+ result.mrs_updated
+ result.documents_regenerated
+ result.documents_embedded;
let total_errors = result.documents_errored
+ result.embedding_failed
+ entity_results
.iter()
.filter(|e| e.outcome != "synced" && e.outcome != "skipped_stale")
.count();
if let Some(rec) = recorder.take() {
rec.succeed(&recorder_conn, &[], total_items, total_errors)?;
}
}
Err(ref e) => {
if let Some(rec) = recorder.take() {
let _ = rec.fail(&recorder_conn, &e.to_string(), None);
}
}
}
lock.release();
// Propagate error after cleanup
pipeline_result?;
result.entity_results = Some(entity_results);
let elapsed = pipeline_start.elapsed();
debug!(
elapsed_ms = elapsed.as_millis(),
issues = result.issues_updated,
mrs = result.mrs_updated,
docs = result.documents_regenerated,
embedded = result.documents_embedded,
"Surgical sync pipeline complete"
);
Ok(result)
}
.instrument(span)
.await
}
#[allow(clippy::too_many_arguments)]
async fn run_pipeline_stages(
conn: &rusqlite::Connection,
recorder_conn: &rusqlite::Connection,
config: &Config,
client: &GitLabClient,
options: &SyncOptions,
preflight: &crate::ingestion::surgical::PreflightResult,
project_id: i64,
gitlab_project_id: i64,
entity_results: &mut Vec<EntitySyncResult>,
result: &mut SyncResult,
recorder: Option<&SyncRunRecorder>,
signal: &ShutdownSignal,
) -> Result<()> {
let mut all_dirty_source_keys: Vec<(SourceType, i64)> = Vec::new();
// ── Stage: Ingest ──
if let Some(rec) = recorder {
rec.update_phase(recorder_conn, "ingest")?;
}
let stage_start = Instant::now();
let spinner = stage_spinner_v2(Icons::sync(), "Ingest", "processing...", options.robot_mode);
// Ingest issues
for issue in &preflight.issues {
match ingest_issue_by_iid(conn, config, project_id, issue) {
Ok(ingest_result) => {
if ingest_result.skipped_stale {
entity_results.push(EntitySyncResult {
entity_type: "issue".to_string(),
iid: issue.iid as u64,
outcome: "skipped_stale".to_string(),
error: None,
toctou_reason: Some("updated_at not newer than DB".to_string()),
});
if let Some(rec) = recorder {
let _ = rec.record_entity_result(recorder_conn, "issue", "skipped_stale");
}
} else {
result.issues_updated += 1;
all_dirty_source_keys.extend(ingest_result.dirty_source_keys);
entity_results.push(EntitySyncResult {
entity_type: "issue".to_string(),
iid: issue.iid as u64,
outcome: "synced".to_string(),
error: None,
toctou_reason: None,
});
if let Some(rec) = recorder {
let _ = rec.record_entity_result(recorder_conn, "issue", "ingested");
}
}
}
Err(e) => {
warn!(iid = issue.iid, error = %e, "Failed to ingest issue");
entity_results.push(EntitySyncResult {
entity_type: "issue".to_string(),
iid: issue.iid as u64,
outcome: "error".to_string(),
error: Some(e.to_string()),
toctou_reason: None,
});
if let Some(rec) = recorder {
let _ = rec.record_entity_result(recorder_conn, "issue", "warning");
}
}
}
}
// Ingest MRs
for mr in &preflight.merge_requests {
match ingest_mr_by_iid(conn, config, project_id, mr) {
Ok(ingest_result) => {
if ingest_result.skipped_stale {
entity_results.push(EntitySyncResult {
entity_type: "merge_request".to_string(),
iid: mr.iid as u64,
outcome: "skipped_stale".to_string(),
error: None,
toctou_reason: Some("updated_at not newer than DB".to_string()),
});
if let Some(rec) = recorder {
let _ = rec.record_entity_result(recorder_conn, "mr", "skipped_stale");
}
} else {
result.mrs_updated += 1;
all_dirty_source_keys.extend(ingest_result.dirty_source_keys);
entity_results.push(EntitySyncResult {
entity_type: "merge_request".to_string(),
iid: mr.iid as u64,
outcome: "synced".to_string(),
error: None,
toctou_reason: None,
});
if let Some(rec) = recorder {
let _ = rec.record_entity_result(recorder_conn, "mr", "ingested");
}
}
}
Err(e) => {
warn!(iid = mr.iid, error = %e, "Failed to ingest MR");
entity_results.push(EntitySyncResult {
entity_type: "merge_request".to_string(),
iid: mr.iid as u64,
outcome: "error".to_string(),
error: Some(e.to_string()),
toctou_reason: None,
});
if let Some(rec) = recorder {
let _ = rec.record_entity_result(recorder_conn, "mr", "warning");
}
}
}
}
let ingest_summary = format!(
"{} issues, {} MRs ingested",
result.issues_updated, result.mrs_updated
);
let ingest_icon = color_icon(Icons::success(), false);
emit_stage_line(
&spinner,
&ingest_icon,
"Ingest",
&ingest_summary,
stage_start.elapsed(),
options.robot_mode,
);
// ── Check cancellation ──
if signal.is_cancelled() {
debug!("Shutdown requested after ingest stage");
return Ok(());
}
// ── Stage: Dependents ──
if let Some(rec) = recorder {
rec.update_phase(recorder_conn, "dependents")?;
}
let stage_start = Instant::now();
let spinner = stage_spinner_v2(
Icons::sync(),
"Dependents",
"fetching...",
options.robot_mode,
);
let mut total_discussions: usize = 0;
let mut total_events: usize = 0;
// Fetch dependents for successfully ingested issues
for issue in &preflight.issues {
// Only fetch dependents for entities that were actually ingested
let was_ingested = entity_results.iter().any(|e| {
e.entity_type == "issue" && e.iid == issue.iid as u64 && e.outcome == "synced"
});
if !was_ingested {
continue;
}
let local_id: i64 = match conn.query_row(
"SELECT id FROM issues WHERE project_id = ?1 AND iid = ?2",
(project_id, issue.iid),
|row| row.get(0),
) {
Ok(id) => id,
Err(e) => {
warn!(iid = issue.iid, error = %e, "Could not find local issue ID for dependents");
continue;
}
};
match fetch_dependents_for_issue(
client,
conn,
project_id,
gitlab_project_id,
issue.iid,
local_id,
config,
)
.await
{
Ok(dep_result) => {
total_discussions += dep_result.discussions_fetched;
total_events += dep_result.resource_events_fetched;
result.discussions_fetched += dep_result.discussions_fetched;
result.resource_events_fetched += dep_result.resource_events_fetched;
}
Err(e) => {
warn!(iid = issue.iid, error = %e, "Failed to fetch dependents for issue");
}
}
}
// Fetch dependents for successfully ingested MRs
for mr in &preflight.merge_requests {
let was_ingested = entity_results.iter().any(|e| {
e.entity_type == "merge_request" && e.iid == mr.iid as u64 && e.outcome == "synced"
});
if !was_ingested {
continue;
}
let local_id: i64 = match conn.query_row(
"SELECT id FROM merge_requests WHERE project_id = ?1 AND iid = ?2",
(project_id, mr.iid),
|row| row.get(0),
) {
Ok(id) => id,
Err(e) => {
warn!(iid = mr.iid, error = %e, "Could not find local MR ID for dependents");
continue;
}
};
match fetch_dependents_for_mr(
client,
conn,
project_id,
gitlab_project_id,
mr.iid,
local_id,
config,
)
.await
{
Ok(dep_result) => {
total_discussions += dep_result.discussions_fetched;
total_events += dep_result.resource_events_fetched;
result.discussions_fetched += dep_result.discussions_fetched;
result.resource_events_fetched += dep_result.resource_events_fetched;
result.mr_diffs_fetched += dep_result.file_changes_stored;
}
Err(e) => {
warn!(iid = mr.iid, error = %e, "Failed to fetch dependents for MR");
}
}
}
let dep_summary = format!("{} discussions, {} events", total_discussions, total_events);
let dep_icon = color_icon(Icons::success(), false);
emit_stage_line(
&spinner,
&dep_icon,
"Dependents",
&dep_summary,
stage_start.elapsed(),
options.robot_mode,
);
// ── Check cancellation ──
if signal.is_cancelled() {
debug!("Shutdown requested after dependents stage");
return Ok(());
}
// ── Stage: Docs ──
if !options.no_docs && !all_dirty_source_keys.is_empty() {
if let Some(rec) = recorder {
rec.update_phase(recorder_conn, "docs")?;
}
let stage_start = Instant::now();
let spinner =
stage_spinner_v2(Icons::sync(), "Docs", "regenerating...", options.robot_mode);
let docs_result = regenerate_dirty_documents_for_sources(conn, &all_dirty_source_keys)?;
result.documents_regenerated = docs_result.regenerated;
result.documents_errored = docs_result.errored;
for _ in 0..docs_result.regenerated {
if let Some(rec) = recorder {
let _ = rec.record_entity_result(recorder_conn, "doc", "regenerated");
}
}
let docs_summary = format!("{} documents regenerated", result.documents_regenerated);
let docs_icon = color_icon(
if docs_result.errored > 0 {
Icons::warning()
} else {
Icons::success()
},
docs_result.errored > 0,
);
emit_stage_line(
&spinner,
&docs_icon,
"Docs",
&docs_summary,
stage_start.elapsed(),
options.robot_mode,
);
// ── Check cancellation ──
if signal.is_cancelled() {
debug!("Shutdown requested after docs stage");
return Ok(());
}
// ── Stage: Embed ──
if !options.no_embed && !docs_result.document_ids.is_empty() {
if let Some(rec) = recorder {
rec.update_phase(recorder_conn, "embed")?;
}
let stage_start = Instant::now();
let spinner =
stage_spinner_v2(Icons::sync(), "Embed", "embedding...", options.robot_mode);
let ollama_config = OllamaConfig {
base_url: config.embedding.base_url.clone(),
model: config.embedding.model.clone(),
..OllamaConfig::default()
};
let ollama_client = OllamaClient::new(ollama_config);
let model_name = &config.embedding.model;
let concurrency = if config.embedding.concurrency > 0 {
config.embedding.concurrency as usize
} else {
DEFAULT_EMBED_CONCURRENCY
};
match embed_documents_by_ids(
conn,
&ollama_client,
model_name,
concurrency,
&docs_result.document_ids,
signal,
)
.await
{
Ok(embed_result) => {
result.documents_embedded = embed_result.docs_embedded;
result.embedding_failed = embed_result.failed;
for _ in 0..embed_result.docs_embedded {
if let Some(rec) = recorder {
let _ = rec.record_entity_result(recorder_conn, "doc", "embedded");
}
}
let embed_summary = format!("{} chunks embedded", embed_result.chunks_embedded);
let embed_icon = color_icon(
if embed_result.failed > 0 {
Icons::warning()
} else {
Icons::success()
},
embed_result.failed > 0,
);
emit_stage_line(
&spinner,
&embed_icon,
"Embed",
&embed_summary,
stage_start.elapsed(),
options.robot_mode,
);
}
Err(e) => {
let warn_summary = format!("skipped ({})", e);
let warn_icon = color_icon(Icons::warning(), true);
emit_stage_line(
&spinner,
&warn_icon,
"Embed",
&warn_summary,
stage_start.elapsed(),
options.robot_mode,
);
warn!(error = %e, "Embedding stage failed (Ollama may be unavailable), continuing");
}
}
}
}
Ok(())
}
/// Alias for [`Theme::color_icon`] to keep call sites concise.
fn color_icon(icon: &str, has_errors: bool) -> String {
Theme::color_icon(icon, has_errors)
}
fn emit_stage_line(
pb: &indicatif::ProgressBar,
icon: &str,
label: &str,
summary: &str,
elapsed: std::time::Duration,
robot_mode: bool,
) {
pb.finish_and_clear();
if !robot_mode {
crate::cli::progress::multi().suspend(|| {
println!("{}", format_stage_line(icon, label, summary, elapsed));
});
}
}
#[cfg(test)]
mod tests {
use crate::cli::commands::sync::SyncOptions;
#[test]
fn sync_options_is_surgical_required() {
let opts = SyncOptions {
issue_iids: vec![1],
project: Some("group/repo".to_string()),
..SyncOptions::default()
};
assert!(opts.is_surgical());
}
#[test]
fn sync_options_surgical_with_mrs() {
let opts = SyncOptions {
mr_iids: vec![10, 20],
project: Some("group/repo".to_string()),
..SyncOptions::default()
};
assert!(opts.is_surgical());
}
#[test]
fn sync_options_not_surgical_without_iids() {
let opts = SyncOptions {
project: Some("group/repo".to_string()),
..SyncOptions::default()
};
assert!(!opts.is_surgical());
}
}