feat(events): Wire resource event fetching into sync pipeline (bd-1ep)
Integrate resource event fetching as Step 4 of both issue and MR ingestion, gated behind the fetch_resource_events config flag. Orchestrator changes: - Add ProgressEvent variants: ResourceEventsFetchStarted, ResourceEventFetched, ResourceEventsFetchComplete - Add resource_events_fetched/failed fields to IngestProjectResult and IngestMrProjectResult - New enqueue_resource_events_for_entity_type() queries all issues/MRs for a project and enqueues resource_events jobs via the dependent queue (INSERT OR IGNORE for idempotency) - New drain_resource_events() claims jobs in batches, fetches state/label/milestone events from GitLab API, stores them atomically via unchecked_transaction, and handles failures with exponential backoff via fail_job() - Max-iterations guard prevents infinite retry loops within a single drain run - New store_resource_events() + per-type _tx helpers write events using prepared statements inside a single transaction - DrainResult struct tracks fetched/failed counts CLI ingest changes: - IngestResult gains resource_events_fetched/failed fields - Progress bar repurposed for resource event fetch phase (reuses discussion bar with updated template) - Accumulates event counts from both issue and MR ingestion CLI sync changes: - SyncResult gains resource_events_fetched/failed fields - Accumulates counts from both ingest stages - print_sync() conditionally displays event counts - Structured logging includes event counts Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -39,6 +39,9 @@ pub struct IngestResult {
|
||||
pub labels_created: usize,
|
||||
pub discussions_fetched: usize,
|
||||
pub notes_upserted: usize,
|
||||
// Resource events
|
||||
pub resource_events_fetched: usize,
|
||||
pub resource_events_failed: usize,
|
||||
}
|
||||
|
||||
/// Controls what interactive UI elements `run_ingest` displays.
|
||||
@@ -57,17 +60,26 @@ pub struct IngestDisplay {
|
||||
impl IngestDisplay {
|
||||
/// Interactive mode: everything visible.
|
||||
pub fn interactive() -> Self {
|
||||
Self { show_progress: true, show_text: true }
|
||||
Self {
|
||||
show_progress: true,
|
||||
show_text: true,
|
||||
}
|
||||
}
|
||||
|
||||
/// Robot/JSON mode: everything hidden.
|
||||
pub fn silent() -> Self {
|
||||
Self { show_progress: false, show_text: false }
|
||||
Self {
|
||||
show_progress: false,
|
||||
show_text: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Progress only (used by sync in interactive mode).
|
||||
pub fn progress_only() -> Self {
|
||||
Self { show_progress: true, show_text: false }
|
||||
Self {
|
||||
show_progress: true,
|
||||
show_text: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -105,9 +117,10 @@ pub async fn run_ingest(
|
||||
lock.acquire(force)?;
|
||||
|
||||
// Get token from environment
|
||||
let token = std::env::var(&config.gitlab.token_env_var).map_err(|_| LoreError::TokenNotSet {
|
||||
env_var: config.gitlab.token_env_var.clone(),
|
||||
})?;
|
||||
let token =
|
||||
std::env::var(&config.gitlab.token_env_var).map_err(|_| LoreError::TokenNotSet {
|
||||
env_var: config.gitlab.token_env_var.clone(),
|
||||
})?;
|
||||
|
||||
// Create GitLab client
|
||||
let client = GitLabClient::new(&config.gitlab.base_url, &token, None);
|
||||
@@ -199,7 +212,9 @@ pub async fn run_ingest(
|
||||
let b = ProgressBar::new(0);
|
||||
b.set_style(
|
||||
ProgressStyle::default_bar()
|
||||
.template(" {spinner:.blue} Syncing discussions [{bar:30.cyan/dim}] {pos}/{len}")
|
||||
.template(
|
||||
" {spinner:.blue} Syncing discussions [{bar:30.cyan/dim}] {pos}/{len}",
|
||||
)
|
||||
.unwrap()
|
||||
.progress_chars("=> "),
|
||||
);
|
||||
@@ -237,6 +252,23 @@ pub async fn run_ingest(
|
||||
ProgressEvent::MrDiscussionSyncComplete => {
|
||||
disc_bar_clone.finish_and_clear();
|
||||
}
|
||||
ProgressEvent::ResourceEventsFetchStarted { total } => {
|
||||
disc_bar_clone.set_length(total as u64);
|
||||
disc_bar_clone.set_position(0);
|
||||
disc_bar_clone.set_style(
|
||||
ProgressStyle::default_bar()
|
||||
.template(" {spinner:.blue} Fetching resource events [{bar:30.cyan/dim}] {pos}/{len}")
|
||||
.unwrap()
|
||||
.progress_chars("=> "),
|
||||
);
|
||||
disc_bar_clone.enable_steady_tick(std::time::Duration::from_millis(100));
|
||||
}
|
||||
ProgressEvent::ResourceEventFetched { current, total: _ } => {
|
||||
disc_bar_clone.set_position(current as u64);
|
||||
}
|
||||
ProgressEvent::ResourceEventsFetchComplete { .. } => {
|
||||
disc_bar_clone.finish_and_clear();
|
||||
}
|
||||
_ => {}
|
||||
})
|
||||
};
|
||||
@@ -269,6 +301,8 @@ pub async fn run_ingest(
|
||||
total.notes_upserted += result.notes_upserted;
|
||||
total.issues_synced_discussions += result.issues_synced_discussions;
|
||||
total.issues_skipped_discussion_sync += result.issues_skipped_discussion_sync;
|
||||
total.resource_events_fetched += result.resource_events_fetched;
|
||||
total.resource_events_failed += result.resource_events_failed;
|
||||
} else {
|
||||
let result = ingest_project_merge_requests_with_progress(
|
||||
&conn,
|
||||
@@ -301,6 +335,8 @@ pub async fn run_ingest(
|
||||
total.diffnotes_count += result.diffnotes_count;
|
||||
total.mrs_synced_discussions += result.mrs_synced_discussions;
|
||||
total.mrs_skipped_discussion_sync += result.mrs_skipped_discussion_sync;
|
||||
total.resource_events_fetched += result.resource_events_fetched;
|
||||
total.resource_events_failed += result.resource_events_failed;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user