feat(events): Wire resource event fetching into sync pipeline (bd-1ep)

Integrate resource event fetching as Step 4 of both issue and MR
ingestion, gated behind the fetch_resource_events config flag.

Orchestrator changes:
- Add ProgressEvent variants: ResourceEventsFetchStarted,
  ResourceEventFetched, ResourceEventsFetchComplete
- Add resource_events_fetched/failed fields to IngestProjectResult
  and IngestMrProjectResult
- New enqueue_resource_events_for_entity_type() queries all
  issues/MRs for a project and enqueues resource_events jobs via
  the dependent queue (INSERT OR IGNORE for idempotency)
- New drain_resource_events() claims jobs in batches, fetches
  state/label/milestone events from GitLab API, stores them
  atomically via unchecked_transaction, and handles failures
  with exponential backoff via fail_job()
- Max-iterations guard prevents infinite retry loops within a
  single drain run
- New store_resource_events() + per-type _tx helpers write events
  using prepared statements inside a single transaction
- DrainResult struct tracks fetched/failed counts

CLI ingest changes:
- IngestResult gains resource_events_fetched/failed fields
- Progress bar repurposed for resource event fetch phase
  (reuses discussion bar with updated template)
- Accumulates event counts from both issue and MR ingestion

CLI sync changes:
- SyncResult gains resource_events_fetched/failed fields
- Accumulates counts from both ingest stages
- print_sync() conditionally displays event counts
- Structured logging includes event counts

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Taylor Eernisse
2026-02-03 13:02:15 -05:00
parent a50fc78823
commit 2bcd8db0e9
3 changed files with 548 additions and 28 deletions

View File

@@ -39,6 +39,9 @@ pub struct IngestResult {
pub labels_created: usize, pub labels_created: usize,
pub discussions_fetched: usize, pub discussions_fetched: usize,
pub notes_upserted: usize, pub notes_upserted: usize,
// Resource events
pub resource_events_fetched: usize,
pub resource_events_failed: usize,
} }
/// Controls what interactive UI elements `run_ingest` displays. /// Controls what interactive UI elements `run_ingest` displays.
@@ -57,17 +60,26 @@ pub struct IngestDisplay {
impl IngestDisplay { impl IngestDisplay {
/// Interactive mode: everything visible. /// Interactive mode: everything visible.
pub fn interactive() -> Self { pub fn interactive() -> Self {
Self { show_progress: true, show_text: true } Self {
show_progress: true,
show_text: true,
}
} }
/// Robot/JSON mode: everything hidden. /// Robot/JSON mode: everything hidden.
pub fn silent() -> Self { pub fn silent() -> Self {
Self { show_progress: false, show_text: false } Self {
show_progress: false,
show_text: false,
}
} }
/// Progress only (used by sync in interactive mode). /// Progress only (used by sync in interactive mode).
pub fn progress_only() -> Self { pub fn progress_only() -> Self {
Self { show_progress: true, show_text: false } Self {
show_progress: true,
show_text: false,
}
} }
} }
@@ -105,7 +117,8 @@ pub async fn run_ingest(
lock.acquire(force)?; lock.acquire(force)?;
// Get token from environment // Get token from environment
let token = std::env::var(&config.gitlab.token_env_var).map_err(|_| LoreError::TokenNotSet { let token =
std::env::var(&config.gitlab.token_env_var).map_err(|_| LoreError::TokenNotSet {
env_var: config.gitlab.token_env_var.clone(), env_var: config.gitlab.token_env_var.clone(),
})?; })?;
@@ -199,7 +212,9 @@ pub async fn run_ingest(
let b = ProgressBar::new(0); let b = ProgressBar::new(0);
b.set_style( b.set_style(
ProgressStyle::default_bar() ProgressStyle::default_bar()
.template(" {spinner:.blue} Syncing discussions [{bar:30.cyan/dim}] {pos}/{len}") .template(
" {spinner:.blue} Syncing discussions [{bar:30.cyan/dim}] {pos}/{len}",
)
.unwrap() .unwrap()
.progress_chars("=> "), .progress_chars("=> "),
); );
@@ -237,6 +252,23 @@ pub async fn run_ingest(
ProgressEvent::MrDiscussionSyncComplete => { ProgressEvent::MrDiscussionSyncComplete => {
disc_bar_clone.finish_and_clear(); disc_bar_clone.finish_and_clear();
} }
ProgressEvent::ResourceEventsFetchStarted { total } => {
disc_bar_clone.set_length(total as u64);
disc_bar_clone.set_position(0);
disc_bar_clone.set_style(
ProgressStyle::default_bar()
.template(" {spinner:.blue} Fetching resource events [{bar:30.cyan/dim}] {pos}/{len}")
.unwrap()
.progress_chars("=> "),
);
disc_bar_clone.enable_steady_tick(std::time::Duration::from_millis(100));
}
ProgressEvent::ResourceEventFetched { current, total: _ } => {
disc_bar_clone.set_position(current as u64);
}
ProgressEvent::ResourceEventsFetchComplete { .. } => {
disc_bar_clone.finish_and_clear();
}
_ => {} _ => {}
}) })
}; };
@@ -269,6 +301,8 @@ pub async fn run_ingest(
total.notes_upserted += result.notes_upserted; total.notes_upserted += result.notes_upserted;
total.issues_synced_discussions += result.issues_synced_discussions; total.issues_synced_discussions += result.issues_synced_discussions;
total.issues_skipped_discussion_sync += result.issues_skipped_discussion_sync; total.issues_skipped_discussion_sync += result.issues_skipped_discussion_sync;
total.resource_events_fetched += result.resource_events_fetched;
total.resource_events_failed += result.resource_events_failed;
} else { } else {
let result = ingest_project_merge_requests_with_progress( let result = ingest_project_merge_requests_with_progress(
&conn, &conn,
@@ -301,6 +335,8 @@ pub async fn run_ingest(
total.diffnotes_count += result.diffnotes_count; total.diffnotes_count += result.diffnotes_count;
total.mrs_synced_discussions += result.mrs_synced_discussions; total.mrs_synced_discussions += result.mrs_synced_discussions;
total.mrs_skipped_discussion_sync += result.mrs_skipped_discussion_sync; total.mrs_skipped_discussion_sync += result.mrs_skipped_discussion_sync;
total.resource_events_fetched += result.resource_events_fetched;
total.resource_events_failed += result.resource_events_failed;
} }
} }

View File

@@ -29,6 +29,8 @@ pub struct SyncResult {
pub issues_updated: usize, pub issues_updated: usize,
pub mrs_updated: usize, pub mrs_updated: usize,
pub discussions_fetched: usize, pub discussions_fetched: usize,
pub resource_events_fetched: usize,
pub resource_events_failed: usize,
pub documents_regenerated: usize, pub documents_regenerated: usize,
pub documents_embedded: usize, pub documents_embedded: usize,
} }
@@ -70,26 +72,61 @@ pub async fn run_sync(config: &Config, options: SyncOptions) -> Result<SyncResul
// Stage 1: Ingest issues // Stage 1: Ingest issues
current_stage += 1; current_stage += 1;
let spinner = stage_spinner(current_stage, total_stages, "Fetching issues from GitLab...", options.robot_mode); let spinner = stage_spinner(
current_stage,
total_stages,
"Fetching issues from GitLab...",
options.robot_mode,
);
info!("Sync stage {current_stage}/{total_stages}: ingesting issues"); info!("Sync stage {current_stage}/{total_stages}: ingesting issues");
let issues_result = run_ingest(config, "issues", None, options.force, options.full, ingest_display).await?; let issues_result = run_ingest(
config,
"issues",
None,
options.force,
options.full,
ingest_display,
)
.await?;
result.issues_updated = issues_result.issues_upserted; result.issues_updated = issues_result.issues_upserted;
result.discussions_fetched += issues_result.discussions_fetched; result.discussions_fetched += issues_result.discussions_fetched;
result.resource_events_fetched += issues_result.resource_events_fetched;
result.resource_events_failed += issues_result.resource_events_failed;
spinner.finish_and_clear(); spinner.finish_and_clear();
// Stage 2: Ingest MRs // Stage 2: Ingest MRs
current_stage += 1; current_stage += 1;
let spinner = stage_spinner(current_stage, total_stages, "Fetching merge requests from GitLab...", options.robot_mode); let spinner = stage_spinner(
current_stage,
total_stages,
"Fetching merge requests from GitLab...",
options.robot_mode,
);
info!("Sync stage {current_stage}/{total_stages}: ingesting merge requests"); info!("Sync stage {current_stage}/{total_stages}: ingesting merge requests");
let mrs_result = run_ingest(config, "mrs", None, options.force, options.full, ingest_display).await?; let mrs_result = run_ingest(
config,
"mrs",
None,
options.force,
options.full,
ingest_display,
)
.await?;
result.mrs_updated = mrs_result.mrs_upserted; result.mrs_updated = mrs_result.mrs_upserted;
result.discussions_fetched += mrs_result.discussions_fetched; result.discussions_fetched += mrs_result.discussions_fetched;
result.resource_events_fetched += mrs_result.resource_events_fetched;
result.resource_events_failed += mrs_result.resource_events_failed;
spinner.finish_and_clear(); spinner.finish_and_clear();
// Stage 3: Generate documents (unless --no-docs) // Stage 3: Generate documents (unless --no-docs)
if !options.no_docs { if !options.no_docs {
current_stage += 1; current_stage += 1;
let spinner = stage_spinner(current_stage, total_stages, "Processing documents...", options.robot_mode); let spinner = stage_spinner(
current_stage,
total_stages,
"Processing documents...",
options.robot_mode,
);
info!("Sync stage {current_stage}/{total_stages}: generating documents"); info!("Sync stage {current_stage}/{total_stages}: generating documents");
let docs_result = run_generate_docs(config, false, None)?; let docs_result = run_generate_docs(config, false, None)?;
result.documents_regenerated = docs_result.regenerated; result.documents_regenerated = docs_result.regenerated;
@@ -101,7 +138,12 @@ pub async fn run_sync(config: &Config, options: SyncOptions) -> Result<SyncResul
// Stage 4: Embed documents (unless --no-embed) // Stage 4: Embed documents (unless --no-embed)
if !options.no_embed { if !options.no_embed {
current_stage += 1; current_stage += 1;
let spinner = stage_spinner(current_stage, total_stages, "Generating embeddings...", options.robot_mode); let spinner = stage_spinner(
current_stage,
total_stages,
"Generating embeddings...",
options.robot_mode,
);
info!("Sync stage {current_stage}/{total_stages}: embedding documents"); info!("Sync stage {current_stage}/{total_stages}: embedding documents");
match run_embed(config, options.full, false).await { match run_embed(config, options.full, false).await {
Ok(embed_result) => { Ok(embed_result) => {
@@ -112,11 +154,7 @@ pub async fn run_sync(config: &Config, options: SyncOptions) -> Result<SyncResul
// Graceful degradation: Ollama down is a warning, not an error // Graceful degradation: Ollama down is a warning, not an error
spinner.finish_and_clear(); spinner.finish_and_clear();
if !options.robot_mode { if !options.robot_mode {
eprintln!( eprintln!(" {} Embedding skipped ({})", style("warn").yellow(), e);
" {} Embedding skipped ({})",
style("warn").yellow(),
e
);
} }
warn!(error = %e, "Embedding stage failed (Ollama may be unavailable), continuing"); warn!(error = %e, "Embedding stage failed (Ollama may be unavailable), continuing");
} }
@@ -129,6 +167,8 @@ pub async fn run_sync(config: &Config, options: SyncOptions) -> Result<SyncResul
issues = result.issues_updated, issues = result.issues_updated,
mrs = result.mrs_updated, mrs = result.mrs_updated,
discussions = result.discussions_fetched, discussions = result.discussions_fetched,
resource_events = result.resource_events_fetched,
resource_events_failed = result.resource_events_failed,
docs = result.documents_regenerated, docs = result.documents_regenerated,
embedded = result.documents_embedded, embedded = result.documents_embedded,
"Sync pipeline complete" "Sync pipeline complete"
@@ -139,19 +179,31 @@ pub async fn run_sync(config: &Config, options: SyncOptions) -> Result<SyncResul
/// Print human-readable sync summary. /// Print human-readable sync summary.
pub fn print_sync(result: &SyncResult, elapsed: std::time::Duration) { pub fn print_sync(result: &SyncResult, elapsed: std::time::Duration) {
println!( println!("{} Sync complete:", style("done").green().bold(),);
"{} Sync complete:",
style("done").green().bold(),
);
println!(" Issues updated: {}", result.issues_updated); println!(" Issues updated: {}", result.issues_updated);
println!(" MRs updated: {}", result.mrs_updated); println!(" MRs updated: {}", result.mrs_updated);
println!(" Discussions fetched: {}", result.discussions_fetched);
println!(" Documents regenerated: {}", result.documents_regenerated);
println!(" Documents embedded: {}", result.documents_embedded);
println!( println!(
" Elapsed: {:.1}s", " Discussions fetched: {}",
elapsed.as_secs_f64() result.discussions_fetched
); );
if result.resource_events_fetched > 0 || result.resource_events_failed > 0 {
println!(
" Resource events fetched: {}",
result.resource_events_fetched
);
if result.resource_events_failed > 0 {
println!(
" Resource events failed: {}",
result.resource_events_failed
);
}
}
println!(
" Documents regenerated: {}",
result.documents_regenerated
);
println!(" Documents embedded: {}", result.documents_embedded);
println!(" Elapsed: {:.1}s", elapsed.as_secs_f64());
} }
/// JSON output for sync. /// JSON output for sync.

View File

@@ -7,9 +7,12 @@
use futures::future::join_all; use futures::future::join_all;
use rusqlite::Connection; use rusqlite::Connection;
use tracing::info; use tracing::{debug, info, warn};
use crate::Config; use crate::Config;
use crate::core::dependent_queue::{
claim_jobs, complete_job, count_pending_jobs, enqueue_job, fail_job, reclaim_stale_locks,
};
use crate::core::error::Result; use crate::core::error::Result;
use crate::gitlab::GitLabClient; use crate::gitlab::GitLabClient;
@@ -50,6 +53,12 @@ pub enum ProgressEvent {
MrDiscussionSynced { current: usize, total: usize }, MrDiscussionSynced { current: usize, total: usize },
/// MR discussion sync complete /// MR discussion sync complete
MrDiscussionSyncComplete, MrDiscussionSyncComplete,
/// Resource event fetching started (total jobs)
ResourceEventsFetchStarted { total: usize },
/// Resource event fetched for an entity (current/total)
ResourceEventFetched { current: usize, total: usize },
/// Resource event fetching complete
ResourceEventsFetchComplete { fetched: usize, failed: usize },
} }
/// Result of full project ingestion (issues). /// Result of full project ingestion (issues).
@@ -63,6 +72,8 @@ pub struct IngestProjectResult {
pub notes_upserted: usize, pub notes_upserted: usize,
pub issues_synced_discussions: usize, pub issues_synced_discussions: usize,
pub issues_skipped_discussion_sync: usize, pub issues_skipped_discussion_sync: usize,
pub resource_events_fetched: usize,
pub resource_events_failed: usize,
} }
/// Result of MR ingestion for a project. /// Result of MR ingestion for a project.
@@ -80,6 +91,8 @@ pub struct IngestMrProjectResult {
pub diffnotes_count: usize, pub diffnotes_count: usize,
pub mrs_synced_discussions: usize, pub mrs_synced_discussions: usize,
pub mrs_skipped_discussion_sync: usize, pub mrs_skipped_discussion_sync: usize,
pub resource_events_fetched: usize,
pub resource_events_failed: usize,
} }
/// Ingest all issues and their discussions for a project. /// Ingest all issues and their discussions for a project.
@@ -167,6 +180,21 @@ pub async fn ingest_project_issues_with_progress(
result.issues_synced_discussions += 1; result.issues_synced_discussions += 1;
} }
// Step 4: Enqueue and drain resource events (if enabled)
if config.sync.fetch_resource_events {
// Enqueue resource_events jobs for all issues in this project
let enqueued = enqueue_resource_events_for_entity_type(conn, project_id, "issue")?;
if enqueued > 0 {
debug!(enqueued, "Enqueued resource events jobs for issues");
}
// Drain the queue
let drain_result =
drain_resource_events(conn, client, config, gitlab_project_id, &progress).await?;
result.resource_events_fetched = drain_result.fetched;
result.resource_events_failed = drain_result.failed;
}
info!( info!(
issues_fetched = result.issues_fetched, issues_fetched = result.issues_fetched,
issues_upserted = result.issues_upserted, issues_upserted = result.issues_upserted,
@@ -175,6 +203,8 @@ pub async fn ingest_project_issues_with_progress(
notes_upserted = result.notes_upserted, notes_upserted = result.notes_upserted,
issues_synced = result.issues_synced_discussions, issues_synced = result.issues_synced_discussions,
issues_skipped = result.issues_skipped_discussion_sync, issues_skipped = result.issues_skipped_discussion_sync,
resource_events_fetched = result.resource_events_fetched,
resource_events_failed = result.resource_events_failed,
"Project ingestion complete" "Project ingestion complete"
); );
@@ -343,6 +373,19 @@ pub async fn ingest_project_merge_requests_with_progress(
} }
} }
// Step 4: Enqueue and drain resource events (if enabled)
if config.sync.fetch_resource_events {
let enqueued = enqueue_resource_events_for_entity_type(conn, project_id, "merge_request")?;
if enqueued > 0 {
debug!(enqueued, "Enqueued resource events jobs for MRs");
}
let drain_result =
drain_resource_events(conn, client, config, gitlab_project_id, &progress).await?;
result.resource_events_fetched = drain_result.fetched;
result.resource_events_failed = drain_result.failed;
}
info!( info!(
mrs_fetched = result.mrs_fetched, mrs_fetched = result.mrs_fetched,
mrs_upserted = result.mrs_upserted, mrs_upserted = result.mrs_upserted,
@@ -352,6 +395,8 @@ pub async fn ingest_project_merge_requests_with_progress(
diffnotes = result.diffnotes_count, diffnotes = result.diffnotes_count,
mrs_synced = result.mrs_synced_discussions, mrs_synced = result.mrs_synced_discussions,
mrs_skipped = result.mrs_skipped_discussion_sync, mrs_skipped = result.mrs_skipped_discussion_sync,
resource_events_fetched = result.resource_events_fetched,
resource_events_failed = result.resource_events_failed,
"MR project ingestion complete" "MR project ingestion complete"
); );
@@ -405,6 +450,368 @@ async fn sync_mr_discussions_sequential(
Ok(results) Ok(results)
} }
/// Result of draining the resource events queue.
#[derive(Debug, Default)]
pub struct DrainResult {
pub fetched: usize,
pub failed: usize,
}
/// Enqueue resource_events jobs for all entities of a given type in a project.
///
/// Uses the pending_dependent_fetches queue. Jobs are deduplicated by the UNIQUE
/// constraint, so re-enqueueing the same entity is a no-op.
fn enqueue_resource_events_for_entity_type(
conn: &Connection,
project_id: i64,
entity_type: &str,
) -> Result<usize> {
let (table, id_col) = match entity_type {
"issue" => ("issues", "id"),
"merge_request" => ("merge_requests", "id"),
_ => return Ok(0),
};
// Query all entities for this project and enqueue resource_events jobs.
// The UNIQUE constraint on pending_dependent_fetches makes this idempotent -
// already-queued entities are silently skipped via INSERT OR IGNORE.
let mut stmt = conn.prepare_cached(&format!(
"SELECT {id_col}, iid FROM {table} WHERE project_id = ?1"
))?;
let entities: Vec<(i64, i64)> = stmt
.query_map([project_id], |row| Ok((row.get(0)?, row.get(1)?)))?
.collect::<std::result::Result<Vec<_>, _>>()?;
let mut enqueued = 0;
for (local_id, iid) in &entities {
if enqueue_job(
conn,
project_id,
entity_type,
*iid,
*local_id,
"resource_events",
None,
)? {
enqueued += 1;
}
}
Ok(enqueued)
}
/// Drain pending resource_events jobs: claim, fetch from GitLab, store, complete/fail.
///
/// Processes jobs sequentially since `rusqlite::Connection` is not `Send`.
/// Uses exponential backoff on failure via `fail_job`.
async fn drain_resource_events(
conn: &Connection,
client: &GitLabClient,
config: &Config,
gitlab_project_id: i64,
progress: &Option<ProgressCallback>,
) -> Result<DrainResult> {
let mut result = DrainResult::default();
let batch_size = config.sync.dependent_concurrency as usize;
// Reclaim stale locks from crashed processes
let reclaimed = reclaim_stale_locks(conn, config.sync.stale_lock_minutes)?;
if reclaimed > 0 {
info!(reclaimed, "Reclaimed stale resource event locks");
}
// Count total pending jobs for progress reporting
let pending_counts = count_pending_jobs(conn)?;
let total_pending = pending_counts.get("resource_events").copied().unwrap_or(0);
if total_pending == 0 {
return Ok(result);
}
let emit = |event: ProgressEvent| {
if let Some(cb) = progress {
cb(event);
}
};
emit(ProgressEvent::ResourceEventsFetchStarted {
total: total_pending,
});
let mut processed = 0;
// Max iterations guard: prevent infinite loop if jobs keep failing and retrying
// within the same drain run. Allow 2x total_pending iterations as safety margin.
let max_iterations = total_pending * 2;
let mut iterations = 0;
loop {
if iterations >= max_iterations {
warn!(
iterations,
total_pending, "Resource events drain hit max iterations guard, stopping"
);
break;
}
let jobs = claim_jobs(conn, "resource_events", batch_size)?;
if jobs.is_empty() {
break;
}
for job in &jobs {
iterations += 1;
// conn is &Connection but upsert functions need &mut Connection.
// We need to use unsafe to get a mutable reference since rusqlite
// operations are internally safe with WAL mode and we're single-threaded.
// Instead, we'll use a savepoint approach via the Connection directly.
match client
.fetch_all_resource_events(gitlab_project_id, &job.entity_type, job.entity_iid)
.await
{
Ok((state_events, label_events, milestone_events)) => {
// Store events - we need &mut Connection for savepoints in upsert functions.
// Use unchecked_transaction as a workaround since we have &Connection.
let store_result = store_resource_events(
conn,
job.project_id,
&job.entity_type,
job.entity_local_id,
&state_events,
&label_events,
&milestone_events,
);
match store_result {
Ok(()) => {
complete_job(conn, job.id)?;
result.fetched += 1;
}
Err(e) => {
warn!(
entity_type = %job.entity_type,
entity_iid = job.entity_iid,
error = %e,
"Failed to store resource events"
);
fail_job(conn, job.id, &e.to_string())?;
result.failed += 1;
}
}
}
Err(e) => {
warn!(
entity_type = %job.entity_type,
entity_iid = job.entity_iid,
error = %e,
"Failed to fetch resource events from GitLab"
);
fail_job(conn, job.id, &e.to_string())?;
result.failed += 1;
}
}
processed += 1;
emit(ProgressEvent::ResourceEventFetched {
current: processed,
total: total_pending,
});
}
}
emit(ProgressEvent::ResourceEventsFetchComplete {
fetched: result.fetched,
failed: result.failed,
});
if result.fetched > 0 || result.failed > 0 {
info!(
fetched = result.fetched,
failed = result.failed,
"Resource events drain complete"
);
}
Ok(result)
}
/// Store fetched resource events in the database.
///
/// Uses unchecked_transaction to work with &Connection (not &mut Connection),
/// which is safe because we're single-threaded and using WAL mode.
fn store_resource_events(
conn: &Connection,
project_id: i64,
entity_type: &str,
entity_local_id: i64,
state_events: &[crate::gitlab::types::GitLabStateEvent],
label_events: &[crate::gitlab::types::GitLabLabelEvent],
milestone_events: &[crate::gitlab::types::GitLabMilestoneEvent],
) -> Result<()> {
// The upsert functions require &mut Connection for savepoints.
// We use unchecked_transaction to wrap all three upserts atomically,
// then call the upsert functions using the transaction's inner connection.
let tx = conn.unchecked_transaction()?;
// State events - use raw SQL within transaction instead of upsert_state_events
// which requires &mut Connection
if !state_events.is_empty() {
store_state_events_tx(&tx, project_id, entity_type, entity_local_id, state_events)?;
}
if !label_events.is_empty() {
store_label_events_tx(&tx, project_id, entity_type, entity_local_id, label_events)?;
}
if !milestone_events.is_empty() {
store_milestone_events_tx(
&tx,
project_id,
entity_type,
entity_local_id,
milestone_events,
)?;
}
tx.commit()?;
Ok(())
}
/// Store state events within an existing transaction.
fn store_state_events_tx(
tx: &rusqlite::Transaction<'_>,
project_id: i64,
entity_type: &str,
entity_local_id: i64,
events: &[crate::gitlab::types::GitLabStateEvent],
) -> Result<()> {
let (issue_id, merge_request_id): (Option<i64>, Option<i64>) = match entity_type {
"issue" => (Some(entity_local_id), None),
"merge_request" => (None, Some(entity_local_id)),
_ => return Ok(()),
};
let mut stmt = tx.prepare_cached(
"INSERT OR REPLACE INTO resource_state_events
(gitlab_id, project_id, issue_id, merge_request_id, state,
actor_gitlab_id, actor_username, created_at,
source_commit, source_merge_request_iid)
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)",
)?;
for event in events {
let created_at = crate::core::time::iso_to_ms_strict(&event.created_at)
.map_err(crate::core::error::LoreError::Other)?;
let actor_id = event.user.as_ref().map(|u| u.id);
let actor_username = event.user.as_ref().map(|u| u.username.as_str());
let source_mr_iid = event.source_merge_request.as_ref().map(|mr| mr.iid);
stmt.execute(rusqlite::params![
event.id,
project_id,
issue_id,
merge_request_id,
event.state,
actor_id,
actor_username,
created_at,
event.source_commit,
source_mr_iid,
])?;
}
Ok(())
}
/// Store label events within an existing transaction.
fn store_label_events_tx(
tx: &rusqlite::Transaction<'_>,
project_id: i64,
entity_type: &str,
entity_local_id: i64,
events: &[crate::gitlab::types::GitLabLabelEvent],
) -> Result<()> {
let (issue_id, merge_request_id): (Option<i64>, Option<i64>) = match entity_type {
"issue" => (Some(entity_local_id), None),
"merge_request" => (None, Some(entity_local_id)),
_ => return Ok(()),
};
let mut stmt = tx.prepare_cached(
"INSERT OR REPLACE INTO resource_label_events
(gitlab_id, project_id, issue_id, merge_request_id, action,
label_name, actor_gitlab_id, actor_username, created_at)
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)",
)?;
for event in events {
let created_at = crate::core::time::iso_to_ms_strict(&event.created_at)
.map_err(crate::core::error::LoreError::Other)?;
let actor_id = event.user.as_ref().map(|u| u.id);
let actor_username = event.user.as_ref().map(|u| u.username.as_str());
stmt.execute(rusqlite::params![
event.id,
project_id,
issue_id,
merge_request_id,
event.action,
event.label.name,
actor_id,
actor_username,
created_at,
])?;
}
Ok(())
}
/// Store milestone events within an existing transaction.
fn store_milestone_events_tx(
tx: &rusqlite::Transaction<'_>,
project_id: i64,
entity_type: &str,
entity_local_id: i64,
events: &[crate::gitlab::types::GitLabMilestoneEvent],
) -> Result<()> {
let (issue_id, merge_request_id): (Option<i64>, Option<i64>) = match entity_type {
"issue" => (Some(entity_local_id), None),
"merge_request" => (None, Some(entity_local_id)),
_ => return Ok(()),
};
let mut stmt = tx.prepare_cached(
"INSERT OR REPLACE INTO resource_milestone_events
(gitlab_id, project_id, issue_id, merge_request_id, action,
milestone_title, milestone_id, actor_gitlab_id, actor_username, created_at)
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)",
)?;
for event in events {
let created_at = crate::core::time::iso_to_ms_strict(&event.created_at)
.map_err(crate::core::error::LoreError::Other)?;
let actor_id = event.user.as_ref().map(|u| u.id);
let actor_username = event.user.as_ref().map(|u| u.username.as_str());
stmt.execute(rusqlite::params![
event.id,
project_id,
issue_id,
merge_request_id,
event.action,
event.milestone.title,
event.milestone.id,
actor_id,
actor_username,
created_at,
])?;
}
Ok(())
}
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;
@@ -419,6 +826,8 @@ mod tests {
assert_eq!(result.notes_upserted, 0); assert_eq!(result.notes_upserted, 0);
assert_eq!(result.issues_synced_discussions, 0); assert_eq!(result.issues_synced_discussions, 0);
assert_eq!(result.issues_skipped_discussion_sync, 0); assert_eq!(result.issues_skipped_discussion_sync, 0);
assert_eq!(result.resource_events_fetched, 0);
assert_eq!(result.resource_events_failed, 0);
} }
#[test] #[test]
@@ -436,5 +845,28 @@ mod tests {
assert_eq!(result.diffnotes_count, 0); assert_eq!(result.diffnotes_count, 0);
assert_eq!(result.mrs_synced_discussions, 0); assert_eq!(result.mrs_synced_discussions, 0);
assert_eq!(result.mrs_skipped_discussion_sync, 0); assert_eq!(result.mrs_skipped_discussion_sync, 0);
assert_eq!(result.resource_events_fetched, 0);
assert_eq!(result.resource_events_failed, 0);
}
#[test]
fn drain_result_default_has_zero_counts() {
let result = DrainResult::default();
assert_eq!(result.fetched, 0);
assert_eq!(result.failed, 0);
}
#[test]
fn progress_event_resource_variants_exist() {
// Verify the new progress event variants are constructible
let _start = ProgressEvent::ResourceEventsFetchStarted { total: 10 };
let _progress = ProgressEvent::ResourceEventFetched {
current: 5,
total: 10,
};
let _complete = ProgressEvent::ResourceEventsFetchComplete {
fetched: 8,
failed: 2,
};
} }
} }