theirongolddev/gitlore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(ingestion): Move counter increments after transaction commit

Browse Source 
Ingestion counters (discussions_upserted, notes_upserted,
discussions_fetched, diffnotes_count) were incremented before
tx.commit(), meaning a failed commit would report inflated
metrics. Counters now increment only after successful commit
so reported numbers accurately reflect persisted state.

Also simplifies the stale-removal guard in issue discussions:
the received_first_response flag was unnecessary since an empty
seen_discussion_ids list is safe to pass to remove_stale -- if
there were no discussions, stale removal correctly sweeps all
previously-stored discussions. The two separate code paths
(empty vs populated) are collapsed into a single branch.

Derives Default on IngestResult to eliminate verbose zero-init.

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Taylor Eernisse
2026-01-29 08:42:11 -05:00
parent 753ff46bb4
commit 8fe5feda7e
3 changed files with 29 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
src/ingestion/mr_discussions.rs
View File

@@ -155,14 +155,13 @@ pub fn write_prefetched_mr_discussions(
// Write each discussion
for disc in &prefetched.discussions {
result.discussions_fetched += 1;
// Count DiffNotes
result.diffnotes_count += disc
// Count DiffNotes upfront (independent of transaction)
let diffnotes_in_disc = disc
.notes
.iter()
.filter(|n| n.position_new_path.is_some() || n.position_old_path.is_some())
.count();
let notes_in_disc = disc.notes.len();
// Start transaction
let tx = conn.unchecked_transaction()?;
@@ -182,7 +181,6 @@ pub fn write_prefetched_mr_discussions(
// Upsert discussion
upsert_discussion(&tx, &disc.normalized, run_seen_at, payload_id)?;
result.discussions_upserted += 1;
// Get local discussion ID
let local_discussion_id: i64 = tx.query_row(
@@ -219,10 +217,15 @@ pub fn write_prefetched_mr_discussions(
};
upsert_note(&tx, local_discussion_id, note, run_seen_at, note_payload_id)?;
result.notes_upserted += 1;
}
tx.commit()?;
// Increment counters AFTER successful commit to keep metrics honest
result.discussions_fetched += 1;
result.discussions_upserted += 1;
result.notes_upserted += notes_in_disc;
result.diffnotes_count += diffnotes_in_disc;
}
// Only sweep stale data and advance watermark on full success
@@ -343,8 +346,6 @@ async fn ingest_discussions_for_mr(
break;
}
};
result.discussions_fetched += 1;
// CRITICAL: Parse notes BEFORE any destructive DB operations
let notes = match transform_notes_with_diff_position(&gitlab_discussion, local_project_id) {
Ok(notes) => notes,
@@ -361,11 +362,12 @@ async fn ingest_discussions_for_mr(
}
};
// Count DiffNotes
result.diffnotes_count += notes
// Count DiffNotes upfront (independent of transaction)
let diffnotes_in_disc = notes
.iter()
.filter(|n| n.position_new_path.is_some() || n.position_old_path.is_some())
.count();
let notes_count = notes.len();
// Transform discussion
let normalized_discussion =
@@ -389,7 +391,6 @@ async fn ingest_discussions_for_mr(
// Upsert discussion with run_seen_at
upsert_discussion(&tx, &normalized_discussion, run_seen_at, payload_id)?;
result.discussions_upserted += 1;
// Get local discussion ID
let local_discussion_id: i64 = tx.query_row(
@@ -433,10 +434,15 @@ async fn ingest_discussions_for_mr(
};
upsert_note(&tx, local_discussion_id, note, run_seen_at, note_payload_id)?;
result.notes_upserted += 1;
}
tx.commit()?;
// Increment counters AFTER successful commit to keep metrics honest
result.discussions_fetched += 1;
result.discussions_upserted += 1;
result.notes_upserted += notes_count;
result.diffnotes_count += diffnotes_in_disc;
}
// Only sweep stale data and advance watermark on full success