theirongolddev/gitlore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(ingestion): Mark entities dirty on ingest for document regeneration

Browse Source 
Integrates the dirty tracking system into all four ingestion paths
(issues, MRs, issue discussions, MR discussions). After each entity
is upserted within its transaction, a corresponding dirty_queue entry
is inserted so the document regenerator knows which documents need
rebuilding.

This ensures that document generation stays transactionally consistent
with data changes: if the ingest transaction rolls back, the dirty
marker rolls back too, preventing stale document regeneration attempts.

Also updates GiError references to LoreError in these files as part
of the codebase-wide rename, and adjusts issue discussion logging
from info to debug level to reduce noise during normal sync runs.

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Taylor Eernisse
2026-01-30 15:46:51 -05:00
parent d5bdb24b0f
commit 559f0702ad
4 changed files with 31 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
src/ingestion/merge_requests.rs
View File

@@ -13,9 +13,11 @@ use rusqlite::{Connection, Transaction, params};
use tracing::{debug, info, warn};
use crate::Config;
use crate::core::error::{GiError, Result};
use crate::core::error::{LoreError, Result};
use crate::core::payloads::{StorePayloadOptions, store_payload};
use crate::core::time::now_ms;
use crate::documents::SourceType;
use crate::ingestion::dirty_tracker;
use crate::gitlab::GitLabClient;
use crate::gitlab::transformers::merge_request::transform_merge_request;
use crate::gitlab::types::GitLabMergeRequest;
@@ -166,7 +168,7 @@ fn process_single_mr(
// Transform MR first (outside transaction - no DB access)
let payload_json = serde_json::to_value(mr)?;
let transformed = transform_merge_request(mr, project_id)
.map_err(|e| GiError::Other(format!("MR transform failed: {}", e)))?;
.map_err(|e| LoreError::Other(format!("MR transform failed: {}", e)))?;
// Wrap all DB operations in a transaction for atomicity
let tx = conn.unchecked_transaction()?;
@@ -263,6 +265,9 @@ fn process_mr_in_transaction(
|row| row.get(0),
)?;
// Mark dirty for document regeneration (inside transaction)
dirty_tracker::mark_dirty_tx(tx, SourceType::MergeRequest, local_mr_id)?;
// Clear-and-relink labels
tx.execute(
"DELETE FROM mr_labels WHERE merge_request_id = ?",
@@ -448,7 +453,7 @@ pub fn get_mrs_needing_discussion_sync(
fn parse_timestamp(ts: &str) -> Result<i64> {
chrono::DateTime::parse_from_rfc3339(ts)
.map(|dt| dt.timestamp_millis())
.map_err(|e| GiError::Other(format!("Failed to parse timestamp '{}': {}", ts, e)))
.map_err(|e| LoreError::Other(format!("Failed to parse timestamp '{}': {}", ts, e)))
}
#[cfg(test)]