feat(ingestion): Mark entities dirty on ingest for document regeneration
Integrates the dirty tracking system into all four ingestion paths (issues, MRs, issue discussions, MR discussions). After each entity is upserted within its transaction, a corresponding dirty_queue entry is inserted so the document regenerator knows which documents need rebuilding. This ensures that document generation stays transactionally consistent with data changes: if the ingest transaction rolls back, the dirty marker rolls back too, preventing stale document regeneration attempts. Also updates GiError references to LoreError in these files as part of the codebase-wide rename, and adjusts issue discussion logging from info to debug level to reduce noise during normal sync runs. Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -8,11 +8,13 @@
|
||||
|
||||
use futures::StreamExt;
|
||||
use rusqlite::Connection;
|
||||
use tracing::{debug, info, warn};
|
||||
use tracing::{debug, warn};
|
||||
|
||||
use crate::Config;
|
||||
use crate::core::error::Result;
|
||||
use crate::core::payloads::{StorePayloadOptions, store_payload};
|
||||
use crate::documents::SourceType;
|
||||
use crate::ingestion::dirty_tracker;
|
||||
use crate::gitlab::GitLabClient;
|
||||
use crate::gitlab::transformers::{NoteableRef, transform_discussion, transform_notes};
|
||||
|
||||
@@ -55,7 +57,7 @@ pub async fn ingest_issue_discussions(
|
||||
total_result.stale_discussions_removed += result.stale_discussions_removed;
|
||||
}
|
||||
|
||||
info!(
|
||||
debug!(
|
||||
issues_processed = issues.len(),
|
||||
discussions_fetched = total_result.discussions_fetched,
|
||||
discussions_upserted = total_result.discussions_upserted,
|
||||
@@ -90,7 +92,7 @@ async fn ingest_discussions_for_issue(
|
||||
// Track discussions we've seen for stale removal
|
||||
let mut seen_discussion_ids: Vec<String> = Vec::new();
|
||||
// Track if any error occurred during pagination
|
||||
let mut pagination_error: Option<crate::core::error::GiError> = None;
|
||||
let mut pagination_error: Option<crate::core::error::LoreError> = None;
|
||||
|
||||
while let Some(disc_result) = discussions_stream.next().await {
|
||||
|
||||
@@ -141,6 +143,9 @@ async fn ingest_discussions_for_issue(
|
||||
|row| row.get(0),
|
||||
)?;
|
||||
|
||||
// Mark dirty for document regeneration (inside transaction)
|
||||
dirty_tracker::mark_dirty_tx(&tx, SourceType::Discussion, local_discussion_id)?;
|
||||
|
||||
// Transform and store notes
|
||||
let notes = transform_notes(&gitlab_discussion, local_project_id);
|
||||
let notes_count = notes.len();
|
||||
|
||||
Reference in New Issue
Block a user