diff --git a/src/ingestion/discussions.rs b/src/ingestion/discussions.rs index 12c6090..18ebe77 100644 --- a/src/ingestion/discussions.rs +++ b/src/ingestion/discussions.rs @@ -8,11 +8,13 @@ use futures::StreamExt; use rusqlite::Connection; -use tracing::{debug, info, warn}; +use tracing::{debug, warn}; use crate::Config; use crate::core::error::Result; use crate::core::payloads::{StorePayloadOptions, store_payload}; +use crate::documents::SourceType; +use crate::ingestion::dirty_tracker; use crate::gitlab::GitLabClient; use crate::gitlab::transformers::{NoteableRef, transform_discussion, transform_notes}; @@ -55,7 +57,7 @@ pub async fn ingest_issue_discussions( total_result.stale_discussions_removed += result.stale_discussions_removed; } - info!( + debug!( issues_processed = issues.len(), discussions_fetched = total_result.discussions_fetched, discussions_upserted = total_result.discussions_upserted, @@ -90,7 +92,7 @@ async fn ingest_discussions_for_issue( // Track discussions we've seen for stale removal let mut seen_discussion_ids: Vec = Vec::new(); // Track if any error occurred during pagination - let mut pagination_error: Option = None; + let mut pagination_error: Option = None; while let Some(disc_result) = discussions_stream.next().await { @@ -141,6 +143,9 @@ async fn ingest_discussions_for_issue( |row| row.get(0), )?; + // Mark dirty for document regeneration (inside transaction) + dirty_tracker::mark_dirty_tx(&tx, SourceType::Discussion, local_discussion_id)?; + // Transform and store notes let notes = transform_notes(&gitlab_discussion, local_project_id); let notes_count = notes.len(); diff --git a/src/ingestion/issues.rs b/src/ingestion/issues.rs index 63ffe7c..8a19523 100644 --- a/src/ingestion/issues.rs +++ b/src/ingestion/issues.rs @@ -14,9 +14,11 @@ use rusqlite::{Connection, Transaction}; use tracing::{debug, info, warn}; use crate::Config; -use crate::core::error::{GiError, Result}; +use crate::core::error::{LoreError, Result}; use crate::core::payloads::{StorePayloadOptions, store_payload}; use crate::core::time::now_ms; +use crate::documents::SourceType; +use crate::ingestion::dirty_tracker; use crate::gitlab::GitLabClient; use crate::gitlab::transformers::{MilestoneRow, transform_issue}; use crate::gitlab::types::GitLabIssue; @@ -297,6 +299,9 @@ fn process_issue_in_transaction( |row| row.get(0), )?; + // Mark dirty for document regeneration (inside transaction) + dirty_tracker::mark_dirty_tx(tx, SourceType::Issue, local_issue_id)?; + // Clear existing label links (stale removal) tx.execute( "DELETE FROM issue_labels WHERE issue_id = ?", @@ -470,7 +475,7 @@ fn get_issues_needing_discussion_sync( fn parse_timestamp(ts: &str) -> Result { chrono::DateTime::parse_from_rfc3339(ts) .map(|dt| dt.timestamp_millis()) - .map_err(|e| GiError::Other(format!("Failed to parse timestamp '{}': {}", ts, e))) + .map_err(|e| LoreError::Other(format!("Failed to parse timestamp '{}': {}", ts, e))) } #[cfg(test)] diff --git a/src/ingestion/merge_requests.rs b/src/ingestion/merge_requests.rs index 4172611..a195774 100644 --- a/src/ingestion/merge_requests.rs +++ b/src/ingestion/merge_requests.rs @@ -13,9 +13,11 @@ use rusqlite::{Connection, Transaction, params}; use tracing::{debug, info, warn}; use crate::Config; -use crate::core::error::{GiError, Result}; +use crate::core::error::{LoreError, Result}; use crate::core::payloads::{StorePayloadOptions, store_payload}; use crate::core::time::now_ms; +use crate::documents::SourceType; +use crate::ingestion::dirty_tracker; use crate::gitlab::GitLabClient; use crate::gitlab::transformers::merge_request::transform_merge_request; use crate::gitlab::types::GitLabMergeRequest; @@ -166,7 +168,7 @@ fn process_single_mr( // Transform MR first (outside transaction - no DB access) let payload_json = serde_json::to_value(mr)?; let transformed = transform_merge_request(mr, project_id) - .map_err(|e| GiError::Other(format!("MR transform failed: {}", e)))?; + .map_err(|e| LoreError::Other(format!("MR transform failed: {}", e)))?; // Wrap all DB operations in a transaction for atomicity let tx = conn.unchecked_transaction()?; @@ -263,6 +265,9 @@ fn process_mr_in_transaction( |row| row.get(0), )?; + // Mark dirty for document regeneration (inside transaction) + dirty_tracker::mark_dirty_tx(tx, SourceType::MergeRequest, local_mr_id)?; + // Clear-and-relink labels tx.execute( "DELETE FROM mr_labels WHERE merge_request_id = ?", @@ -448,7 +453,7 @@ pub fn get_mrs_needing_discussion_sync( fn parse_timestamp(ts: &str) -> Result { chrono::DateTime::parse_from_rfc3339(ts) .map(|dt| dt.timestamp_millis()) - .map_err(|e| GiError::Other(format!("Failed to parse timestamp '{}': {}", ts, e))) + .map_err(|e| LoreError::Other(format!("Failed to parse timestamp '{}': {}", ts, e))) } #[cfg(test)] diff --git a/src/ingestion/mr_discussions.rs b/src/ingestion/mr_discussions.rs index 9dc34f1..9a5add0 100644 --- a/src/ingestion/mr_discussions.rs +++ b/src/ingestion/mr_discussions.rs @@ -18,6 +18,8 @@ use crate::Config; use crate::core::error::Result; use crate::core::payloads::{StorePayloadOptions, store_payload}; use crate::core::time::now_ms; +use crate::documents::SourceType; +use crate::ingestion::dirty_tracker; use crate::gitlab::GitLabClient; use crate::gitlab::transformers::{ NormalizedDiscussion, NormalizedNote, transform_mr_discussion, @@ -189,6 +191,9 @@ pub fn write_prefetched_mr_discussions( |row| row.get(0), )?; + // Mark dirty for document regeneration (inside transaction) + dirty_tracker::mark_dirty_tx(&tx, SourceType::Discussion, local_discussion_id)?; + // Upsert notes for note in &disc.notes { let should_store_payload = !note.is_system @@ -402,6 +407,9 @@ async fn ingest_discussions_for_mr( |row| row.get(0), )?; + // Mark dirty for document regeneration (inside transaction) + dirty_tracker::mark_dirty_tx(&tx, SourceType::Discussion, local_discussion_id)?; + // Upsert notes (not delete-all-then-insert) for note in ¬es { // Selective payload storage: skip system notes without position