feat: implement per-note search and document pipeline
- Add SourceType::Note with extract_note_document() and ParentMetadataCache - Migration 022: composite indexes for notes queries + author_id column - Migration 024: table rebuild adding 'note' to CHECK constraints, defense triggers - Migration 025: backfill existing non-system notes into dirty queue - Add lore notes CLI command with 17 filter options (author, path, resolution, etc.) - Support table/json/jsonl/csv output formats with field selection - Wire note dirty tracking through discussion and MR discussion ingestion - Fix test_migration_024_preserves_existing_data off-by-one (tested wrong migration) - Fix upsert_document_inner returning false for label/path-only changes
This commit is contained in:
teernisse
@@ -1,17 +1,26 @@
|
||||
use futures::StreamExt;
|
||||
use rusqlite::Connection;
|
||||
use rusqlite::{Connection, params};
|
||||
use tracing::{debug, warn};
|
||||
|
||||
use crate::Config;
|
||||
use crate::core::error::Result;
|
||||
use crate::core::payloads::{StorePayloadOptions, store_payload};
|
||||
use crate::core::time::now_ms;
|
||||
use crate::documents::SourceType;
|
||||
use crate::gitlab::GitLabClient;
|
||||
use crate::gitlab::transformers::{NoteableRef, transform_discussion, transform_notes};
|
||||
use crate::gitlab::transformers::{
|
||||
NormalizedNote, NoteableRef, transform_discussion, transform_notes,
|
||||
};
|
||||
use crate::ingestion::dirty_tracker;
|
||||
|
||||
use super::issues::IssueForDiscussionSync;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct NoteUpsertOutcome {
|
||||
pub local_note_id: i64,
|
||||
pub changed_semantics: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct IngestDiscussionsResult {
|
||||
pub discussions_fetched: usize,
|
||||
@@ -80,6 +89,8 @@ async fn ingest_discussions_for_issue(
|
||||
let mut seen_discussion_ids: Vec<String> = Vec::new();
|
||||
let mut pagination_error: Option<crate::core::error::LoreError> = None;
|
||||
|
||||
let run_seen_at = now_ms();
|
||||
|
||||
while let Some(disc_result) = discussions_stream.next().await {
|
||||
let gitlab_discussion = match disc_result {
|
||||
Ok(d) => d,
|
||||
@@ -126,18 +137,29 @@ async fn ingest_discussions_for_issue(
|
||||
|
||||
dirty_tracker::mark_dirty_tx(&tx, SourceType::Discussion, local_discussion_id)?;
|
||||
|
||||
// Mark child note documents dirty (they inherit parent metadata)
|
||||
tx.execute(
|
||||
"INSERT INTO dirty_sources (source_type, source_id, queued_at)
|
||||
SELECT 'note', n.id, ?1
|
||||
FROM notes n
|
||||
WHERE n.discussion_id = ?2 AND n.is_system = 0
|
||||
ON CONFLICT(source_type, source_id) DO UPDATE SET queued_at = excluded.queued_at, attempt_count = 0",
|
||||
params![now_ms(), local_discussion_id],
|
||||
)?;
|
||||
|
||||
let notes = transform_notes(&gitlab_discussion, local_project_id);
|
||||
let notes_count = notes.len();
|
||||
|
||||
tx.execute(
|
||||
"DELETE FROM notes WHERE discussion_id = ?",
|
||||
[local_discussion_id],
|
||||
)?;
|
||||
|
||||
for note in notes {
|
||||
insert_note(&tx, local_discussion_id, ¬e, None)?;
|
||||
let outcome =
|
||||
upsert_note_for_issue(&tx, local_discussion_id, ¬e, run_seen_at, None)?;
|
||||
if !note.is_system && outcome.changed_semantics {
|
||||
dirty_tracker::mark_dirty_tx(&tx, SourceType::Note, outcome.local_note_id)?;
|
||||
}
|
||||
}
|
||||
|
||||
sweep_stale_issue_notes(&tx, local_discussion_id, run_seen_at)?;
|
||||
|
||||
tx.commit()?;
|
||||
|
||||
result.discussions_upserted += 1;
|
||||
@@ -198,38 +220,182 @@ fn upsert_discussion(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn insert_note(
|
||||
fn upsert_note_for_issue(
|
||||
conn: &Connection,
|
||||
discussion_id: i64,
|
||||
note: &crate::gitlab::transformers::NormalizedNote,
|
||||
note: &NormalizedNote,
|
||||
last_seen_at: i64,
|
||||
payload_id: Option<i64>,
|
||||
) -> Result<()> {
|
||||
) -> Result<NoteUpsertOutcome> {
|
||||
// Pre-read for semantic change detection
|
||||
let existing = conn
|
||||
.query_row(
|
||||
"SELECT id, body, note_type, resolved, resolved_by,
|
||||
position_old_path, position_new_path, position_old_line, position_new_line,
|
||||
position_type, position_line_range_start, position_line_range_end,
|
||||
position_base_sha, position_start_sha, position_head_sha
|
||||
FROM notes WHERE gitlab_id = ?",
|
||||
params![note.gitlab_id],
|
||||
|row| {
|
||||
Ok((
|
||||
row.get::<_, i64>(0)?,
|
||||
row.get::<_, String>(1)?,
|
||||
row.get::<_, Option<String>>(2)?,
|
||||
row.get::<_, bool>(3)?,
|
||||
row.get::<_, Option<String>>(4)?,
|
||||
row.get::<_, Option<String>>(5)?,
|
||||
row.get::<_, Option<String>>(6)?,
|
||||
row.get::<_, Option<i32>>(7)?,
|
||||
row.get::<_, Option<i32>>(8)?,
|
||||
row.get::<_, Option<String>>(9)?,
|
||||
row.get::<_, Option<i32>>(10)?,
|
||||
row.get::<_, Option<i32>>(11)?,
|
||||
row.get::<_, Option<String>>(12)?,
|
||||
row.get::<_, Option<String>>(13)?,
|
||||
row.get::<_, Option<String>>(14)?,
|
||||
))
|
||||
},
|
||||
)
|
||||
.ok();
|
||||
|
||||
let changed_semantics = match &existing {
|
||||
Some((
|
||||
_id,
|
||||
body,
|
||||
note_type,
|
||||
resolved,
|
||||
resolved_by,
|
||||
pos_old_path,
|
||||
pos_new_path,
|
||||
pos_old_line,
|
||||
pos_new_line,
|
||||
pos_type,
|
||||
pos_range_start,
|
||||
pos_range_end,
|
||||
pos_base_sha,
|
||||
pos_start_sha,
|
||||
pos_head_sha,
|
||||
)) => {
|
||||
*body != note.body
|
||||
|| *note_type != note.note_type
|
||||
|| *resolved != note.resolved
|
||||
|| *resolved_by != note.resolved_by
|
||||
|| *pos_old_path != note.position_old_path
|
||||
|| *pos_new_path != note.position_new_path
|
||||
|| *pos_old_line != note.position_old_line
|
||||
|| *pos_new_line != note.position_new_line
|
||||
|| *pos_type != note.position_type
|
||||
|| *pos_range_start != note.position_line_range_start
|
||||
|| *pos_range_end != note.position_line_range_end
|
||||
|| *pos_base_sha != note.position_base_sha
|
||||
|| *pos_start_sha != note.position_start_sha
|
||||
|| *pos_head_sha != note.position_head_sha
|
||||
}
|
||||
None => true,
|
||||
};
|
||||
|
||||
conn.execute(
|
||||
"INSERT INTO notes (
|
||||
gitlab_id, discussion_id, project_id, note_type, is_system,
|
||||
author_username, body, created_at, updated_at, last_seen_at,
|
||||
position, resolvable, resolved, resolved_by, resolved_at, raw_payload_id
|
||||
) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16)",
|
||||
(
|
||||
author_id, author_username, body, created_at, updated_at, last_seen_at,
|
||||
position, resolvable, resolved, resolved_by, resolved_at,
|
||||
position_old_path, position_new_path, position_old_line, position_new_line,
|
||||
position_type, position_line_range_start, position_line_range_end,
|
||||
position_base_sha, position_start_sha, position_head_sha,
|
||||
raw_payload_id
|
||||
) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17, ?18, ?19, ?20, ?21, ?22, ?23, ?24, ?25, ?26, ?27)
|
||||
ON CONFLICT(gitlab_id) DO UPDATE SET
|
||||
body = excluded.body,
|
||||
note_type = excluded.note_type,
|
||||
author_id = excluded.author_id,
|
||||
updated_at = excluded.updated_at,
|
||||
last_seen_at = excluded.last_seen_at,
|
||||
resolvable = excluded.resolvable,
|
||||
resolved = excluded.resolved,
|
||||
resolved_by = excluded.resolved_by,
|
||||
resolved_at = excluded.resolved_at,
|
||||
position_old_path = excluded.position_old_path,
|
||||
position_new_path = excluded.position_new_path,
|
||||
position_old_line = excluded.position_old_line,
|
||||
position_new_line = excluded.position_new_line,
|
||||
position_type = excluded.position_type,
|
||||
position_line_range_start = excluded.position_line_range_start,
|
||||
position_line_range_end = excluded.position_line_range_end,
|
||||
position_base_sha = excluded.position_base_sha,
|
||||
position_start_sha = excluded.position_start_sha,
|
||||
position_head_sha = excluded.position_head_sha,
|
||||
raw_payload_id = COALESCE(excluded.raw_payload_id, raw_payload_id)",
|
||||
params![
|
||||
note.gitlab_id,
|
||||
discussion_id,
|
||||
note.project_id,
|
||||
¬e.note_type,
|
||||
note.is_system,
|
||||
note.author_id,
|
||||
¬e.author_username,
|
||||
¬e.body,
|
||||
note.created_at,
|
||||
note.updated_at,
|
||||
note.last_seen_at,
|
||||
last_seen_at,
|
||||
note.position,
|
||||
note.resolvable,
|
||||
note.resolved,
|
||||
¬e.resolved_by,
|
||||
note.resolved_at,
|
||||
¬e.position_old_path,
|
||||
¬e.position_new_path,
|
||||
note.position_old_line,
|
||||
note.position_new_line,
|
||||
¬e.position_type,
|
||||
note.position_line_range_start,
|
||||
note.position_line_range_end,
|
||||
¬e.position_base_sha,
|
||||
¬e.position_start_sha,
|
||||
¬e.position_head_sha,
|
||||
payload_id,
|
||||
),
|
||||
],
|
||||
)?;
|
||||
Ok(())
|
||||
|
||||
let local_note_id: i64 = conn.query_row(
|
||||
"SELECT id FROM notes WHERE gitlab_id = ?",
|
||||
params![note.gitlab_id],
|
||||
|row| row.get(0),
|
||||
)?;
|
||||
|
||||
Ok(NoteUpsertOutcome {
|
||||
local_note_id,
|
||||
changed_semantics,
|
||||
})
|
||||
}
|
||||
|
||||
fn sweep_stale_issue_notes(
|
||||
conn: &Connection,
|
||||
discussion_id: i64,
|
||||
last_seen_at: i64,
|
||||
) -> Result<usize> {
|
||||
// Step 1: Delete note documents for stale notes
|
||||
conn.execute(
|
||||
"DELETE FROM documents WHERE source_type = 'note' AND source_id IN
|
||||
(SELECT id FROM notes WHERE discussion_id = ?1 AND last_seen_at < ?2 AND is_system = 0)",
|
||||
params![discussion_id, last_seen_at],
|
||||
)?;
|
||||
|
||||
// Step 2: Delete dirty_sources entries for stale notes
|
||||
conn.execute(
|
||||
"DELETE FROM dirty_sources WHERE source_type = 'note' AND source_id IN
|
||||
(SELECT id FROM notes WHERE discussion_id = ?1 AND last_seen_at < ?2 AND is_system = 0)",
|
||||
params![discussion_id, last_seen_at],
|
||||
)?;
|
||||
|
||||
// Step 3: Delete the stale notes themselves
|
||||
let deleted = conn.execute(
|
||||
"DELETE FROM notes WHERE discussion_id = ?1 AND last_seen_at < ?2",
|
||||
params![discussion_id, last_seen_at],
|
||||
)?;
|
||||
if deleted > 0 {
|
||||
debug!(discussion_id, deleted, "Swept stale issue notes");
|
||||
}
|
||||
Ok(deleted)
|
||||
}
|
||||
|
||||
fn remove_stale_discussions(
|
||||
@@ -303,6 +469,9 @@ fn update_issue_sync_timestamp(conn: &Connection, issue_id: i64, updated_at: i64
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::core::db::{create_connection, run_migrations};
|
||||
use crate::gitlab::transformers::NormalizedNote;
|
||||
use std::path::Path;
|
||||
|
||||
#[test]
|
||||
fn result_default_has_zero_counts() {
|
||||
@@ -311,4 +480,462 @@ mod tests {
|
||||
assert_eq!(result.discussions_upserted, 0);
|
||||
assert_eq!(result.notes_upserted, 0);
|
||||
}
|
||||
|
||||
fn setup() -> Connection {
|
||||
let conn = create_connection(Path::new(":memory:")).unwrap();
|
||||
run_migrations(&conn).unwrap();
|
||||
|
||||
conn.execute(
|
||||
"INSERT INTO projects (gitlab_project_id, path_with_namespace, web_url) \
|
||||
VALUES (1, 'group/repo', 'https://gitlab.com/group/repo')",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
conn.execute(
|
||||
"INSERT INTO issues (gitlab_id, iid, project_id, title, state, author_username, created_at, updated_at, last_seen_at) \
|
||||
VALUES (100, 1, 1, 'Test Issue', 'opened', 'testuser', 1000, 2000, 3000)",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
conn.execute(
|
||||
"INSERT INTO discussions (gitlab_discussion_id, project_id, issue_id, noteable_type, individual_note, last_seen_at, resolvable, resolved) \
|
||||
VALUES ('disc-1', 1, 1, 'Issue', 0, 3000, 0, 0)",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
conn
|
||||
}
|
||||
|
||||
fn get_discussion_id(conn: &Connection) -> i64 {
|
||||
conn.query_row("SELECT id FROM discussions LIMIT 1", [], |row| row.get(0))
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn make_note(
|
||||
gitlab_id: i64,
|
||||
project_id: i64,
|
||||
body: &str,
|
||||
note_type: Option<&str>,
|
||||
created_at: i64,
|
||||
updated_at: i64,
|
||||
resolved: bool,
|
||||
resolved_by: Option<&str>,
|
||||
) -> NormalizedNote {
|
||||
NormalizedNote {
|
||||
gitlab_id,
|
||||
project_id,
|
||||
note_type: note_type.map(String::from),
|
||||
is_system: false,
|
||||
author_id: None,
|
||||
author_username: "testuser".to_string(),
|
||||
body: body.to_string(),
|
||||
created_at,
|
||||
updated_at,
|
||||
last_seen_at: updated_at,
|
||||
position: 0,
|
||||
resolvable: false,
|
||||
resolved,
|
||||
resolved_by: resolved_by.map(String::from),
|
||||
resolved_at: None,
|
||||
position_old_path: None,
|
||||
position_new_path: None,
|
||||
position_old_line: None,
|
||||
position_new_line: None,
|
||||
position_type: None,
|
||||
position_line_range_start: None,
|
||||
position_line_range_end: None,
|
||||
position_base_sha: None,
|
||||
position_start_sha: None,
|
||||
position_head_sha: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_issue_note_upsert_stable_id() {
|
||||
let conn = setup();
|
||||
let disc_id = get_discussion_id(&conn);
|
||||
let last_seen_at = 5000;
|
||||
|
||||
let note1 = make_note(1001, 1, "First note", None, 1000, 2000, false, None);
|
||||
let note2 = make_note(1002, 1, "Second note", None, 1000, 2000, false, None);
|
||||
|
||||
let out1 = upsert_note_for_issue(&conn, disc_id, ¬e1, last_seen_at, None).unwrap();
|
||||
let out2 = upsert_note_for_issue(&conn, disc_id, ¬e2, last_seen_at, None).unwrap();
|
||||
let id1 = out1.local_note_id;
|
||||
let id2 = out2.local_note_id;
|
||||
|
||||
// Re-sync same gitlab_ids
|
||||
let out1b = upsert_note_for_issue(&conn, disc_id, ¬e1, last_seen_at + 1, None).unwrap();
|
||||
let out2b = upsert_note_for_issue(&conn, disc_id, ¬e2, last_seen_at + 1, None).unwrap();
|
||||
|
||||
assert_eq!(id1, out1b.local_note_id);
|
||||
assert_eq!(id2, out2b.local_note_id);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_issue_note_upsert_detects_body_change() {
|
||||
let conn = setup();
|
||||
let disc_id = get_discussion_id(&conn);
|
||||
|
||||
let note = make_note(2001, 1, "Original body", None, 1000, 2000, false, None);
|
||||
upsert_note_for_issue(&conn, disc_id, ¬e, 5000, None).unwrap();
|
||||
|
||||
let mut changed = make_note(2001, 1, "Updated body", None, 1000, 3000, false, None);
|
||||
changed.updated_at = 3000;
|
||||
let outcome = upsert_note_for_issue(&conn, disc_id, &changed, 5001, None).unwrap();
|
||||
assert!(outcome.changed_semantics);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_issue_note_upsert_unchanged_returns_false() {
|
||||
let conn = setup();
|
||||
let disc_id = get_discussion_id(&conn);
|
||||
|
||||
let note = make_note(3001, 1, "Same body", None, 1000, 2000, false, None);
|
||||
upsert_note_for_issue(&conn, disc_id, ¬e, 5000, None).unwrap();
|
||||
|
||||
// Re-sync identical note
|
||||
let outcome = upsert_note_for_issue(&conn, disc_id, ¬e, 5001, None).unwrap();
|
||||
assert!(!outcome.changed_semantics);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_issue_note_upsert_updated_at_only_does_not_mark_semantic_change() {
|
||||
let conn = setup();
|
||||
let disc_id = get_discussion_id(&conn);
|
||||
|
||||
let note = make_note(4001, 1, "Body stays", None, 1000, 2000, false, None);
|
||||
upsert_note_for_issue(&conn, disc_id, ¬e, 5000, None).unwrap();
|
||||
|
||||
// Only change updated_at (non-semantic field)
|
||||
let mut same = make_note(4001, 1, "Body stays", None, 1000, 9999, false, None);
|
||||
same.updated_at = 9999;
|
||||
let outcome = upsert_note_for_issue(&conn, disc_id, &same, 5001, None).unwrap();
|
||||
assert!(!outcome.changed_semantics);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_issue_note_sweep_removes_stale() {
|
||||
let conn = setup();
|
||||
let disc_id = get_discussion_id(&conn);
|
||||
|
||||
let note1 = make_note(5001, 1, "Keep me", None, 1000, 2000, false, None);
|
||||
let note2 = make_note(5002, 1, "Stale me", None, 1000, 2000, false, None);
|
||||
|
||||
upsert_note_for_issue(&conn, disc_id, ¬e1, 5000, None).unwrap();
|
||||
upsert_note_for_issue(&conn, disc_id, ¬e2, 5000, None).unwrap();
|
||||
|
||||
// Re-sync only note1 with newer timestamp
|
||||
upsert_note_for_issue(&conn, disc_id, ¬e1, 6000, None).unwrap();
|
||||
|
||||
// Sweep should remove note2 (last_seen_at=5000 < 6000)
|
||||
let swept = sweep_stale_issue_notes(&conn, disc_id, 6000).unwrap();
|
||||
assert_eq!(swept, 1);
|
||||
|
||||
let count: i64 = conn
|
||||
.query_row(
|
||||
"SELECT COUNT(*) FROM notes WHERE discussion_id = ?",
|
||||
[disc_id],
|
||||
|row| row.get(0),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(count, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_issue_note_upsert_returns_local_id() {
|
||||
let conn = setup();
|
||||
let disc_id = get_discussion_id(&conn);
|
||||
|
||||
let note = make_note(6001, 1, "Check my ID", None, 1000, 2000, false, None);
|
||||
let outcome = upsert_note_for_issue(&conn, disc_id, ¬e, 5000, None).unwrap();
|
||||
|
||||
// Verify the local_note_id matches what's in the DB
|
||||
let db_id: i64 = conn
|
||||
.query_row(
|
||||
"SELECT id FROM notes WHERE gitlab_id = ?",
|
||||
[6001_i64],
|
||||
|row| row.get(0),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(outcome.local_note_id, db_id);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_issue_note_upsert_captures_author_id() {
|
||||
let conn = setup();
|
||||
let disc_id = get_discussion_id(&conn);
|
||||
|
||||
let mut note = make_note(7001, 1, "With author", None, 1000, 2000, false, None);
|
||||
note.author_id = Some(12345);
|
||||
|
||||
upsert_note_for_issue(&conn, disc_id, ¬e, 5000, None).unwrap();
|
||||
|
||||
let stored: Option<i64> = conn
|
||||
.query_row(
|
||||
"SELECT author_id FROM notes WHERE gitlab_id = ?",
|
||||
[7001_i64],
|
||||
|row| row.get(0),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(stored, Some(12345));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_note_upsert_author_id_nullable() {
|
||||
let conn = setup();
|
||||
let disc_id = get_discussion_id(&conn);
|
||||
|
||||
let note = make_note(7002, 1, "No author id", None, 1000, 2000, false, None);
|
||||
// author_id defaults to None in make_note
|
||||
|
||||
upsert_note_for_issue(&conn, disc_id, ¬e, 5000, None).unwrap();
|
||||
|
||||
let stored: Option<i64> = conn
|
||||
.query_row(
|
||||
"SELECT author_id FROM notes WHERE gitlab_id = ?",
|
||||
[7002_i64],
|
||||
|row| row.get(0),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(stored, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_note_author_id_survives_username_change() {
|
||||
let conn = setup();
|
||||
let disc_id = get_discussion_id(&conn);
|
||||
|
||||
let mut note = make_note(7003, 1, "Original body", None, 1000, 2000, false, None);
|
||||
note.author_id = Some(99999);
|
||||
note.author_username = "oldname".to_string();
|
||||
|
||||
upsert_note_for_issue(&conn, disc_id, ¬e, 5000, None).unwrap();
|
||||
|
||||
// Re-sync with changed username, changed body, same author_id
|
||||
let mut updated = make_note(7003, 1, "Updated body", None, 1000, 3000, false, None);
|
||||
updated.author_id = Some(99999);
|
||||
updated.author_username = "newname".to_string();
|
||||
|
||||
upsert_note_for_issue(&conn, disc_id, &updated, 5001, None).unwrap();
|
||||
|
||||
// author_id must survive the re-sync intact
|
||||
let stored_id: Option<i64> = conn
|
||||
.query_row(
|
||||
"SELECT author_id FROM notes WHERE gitlab_id = ?",
|
||||
[7003_i64],
|
||||
|row| row.get(0),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(stored_id, Some(99999));
|
||||
}
|
||||
|
||||
fn insert_note_document(conn: &Connection, note_local_id: i64) {
|
||||
conn.execute(
|
||||
"INSERT INTO documents (source_type, source_id, project_id, content_text, content_hash) \
|
||||
VALUES ('note', ?1, 1, 'note content', 'hash123')",
|
||||
[note_local_id],
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
fn insert_note_dirty_source(conn: &Connection, note_local_id: i64) {
|
||||
conn.execute(
|
||||
"INSERT INTO dirty_sources (source_type, source_id, queued_at) \
|
||||
VALUES ('note', ?1, 1000)",
|
||||
[note_local_id],
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
fn count_note_documents(conn: &Connection, note_local_id: i64) -> i64 {
|
||||
conn.query_row(
|
||||
"SELECT COUNT(*) FROM documents WHERE source_type = 'note' AND source_id = ?",
|
||||
[note_local_id],
|
||||
|row| row.get(0),
|
||||
)
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn count_note_dirty_sources(conn: &Connection, note_local_id: i64) -> i64 {
|
||||
conn.query_row(
|
||||
"SELECT COUNT(*) FROM dirty_sources WHERE source_type = 'note' AND source_id = ?",
|
||||
[note_local_id],
|
||||
|row| row.get(0),
|
||||
)
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_issue_note_sweep_deletes_note_documents_immediately() {
|
||||
let conn = setup();
|
||||
let disc_id = get_discussion_id(&conn);
|
||||
|
||||
// Insert 3 notes
|
||||
let note1 = make_note(9001, 1, "Keep me", None, 1000, 2000, false, None);
|
||||
let note2 = make_note(9002, 1, "Keep me too", None, 1000, 2000, false, None);
|
||||
let note3 = make_note(9003, 1, "Stale me", None, 1000, 2000, false, None);
|
||||
|
||||
let out1 = upsert_note_for_issue(&conn, disc_id, ¬e1, 5000, None).unwrap();
|
||||
let out2 = upsert_note_for_issue(&conn, disc_id, ¬e2, 5000, None).unwrap();
|
||||
let out3 = upsert_note_for_issue(&conn, disc_id, ¬e3, 5000, None).unwrap();
|
||||
|
||||
// Add documents for all 3
|
||||
insert_note_document(&conn, out1.local_note_id);
|
||||
insert_note_document(&conn, out2.local_note_id);
|
||||
insert_note_document(&conn, out3.local_note_id);
|
||||
|
||||
// Add dirty_sources for note3
|
||||
insert_note_dirty_source(&conn, out3.local_note_id);
|
||||
|
||||
// Re-sync only notes 1 and 2 with newer timestamp
|
||||
upsert_note_for_issue(&conn, disc_id, ¬e1, 6000, None).unwrap();
|
||||
upsert_note_for_issue(&conn, disc_id, ¬e2, 6000, None).unwrap();
|
||||
|
||||
// Sweep should remove note3 and its document + dirty_source
|
||||
sweep_stale_issue_notes(&conn, disc_id, 6000).unwrap();
|
||||
|
||||
// Stale note's document should be gone
|
||||
assert_eq!(count_note_documents(&conn, out3.local_note_id), 0);
|
||||
assert_eq!(count_note_dirty_sources(&conn, out3.local_note_id), 0);
|
||||
|
||||
// Kept notes' documents should survive
|
||||
assert_eq!(count_note_documents(&conn, out1.local_note_id), 1);
|
||||
assert_eq!(count_note_documents(&conn, out2.local_note_id), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_sweep_deletion_handles_note_without_document() {
|
||||
let conn = setup();
|
||||
let disc_id = get_discussion_id(&conn);
|
||||
|
||||
let note = make_note(9004, 1, "No doc", None, 1000, 2000, false, None);
|
||||
upsert_note_for_issue(&conn, disc_id, ¬e, 5000, None).unwrap();
|
||||
|
||||
// Don't insert any document -- sweep should still work without error
|
||||
let swept = sweep_stale_issue_notes(&conn, disc_id, 6000).unwrap();
|
||||
assert_eq!(swept, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_set_based_deletion_atomicity() {
|
||||
let conn = setup();
|
||||
let disc_id = get_discussion_id(&conn);
|
||||
|
||||
// Insert a stale note with both document and dirty_source
|
||||
let note = make_note(9005, 1, "Stale with deps", None, 1000, 2000, false, None);
|
||||
let out = upsert_note_for_issue(&conn, disc_id, ¬e, 5000, None).unwrap();
|
||||
insert_note_document(&conn, out.local_note_id);
|
||||
insert_note_dirty_source(&conn, out.local_note_id);
|
||||
|
||||
// Verify they exist before sweep
|
||||
assert_eq!(count_note_documents(&conn, out.local_note_id), 1);
|
||||
assert_eq!(count_note_dirty_sources(&conn, out.local_note_id), 1);
|
||||
|
||||
// The sweep function already runs inside a transaction (called from
|
||||
// ingest_discussions_for_issue's tx). Simulate by wrapping in a transaction.
|
||||
let tx = conn.unchecked_transaction().unwrap();
|
||||
sweep_stale_issue_notes(&tx, disc_id, 6000).unwrap();
|
||||
tx.commit().unwrap();
|
||||
|
||||
// All three DELETEs must have happened
|
||||
assert_eq!(count_note_documents(&conn, out.local_note_id), 0);
|
||||
assert_eq!(count_note_dirty_sources(&conn, out.local_note_id), 0);
|
||||
|
||||
let note_count: i64 = conn
|
||||
.query_row(
|
||||
"SELECT COUNT(*) FROM notes WHERE gitlab_id = ?",
|
||||
[9005_i64],
|
||||
|row| row.get(0),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(note_count, 0);
|
||||
}
|
||||
|
||||
fn count_dirty_notes(conn: &Connection) -> i64 {
|
||||
conn.query_row(
|
||||
"SELECT COUNT(*) FROM dirty_sources WHERE source_type = 'note'",
|
||||
[],
|
||||
|row| row.get(0),
|
||||
)
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parent_title_change_marks_notes_dirty() {
|
||||
let conn = setup();
|
||||
let disc_id = get_discussion_id(&conn);
|
||||
|
||||
// Insert two user notes and one system note
|
||||
let note1 = make_note(10001, 1, "User note 1", None, 1000, 2000, false, None);
|
||||
let note2 = make_note(10002, 1, "User note 2", None, 1000, 2000, false, None);
|
||||
let mut sys_note = make_note(10003, 1, "System note", None, 1000, 2000, false, None);
|
||||
sys_note.is_system = true;
|
||||
|
||||
let out1 = upsert_note_for_issue(&conn, disc_id, ¬e1, 5000, None).unwrap();
|
||||
let out2 = upsert_note_for_issue(&conn, disc_id, ¬e2, 5000, None).unwrap();
|
||||
upsert_note_for_issue(&conn, disc_id, &sys_note, 5000, None).unwrap();
|
||||
|
||||
// Clear any dirty_sources from individual note upserts
|
||||
conn.execute("DELETE FROM dirty_sources WHERE source_type = 'note'", [])
|
||||
.unwrap();
|
||||
assert_eq!(count_dirty_notes(&conn), 0);
|
||||
|
||||
// Simulate parent title change triggering discussion re-ingest:
|
||||
// update the issue title, then run the propagation SQL
|
||||
conn.execute("UPDATE issues SET title = 'Changed Title' WHERE id = 1", [])
|
||||
.unwrap();
|
||||
|
||||
// Run the propagation query (same as in ingestion code)
|
||||
conn.execute(
|
||||
"INSERT INTO dirty_sources (source_type, source_id, queued_at)
|
||||
SELECT 'note', n.id, ?1
|
||||
FROM notes n
|
||||
WHERE n.discussion_id = ?2 AND n.is_system = 0
|
||||
ON CONFLICT(source_type, source_id) DO UPDATE SET queued_at = excluded.queued_at, attempt_count = 0",
|
||||
params![now_ms(), disc_id],
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
// Both user notes should be dirty, system note should not
|
||||
assert_eq!(count_dirty_notes(&conn), 2);
|
||||
assert_eq!(count_note_dirty_sources(&conn, out1.local_note_id), 1);
|
||||
assert_eq!(count_note_dirty_sources(&conn, out2.local_note_id), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parent_label_change_marks_notes_dirty() {
|
||||
let conn = setup();
|
||||
let disc_id = get_discussion_id(&conn);
|
||||
|
||||
// Insert one user note
|
||||
let note = make_note(11001, 1, "User note", None, 1000, 2000, false, None);
|
||||
let out = upsert_note_for_issue(&conn, disc_id, ¬e, 5000, None).unwrap();
|
||||
|
||||
// Clear dirty_sources
|
||||
conn.execute("DELETE FROM dirty_sources WHERE source_type = 'note'", [])
|
||||
.unwrap();
|
||||
|
||||
// Simulate label change on parent issue (labels are part of issue metadata)
|
||||
conn.execute("UPDATE issues SET updated_at = 9999 WHERE id = 1", [])
|
||||
.unwrap();
|
||||
|
||||
// Run propagation query
|
||||
conn.execute(
|
||||
"INSERT INTO dirty_sources (source_type, source_id, queued_at)
|
||||
SELECT 'note', n.id, ?1
|
||||
FROM notes n
|
||||
WHERE n.discussion_id = ?2 AND n.is_system = 0
|
||||
ON CONFLICT(source_type, source_id) DO UPDATE SET queued_at = excluded.queued_at, attempt_count = 0",
|
||||
params![now_ms(), disc_id],
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(count_dirty_notes(&conn), 1);
|
||||
assert_eq!(count_note_dirty_sources(&conn, out.local_note_id), 1);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user