feat(core): Add cross-reference extraction infrastructure
Introduces two new modules for extracting and storing entity cross-references from GitLab data: note_parser.rs: - Parses system notes for "mentioned in" and "closed by" patterns - Extracts cross-project references (group/project#42, group/project!123) - Uses lazy-compiled regexes for performance - Handles both issue (#) and MR (!) sigils - Provides extract_refs_from_system_notes() for batch processing references.rs: - Extracts refs from resource_state_events table (API-sourced closes links) - Provides insert_entity_reference() for storing discovered references - Includes resolution helpers: resolve_issue_local_id, resolve_mr_local_id, resolve_project_path for converting iids to internal IDs - Enables cross-project reference resolution These modules power the entity_references table, enabling features like "find all MRs that close this issue" and "find all issues mentioned in this MR". Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -1,5 +1,3 @@
|
||||
//! Core infrastructure modules.
|
||||
|
||||
pub mod backoff;
|
||||
pub mod config;
|
||||
pub mod db;
|
||||
@@ -9,9 +7,11 @@ pub mod events_db;
|
||||
pub mod lock;
|
||||
pub mod logging;
|
||||
pub mod metrics;
|
||||
pub mod note_parser;
|
||||
pub mod paths;
|
||||
pub mod payloads;
|
||||
pub mod project;
|
||||
pub mod references;
|
||||
pub mod sync_run;
|
||||
pub mod time;
|
||||
|
||||
|
||||
561
src/core/note_parser.rs
Normal file
561
src/core/note_parser.rs
Normal file
@@ -0,0 +1,561 @@
|
||||
use std::sync::LazyLock;
|
||||
|
||||
use regex::Regex;
|
||||
use rusqlite::Connection;
|
||||
use tracing::debug;
|
||||
|
||||
use super::error::Result;
|
||||
use super::time::now_ms;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ParsedCrossRef {
|
||||
pub reference_type: String,
|
||||
pub target_entity_type: String,
|
||||
pub target_iid: i64,
|
||||
pub target_project_path: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct ExtractResult {
|
||||
pub inserted: usize,
|
||||
pub skipped_unresolvable: usize,
|
||||
pub parse_failures: usize,
|
||||
}
|
||||
|
||||
static MENTIONED_RE: LazyLock<Regex> = LazyLock::new(|| {
|
||||
Regex::new(
|
||||
r"mentioned in (?:(?P<project>[\w][\w.\-]*(?:/[\w][\w.\-]*)+))?(?P<sigil>[#!])(?P<iid>\d+)",
|
||||
)
|
||||
.expect("mentioned regex is valid")
|
||||
});
|
||||
|
||||
static CLOSED_BY_RE: LazyLock<Regex> = LazyLock::new(|| {
|
||||
Regex::new(
|
||||
r"closed by (?:(?P<project>[\w][\w.\-]*(?:/[\w][\w.\-]*)+))?(?P<sigil>[#!])(?P<iid>\d+)",
|
||||
)
|
||||
.expect("closed_by regex is valid")
|
||||
});
|
||||
|
||||
pub fn parse_cross_refs(body: &str) -> Vec<ParsedCrossRef> {
|
||||
let mut refs = Vec::new();
|
||||
|
||||
for caps in MENTIONED_RE.captures_iter(body) {
|
||||
if let Some(parsed) = capture_to_cross_ref(&caps, "mentioned") {
|
||||
refs.push(parsed);
|
||||
}
|
||||
}
|
||||
|
||||
for caps in CLOSED_BY_RE.captures_iter(body) {
|
||||
if let Some(parsed) = capture_to_cross_ref(&caps, "closes") {
|
||||
refs.push(parsed);
|
||||
}
|
||||
}
|
||||
|
||||
refs
|
||||
}
|
||||
|
||||
fn capture_to_cross_ref(
|
||||
caps: ®ex::Captures<'_>,
|
||||
reference_type: &str,
|
||||
) -> Option<ParsedCrossRef> {
|
||||
let sigil = caps.name("sigil")?.as_str();
|
||||
let iid_str = caps.name("iid")?.as_str();
|
||||
let iid: i64 = iid_str.parse().ok()?;
|
||||
let project = caps.name("project").map(|m| m.as_str().to_owned());
|
||||
|
||||
let target_entity_type = match sigil {
|
||||
"#" => "issue",
|
||||
"!" => "merge_request",
|
||||
_ => return None,
|
||||
};
|
||||
|
||||
Some(ParsedCrossRef {
|
||||
reference_type: reference_type.to_owned(),
|
||||
target_entity_type: target_entity_type.to_owned(),
|
||||
target_iid: iid,
|
||||
target_project_path: project,
|
||||
})
|
||||
}
|
||||
|
||||
struct SystemNote {
|
||||
note_id: i64,
|
||||
body: String,
|
||||
noteable_type: String,
|
||||
entity_id: i64,
|
||||
}
|
||||
|
||||
pub fn extract_refs_from_system_notes(conn: &Connection, project_id: i64) -> Result<ExtractResult> {
|
||||
let mut result = ExtractResult::default();
|
||||
|
||||
let mut stmt = conn.prepare_cached(
|
||||
"SELECT n.id, n.body, d.noteable_type,
|
||||
COALESCE(d.issue_id, d.merge_request_id) AS entity_id
|
||||
FROM notes n
|
||||
JOIN discussions d ON n.discussion_id = d.id
|
||||
WHERE n.is_system = 1
|
||||
AND n.project_id = ?1
|
||||
AND n.body IS NOT NULL",
|
||||
)?;
|
||||
|
||||
let notes: Vec<SystemNote> = stmt
|
||||
.query_map([project_id], |row| {
|
||||
Ok(SystemNote {
|
||||
note_id: row.get(0)?,
|
||||
body: row.get(1)?,
|
||||
noteable_type: row.get(2)?,
|
||||
entity_id: row.get(3)?,
|
||||
})
|
||||
})?
|
||||
.filter_map(|r| r.ok())
|
||||
.collect();
|
||||
|
||||
if notes.is_empty() {
|
||||
return Ok(result);
|
||||
}
|
||||
|
||||
let mut insert_stmt = conn.prepare_cached(
|
||||
"INSERT OR IGNORE INTO entity_references
|
||||
(project_id, source_entity_type, source_entity_id,
|
||||
target_entity_type, target_entity_id,
|
||||
target_project_path, target_entity_iid,
|
||||
reference_type, source_method, created_at)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, 'note_parse', ?9)",
|
||||
)?;
|
||||
|
||||
let now = now_ms();
|
||||
|
||||
for note in ¬es {
|
||||
let cross_refs = parse_cross_refs(¬e.body);
|
||||
|
||||
if cross_refs.is_empty() {
|
||||
debug!(
|
||||
note_id = note.note_id,
|
||||
body = %note.body,
|
||||
"System note did not match any cross-reference pattern"
|
||||
);
|
||||
result.parse_failures += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
let source_entity_type = noteable_type_to_entity_type(¬e.noteable_type);
|
||||
|
||||
for xref in &cross_refs {
|
||||
let target_entity_id = if xref.target_project_path.is_none() {
|
||||
resolve_entity_id(conn, project_id, &xref.target_entity_type, xref.target_iid)
|
||||
} else {
|
||||
resolve_cross_project_entity(
|
||||
conn,
|
||||
xref.target_project_path.as_deref().unwrap_or_default(),
|
||||
&xref.target_entity_type,
|
||||
xref.target_iid,
|
||||
)
|
||||
};
|
||||
|
||||
let rows_changed = insert_stmt.execute(rusqlite::params![
|
||||
project_id,
|
||||
source_entity_type,
|
||||
note.entity_id,
|
||||
xref.target_entity_type,
|
||||
target_entity_id,
|
||||
xref.target_project_path,
|
||||
if target_entity_id.is_none() {
|
||||
Some(xref.target_iid)
|
||||
} else {
|
||||
None
|
||||
},
|
||||
xref.reference_type,
|
||||
now,
|
||||
])?;
|
||||
|
||||
if rows_changed > 0 {
|
||||
if target_entity_id.is_none() {
|
||||
result.skipped_unresolvable += 1;
|
||||
} else {
|
||||
result.inserted += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if result.inserted > 0 || result.skipped_unresolvable > 0 {
|
||||
debug!(
|
||||
inserted = result.inserted,
|
||||
unresolvable = result.skipped_unresolvable,
|
||||
parse_failures = result.parse_failures,
|
||||
"System note cross-reference extraction complete"
|
||||
);
|
||||
}
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
fn noteable_type_to_entity_type(noteable_type: &str) -> &str {
|
||||
match noteable_type {
|
||||
"Issue" => "issue",
|
||||
"MergeRequest" => "merge_request",
|
||||
_ => "issue",
|
||||
}
|
||||
}
|
||||
|
||||
fn resolve_entity_id(
|
||||
conn: &Connection,
|
||||
project_id: i64,
|
||||
entity_type: &str,
|
||||
iid: i64,
|
||||
) -> Option<i64> {
|
||||
let (table, id_col) = match entity_type {
|
||||
"issue" => ("issues", "id"),
|
||||
"merge_request" => ("merge_requests", "id"),
|
||||
_ => return None,
|
||||
};
|
||||
|
||||
let sql = format!("SELECT {id_col} FROM {table} WHERE project_id = ?1 AND iid = ?2");
|
||||
conn.query_row(&sql, rusqlite::params![project_id, iid], |row| row.get(0))
|
||||
.ok()
|
||||
}
|
||||
|
||||
fn resolve_cross_project_entity(
|
||||
conn: &Connection,
|
||||
project_path: &str,
|
||||
entity_type: &str,
|
||||
iid: i64,
|
||||
) -> Option<i64> {
|
||||
let project_id: i64 = conn
|
||||
.query_row(
|
||||
"SELECT id FROM projects WHERE path_with_namespace = ?1",
|
||||
[project_path],
|
||||
|row| row.get(0),
|
||||
)
|
||||
.ok()?;
|
||||
|
||||
resolve_entity_id(conn, project_id, entity_type, iid)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_parse_mentioned_in_mr() {
|
||||
let refs = parse_cross_refs("mentioned in !567");
|
||||
assert_eq!(refs.len(), 1);
|
||||
assert_eq!(refs[0].reference_type, "mentioned");
|
||||
assert_eq!(refs[0].target_entity_type, "merge_request");
|
||||
assert_eq!(refs[0].target_iid, 567);
|
||||
assert!(refs[0].target_project_path.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_mentioned_in_issue() {
|
||||
let refs = parse_cross_refs("mentioned in #234");
|
||||
assert_eq!(refs.len(), 1);
|
||||
assert_eq!(refs[0].reference_type, "mentioned");
|
||||
assert_eq!(refs[0].target_entity_type, "issue");
|
||||
assert_eq!(refs[0].target_iid, 234);
|
||||
assert!(refs[0].target_project_path.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_mentioned_cross_project() {
|
||||
let refs = parse_cross_refs("mentioned in group/repo!789");
|
||||
assert_eq!(refs.len(), 1);
|
||||
assert_eq!(refs[0].reference_type, "mentioned");
|
||||
assert_eq!(refs[0].target_entity_type, "merge_request");
|
||||
assert_eq!(refs[0].target_iid, 789);
|
||||
assert_eq!(refs[0].target_project_path.as_deref(), Some("group/repo"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_mentioned_cross_project_issue() {
|
||||
let refs = parse_cross_refs("mentioned in group/repo#123");
|
||||
assert_eq!(refs.len(), 1);
|
||||
assert_eq!(refs[0].reference_type, "mentioned");
|
||||
assert_eq!(refs[0].target_entity_type, "issue");
|
||||
assert_eq!(refs[0].target_iid, 123);
|
||||
assert_eq!(refs[0].target_project_path.as_deref(), Some("group/repo"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_closed_by_mr() {
|
||||
let refs = parse_cross_refs("closed by !567");
|
||||
assert_eq!(refs.len(), 1);
|
||||
assert_eq!(refs[0].reference_type, "closes");
|
||||
assert_eq!(refs[0].target_entity_type, "merge_request");
|
||||
assert_eq!(refs[0].target_iid, 567);
|
||||
assert!(refs[0].target_project_path.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_closed_by_cross_project() {
|
||||
let refs = parse_cross_refs("closed by group/repo!789");
|
||||
assert_eq!(refs.len(), 1);
|
||||
assert_eq!(refs[0].reference_type, "closes");
|
||||
assert_eq!(refs[0].target_entity_type, "merge_request");
|
||||
assert_eq!(refs[0].target_iid, 789);
|
||||
assert_eq!(refs[0].target_project_path.as_deref(), Some("group/repo"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_multiple_refs() {
|
||||
let refs = parse_cross_refs("mentioned in !123 and mentioned in #456");
|
||||
assert_eq!(refs.len(), 2);
|
||||
assert_eq!(refs[0].target_entity_type, "merge_request");
|
||||
assert_eq!(refs[0].target_iid, 123);
|
||||
assert_eq!(refs[1].target_entity_type, "issue");
|
||||
assert_eq!(refs[1].target_iid, 456);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_no_refs() {
|
||||
let refs = parse_cross_refs("Updated the description");
|
||||
assert!(refs.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_non_english_note() {
|
||||
let refs = parse_cross_refs("a ajout\u{00e9} l'\u{00e9}tiquette ~bug");
|
||||
assert!(refs.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_multi_level_group_path() {
|
||||
let refs = parse_cross_refs("mentioned in top/sub/project#123");
|
||||
assert_eq!(refs.len(), 1);
|
||||
assert_eq!(
|
||||
refs[0].target_project_path.as_deref(),
|
||||
Some("top/sub/project")
|
||||
);
|
||||
assert_eq!(refs[0].target_iid, 123);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_deeply_nested_group_path() {
|
||||
let refs = parse_cross_refs("mentioned in a/b/c/d/e!42");
|
||||
assert_eq!(refs.len(), 1);
|
||||
assert_eq!(refs[0].target_project_path.as_deref(), Some("a/b/c/d/e"));
|
||||
assert_eq!(refs[0].target_iid, 42);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_hyphenated_project_path() {
|
||||
let refs = parse_cross_refs("mentioned in my-group/my-project#99");
|
||||
assert_eq!(refs.len(), 1);
|
||||
assert_eq!(
|
||||
refs[0].target_project_path.as_deref(),
|
||||
Some("my-group/my-project")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_dotted_project_path() {
|
||||
let refs = parse_cross_refs("mentioned in visiostack.io/backend#123");
|
||||
assert_eq!(refs.len(), 1);
|
||||
assert_eq!(
|
||||
refs[0].target_project_path.as_deref(),
|
||||
Some("visiostack.io/backend")
|
||||
);
|
||||
assert_eq!(refs[0].target_iid, 123);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_dotted_nested_project_path() {
|
||||
let refs = parse_cross_refs("closed by my.org/sub.group/my.project!42");
|
||||
assert_eq!(refs.len(), 1);
|
||||
assert_eq!(
|
||||
refs[0].target_project_path.as_deref(),
|
||||
Some("my.org/sub.group/my.project")
|
||||
);
|
||||
assert_eq!(refs[0].target_entity_type, "merge_request");
|
||||
assert_eq!(refs[0].target_iid, 42);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_self_reference_is_valid() {
|
||||
let refs = parse_cross_refs("mentioned in #123");
|
||||
assert_eq!(refs.len(), 1);
|
||||
assert_eq!(refs[0].target_iid, 123);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_mixed_mentioned_and_closed() {
|
||||
let refs = parse_cross_refs("mentioned in !10 and closed by !20");
|
||||
assert_eq!(refs.len(), 2);
|
||||
assert_eq!(refs[0].reference_type, "mentioned");
|
||||
assert_eq!(refs[0].target_iid, 10);
|
||||
assert_eq!(refs[1].reference_type, "closes");
|
||||
assert_eq!(refs[1].target_iid, 20);
|
||||
}
|
||||
|
||||
fn setup_test_db() -> Connection {
|
||||
use crate::core::db::{create_connection, run_migrations};
|
||||
|
||||
let conn = create_connection(std::path::Path::new(":memory:")).unwrap();
|
||||
run_migrations(&conn).unwrap();
|
||||
conn
|
||||
}
|
||||
|
||||
fn seed_test_data(conn: &Connection) -> i64 {
|
||||
let now = now_ms();
|
||||
|
||||
conn.execute(
|
||||
"INSERT INTO projects (id, gitlab_project_id, path_with_namespace, web_url, created_at, updated_at)
|
||||
VALUES (1, 100, 'group/test-project', 'https://gitlab.com/group/test-project', ?1, ?1)",
|
||||
[now],
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
conn.execute(
|
||||
"INSERT INTO issues (id, gitlab_id, project_id, iid, title, state, created_at, updated_at, last_seen_at)
|
||||
VALUES (10, 1000, 1, 123, 'Test Issue', 'opened', ?1, ?1, ?1)",
|
||||
[now],
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
conn.execute(
|
||||
"INSERT INTO issues (id, gitlab_id, project_id, iid, title, state, created_at, updated_at, last_seen_at)
|
||||
VALUES (11, 1001, 1, 456, 'Another Issue', 'opened', ?1, ?1, ?1)",
|
||||
[now],
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
conn.execute(
|
||||
"INSERT INTO merge_requests (id, gitlab_id, project_id, iid, title, state, source_branch, target_branch, author_username, created_at, updated_at, last_seen_at)
|
||||
VALUES (20, 2000, 1, 789, 'Test MR', 'opened', 'feat', 'main', 'dev', ?1, ?1, ?1)",
|
||||
[now],
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
conn.execute(
|
||||
"INSERT INTO discussions (id, gitlab_discussion_id, project_id, issue_id, noteable_type, last_seen_at)
|
||||
VALUES (30, 'disc-aaa', 1, 10, 'Issue', ?1)",
|
||||
[now],
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
conn.execute(
|
||||
"INSERT INTO discussions (id, gitlab_discussion_id, project_id, merge_request_id, noteable_type, last_seen_at)
|
||||
VALUES (31, 'disc-bbb', 1, 20, 'MergeRequest', ?1)",
|
||||
[now],
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
conn.execute(
|
||||
"INSERT INTO notes (id, gitlab_id, discussion_id, project_id, is_system, body, created_at, updated_at, last_seen_at)
|
||||
VALUES (40, 4000, 30, 1, 1, 'mentioned in !789', ?1, ?1, ?1)",
|
||||
[now],
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
conn.execute(
|
||||
"INSERT INTO notes (id, gitlab_id, discussion_id, project_id, is_system, body, created_at, updated_at, last_seen_at)
|
||||
VALUES (41, 4001, 31, 1, 1, 'mentioned in #456', ?1, ?1, ?1)",
|
||||
[now],
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
conn.execute(
|
||||
"INSERT INTO notes (id, gitlab_id, discussion_id, project_id, is_system, body, created_at, updated_at, last_seen_at)
|
||||
VALUES (42, 4002, 30, 1, 0, 'mentioned in !999', ?1, ?1, ?1)",
|
||||
[now],
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
conn.execute(
|
||||
"INSERT INTO notes (id, gitlab_id, discussion_id, project_id, is_system, body, created_at, updated_at, last_seen_at)
|
||||
VALUES (43, 4003, 30, 1, 1, 'added label ~bug', ?1, ?1, ?1)",
|
||||
[now],
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
conn.execute(
|
||||
"INSERT INTO notes (id, gitlab_id, discussion_id, project_id, is_system, body, created_at, updated_at, last_seen_at)
|
||||
VALUES (44, 4004, 30, 1, 1, 'mentioned in other/project#999', ?1, ?1, ?1)",
|
||||
[now],
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
1
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_refs_from_system_notes_integration() {
|
||||
let conn = setup_test_db();
|
||||
let project_id = seed_test_data(&conn);
|
||||
|
||||
let result = extract_refs_from_system_notes(&conn, project_id).unwrap();
|
||||
|
||||
assert_eq!(result.inserted, 2, "Two same-project refs should resolve");
|
||||
assert_eq!(
|
||||
result.skipped_unresolvable, 1,
|
||||
"One cross-project ref should be unresolvable"
|
||||
);
|
||||
assert_eq!(
|
||||
result.parse_failures, 1,
|
||||
"One system note has no cross-ref pattern"
|
||||
);
|
||||
|
||||
let ref_count: i64 = conn
|
||||
.query_row(
|
||||
"SELECT COUNT(*) FROM entity_references WHERE project_id = ?1 AND source_method = 'note_parse'",
|
||||
[project_id],
|
||||
|row| row.get(0),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(ref_count, 3, "Should have 3 entity_references rows total");
|
||||
|
||||
let unresolved_count: i64 = conn
|
||||
.query_row(
|
||||
"SELECT COUNT(*) FROM entity_references WHERE target_entity_id IS NULL AND source_method = 'note_parse'",
|
||||
[],
|
||||
|row| row.get(0),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
unresolved_count, 1,
|
||||
"Should have 1 unresolved cross-project ref"
|
||||
);
|
||||
|
||||
let (path, iid): (String, i64) = conn
|
||||
.query_row(
|
||||
"SELECT target_project_path, target_entity_iid FROM entity_references WHERE target_entity_id IS NULL",
|
||||
[],
|
||||
|row| Ok((row.get(0)?, row.get(1)?)),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(path, "other/project");
|
||||
assert_eq!(iid, 999);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_refs_idempotent() {
|
||||
let conn = setup_test_db();
|
||||
let project_id = seed_test_data(&conn);
|
||||
|
||||
let result1 = extract_refs_from_system_notes(&conn, project_id).unwrap();
|
||||
let result2 = extract_refs_from_system_notes(&conn, project_id).unwrap();
|
||||
|
||||
assert_eq!(result2.inserted, 0);
|
||||
assert_eq!(result2.skipped_unresolvable, 0);
|
||||
|
||||
let total: i64 = conn
|
||||
.query_row(
|
||||
"SELECT COUNT(*) FROM entity_references WHERE source_method = 'note_parse'",
|
||||
[],
|
||||
|row| row.get(0),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
total,
|
||||
(result1.inserted + result1.skipped_unresolvable) as i64
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_refs_empty_project() {
|
||||
let conn = setup_test_db();
|
||||
let result = extract_refs_from_system_notes(&conn, 999).unwrap();
|
||||
assert_eq!(result.inserted, 0);
|
||||
assert_eq!(result.skipped_unresolvable, 0);
|
||||
assert_eq!(result.parse_failures, 0);
|
||||
}
|
||||
}
|
||||
551
src/core/references.rs
Normal file
551
src/core/references.rs
Normal file
@@ -0,0 +1,551 @@
|
||||
use rusqlite::{Connection, OptionalExtension};
|
||||
use tracing::info;
|
||||
|
||||
use super::error::Result;
|
||||
use super::time::now_ms;
|
||||
|
||||
pub fn extract_refs_from_state_events(conn: &Connection, project_id: i64) -> Result<usize> {
|
||||
let changes = conn.execute(
|
||||
"INSERT OR IGNORE INTO entity_references (
|
||||
project_id,
|
||||
source_entity_type, source_entity_id,
|
||||
target_entity_type, target_entity_id,
|
||||
reference_type, source_method, created_at
|
||||
)
|
||||
SELECT
|
||||
rse.project_id,
|
||||
'merge_request',
|
||||
mr.id,
|
||||
'issue',
|
||||
rse.issue_id,
|
||||
'closes',
|
||||
'api',
|
||||
rse.created_at
|
||||
FROM resource_state_events rse
|
||||
JOIN merge_requests mr
|
||||
ON mr.project_id = rse.project_id
|
||||
AND mr.iid = rse.source_merge_request_iid
|
||||
WHERE rse.source_merge_request_iid IS NOT NULL
|
||||
AND rse.issue_id IS NOT NULL
|
||||
AND rse.project_id = ?1",
|
||||
rusqlite::params![project_id],
|
||||
)?;
|
||||
|
||||
if changes > 0 {
|
||||
info!(
|
||||
project_id,
|
||||
references_inserted = changes,
|
||||
"Extracted cross-references from state events"
|
||||
);
|
||||
}
|
||||
|
||||
Ok(changes)
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct EntityReference<'a> {
|
||||
pub project_id: i64,
|
||||
pub source_entity_type: &'a str,
|
||||
pub source_entity_id: i64,
|
||||
pub target_entity_type: &'a str,
|
||||
pub target_entity_id: Option<i64>,
|
||||
pub target_project_path: Option<&'a str>,
|
||||
pub target_entity_iid: Option<i64>,
|
||||
pub reference_type: &'a str,
|
||||
pub source_method: &'a str,
|
||||
}
|
||||
|
||||
pub fn insert_entity_reference(conn: &Connection, ref_: &EntityReference<'_>) -> Result<bool> {
|
||||
let now = now_ms();
|
||||
let changes = conn.execute(
|
||||
"INSERT OR IGNORE INTO entity_references \
|
||||
(project_id, source_entity_type, source_entity_id, \
|
||||
target_entity_type, target_entity_id, target_project_path, target_entity_iid, \
|
||||
reference_type, source_method, created_at) \
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)",
|
||||
rusqlite::params![
|
||||
ref_.project_id,
|
||||
ref_.source_entity_type,
|
||||
ref_.source_entity_id,
|
||||
ref_.target_entity_type,
|
||||
ref_.target_entity_id,
|
||||
ref_.target_project_path,
|
||||
ref_.target_entity_iid,
|
||||
ref_.reference_type,
|
||||
ref_.source_method,
|
||||
now,
|
||||
],
|
||||
)?;
|
||||
|
||||
Ok(changes > 0)
|
||||
}
|
||||
|
||||
pub fn resolve_issue_local_id(
|
||||
conn: &Connection,
|
||||
project_id: i64,
|
||||
issue_iid: i64,
|
||||
) -> Result<Option<i64>> {
|
||||
let mut stmt =
|
||||
conn.prepare_cached("SELECT id FROM issues WHERE project_id = ?1 AND iid = ?2")?;
|
||||
|
||||
let result = stmt
|
||||
.query_row(rusqlite::params![project_id, issue_iid], |row| row.get(0))
|
||||
.optional()?;
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
pub fn resolve_project_path(conn: &Connection, gitlab_project_id: i64) -> Result<Option<String>> {
|
||||
let mut stmt = conn
|
||||
.prepare_cached("SELECT path_with_namespace FROM projects WHERE gitlab_project_id = ?1")?;
|
||||
|
||||
let result = stmt
|
||||
.query_row(rusqlite::params![gitlab_project_id], |row| row.get(0))
|
||||
.optional()?;
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
pub fn count_references_for_source(
|
||||
conn: &Connection,
|
||||
source_entity_type: &str,
|
||||
source_entity_id: i64,
|
||||
) -> Result<usize> {
|
||||
let count: i64 = conn.query_row(
|
||||
"SELECT COUNT(*) FROM entity_references \
|
||||
WHERE source_entity_type = ?1 AND source_entity_id = ?2",
|
||||
rusqlite::params![source_entity_type, source_entity_id],
|
||||
|row| row.get(0),
|
||||
)?;
|
||||
|
||||
Ok(count as usize)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::core::db::{create_connection, run_migrations};
|
||||
use std::path::Path;
|
||||
|
||||
fn setup_test_db() -> Connection {
|
||||
let conn = create_connection(Path::new(":memory:")).unwrap();
|
||||
run_migrations(&conn).unwrap();
|
||||
conn
|
||||
}
|
||||
|
||||
fn seed_project_issue_mr(conn: &Connection) -> (i64, i64, i64) {
|
||||
conn.execute(
|
||||
"INSERT INTO projects (id, gitlab_project_id, path_with_namespace, web_url, created_at, updated_at)
|
||||
VALUES (1, 100, 'group/repo', 'https://gitlab.example.com/group/repo', 1000, 2000)",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
conn.execute(
|
||||
"INSERT INTO issues (id, gitlab_id, iid, project_id, title, state, created_at, updated_at, last_seen_at)
|
||||
VALUES (1, 200, 10, 1, 'Test issue', 'closed', 1000, 2000, 2000)",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
conn.execute(
|
||||
"INSERT INTO merge_requests (id, gitlab_id, iid, project_id, title, state, created_at, updated_at, last_seen_at, source_branch, target_branch)
|
||||
VALUES (1, 300, 5, 1, 'Test MR', 'merged', 1000, 2000, 2000, 'feature', 'main')",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
(1, 1, 1)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_refs_from_state_events_basic() {
|
||||
let conn = setup_test_db();
|
||||
let (project_id, issue_id, mr_id) = seed_project_issue_mr(&conn);
|
||||
|
||||
conn.execute(
|
||||
"INSERT INTO resource_state_events
|
||||
(gitlab_id, project_id, issue_id, merge_request_id, state,
|
||||
created_at, source_merge_request_iid)
|
||||
VALUES (1, ?1, ?2, NULL, 'closed', 3000, 5)",
|
||||
rusqlite::params![project_id, issue_id],
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let count = extract_refs_from_state_events(&conn, project_id).unwrap();
|
||||
assert_eq!(count, 1, "Should insert exactly one reference");
|
||||
|
||||
let (src_type, src_id, tgt_type, tgt_id, ref_type, method): (
|
||||
String,
|
||||
i64,
|
||||
String,
|
||||
i64,
|
||||
String,
|
||||
String,
|
||||
) = conn
|
||||
.query_row(
|
||||
"SELECT source_entity_type, source_entity_id,
|
||||
target_entity_type, target_entity_id,
|
||||
reference_type, source_method
|
||||
FROM entity_references WHERE project_id = ?1",
|
||||
[project_id],
|
||||
|row| {
|
||||
Ok((
|
||||
row.get(0)?,
|
||||
row.get(1)?,
|
||||
row.get(2)?,
|
||||
row.get(3)?,
|
||||
row.get(4)?,
|
||||
row.get(5)?,
|
||||
))
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(src_type, "merge_request");
|
||||
assert_eq!(src_id, mr_id, "Source should be the MR's local DB id");
|
||||
assert_eq!(tgt_type, "issue");
|
||||
assert_eq!(tgt_id, issue_id, "Target should be the issue's local DB id");
|
||||
assert_eq!(ref_type, "closes");
|
||||
assert_eq!(method, "api");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_refs_dedup_with_closes_issues() {
|
||||
let conn = setup_test_db();
|
||||
let (project_id, issue_id, mr_id) = seed_project_issue_mr(&conn);
|
||||
|
||||
conn.execute(
|
||||
"INSERT INTO entity_references
|
||||
(project_id, source_entity_type, source_entity_id,
|
||||
target_entity_type, target_entity_id,
|
||||
reference_type, source_method, created_at)
|
||||
VALUES (?1, 'merge_request', ?2, 'issue', ?3, 'closes', 'api', 3000)",
|
||||
rusqlite::params![project_id, mr_id, issue_id],
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
conn.execute(
|
||||
"INSERT INTO resource_state_events
|
||||
(gitlab_id, project_id, issue_id, merge_request_id, state,
|
||||
created_at, source_merge_request_iid)
|
||||
VALUES (1, ?1, ?2, NULL, 'closed', 3000, 5)",
|
||||
rusqlite::params![project_id, issue_id],
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let count = extract_refs_from_state_events(&conn, project_id).unwrap();
|
||||
assert_eq!(count, 0, "Should not insert duplicate reference");
|
||||
|
||||
let total: i64 = conn
|
||||
.query_row(
|
||||
"SELECT COUNT(*) FROM entity_references WHERE project_id = ?1",
|
||||
[project_id],
|
||||
|row| row.get(0),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(total, 1, "Should still have exactly one reference");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_refs_no_source_mr() {
|
||||
let conn = setup_test_db();
|
||||
let (project_id, issue_id, _mr_id) = seed_project_issue_mr(&conn);
|
||||
|
||||
conn.execute(
|
||||
"INSERT INTO resource_state_events
|
||||
(gitlab_id, project_id, issue_id, merge_request_id, state,
|
||||
created_at, source_merge_request_iid)
|
||||
VALUES (1, ?1, ?2, NULL, 'closed', 3000, NULL)",
|
||||
rusqlite::params![project_id, issue_id],
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let count = extract_refs_from_state_events(&conn, project_id).unwrap();
|
||||
assert_eq!(count, 0, "Should not create refs when no source MR");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_refs_mr_not_synced() {
|
||||
let conn = setup_test_db();
|
||||
let (project_id, issue_id, _mr_id) = seed_project_issue_mr(&conn);
|
||||
|
||||
conn.execute(
|
||||
"INSERT INTO resource_state_events
|
||||
(gitlab_id, project_id, issue_id, merge_request_id, state,
|
||||
created_at, source_merge_request_iid)
|
||||
VALUES (2, ?1, ?2, NULL, 'closed', 3000, 999)",
|
||||
rusqlite::params![project_id, issue_id],
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let count = extract_refs_from_state_events(&conn, project_id).unwrap();
|
||||
assert_eq!(
|
||||
count, 0,
|
||||
"Should not create ref when MR is not synced locally"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_refs_idempotent() {
|
||||
let conn = setup_test_db();
|
||||
let (project_id, issue_id, _mr_id) = seed_project_issue_mr(&conn);
|
||||
|
||||
conn.execute(
|
||||
"INSERT INTO resource_state_events
|
||||
(gitlab_id, project_id, issue_id, merge_request_id, state,
|
||||
created_at, source_merge_request_iid)
|
||||
VALUES (1, ?1, ?2, NULL, 'closed', 3000, 5)",
|
||||
rusqlite::params![project_id, issue_id],
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let count1 = extract_refs_from_state_events(&conn, project_id).unwrap();
|
||||
assert_eq!(count1, 1);
|
||||
|
||||
let count2 = extract_refs_from_state_events(&conn, project_id).unwrap();
|
||||
assert_eq!(count2, 0, "Second run should insert nothing (idempotent)");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_refs_multiple_events_same_mr_issue() {
|
||||
let conn = setup_test_db();
|
||||
let (project_id, issue_id, _mr_id) = seed_project_issue_mr(&conn);
|
||||
|
||||
conn.execute(
|
||||
"INSERT INTO resource_state_events
|
||||
(gitlab_id, project_id, issue_id, merge_request_id, state,
|
||||
created_at, source_merge_request_iid)
|
||||
VALUES (1, ?1, ?2, NULL, 'closed', 3000, 5)",
|
||||
rusqlite::params![project_id, issue_id],
|
||||
)
|
||||
.unwrap();
|
||||
conn.execute(
|
||||
"INSERT INTO resource_state_events
|
||||
(gitlab_id, project_id, issue_id, merge_request_id, state,
|
||||
created_at, source_merge_request_iid)
|
||||
VALUES (2, ?1, ?2, NULL, 'closed', 4000, 5)",
|
||||
rusqlite::params![project_id, issue_id],
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let count = extract_refs_from_state_events(&conn, project_id).unwrap();
|
||||
assert!(count <= 2, "At most 2 inserts attempted");
|
||||
|
||||
let total: i64 = conn
|
||||
.query_row(
|
||||
"SELECT COUNT(*) FROM entity_references WHERE project_id = ?1",
|
||||
[project_id],
|
||||
|row| row.get(0),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
total, 1,
|
||||
"Only one unique reference should exist for same MR->issue pair"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_refs_scoped_to_project() {
|
||||
let conn = setup_test_db();
|
||||
seed_project_issue_mr(&conn);
|
||||
|
||||
conn.execute(
|
||||
"INSERT INTO projects (id, gitlab_project_id, path_with_namespace, web_url, created_at, updated_at)
|
||||
VALUES (2, 101, 'group/other', 'https://gitlab.example.com/group/other', 1000, 2000)",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
conn.execute(
|
||||
"INSERT INTO issues (id, gitlab_id, iid, project_id, title, state, created_at, updated_at, last_seen_at)
|
||||
VALUES (2, 201, 10, 2, 'Other issue', 'closed', 1000, 2000, 2000)",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
conn.execute(
|
||||
"INSERT INTO merge_requests (id, gitlab_id, iid, project_id, title, state, created_at, updated_at, last_seen_at, source_branch, target_branch)
|
||||
VALUES (2, 301, 5, 2, 'Other MR', 'merged', 1000, 2000, 2000, 'feature', 'main')",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
conn.execute(
|
||||
"INSERT INTO resource_state_events
|
||||
(gitlab_id, project_id, issue_id, merge_request_id, state,
|
||||
created_at, source_merge_request_iid)
|
||||
VALUES (1, 1, 1, NULL, 'closed', 3000, 5)",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
conn.execute(
|
||||
"INSERT INTO resource_state_events
|
||||
(gitlab_id, project_id, issue_id, merge_request_id, state,
|
||||
created_at, source_merge_request_iid)
|
||||
VALUES (2, 2, 2, NULL, 'closed', 3000, 5)",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let count = extract_refs_from_state_events(&conn, 1).unwrap();
|
||||
assert_eq!(count, 1);
|
||||
|
||||
let total: i64 = conn
|
||||
.query_row("SELECT COUNT(*) FROM entity_references", [], |row| {
|
||||
row.get(0)
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(total, 1, "Only project 1 refs should be created");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_insert_entity_reference_creates_row() {
|
||||
let conn = setup_test_db();
|
||||
let (project_id, issue_id, mr_id) = seed_project_issue_mr(&conn);
|
||||
|
||||
let ref_ = EntityReference {
|
||||
project_id,
|
||||
source_entity_type: "merge_request",
|
||||
source_entity_id: mr_id,
|
||||
target_entity_type: "issue",
|
||||
target_entity_id: Some(issue_id),
|
||||
target_project_path: None,
|
||||
target_entity_iid: None,
|
||||
reference_type: "closes",
|
||||
source_method: "api",
|
||||
};
|
||||
|
||||
let inserted = insert_entity_reference(&conn, &ref_).unwrap();
|
||||
assert!(inserted);
|
||||
|
||||
let count = count_references_for_source(&conn, "merge_request", mr_id).unwrap();
|
||||
assert_eq!(count, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_insert_entity_reference_idempotent() {
|
||||
let conn = setup_test_db();
|
||||
let (project_id, issue_id, mr_id) = seed_project_issue_mr(&conn);
|
||||
|
||||
let ref_ = EntityReference {
|
||||
project_id,
|
||||
source_entity_type: "merge_request",
|
||||
source_entity_id: mr_id,
|
||||
target_entity_type: "issue",
|
||||
target_entity_id: Some(issue_id),
|
||||
target_project_path: None,
|
||||
target_entity_iid: None,
|
||||
reference_type: "closes",
|
||||
source_method: "api",
|
||||
};
|
||||
|
||||
let first = insert_entity_reference(&conn, &ref_).unwrap();
|
||||
assert!(first);
|
||||
|
||||
let second = insert_entity_reference(&conn, &ref_).unwrap();
|
||||
assert!(!second, "Duplicate insert should be ignored");
|
||||
|
||||
let count = count_references_for_source(&conn, "merge_request", mr_id).unwrap();
|
||||
assert_eq!(count, 1, "Still just one reference");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_insert_entity_reference_cross_project_unresolved() {
|
||||
let conn = setup_test_db();
|
||||
let (project_id, _issue_id, mr_id) = seed_project_issue_mr(&conn);
|
||||
|
||||
let ref_ = EntityReference {
|
||||
project_id,
|
||||
source_entity_type: "merge_request",
|
||||
source_entity_id: mr_id,
|
||||
target_entity_type: "issue",
|
||||
target_entity_id: None,
|
||||
target_project_path: Some("other-group/other-project"),
|
||||
target_entity_iid: Some(99),
|
||||
reference_type: "closes",
|
||||
source_method: "api",
|
||||
};
|
||||
|
||||
let inserted = insert_entity_reference(&conn, &ref_).unwrap();
|
||||
assert!(inserted);
|
||||
|
||||
let (target_id, target_path, target_iid): (Option<i64>, Option<String>, Option<i64>) = conn
|
||||
.query_row(
|
||||
"SELECT target_entity_id, target_project_path, target_entity_iid \
|
||||
FROM entity_references WHERE source_entity_id = ?1",
|
||||
[mr_id],
|
||||
|row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert!(target_id.is_none());
|
||||
assert_eq!(target_path, Some("other-group/other-project".to_string()));
|
||||
assert_eq!(target_iid, Some(99));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_insert_multiple_closes_references() {
|
||||
let conn = setup_test_db();
|
||||
let (project_id, issue_id, mr_id) = seed_project_issue_mr(&conn);
|
||||
|
||||
conn.execute(
|
||||
"INSERT INTO issues (id, gitlab_id, iid, project_id, title, state, created_at, updated_at, last_seen_at)
|
||||
VALUES (10, 210, 11, ?1, 'Second issue', 'opened', 1000, 2000, 2000)",
|
||||
rusqlite::params![project_id],
|
||||
)
|
||||
.unwrap();
|
||||
let issue_id_2 = 10i64;
|
||||
|
||||
for target_id in [issue_id, issue_id_2] {
|
||||
let ref_ = EntityReference {
|
||||
project_id,
|
||||
source_entity_type: "merge_request",
|
||||
source_entity_id: mr_id,
|
||||
target_entity_type: "issue",
|
||||
target_entity_id: Some(target_id),
|
||||
target_project_path: None,
|
||||
target_entity_iid: None,
|
||||
reference_type: "closes",
|
||||
source_method: "api",
|
||||
};
|
||||
insert_entity_reference(&conn, &ref_).unwrap();
|
||||
}
|
||||
|
||||
let count = count_references_for_source(&conn, "merge_request", mr_id).unwrap();
|
||||
assert_eq!(count, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_resolve_issue_local_id_found() {
|
||||
let conn = setup_test_db();
|
||||
let (project_id, issue_id, _mr_id) = seed_project_issue_mr(&conn);
|
||||
|
||||
let resolved = resolve_issue_local_id(&conn, project_id, 10).unwrap();
|
||||
assert_eq!(resolved, Some(issue_id));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_resolve_issue_local_id_not_found() {
|
||||
let conn = setup_test_db();
|
||||
let (project_id, _issue_id, _mr_id) = seed_project_issue_mr(&conn);
|
||||
|
||||
let resolved = resolve_issue_local_id(&conn, project_id, 999).unwrap();
|
||||
assert!(resolved.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_resolve_project_path_found() {
|
||||
let conn = setup_test_db();
|
||||
seed_project_issue_mr(&conn);
|
||||
|
||||
let path = resolve_project_path(&conn, 100).unwrap();
|
||||
assert_eq!(path, Some("group/repo".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_resolve_project_path_not_found() {
|
||||
let conn = setup_test_db();
|
||||
|
||||
let path = resolve_project_path(&conn, 999).unwrap();
|
||||
assert!(path.is_none());
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user