Introduces NormalizedMergeRequest transformer and updates discussion
normalization to handle both issue and MR discussions polymorphically.
New transformers:
- NormalizedMergeRequest: Transforms API MergeRequest to database row,
extracting labels/assignees/reviewers into separate collections for
junction table insertion. Handles draft detection, detailed_merge_status
preference over deprecated merge_status, and merge_user over merged_by.
Discussion transformer updates:
- NormalizedDiscussion now takes noteable_type ("Issue" | "MergeRequest")
and noteable_id for polymorphic FK binding
- normalize_discussions_for_issue(): Convenience wrapper for issues
- normalize_discussions_for_mr(): Convenience wrapper for MRs
- DiffNote position fields (type, line_range, SHA triplet) now extracted
from API position object for code review context
Design decisions:
- Transformer returns (normalized_item, labels, assignees, reviewers)
tuple for efficient batch insertion without re-querying
- Timestamps converted to ms epoch for SQLite storage consistency
- Optional fields use map() chains for clean null handling
The polymorphic discussion approach allows reusing the same discussions
and notes tables for both issues and MRs, with noteable_type + FK
determining the parent relationship.
Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
588 lines
18 KiB
Rust
588 lines
18 KiB
Rust
//! Discussion and note transformers: convert GitLab discussions to local schema.
|
|
|
|
use chrono::DateTime;
|
|
|
|
use crate::core::time::now_ms;
|
|
use crate::gitlab::types::{GitLabDiscussion, GitLabNote};
|
|
|
|
/// Reference to the parent noteable (Issue or MergeRequest).
|
|
/// Uses an enum to prevent accidentally mixing up issue vs MR IDs at compile time.
|
|
#[derive(Debug, Clone, Copy)]
|
|
pub enum NoteableRef {
|
|
Issue(i64),
|
|
MergeRequest(i64),
|
|
}
|
|
|
|
/// Normalized discussion for local storage.
|
|
#[derive(Debug, Clone)]
|
|
pub struct NormalizedDiscussion {
|
|
pub gitlab_discussion_id: String,
|
|
pub project_id: i64,
|
|
pub issue_id: Option<i64>,
|
|
pub merge_request_id: Option<i64>,
|
|
pub noteable_type: String, // "Issue" or "MergeRequest"
|
|
pub individual_note: bool,
|
|
pub first_note_at: Option<i64>, // min(note.created_at) in ms epoch
|
|
pub last_note_at: Option<i64>, // max(note.created_at) in ms epoch
|
|
pub last_seen_at: i64,
|
|
pub resolvable: bool, // any note is resolvable
|
|
pub resolved: bool, // all resolvable notes are resolved
|
|
}
|
|
|
|
/// Normalized note for local storage.
|
|
#[derive(Debug, Clone)]
|
|
pub struct NormalizedNote {
|
|
pub gitlab_id: i64,
|
|
pub project_id: i64,
|
|
pub note_type: Option<String>, // "DiscussionNote" | "DiffNote" | null
|
|
pub is_system: bool,
|
|
pub author_username: String,
|
|
pub body: String,
|
|
pub created_at: i64, // ms epoch
|
|
pub updated_at: i64, // ms epoch
|
|
pub last_seen_at: i64,
|
|
pub position: i32, // 0-indexed array position
|
|
pub resolvable: bool,
|
|
pub resolved: bool,
|
|
pub resolved_by: Option<String>,
|
|
pub resolved_at: Option<i64>,
|
|
// DiffNote position fields (CP1 - basic path/line)
|
|
pub position_old_path: Option<String>,
|
|
pub position_new_path: Option<String>,
|
|
pub position_old_line: Option<i32>,
|
|
pub position_new_line: Option<i32>,
|
|
// DiffNote extended position fields (CP2)
|
|
pub position_type: Option<String>, // "text" | "image" | "file"
|
|
pub position_line_range_start: Option<i32>, // multi-line comment start
|
|
pub position_line_range_end: Option<i32>, // multi-line comment end
|
|
pub position_base_sha: Option<String>, // Base commit SHA for diff
|
|
pub position_start_sha: Option<String>, // Start commit SHA for diff
|
|
pub position_head_sha: Option<String>, // Head commit SHA for diff
|
|
}
|
|
|
|
/// Parse ISO 8601 timestamp to milliseconds, returning None on failure.
|
|
fn parse_timestamp_opt(ts: &str) -> Option<i64> {
|
|
DateTime::parse_from_rfc3339(ts)
|
|
.ok()
|
|
.map(|dt| dt.timestamp_millis())
|
|
}
|
|
|
|
/// Parse ISO 8601 timestamp to milliseconds, defaulting to 0 on failure.
|
|
fn parse_timestamp(ts: &str) -> i64 {
|
|
parse_timestamp_opt(ts).unwrap_or(0)
|
|
}
|
|
|
|
/// Transform a GitLab discussion into normalized schema.
|
|
pub fn transform_discussion(
|
|
gitlab_discussion: &GitLabDiscussion,
|
|
local_project_id: i64,
|
|
noteable: NoteableRef,
|
|
) -> NormalizedDiscussion {
|
|
let now = now_ms();
|
|
|
|
// Derive issue_id, merge_request_id, and noteable_type from the enum
|
|
let (issue_id, merge_request_id, noteable_type) = match noteable {
|
|
NoteableRef::Issue(id) => (Some(id), None, "Issue"),
|
|
NoteableRef::MergeRequest(id) => (None, Some(id), "MergeRequest"),
|
|
};
|
|
|
|
// Compute first_note_at and last_note_at from notes
|
|
let note_timestamps: Vec<i64> = gitlab_discussion
|
|
.notes
|
|
.iter()
|
|
.filter_map(|n| parse_timestamp_opt(&n.created_at))
|
|
.collect();
|
|
|
|
let first_note_at = note_timestamps.iter().min().copied();
|
|
let last_note_at = note_timestamps.iter().max().copied();
|
|
|
|
// Compute resolvable: any note is resolvable
|
|
let resolvable = gitlab_discussion.notes.iter().any(|n| n.resolvable);
|
|
|
|
// Compute resolved: all resolvable notes are resolved
|
|
let resolved = if resolvable {
|
|
gitlab_discussion
|
|
.notes
|
|
.iter()
|
|
.filter(|n| n.resolvable)
|
|
.all(|n| n.resolved)
|
|
} else {
|
|
false
|
|
};
|
|
|
|
NormalizedDiscussion {
|
|
gitlab_discussion_id: gitlab_discussion.id.clone(),
|
|
project_id: local_project_id,
|
|
issue_id,
|
|
merge_request_id,
|
|
noteable_type: noteable_type.to_string(),
|
|
individual_note: gitlab_discussion.individual_note,
|
|
first_note_at,
|
|
last_note_at,
|
|
last_seen_at: now,
|
|
resolvable,
|
|
resolved,
|
|
}
|
|
}
|
|
|
|
/// Transform a GitLab discussion for MR context.
|
|
/// Convenience wrapper that uses NoteableRef::MergeRequest internally.
|
|
pub fn transform_mr_discussion(
|
|
gitlab_discussion: &GitLabDiscussion,
|
|
local_project_id: i64,
|
|
local_mr_id: i64,
|
|
) -> NormalizedDiscussion {
|
|
transform_discussion(
|
|
gitlab_discussion,
|
|
local_project_id,
|
|
NoteableRef::MergeRequest(local_mr_id),
|
|
)
|
|
}
|
|
|
|
/// Transform notes from a GitLab discussion into normalized schema.
|
|
pub fn transform_notes(
|
|
gitlab_discussion: &GitLabDiscussion,
|
|
local_project_id: i64,
|
|
) -> Vec<NormalizedNote> {
|
|
let now = now_ms();
|
|
|
|
gitlab_discussion
|
|
.notes
|
|
.iter()
|
|
.enumerate()
|
|
.map(|(idx, note)| transform_single_note(note, local_project_id, idx as i32, now))
|
|
.collect()
|
|
}
|
|
|
|
fn transform_single_note(
|
|
note: &GitLabNote,
|
|
local_project_id: i64,
|
|
position: i32,
|
|
now: i64,
|
|
) -> NormalizedNote {
|
|
// Extract DiffNote position fields if present
|
|
let (
|
|
position_old_path,
|
|
position_new_path,
|
|
position_old_line,
|
|
position_new_line,
|
|
position_type,
|
|
position_line_range_start,
|
|
position_line_range_end,
|
|
position_base_sha,
|
|
position_start_sha,
|
|
position_head_sha,
|
|
) = extract_position_fields(¬e.position);
|
|
|
|
NormalizedNote {
|
|
gitlab_id: note.id,
|
|
project_id: local_project_id,
|
|
note_type: note.note_type.clone(),
|
|
is_system: note.system,
|
|
author_username: note.author.username.clone(),
|
|
body: note.body.clone(),
|
|
created_at: parse_timestamp(¬e.created_at),
|
|
updated_at: parse_timestamp(¬e.updated_at),
|
|
last_seen_at: now,
|
|
position,
|
|
resolvable: note.resolvable,
|
|
resolved: note.resolved,
|
|
resolved_by: note.resolved_by.as_ref().map(|a| a.username.clone()),
|
|
resolved_at: note
|
|
.resolved_at
|
|
.as_ref()
|
|
.and_then(|ts| parse_timestamp_opt(ts)),
|
|
position_old_path,
|
|
position_new_path,
|
|
position_old_line,
|
|
position_new_line,
|
|
position_type,
|
|
position_line_range_start,
|
|
position_line_range_end,
|
|
position_base_sha,
|
|
position_start_sha,
|
|
position_head_sha,
|
|
}
|
|
}
|
|
|
|
/// Extract DiffNote position fields from GitLabNotePosition.
|
|
/// Returns tuple of all position fields (all None if position is None).
|
|
#[allow(clippy::type_complexity)]
|
|
fn extract_position_fields(
|
|
position: &Option<crate::gitlab::types::GitLabNotePosition>,
|
|
) -> (
|
|
Option<String>,
|
|
Option<String>,
|
|
Option<i32>,
|
|
Option<i32>,
|
|
Option<String>,
|
|
Option<i32>,
|
|
Option<i32>,
|
|
Option<String>,
|
|
Option<String>,
|
|
Option<String>,
|
|
) {
|
|
match position {
|
|
Some(pos) => {
|
|
let line_range_start = pos.line_range.as_ref().and_then(|lr| lr.start_line());
|
|
let line_range_end = pos.line_range.as_ref().and_then(|lr| lr.end_line());
|
|
|
|
(
|
|
pos.old_path.clone(),
|
|
pos.new_path.clone(),
|
|
pos.old_line,
|
|
pos.new_line,
|
|
pos.position_type.clone(),
|
|
line_range_start,
|
|
line_range_end,
|
|
pos.base_sha.clone(),
|
|
pos.start_sha.clone(),
|
|
pos.head_sha.clone(),
|
|
)
|
|
}
|
|
None => (None, None, None, None, None, None, None, None, None, None),
|
|
}
|
|
}
|
|
|
|
/// Parse ISO 8601 timestamp to milliseconds with strict error handling.
|
|
/// Returns Err with the invalid timestamp in the error message.
|
|
fn parse_timestamp_strict(ts: &str) -> Result<i64, String> {
|
|
DateTime::parse_from_rfc3339(ts)
|
|
.map(|dt| dt.timestamp_millis())
|
|
.map_err(|_| format!("Invalid timestamp: {}", ts))
|
|
}
|
|
|
|
/// Transform notes from a GitLab discussion with strict timestamp parsing.
|
|
/// Returns Err if any timestamp is invalid - no silent fallback to 0.
|
|
pub fn transform_notes_with_diff_position(
|
|
gitlab_discussion: &GitLabDiscussion,
|
|
local_project_id: i64,
|
|
) -> Result<Vec<NormalizedNote>, String> {
|
|
let now = now_ms();
|
|
|
|
gitlab_discussion
|
|
.notes
|
|
.iter()
|
|
.enumerate()
|
|
.map(|(idx, note)| transform_single_note_strict(note, local_project_id, idx as i32, now))
|
|
.collect()
|
|
}
|
|
|
|
fn transform_single_note_strict(
|
|
note: &GitLabNote,
|
|
local_project_id: i64,
|
|
position: i32,
|
|
now: i64,
|
|
) -> Result<NormalizedNote, String> {
|
|
// Parse timestamps with strict error handling
|
|
let created_at = parse_timestamp_strict(¬e.created_at)?;
|
|
let updated_at = parse_timestamp_strict(¬e.updated_at)?;
|
|
let resolved_at = match ¬e.resolved_at {
|
|
Some(ts) => Some(parse_timestamp_strict(ts)?),
|
|
None => None,
|
|
};
|
|
|
|
// Extract DiffNote position fields if present
|
|
let (
|
|
position_old_path,
|
|
position_new_path,
|
|
position_old_line,
|
|
position_new_line,
|
|
position_type,
|
|
position_line_range_start,
|
|
position_line_range_end,
|
|
position_base_sha,
|
|
position_start_sha,
|
|
position_head_sha,
|
|
) = extract_position_fields(¬e.position);
|
|
|
|
Ok(NormalizedNote {
|
|
gitlab_id: note.id,
|
|
project_id: local_project_id,
|
|
note_type: note.note_type.clone(),
|
|
is_system: note.system,
|
|
author_username: note.author.username.clone(),
|
|
body: note.body.clone(),
|
|
created_at,
|
|
updated_at,
|
|
last_seen_at: now,
|
|
position,
|
|
resolvable: note.resolvable,
|
|
resolved: note.resolved,
|
|
resolved_by: note.resolved_by.as_ref().map(|a| a.username.clone()),
|
|
resolved_at,
|
|
position_old_path,
|
|
position_new_path,
|
|
position_old_line,
|
|
position_new_line,
|
|
position_type,
|
|
position_line_range_start,
|
|
position_line_range_end,
|
|
position_base_sha,
|
|
position_start_sha,
|
|
position_head_sha,
|
|
})
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
use crate::gitlab::types::GitLabAuthor;
|
|
|
|
fn make_test_note(
|
|
id: i64,
|
|
created_at: &str,
|
|
system: bool,
|
|
resolvable: bool,
|
|
resolved: bool,
|
|
) -> GitLabNote {
|
|
GitLabNote {
|
|
id,
|
|
note_type: Some("DiscussionNote".to_string()),
|
|
body: format!("Note {}", id),
|
|
author: GitLabAuthor {
|
|
id: 1,
|
|
username: "testuser".to_string(),
|
|
name: "Test User".to_string(),
|
|
},
|
|
created_at: created_at.to_string(),
|
|
updated_at: created_at.to_string(),
|
|
system,
|
|
resolvable,
|
|
resolved,
|
|
resolved_by: None,
|
|
resolved_at: None,
|
|
position: None,
|
|
}
|
|
}
|
|
|
|
fn make_test_discussion(individual_note: bool, notes: Vec<GitLabNote>) -> GitLabDiscussion {
|
|
GitLabDiscussion {
|
|
id: "6a9c1750b37d513a43987b574953fceb50b03ce7".to_string(),
|
|
individual_note,
|
|
notes,
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn transforms_discussion_payload_to_normalized_schema() {
|
|
let discussion = make_test_discussion(
|
|
false,
|
|
vec![make_test_note(
|
|
1,
|
|
"2024-01-16T09:00:00.000Z",
|
|
false,
|
|
false,
|
|
false,
|
|
)],
|
|
);
|
|
|
|
let result = transform_discussion(&discussion, 100, NoteableRef::Issue(42));
|
|
|
|
assert_eq!(
|
|
result.gitlab_discussion_id,
|
|
"6a9c1750b37d513a43987b574953fceb50b03ce7"
|
|
);
|
|
assert_eq!(result.project_id, 100);
|
|
assert_eq!(result.issue_id, Some(42));
|
|
assert_eq!(result.merge_request_id, None);
|
|
assert_eq!(result.noteable_type, "Issue");
|
|
assert!(!result.individual_note);
|
|
}
|
|
|
|
#[test]
|
|
fn transforms_merge_request_discussion() {
|
|
let discussion = make_test_discussion(
|
|
false,
|
|
vec![make_test_note(
|
|
1,
|
|
"2024-01-16T09:00:00.000Z",
|
|
false,
|
|
false,
|
|
false,
|
|
)],
|
|
);
|
|
|
|
let result = transform_discussion(&discussion, 100, NoteableRef::MergeRequest(99));
|
|
|
|
assert_eq!(result.issue_id, None);
|
|
assert_eq!(result.merge_request_id, Some(99));
|
|
assert_eq!(result.noteable_type, "MergeRequest");
|
|
}
|
|
|
|
#[test]
|
|
fn extracts_notes_array_from_discussion() {
|
|
let discussion = make_test_discussion(
|
|
false,
|
|
vec![
|
|
make_test_note(1, "2024-01-16T09:00:00.000Z", false, false, false),
|
|
make_test_note(2, "2024-01-16T10:00:00.000Z", false, false, false),
|
|
],
|
|
);
|
|
|
|
let notes = transform_notes(&discussion, 100);
|
|
|
|
assert_eq!(notes.len(), 2);
|
|
assert_eq!(notes[0].gitlab_id, 1);
|
|
assert_eq!(notes[1].gitlab_id, 2);
|
|
}
|
|
|
|
#[test]
|
|
fn sets_individual_note_flag_correctly() {
|
|
let threaded = make_test_discussion(
|
|
false,
|
|
vec![make_test_note(
|
|
1,
|
|
"2024-01-16T09:00:00.000Z",
|
|
false,
|
|
false,
|
|
false,
|
|
)],
|
|
);
|
|
let standalone = make_test_discussion(
|
|
true,
|
|
vec![make_test_note(
|
|
1,
|
|
"2024-01-16T09:00:00.000Z",
|
|
false,
|
|
false,
|
|
false,
|
|
)],
|
|
);
|
|
|
|
assert!(!transform_discussion(&threaded, 100, NoteableRef::Issue(42)).individual_note);
|
|
assert!(transform_discussion(&standalone, 100, NoteableRef::Issue(42)).individual_note);
|
|
}
|
|
|
|
#[test]
|
|
fn flags_system_notes_with_is_system_true() {
|
|
let discussion = make_test_discussion(
|
|
false,
|
|
vec![
|
|
make_test_note(1, "2024-01-16T09:00:00.000Z", false, false, false),
|
|
make_test_note(2, "2024-01-16T09:00:00.000Z", true, false, false), // system note
|
|
],
|
|
);
|
|
|
|
let notes = transform_notes(&discussion, 100);
|
|
|
|
assert!(!notes[0].is_system);
|
|
assert!(notes[1].is_system);
|
|
}
|
|
|
|
#[test]
|
|
fn preserves_note_order_via_position_field() {
|
|
let discussion = make_test_discussion(
|
|
false,
|
|
vec![
|
|
make_test_note(1, "2024-01-16T09:00:00.000Z", false, false, false),
|
|
make_test_note(2, "2024-01-16T10:00:00.000Z", false, false, false),
|
|
make_test_note(3, "2024-01-16T11:00:00.000Z", false, false, false),
|
|
],
|
|
);
|
|
|
|
let notes = transform_notes(&discussion, 100);
|
|
|
|
assert_eq!(notes[0].position, 0);
|
|
assert_eq!(notes[1].position, 1);
|
|
assert_eq!(notes[2].position, 2);
|
|
}
|
|
|
|
#[test]
|
|
fn computes_first_note_at_and_last_note_at_correctly() {
|
|
let discussion = make_test_discussion(
|
|
false,
|
|
vec![
|
|
make_test_note(1, "2024-01-16T09:00:00.000Z", false, false, false),
|
|
make_test_note(2, "2024-01-16T11:00:00.000Z", false, false, false), // latest
|
|
make_test_note(3, "2024-01-16T10:00:00.000Z", false, false, false),
|
|
],
|
|
);
|
|
|
|
let result = transform_discussion(&discussion, 100, NoteableRef::Issue(42));
|
|
|
|
// first_note_at should be 09:00 (note 1)
|
|
assert_eq!(result.first_note_at, Some(1705395600000));
|
|
// last_note_at should be 11:00 (note 2)
|
|
assert_eq!(result.last_note_at, Some(1705402800000));
|
|
}
|
|
|
|
#[test]
|
|
fn single_note_has_equal_first_and_last() {
|
|
let discussion = make_test_discussion(
|
|
false,
|
|
vec![make_test_note(
|
|
1,
|
|
"2024-01-16T09:00:00.000Z",
|
|
false,
|
|
false,
|
|
false,
|
|
)],
|
|
);
|
|
|
|
let result = transform_discussion(&discussion, 100, NoteableRef::Issue(42));
|
|
|
|
assert_eq!(result.first_note_at, result.last_note_at);
|
|
assert_eq!(result.first_note_at, Some(1705395600000));
|
|
}
|
|
|
|
#[test]
|
|
fn computes_resolvable_when_any_note_is_resolvable() {
|
|
let not_resolvable = make_test_discussion(
|
|
false,
|
|
vec![
|
|
make_test_note(1, "2024-01-16T09:00:00.000Z", false, false, false),
|
|
make_test_note(2, "2024-01-16T10:00:00.000Z", false, false, false),
|
|
],
|
|
);
|
|
|
|
let resolvable = make_test_discussion(
|
|
false,
|
|
vec![
|
|
make_test_note(1, "2024-01-16T09:00:00.000Z", false, true, false), // resolvable
|
|
make_test_note(2, "2024-01-16T10:00:00.000Z", false, false, false),
|
|
],
|
|
);
|
|
|
|
assert!(!transform_discussion(¬_resolvable, 100, NoteableRef::Issue(42)).resolvable);
|
|
assert!(transform_discussion(&resolvable, 100, NoteableRef::Issue(42)).resolvable);
|
|
}
|
|
|
|
#[test]
|
|
fn computes_resolved_only_when_all_resolvable_notes_resolved() {
|
|
// Mix of resolved/unresolved - not resolved
|
|
let partial = make_test_discussion(
|
|
false,
|
|
vec![
|
|
make_test_note(1, "2024-01-16T09:00:00.000Z", false, true, true), // resolved
|
|
make_test_note(2, "2024-01-16T10:00:00.000Z", false, true, false), // not resolved
|
|
],
|
|
);
|
|
|
|
// All resolvable notes resolved
|
|
let fully_resolved = make_test_discussion(
|
|
false,
|
|
vec![
|
|
make_test_note(1, "2024-01-16T09:00:00.000Z", false, true, true),
|
|
make_test_note(2, "2024-01-16T10:00:00.000Z", false, true, true),
|
|
],
|
|
);
|
|
|
|
// No resolvable notes - resolved should be false
|
|
let no_resolvable = make_test_discussion(
|
|
false,
|
|
vec![make_test_note(
|
|
1,
|
|
"2024-01-16T09:00:00.000Z",
|
|
false,
|
|
false,
|
|
false,
|
|
)],
|
|
);
|
|
|
|
assert!(!transform_discussion(&partial, 100, NoteableRef::Issue(42)).resolved);
|
|
assert!(transform_discussion(&fully_resolved, 100, NoteableRef::Issue(42)).resolved);
|
|
assert!(!transform_discussion(&no_resolvable, 100, NoteableRef::Issue(42)).resolved);
|
|
}
|
|
}
|