feat(embedding): strip GitLab boilerplate from titles before embedding
GitLab auto-generates MR titles like "Draft: Resolve \"Issue Title\"" when creating MRs from issues. This 4-token boilerplate prefix dominated the embedding vectors, causing unrelated MRs with the same title structure to appear as highly similar in "lore related" results (0.667 similarity vs 0.674 for the actual parent issue — a difference of only 0.007). Add normalize_title_for_embedding() which deterministically strips: - "Draft: " prefix (case-insensitive) - "WIP: " prefix (case-insensitive) - "Resolve \"...\"" wrapper (extracts inner title) - Combinations: "Draft: Resolve \"...\"" The normalization is applied in all four document extractors (issues, MRs, discussions, notes) to the content_text field only. DocumentData.title preserves the original title for human-readable display in CLI output. Since content_text changes, content_hash will differ from stored values, triggering automatic re-embedding on the next "lore embed" run. Uses str::get() for all byte-offset slicing to prevent panics on titles containing emoji or other multi-byte UTF-8 characters. 15 new tests covering: all boilerplate patterns, case insensitivity, edge cases (empty inner text, no-op for normal titles), UTF-8 safety, and end-to-end document extraction with boilerplate titles. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -439,6 +439,7 @@ fn build_note_document(
|
||||
let url = parent_web_url.map(|wu| format!("{}#note_{}", wu, gitlab_id));
|
||||
|
||||
let display_title = parent_title.unwrap_or("(untitled)");
|
||||
let embed_title = normalize_title_for_embedding(display_title);
|
||||
let display_note_type = note_type.as_deref().unwrap_or("Note");
|
||||
let display_author = author_username.as_deref().unwrap_or("unknown");
|
||||
let parent_prefix = if parent_type_label == "Issue" {
|
||||
@@ -447,6 +448,7 @@ fn build_note_document(
|
||||
format!("MR !{}", parent_iid)
|
||||
};
|
||||
|
||||
// Display title uses original (for human-readable output)
|
||||
let title = format!(
|
||||
"Note by @{} on {}: {}",
|
||||
display_author, parent_prefix, display_title
|
||||
@@ -461,7 +463,7 @@ fn build_note_document(
|
||||
let _ = writeln!(content, "project: {}", path_with_namespace);
|
||||
let _ = writeln!(content, "parent_type: {}", parent_type_label);
|
||||
let _ = writeln!(content, "parent_iid: {}", parent_iid);
|
||||
let _ = writeln!(content, "parent_title: {}", display_title);
|
||||
let _ = writeln!(content, "parent_title: {}", embed_title);
|
||||
let _ = writeln!(content, "note_type: {}", display_note_type);
|
||||
let _ = writeln!(content, "author: @{}", display_author);
|
||||
let _ = writeln!(content, "created_at: {}", ms_to_iso(created_at));
|
||||
|
||||
Reference in New Issue
Block a user