theirongolddev/gitlore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: Remove redundant doc comments throughout codebase

Browse Source 
Removes module-level doc comments (//! lines) and excessive inline doc
comments that were duplicating information already evident from:
- Function/struct names (self-documenting code)
- Type signatures (the what is clear from types)
- Implementation context (the how is clear from code)

Affected modules:
- cli/* - Removed command descriptions duplicating clap help text
- core/* - Removed module headers and obvious function docs
- documents/* - Removed extractor/regenerator/truncation docs
- embedding/* - Removed pipeline and chunking docs
- gitlab/* - Removed client and transformer docs (kept type definitions)
- ingestion/* - Removed orchestrator and ingestion docs
- search/* - Removed FTS and vector search docs

Philosophy: Code should be self-documenting. Comments should explain
"why" (business decisions, non-obvious constraints) not "what" (which
the code itself shows). This change reduces noise and maintenance burden
while keeping the codebase just as understandable.

Retains comments for:
- Non-obvious business logic
- Important safety invariants
- Complex algorithm explanations
- Public API boundaries where generated docs matter

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Taylor Eernisse
2026-02-05 00:04:32 -05:00
parent 976ad92ef0
commit 65583ed5d6
57 changed files with 143 additions and 1693 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
src/search/filters.rs
View File

@@ -5,14 +5,12 @@ use rusqlite::Connection;
const DEFAULT_LIMIT: usize = 20;
const MAX_LIMIT: usize = 100;
/// Path filter: exact match or prefix match (trailing `/`).
#[derive(Debug, Clone)]
pub enum PathFilter {
Exact(String),
Prefix(String),
}
/// Filters applied to search results post-retrieval.
#[derive(Debug, Clone, Default)]
pub struct SearchFilters {
pub source_type: Option<SourceType>,
@@ -26,7 +24,6 @@ pub struct SearchFilters {
}
impl SearchFilters {
/// Returns true if any filter (besides limit) is set.
pub fn has_any_filter(&self) -> bool {
self.source_type.is_some()
|| self.author.is_some()
@@ -37,7 +34,6 @@ impl SearchFilters {
|| self.path.is_some()
}
/// Clamp limit to [1, 100], defaulting 0 to 20.
pub fn clamp_limit(&self) -> usize {
if self.limit == 0 {
DEFAULT_LIMIT
@@ -47,17 +43,12 @@ impl SearchFilters {
}
}
/// Escape SQL LIKE wildcards in a string.
fn escape_like(s: &str) -> String {
s.replace('\\', "\\\\")
.replace('%', "\\%")
.replace('_', "\\_")
}
/// Apply filters to a ranked list of document IDs, preserving rank order.
///
/// Uses json_each() to pass ranked IDs efficiently and maintain ordering
/// via ORDER BY j.key.
pub fn apply_filters(
conn: &Connection,
document_ids: &[i64],
@@ -216,8 +207,6 @@ mod tests {
#[test]
fn test_empty_ids() {
// Cannot test apply_filters without DB, but we can verify empty returns empty
// by testing the early return path logic
let f = SearchFilters::default();
assert!(!f.has_any_filter());
}

28
src/search/fts.rs
View File

@@ -1,16 +1,12 @@
use crate::core::error::Result;
use rusqlite::Connection;
/// FTS query mode.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum FtsQueryMode {
/// Safe mode: each token wrapped in quotes, trailing * preserved on alphanumeric tokens.
Safe,
/// Raw mode: query passed directly to FTS5 (for advanced users).
Raw,
}
/// A single FTS5 search result.
#[derive(Debug)]
pub struct FtsResult {
pub document_id: i64,
@@ -18,14 +14,6 @@ pub struct FtsResult {
pub snippet: String,
}
/// Convert raw user input into a safe FTS5 query.
///
/// Safe mode:
/// - Splits on whitespace
/// - Wraps each token in double quotes (escaping internal quotes)
/// - Preserves trailing `*` on alphanumeric-only tokens (prefix search)
///
/// Raw mode: passes through unchanged.
pub fn to_fts_query(raw: &str, mode: FtsQueryMode) -> String {
match mode {
FtsQueryMode::Raw => raw.to_string(),
@@ -38,16 +26,13 @@ pub fn to_fts_query(raw: &str, mode: FtsQueryMode) -> String {
let tokens: Vec<String> = trimmed
.split_whitespace()
.map(|token| {
// Check if token ends with * and the rest is alphanumeric
if let Some(stem) = token.strip_suffix('*')
&& !stem.is_empty()
&& stem.chars().all(|c| c.is_alphanumeric() || c == '_')
{
// Preserve prefix search: "stem"*
let escaped = stem.replace('"', "\"\"");
return format!("\"{}\"*", escaped);
}
// Default: wrap in quotes, escape internal quotes
let escaped = token.replace('"', "\"\"");
format!("\"{}\"", escaped)
})
@@ -58,10 +43,6 @@ pub fn to_fts_query(raw: &str, mode: FtsQueryMode) -> String {
}
}
/// Execute an FTS5 search query.
///
/// Returns results ranked by BM25 score (lower = better match) with
/// contextual snippets highlighting matches.
pub fn search_fts(
conn: &Connection,
query: &str,
@@ -97,14 +78,11 @@ pub fn search_fts(
Ok(results)
}
/// Generate a fallback snippet for results without FTS snippets.
/// Truncates at a word boundary and appends "...".
pub fn generate_fallback_snippet(content_text: &str, max_chars: usize) -> String {
if content_text.chars().count() <= max_chars {
return content_text.to_string();
}
// Collect the char boundary at max_chars to slice correctly for multi-byte content
let byte_end = content_text
.char_indices()
.nth(max_chars)
@@ -112,7 +90,6 @@ pub fn generate_fallback_snippet(content_text: &str, max_chars: usize) -> String
.unwrap_or(content_text.len());
let truncated = &content_text[..byte_end];
// Walk backward to find a word boundary (space)
if let Some(last_space) = truncated.rfind(' ') {
format!("{}...", &truncated[..last_space])
} else {
@@ -120,7 +97,6 @@ pub fn generate_fallback_snippet(content_text: &str, max_chars: usize) -> String
}
}
/// Get the best snippet: prefer FTS snippet, fall back to truncated content.
pub fn get_result_snippet(fts_snippet: Option<&str>, content_text: &str) -> String {
match fts_snippet {
Some(s) if !s.is_empty() => s.to_string(),
@@ -179,11 +155,9 @@ mod tests {
#[test]
fn test_prefix_only_alphanumeric() {
// Non-alphanumeric prefix: C++* should NOT be treated as prefix search
let result = to_fts_query("C++*", FtsQueryMode::Safe);
assert_eq!(result, "\"C++*\"");
// Pure alphanumeric prefix: auth* should be prefix search
let result = to_fts_query("auth*", FtsQueryMode::Safe);
assert_eq!(result, "\"auth\"*");
}
@@ -205,7 +179,7 @@ mod tests {
let content = "This is a moderately long piece of text that should be truncated at a word boundary for readability purposes";
let result = generate_fallback_snippet(content, 50);
assert!(result.ends_with("..."));
assert!(result.len() <= 55); // 50 + "..."
assert!(result.len() <= 55);
}
#[test]

15
src/search/hybrid.rs
View File

@@ -1,5 +1,3 @@
//! Hybrid search orchestrator combining FTS5 + sqlite-vec via RRF.
use rusqlite::Connection;
use crate::core::error::Result;
@@ -11,7 +9,6 @@ const BASE_RECALL_MIN: usize = 50;
const FILTERED_RECALL_MIN: usize = 200;
const RECALL_CAP: usize = 1500;
/// Search mode selection.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum SearchMode {
Hybrid,
@@ -38,7 +35,6 @@ impl SearchMode {
}
}
/// Combined search result with provenance from both retrieval lists.
pub struct HybridResult {
pub document_id: i64,
pub score: f64,
@@ -47,11 +43,6 @@ pub struct HybridResult {
pub rrf_score: f64,
}
/// Execute hybrid search, returning ranked results + any warnings.
///
/// `client` is `Option` to enable graceful degradation: when Ollama is
/// unavailable, the caller passes `None` and hybrid mode falls back to
/// FTS-only with a warning.
pub async fn search_hybrid(
conn: &Connection,
client: Option<&OllamaClient>,
@@ -62,7 +53,6 @@ pub async fn search_hybrid(
) -> Result<(Vec<HybridResult>, Vec<String>)> {
let mut warnings: Vec<String> = Vec::new();
// Adaptive recall
let requested = filters.clamp_limit();
let top_k = if filters.has_any_filter() {
(requested * 50).clamp(FILTERED_RECALL_MIN, RECALL_CAP)
@@ -159,7 +149,6 @@ pub async fn search_hybrid(
})
.collect();
// Apply post-retrieval filters and limit
let limit = filters.clamp_limit();
let results = if filters.has_any_filter() {
let all_ids: Vec<i64> = results.iter().map(|r| r.document_id).collect();
@@ -232,7 +221,7 @@ mod tests {
};
let requested = filters.clamp_limit();
let top_k = (requested * 50).clamp(FILTERED_RECALL_MIN, RECALL_CAP);
assert_eq!(top_k, RECALL_CAP); // 5000 capped to 1500
assert_eq!(top_k, RECALL_CAP);
}
#[test]
@@ -243,6 +232,6 @@ mod tests {
};
let requested = filters.clamp_limit();
let top_k = (requested * 10).clamp(BASE_RECALL_MIN, RECALL_CAP);
assert_eq!(top_k, BASE_RECALL_MIN); // 10 -> 50
assert_eq!(top_k, BASE_RECALL_MIN);
}
}

29
src/search/rrf.rs
View File

@@ -2,39 +2,24 @@ use std::collections::HashMap;
const RRF_K: f64 = 60.0;
/// A single result from Reciprocal Rank Fusion, containing both raw and
/// normalized scores plus per-list rank provenance for --explain output.
pub struct RrfResult {
pub document_id: i64,
/// Raw RRF score: sum of 1/(k + rank) across all lists.
pub rrf_score: f64,
/// Normalized to [0, 1] where the best result is 1.0.
pub normalized_score: f64,
/// 1-indexed rank in the vector results list, if present.
pub vector_rank: Option<usize>,
/// 1-indexed rank in the FTS results list, if present.
pub fts_rank: Option<usize>,
}
/// Combine vector and FTS retrieval results using Reciprocal Rank Fusion.
///
/// Input tuples are `(document_id, score/distance)` — already sorted by each retriever.
/// Ranks are 1-indexed (first result = rank 1).
///
/// Score = sum of 1/(k + rank) for each list containing the document.
pub fn rank_rrf(vector_results: &[(i64, f64)], fts_results: &[(i64, f64)]) -> Vec<RrfResult> {
if vector_results.is_empty() && fts_results.is_empty() {
return Vec::new();
}
// (rrf_score, vector_rank, fts_rank)
let mut scores: HashMap<i64, (f64, Option<usize>, Option<usize>)> = HashMap::new();
for (i, &(doc_id, _)) in vector_results.iter().enumerate() {
let rank = i + 1; // 1-indexed
let rank = i + 1;
let entry = scores.entry(doc_id).or_insert((0.0, None, None));
// Only count the first occurrence per list to prevent duplicates
// from inflating the score.
if entry.1.is_none() {
entry.0 += 1.0 / (RRF_K + rank as f64);
entry.1 = Some(rank);
@@ -42,7 +27,7 @@ pub fn rank_rrf(vector_results: &[(i64, f64)], fts_results: &[(i64, f64)]) -> Ve
}
for (i, &(doc_id, _)) in fts_results.iter().enumerate() {
let rank = i + 1; // 1-indexed
let rank = i + 1;
let entry = scores.entry(doc_id).or_insert((0.0, None, None));
if entry.2.is_none() {
entry.0 += 1.0 / (RRF_K + rank as f64);
@@ -55,16 +40,14 @@ pub fn rank_rrf(vector_results: &[(i64, f64)], fts_results: &[(i64, f64)]) -> Ve
.map(|(doc_id, (rrf_score, vector_rank, fts_rank))| RrfResult {
document_id: doc_id,
rrf_score,
normalized_score: 0.0, // filled in below
normalized_score: 0.0,
vector_rank,
fts_rank,
})
.collect();
// Sort descending by rrf_score
results.sort_by(|a, b| b.rrf_score.total_cmp(&a.rrf_score));
// Normalize: best = 1.0
if let Some(max_score) = results.first().map(|r| r.rrf_score).filter(|&s| s > 0.0) {
for result in &mut results {
result.normalized_score = result.rrf_score / max_score;
@@ -84,10 +67,8 @@ mod tests {
let fts = vec![(1, 5.0), (3, 3.0)];
let results = rank_rrf(&vector, &fts);
// Doc 1 appears in both lists, should rank highest
assert_eq!(results[0].document_id, 1);
// Doc 1 score should be higher than doc 2 and doc 3
let doc1 = &results[0];
let doc2_score = results
.iter()
@@ -121,10 +102,8 @@ mod tests {
let fts = vec![(1, 5.0), (3, 3.0)];
let results = rank_rrf(&vector, &fts);
// Best result should have normalized_score = 1.0
assert!((results[0].normalized_score - 1.0).abs() < f64::EPSILON);
// All scores in [0, 1]
for r in &results {
assert!(r.normalized_score >= 0.0);
assert!(r.normalized_score <= 1.0);
@@ -165,7 +144,6 @@ mod tests {
assert_eq!(results.len(), 1);
let r = &results[0];
// RRF score = 1/(60+1) + 1/(60+1) = 2/61
let expected = 2.0 / 61.0;
assert!((r.rrf_score - expected).abs() < 1e-10);
assert!((r.normalized_score - 1.0).abs() < f64::EPSILON);
@@ -177,7 +155,6 @@ mod tests {
let results = rank_rrf(&vector, &[]);
assert_eq!(results.len(), 2);
// Single result should still have normalized_score = 1.0
assert!((results[0].normalized_score - 1.0).abs() < f64::EPSILON);
}
}

29
src/search/vector.rs
View File

@@ -5,16 +5,13 @@ use rusqlite::Connection;
use crate::core::error::Result;
use crate::embedding::chunk_ids::decode_rowid;
/// A single vector search result (document-level, deduplicated).
#[derive(Debug)]
pub struct VectorResult {
pub document_id: i64,
pub distance: f64,
}
/// Query the maximum number of chunks per document for adaptive dedup sizing.
fn max_chunks_per_document(conn: &Connection) -> i64 {
// Fast path: stored chunk_count on sentinel rows (post-migration 010)
let stored: Option<i64> = conn
.query_row(
"SELECT MAX(chunk_count) FROM embedding_metadata
@@ -28,7 +25,6 @@ fn max_chunks_per_document(conn: &Connection) -> i64 {
return max;
}
// Fallback for pre-migration data: count chunks per document
conn.query_row(
"SELECT COALESCE(MAX(cnt), 1) FROM (
SELECT COUNT(*) as cnt FROM embedding_metadata
@@ -40,12 +36,6 @@ fn max_chunks_per_document(conn: &Connection) -> i64 {
.unwrap_or(1)
}
/// Search documents using sqlite-vec KNN query.
///
/// Over-fetches by an adaptive multiplier based on actual max chunks per document
/// to handle chunk deduplication (multiple chunks per document produce multiple
/// KNN results for the same document_id).
/// Returns deduplicated results with best (lowest) distance per document.
pub fn search_vector(
conn: &Connection,
query_embedding: &[f32],
@@ -55,7 +45,6 @@ pub fn search_vector(
return Ok(Vec::new());
}
// Convert to raw little-endian bytes for sqlite-vec
let embedding_bytes: Vec<u8> = query_embedding
.iter()
.flat_map(|f| f.to_le_bytes())
@@ -79,7 +68,6 @@ pub fn search_vector(
})?
.collect::<std::result::Result<Vec<_>, _>>()?;
// Dedup by document_id, keeping best (lowest) distance
let mut best: HashMap<i64, f64> = HashMap::new();
for (rowid, distance) in rows {
let (document_id, _chunk_index) = decode_rowid(rowid);
@@ -92,7 +80,6 @@ pub fn search_vector(
.or_insert(distance);
}
// Sort by distance ascending, take limit
let mut results: Vec<VectorResult> = best
.into_iter()
.map(|(document_id, distance)| VectorResult {
@@ -110,29 +97,20 @@ pub fn search_vector(
mod tests {
use super::*;
// Note: Full integration tests require sqlite-vec loaded, which happens via
// create_connection in db.rs. These are basic unit tests for the dedup logic.
#[test]
fn test_empty_returns_empty() {
// Can't test KNN without sqlite-vec, but we can test edge cases
let result = search_vector_dedup(vec![], 10);
assert!(result.is_empty());
}
#[test]
fn test_dedup_keeps_best_distance() {
// Simulate: doc 1 has chunks at rowid 1000 (idx 0) and 1001 (idx 1)
let rows = vec![
(1000_i64, 0.5_f64), // doc 1, chunk 0
(1001, 0.3), // doc 1, chunk 1 (better)
(2000, 0.4), // doc 2, chunk 0
];
let rows = vec![(1000_i64, 0.5_f64), (1001, 0.3), (2000, 0.4)];
let results = search_vector_dedup(rows, 10);
assert_eq!(results.len(), 2);
assert_eq!(results[0].document_id, 1); // doc 1 best = 0.3
assert_eq!(results[0].document_id, 1);
assert!((results[0].distance - 0.3).abs() < f64::EPSILON);
assert_eq!(results[1].document_id, 2); // doc 2 = 0.4
assert_eq!(results[1].document_id, 2);
}
#[test]
@@ -142,7 +120,6 @@ mod tests {
assert_eq!(results.len(), 2);
}
/// Helper for testing dedup logic without sqlite-vec
fn search_vector_dedup(rows: Vec<(i64, f64)>, limit: usize) -> Vec<VectorResult> {
let mut best: HashMap<i64, f64> = HashMap::new();
for (rowid, distance) in rows {