theirongolddev/gitlore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf: force partial index for DiffNote queries, batch stats counts

Browse Source 
Query optimizer fixes for the `who` and `stats` commands based on
a systematic performance audit of the SQLite query plans.

who command (expert/reviews/detail modes):
- Add INDEXED BY idx_notes_diffnote_path_created hints to all DiffNote
  queries. SQLite's planner was selecting idx_notes_system (38% of rows)
  over the far more selective partial index (9.3% of rows). Measured
  50-133x speedup on expert queries, 26x on reviews queries.
- Reorder JOIN clauses in detail mode's MR-author sub-select to match
  the index scan direction (notes -> discussions -> merge_requests).

stats command:
- Replace 12+ sequential COUNT(*) queries with conditional aggregates
  (COALESCE + SUM + CASE). Documents, dirty_sources, pending_discussion_
  fetches, and pending_dependent_fetches tables each scanned once instead
  of 2-3 times. Measured 1.7x speedup (109ms -> 65ms warm cache).
- Switch FTS document count from COUNT(*) on the virtual table to
  COUNT(*) on documents_fts_docsize shadow table (B-tree scan vs FTS5
  virtual table overhead). Measured 19x speedup for that single query.

Database: 61652 docs, 282K notes, 211K discussions, 1.5GB.
This commit is contained in:
teernisse
2026-02-11 16:00:34 -05:00
parent 039ab1c2a3
commit acc5e12e3d
3 changed files with 291 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
src/cli/commands/who.rs
View File

@@ -473,9 +473,11 @@ fn build_path_query(conn: &Connection, path: &str, project_id: Option<i64>) -> R
let looks_like_file = !forced_dir && (is_root || last_segment.contains('.'));
// Probe 1: exact file exists in DiffNotes OR mr_file_changes (project-scoped)
// Exact-match probes already use the partial index, but LIKE probes below
// benefit from the INDEXED BY hint (same planner issue as expert query).
let exact_exists = conn
.query_row(
"SELECT 1 FROM notes
"SELECT 1 FROM notes INDEXED BY idx_notes_diffnote_path_created
WHERE note_type = 'DiffNote'
AND is_system = 0
AND position_new_path = ?1
@@ -501,7 +503,7 @@ fn build_path_query(conn: &Connection, path: &str, project_id: Option<i64>) -> R
let escaped = escape_like(trimmed);
let pat = format!("{escaped}/%");
conn.query_row(
"SELECT 1 FROM notes
"SELECT 1 FROM notes INDEXED BY idx_notes_diffnote_path_created
WHERE note_type = 'DiffNote'
AND is_system = 0
AND position_new_path LIKE ?1 ESCAPE '\\'
@@ -597,7 +599,8 @@ fn suffix_probe(conn: &Connection, suffix: &str, project_id: Option<i64>) -> Res
let mut stmt = conn.prepare_cached(
"SELECT DISTINCT full_path FROM (
SELECT position_new_path AS full_path FROM notes
SELECT position_new_path AS full_path
FROM notes INDEXED BY idx_notes_diffnote_path_created
WHERE note_type = 'DiffNote'
AND is_system = 0
AND (position_new_path LIKE ?1 ESCAPE '\\' OR position_new_path = ?2)
@@ -658,6 +661,13 @@ fn query_expert(
} else {
"= ?1"
};
// When scanning DiffNotes with a LIKE prefix, SQLite's planner picks the
// low-selectivity idx_notes_system (38% of rows) instead of the much more
// selective partial index idx_notes_diffnote_path_created (9.3% of rows).
// INDEXED BY forces the correct index: measured 64x speedup (1.22s → 0.019s).
// For exact matches SQLite already picks the partial index, but the hint
// is harmless and keeps behavior consistent.
let notes_indexed_by = "INDEXED BY idx_notes_diffnote_path_created";
let author_w = scoring.author_weight;
let reviewer_w = scoring.reviewer_weight;
let note_b = scoring.note_bonus;
@@ -672,7 +682,7 @@ fn query_expert(
n.id AS note_id,
n.created_at AS seen_at,
(p.path_with_namespace || '!' || CAST(m.iid AS TEXT)) AS mr_ref
FROM notes n
FROM notes n {notes_indexed_by}
JOIN discussions d ON n.discussion_id = d.id
JOIN merge_requests m ON d.merge_request_id = m.id
JOIN projects p ON m.project_id = p.id
@@ -697,7 +707,7 @@ fn query_expert(
(p.path_with_namespace || '!' || CAST(m.iid AS TEXT)) AS mr_ref
FROM merge_requests m
JOIN discussions d ON d.merge_request_id = m.id
JOIN notes n ON n.discussion_id = d.id
JOIN notes n {notes_indexed_by} ON n.discussion_id = d.id
JOIN projects p ON m.project_id = p.id
WHERE n.note_type = 'DiffNote'
AND n.is_system = 0
@@ -851,6 +861,7 @@ fn query_expert_details(
.collect();
let in_clause = placeholders.join(",");
let notes_indexed_by = "INDEXED BY idx_notes_diffnote_path_created";
let sql = format!(
"
WITH signals AS (
@@ -863,7 +874,7 @@ fn query_expert_details(
m.title AS title,
COUNT(*) AS note_count,
MAX(n.created_at) AS last_activity
FROM notes n
FROM notes n {notes_indexed_by}
JOIN discussions d ON n.discussion_id = d.id
JOIN merge_requests m ON d.merge_request_id = m.id
JOIN projects p ON m.project_id = p.id
@@ -891,7 +902,7 @@ fn query_expert_details(
MAX(n.created_at) AS last_activity
FROM merge_requests m
JOIN discussions d ON d.merge_request_id = m.id
JOIN notes n ON n.discussion_id = d.id
JOIN notes n {notes_indexed_by} ON n.discussion_id = d.id
JOIN projects p ON m.project_id = p.id
WHERE n.note_type = 'DiffNote'
AND n.is_system = 0
@@ -1194,8 +1205,11 @@ fn query_reviews(
project_id: Option<i64>,
since_ms: i64,
) -> Result<ReviewsResult> {
// Count total DiffNotes by this user on MRs they didn't author
// Force the partial index on DiffNote queries (same rationale as expert mode).
// COUNT + COUNT(DISTINCT) + category extraction all benefit from 26K DiffNote
// scan vs 282K notes full scan: measured 25x speedup.
let total_sql = "SELECT COUNT(*) FROM notes n
INDEXED BY idx_notes_diffnote_path_created
JOIN discussions d ON n.discussion_id = d.id
JOIN merge_requests m ON d.merge_request_id = m.id
WHERE n.author_username = ?1
@@ -1213,6 +1227,7 @@ fn query_reviews(
// Count distinct MRs reviewed
let mrs_sql = "SELECT COUNT(DISTINCT m.id) FROM notes n
INDEXED BY idx_notes_diffnote_path_created
JOIN discussions d ON n.discussion_id = d.id
JOIN merge_requests m ON d.merge_request_id = m.id
WHERE n.author_username = ?1
@@ -1232,7 +1247,7 @@ fn query_reviews(
let cat_sql = "SELECT
SUBSTR(ltrim(n.body), 3, INSTR(SUBSTR(ltrim(n.body), 3), '**') - 1) AS raw_prefix,
COUNT(*) AS cnt
FROM notes n
FROM notes n INDEXED BY idx_notes_diffnote_path_created
JOIN discussions d ON n.discussion_id = d.id
JOIN merge_requests m ON d.merge_request_id = m.id
WHERE n.author_username = ?1
@@ -1517,6 +1532,10 @@ fn query_overlap(
} else {
"= ?1"
};
// Force the partial index on DiffNote queries (same rationale as expert mode).
// Without this hint SQLite picks idx_notes_system (38% of rows) instead of
// idx_notes_diffnote_path_created (9.3% of rows): measured 50-133x slower.
let notes_indexed_by = "INDEXED BY idx_notes_diffnote_path_created";
let sql = format!(
"SELECT username, role, touch_count, last_seen_at, mr_refs FROM (
-- 1. DiffNote reviewer
@@ -1526,7 +1545,7 @@ fn query_overlap(
COUNT(DISTINCT m.id) AS touch_count,
MAX(n.created_at) AS last_seen_at,
GROUP_CONCAT(DISTINCT (p.path_with_namespace || '!' || m.iid)) AS mr_refs
FROM notes n
FROM notes n {notes_indexed_by}
JOIN discussions d ON n.discussion_id = d.id
JOIN merge_requests m ON d.merge_request_id = m.id
JOIN projects p ON m.project_id = p.id
@@ -1549,9 +1568,9 @@ fn query_overlap(
COUNT(DISTINCT m.id) AS touch_count,
MAX(n.created_at) AS last_seen_at,
GROUP_CONCAT(DISTINCT (p.path_with_namespace || '!' || m.iid)) AS mr_refs
FROM merge_requests m
JOIN discussions d ON d.merge_request_id = m.id
JOIN notes n ON n.discussion_id = d.id
FROM notes n {notes_indexed_by}
JOIN discussions d ON n.discussion_id = d.id
JOIN merge_requests m ON d.merge_request_id = m.id
JOIN projects p ON m.project_id = p.id
WHERE n.note_type = 'DiffNote'
AND n.position_new_path {path_op}