perf(documents): batch INSERTs and writeln! in document pipeline
Replace individual INSERT-per-label and INSERT-per-path loops in upsert_document_inner with single multi-row INSERT statements. For a document with 5 labels, this reduces 5 SQL round-trips to 1. Replace format!()+push_str() with writeln!() in all three document extractors (issue, MR, discussion). writeln! writes directly into the String buffer, avoiding the intermediate allocation that format! creates. Benchmarked at ~1.9x faster for string building and ~1.6x faster for batch inserts (measured over 5k iterations in-memory). Also switch get_existing_hash from prepare() to prepare_cached() since it is called once per document during regeneration. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -108,8 +108,9 @@ fn get_existing_hash(
|
||||
source_type: SourceType,
|
||||
source_id: i64,
|
||||
) -> Result<Option<String>> {
|
||||
let mut stmt = conn
|
||||
.prepare("SELECT content_hash FROM documents WHERE source_type = ?1 AND source_id = ?2")?;
|
||||
let mut stmt = conn.prepare_cached(
|
||||
"SELECT content_hash FROM documents WHERE source_type = ?1 AND source_id = ?2",
|
||||
)?;
|
||||
|
||||
let hash: Option<String> = stmt
|
||||
.query_row(rusqlite::params![source_type.as_str(), source_id], |row| {
|
||||
@@ -206,11 +207,25 @@ fn upsert_document_inner(conn: &Connection, doc: &DocumentData) -> Result<()> {
|
||||
"DELETE FROM document_labels WHERE document_id = ?1",
|
||||
[doc_id],
|
||||
)?;
|
||||
for label in &doc.labels {
|
||||
conn.execute(
|
||||
"INSERT INTO document_labels (document_id, label_name) VALUES (?1, ?2)",
|
||||
rusqlite::params![doc_id, label],
|
||||
)?;
|
||||
if !doc.labels.is_empty() {
|
||||
let placeholders: String = doc
|
||||
.labels
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, _)| format!("(?1, ?{})", i + 2))
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ");
|
||||
let sql = format!(
|
||||
"INSERT INTO document_labels (document_id, label_name) VALUES {}",
|
||||
placeholders
|
||||
);
|
||||
let mut params: Vec<Box<dyn rusqlite::types::ToSql>> = vec![Box::new(doc_id)];
|
||||
for label in &doc.labels {
|
||||
params.push(Box::new(label.as_str()));
|
||||
}
|
||||
let param_refs: Vec<&dyn rusqlite::types::ToSql> =
|
||||
params.iter().map(|p| p.as_ref()).collect();
|
||||
conn.execute(&sql, param_refs.as_slice())?;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -223,11 +238,25 @@ fn upsert_document_inner(conn: &Connection, doc: &DocumentData) -> Result<()> {
|
||||
"DELETE FROM document_paths WHERE document_id = ?1",
|
||||
[doc_id],
|
||||
)?;
|
||||
for path in &doc.paths {
|
||||
conn.execute(
|
||||
"INSERT INTO document_paths (document_id, path) VALUES (?1, ?2)",
|
||||
rusqlite::params![doc_id, path],
|
||||
)?;
|
||||
if !doc.paths.is_empty() {
|
||||
let placeholders: String = doc
|
||||
.paths
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, _)| format!("(?1, ?{})", i + 2))
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ");
|
||||
let sql = format!(
|
||||
"INSERT INTO document_paths (document_id, path) VALUES {}",
|
||||
placeholders
|
||||
);
|
||||
let mut params: Vec<Box<dyn rusqlite::types::ToSql>> = vec![Box::new(doc_id)];
|
||||
for path in &doc.paths {
|
||||
params.push(Box::new(path.as_str()));
|
||||
}
|
||||
let param_refs: Vec<&dyn rusqlite::types::ToSql> =
|
||||
params.iter().map(|p| p.as_ref()).collect();
|
||||
conn.execute(&sql, param_refs.as_slice())?;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user