perf: Eliminate double serialization, add SQLite tuning, optimize hot paths
11 isomorphic performance fixes from deep audit (no behavior changes): - Eliminate double serialization: store_payload now accepts pre-serialized bytes (&[u8]) instead of re-serializing from serde_json::Value. Uses Cow<[u8]> for zero-copy when compression is disabled. - Add SQLite cache_size (64MB) and mmap_size (256MB) pragmas - Replace SELECT-then-INSERT label upserts with INSERT...ON CONFLICT RETURNING in both issues.rs and merge_requests.rs - Replace INSERT + SELECT milestone upsert with RETURNING - Use prepare_cached for 5 hot-path queries in extractor.rs - Optimize compute_list_hash: index-sort + incremental SHA-256 instead of clone+sort+join+hash - Pre-allocate embedding float-to-bytes buffer with Vec::with_capacity - Replace RandomState::new() in rand_jitter with atomic counter XOR nanos - Remove redundant per-note payload storage (discussion payload contains all notes already) - Change transform_issue to accept &GitLabIssue (avoids full struct clone) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -166,14 +166,14 @@ fn process_single_mr(
|
||||
mr: &GitLabMergeRequest,
|
||||
) -> Result<ProcessMrResult> {
|
||||
// Transform MR first (outside transaction - no DB access)
|
||||
let payload_json = serde_json::to_value(mr)?;
|
||||
let payload_bytes = serde_json::to_vec(mr)?;
|
||||
let transformed = transform_merge_request(mr, project_id)
|
||||
.map_err(|e| LoreError::Other(format!("MR transform failed: {}", e)))?;
|
||||
|
||||
// Wrap all DB operations in a transaction for atomicity
|
||||
let tx = conn.unchecked_transaction()?;
|
||||
let result =
|
||||
process_mr_in_transaction(&tx, config, project_id, mr, &payload_json, &transformed)?;
|
||||
process_mr_in_transaction(&tx, config, project_id, mr, &payload_bytes, &transformed)?;
|
||||
tx.commit()?;
|
||||
|
||||
Ok(result)
|
||||
@@ -185,7 +185,7 @@ fn process_mr_in_transaction(
|
||||
config: &Config,
|
||||
project_id: i64,
|
||||
mr: &GitLabMergeRequest,
|
||||
payload_json: &serde_json::Value,
|
||||
payload_bytes: &[u8],
|
||||
transformed: &crate::gitlab::transformers::merge_request::MergeRequestWithMetadata,
|
||||
) -> Result<ProcessMrResult> {
|
||||
let mut labels_created = 0;
|
||||
@@ -199,7 +199,7 @@ fn process_mr_in_transaction(
|
||||
project_id: Some(project_id),
|
||||
resource_type: "merge_request",
|
||||
gitlab_id: &mr.id.to_string(),
|
||||
payload: payload_json,
|
||||
json_bytes: payload_bytes,
|
||||
compress: config.storage.compress_raw_payloads,
|
||||
},
|
||||
)?;
|
||||
@@ -315,33 +315,28 @@ fn process_mr_in_transaction(
|
||||
}
|
||||
|
||||
/// Upsert a label within a transaction, returning its ID.
|
||||
/// Upsert a label within a transaction, returning its ID.
|
||||
/// Uses INSERT...ON CONFLICT...RETURNING for a single round-trip.
|
||||
fn upsert_label_tx(
|
||||
tx: &Transaction<'_>,
|
||||
project_id: i64,
|
||||
name: &str,
|
||||
created_count: &mut usize,
|
||||
) -> Result<i64> {
|
||||
// Try to get existing
|
||||
let existing: Option<i64> = tx
|
||||
.query_row(
|
||||
"SELECT id FROM labels WHERE project_id = ? AND name = ?",
|
||||
(project_id, name),
|
||||
|row| row.get(0),
|
||||
)
|
||||
.ok();
|
||||
let id: i64 = tx.query_row(
|
||||
"INSERT INTO labels (project_id, name) VALUES (?1, ?2)
|
||||
ON CONFLICT(project_id, name) DO UPDATE SET name = excluded.name
|
||||
RETURNING id",
|
||||
(project_id, name),
|
||||
|row| row.get(0),
|
||||
)?;
|
||||
|
||||
if let Some(id) = existing {
|
||||
return Ok(id);
|
||||
// If the rowid matches last_insert_rowid, this was a new insert
|
||||
if tx.last_insert_rowid() == id {
|
||||
*created_count += 1;
|
||||
}
|
||||
|
||||
// Insert new
|
||||
tx.execute(
|
||||
"INSERT INTO labels (project_id, name) VALUES (?, ?)",
|
||||
(project_id, name),
|
||||
)?;
|
||||
*created_count += 1;
|
||||
|
||||
Ok(tx.last_insert_rowid())
|
||||
Ok(id)
|
||||
}
|
||||
|
||||
/// Check if an MR passes the cursor filter (not already processed).
|
||||
@@ -412,13 +407,14 @@ fn reset_sync_cursor(conn: &Connection, project_id: i64) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Reset discussion watermarks for all MRs in project (for full sync).
|
||||
/// Reset discussion and resource event watermarks for all MRs in project (for full sync).
|
||||
fn reset_discussion_watermarks(conn: &Connection, project_id: i64) -> Result<()> {
|
||||
conn.execute(
|
||||
"UPDATE merge_requests
|
||||
SET discussions_synced_for_updated_at = NULL,
|
||||
discussions_sync_attempts = 0,
|
||||
discussions_sync_last_error = NULL
|
||||
discussions_sync_last_error = NULL,
|
||||
resource_events_synced_for_updated_at = NULL
|
||||
WHERE project_id = ?",
|
||||
[project_id],
|
||||
)?;
|
||||
|
||||
Reference in New Issue
Block a user