Compare commits
6 Commits
55b895a2eb
...
f4dba386c9
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f4dba386c9 | ||
|
|
856aad1641 | ||
|
|
8fe5feda7e | ||
|
|
753ff46bb4 | ||
|
|
d3a05cfb87 | ||
|
|
390f8a9288 |
@@ -1,7 +1,5 @@
|
||||
# Checkpoint 3: Search & Sync MVP
|
||||
|
||||
> **Note:** The project was renamed from "gitlab-inbox" to "gitlore" and the CLI from "gi" to "lore". References to "gi" in this document should be read as "lore".
|
||||
|
||||
> **Status:** Planning
|
||||
> **Prerequisite:** Checkpoints 0, 1, 2 complete (issues, MRs, discussions ingested)
|
||||
> **Goal:** Deliver working semantic + lexical hybrid search with efficient incremental sync
|
||||
@@ -17,13 +15,27 @@ All code integrates with existing `gitlore` infrastructure:
|
||||
|
||||
---
|
||||
|
||||
## Executive Summary
|
||||
## Executive Summary (Gated Milestones)
|
||||
|
||||
This checkpoint ships in three gates to reduce integration risk. Each gate is independently verifiable and shippable:
|
||||
|
||||
**Gate A (Lexical MVP):** documents + FTS + filters + `lore search --mode=lexical` + `lore stats`
|
||||
**Gate B (Hybrid MVP):** embeddings + vector + RRF fusion + graceful degradation
|
||||
**Gate C (Sync MVP):** `lore sync` orchestration + queues/backoff + integrity check/repair
|
||||
|
||||
**Deliverables:**
|
||||
|
||||
**Gate A**
|
||||
1. Document generation from issues/MRs/discussions with FTS5 indexing
|
||||
2. Ollama-powered embedding pipeline with sqlite-vec storage
|
||||
3. Hybrid search (RRF-ranked vector + lexical) with rich filtering
|
||||
4. Orchestrated `gi sync` command with incremental re-embedding
|
||||
2. Lexical search + filters + snippets + `lore stats`
|
||||
|
||||
**Gate B**
|
||||
3. Ollama-powered embedding pipeline with sqlite-vec storage
|
||||
4. Hybrid search (RRF-ranked vector + lexical) with rich filtering + graceful degradation
|
||||
|
||||
**Gate C**
|
||||
5. Orchestrated `lore sync` command with incremental doc regen + re-embedding
|
||||
6. Integrity checks + repair paths for FTS/embeddings consistency
|
||||
|
||||
**Key Design Decisions:**
|
||||
- Documents are the search unit (not raw entities)
|
||||
@@ -144,15 +156,19 @@ CREATE VIRTUAL TABLE documents_fts USING fts5(
|
||||
prefix='2 3 4'
|
||||
);
|
||||
|
||||
-- Keep FTS in sync via triggers
|
||||
-- Keep FTS in sync via triggers.
|
||||
-- IMPORTANT: COALESCE(title, '') ensures FTS5 external-content table never
|
||||
-- receives NULL values, which can cause inconsistencies with delete operations.
|
||||
-- FTS5 delete requires exact match of original values; NULL != NULL in SQL,
|
||||
-- so a NULL title on insert would make the delete trigger fail silently.
|
||||
CREATE TRIGGER documents_ai AFTER INSERT ON documents BEGIN
|
||||
INSERT INTO documents_fts(rowid, title, content_text)
|
||||
VALUES (new.id, new.title, new.content_text);
|
||||
VALUES (new.id, COALESCE(new.title, ''), new.content_text);
|
||||
END;
|
||||
|
||||
CREATE TRIGGER documents_ad AFTER DELETE ON documents BEGIN
|
||||
INSERT INTO documents_fts(documents_fts, rowid, title, content_text)
|
||||
VALUES('delete', old.id, old.title, old.content_text);
|
||||
VALUES('delete', old.id, COALESCE(old.title, ''), old.content_text);
|
||||
END;
|
||||
|
||||
-- Only rebuild FTS when searchable text actually changes (not metadata-only updates)
|
||||
@@ -160,9 +176,9 @@ CREATE TRIGGER documents_au AFTER UPDATE ON documents
|
||||
WHEN old.title IS NOT new.title OR old.content_text != new.content_text
|
||||
BEGIN
|
||||
INSERT INTO documents_fts(documents_fts, rowid, title, content_text)
|
||||
VALUES('delete', old.id, old.title, old.content_text);
|
||||
VALUES('delete', old.id, COALESCE(old.title, ''), old.content_text);
|
||||
INSERT INTO documents_fts(rowid, title, content_text)
|
||||
VALUES (new.id, new.title, new.content_text);
|
||||
VALUES (new.id, COALESCE(new.title, ''), new.content_text);
|
||||
END;
|
||||
```
|
||||
|
||||
@@ -490,7 +506,7 @@ pub struct NoteContent {
|
||||
|
||||
---
|
||||
|
||||
### 2.4 CLI: `gi generate-docs` (Incremental by Default)
|
||||
### 2.4 CLI: `lore generate-docs` (Incremental by Default)
|
||||
|
||||
**File:** `src/cli/commands/generate_docs.rs`
|
||||
|
||||
@@ -518,84 +534,83 @@ pub struct GenerateDocsResult {
|
||||
pub skipped: usize, // Unchanged documents
|
||||
}
|
||||
|
||||
/// Chunk size for --full mode transactions.
|
||||
/// Chunk size for --full mode dirty queue seeding.
|
||||
/// Balances throughput against WAL file growth and memory pressure.
|
||||
const FULL_MODE_CHUNK_SIZE: usize = 2000;
|
||||
|
||||
/// Run document generation (incremental by default).
|
||||
///
|
||||
/// IMPORTANT: Both modes use the same regenerator codepath to avoid
|
||||
/// logic divergence in label/path hashing, deletion semantics, and
|
||||
/// write-optimization behavior. The only difference is how dirty_sources
|
||||
/// gets populated.
|
||||
///
|
||||
/// Incremental mode (default):
|
||||
/// - Processes only items in dirty_sources queue
|
||||
/// - Processes only items already in dirty_sources queue
|
||||
/// - Fast for routine syncs
|
||||
///
|
||||
/// Full mode (--full):
|
||||
/// - Regenerates ALL documents from scratch
|
||||
/// - Uses chunked transactions (2k docs/tx) to bound WAL growth
|
||||
/// - Seeds dirty_sources with ALL source entities in chunks
|
||||
/// - Drains through the same regenerator pipeline
|
||||
/// - Uses keyset pagination (WHERE id > last_id) to avoid OFFSET degradation
|
||||
/// - Final FTS optimize after all chunks complete
|
||||
/// - Use when schema changes or after migration
|
||||
pub fn run_generate_docs(
|
||||
config: &Config,
|
||||
full: bool,
|
||||
project_filter: Option<&str>,
|
||||
) -> Result<GenerateDocsResult> {
|
||||
if full {
|
||||
// Full mode: regenerate everything using chunked transactions
|
||||
//
|
||||
// Using chunked transactions instead of a single giant transaction:
|
||||
// - Bounds WAL file growth (single 50k-doc tx could balloon WAL)
|
||||
// - Reduces memory pressure from statement caches
|
||||
// - Allows progress reporting between chunks
|
||||
// - Crash partway through leaves partial but consistent state
|
||||
//
|
||||
// Steps per chunk:
|
||||
// 1. BEGIN IMMEDIATE transaction
|
||||
// 2. Query next batch of sources (issues/MRs/discussions)
|
||||
// 3. For each: generate document, compute hash
|
||||
// 4. Upsert into `documents` table (FTS triggers auto-fire)
|
||||
// 5. Populate `document_labels` and `document_paths`
|
||||
// 6. COMMIT
|
||||
// 7. Report progress, loop to next chunk
|
||||
//
|
||||
// After all chunks:
|
||||
// 8. Single final transaction for FTS rebuild:
|
||||
// INSERT INTO documents_fts(documents_fts) VALUES('rebuild')
|
||||
//
|
||||
// Example implementation:
|
||||
let conn = open_db(config)?;
|
||||
let mut result = GenerateDocsResult::default();
|
||||
let mut offset = 0;
|
||||
|
||||
if full {
|
||||
// Full mode: seed dirty_sources with all source entities, then drain.
|
||||
// Uses keyset pagination to avoid O(n²) OFFSET degradation on large tables.
|
||||
//
|
||||
// Seeding is chunked to bound WAL growth:
|
||||
// 1. For each source type (issues, MRs, discussions):
|
||||
// a. Query next chunk WHERE id > last_id ORDER BY id LIMIT chunk_size
|
||||
// b. INSERT OR IGNORE each into dirty_sources
|
||||
// c. Advance last_id = chunk.last().id
|
||||
// d. Loop until chunk is empty
|
||||
// 2. Drain dirty_sources through regenerator (same as incremental)
|
||||
// 3. Final FTS optimize (not full rebuild — triggers handle consistency)
|
||||
//
|
||||
// Benefits of unified codepath:
|
||||
// - No divergence in label/path hash behavior
|
||||
// - No divergence in deletion semantics
|
||||
// - No divergence in write-optimization logic (labels_hash, paths_hash)
|
||||
// - FTS triggers fire identically in both modes
|
||||
|
||||
// Seed issues
|
||||
let mut last_id: i64 = 0;
|
||||
loop {
|
||||
// Process issues in chunks
|
||||
let issues: Vec<Issue> = query_issues(&conn, project_filter, FULL_MODE_CHUNK_SIZE, offset)?;
|
||||
if issues.is_empty() { break; }
|
||||
|
||||
let chunk = query_issue_ids_after(&conn, project_filter, FULL_MODE_CHUNK_SIZE, last_id)?;
|
||||
if chunk.is_empty() { break; }
|
||||
let tx = conn.transaction()?;
|
||||
for issue in &issues {
|
||||
let doc = generate_issue_document(issue)?;
|
||||
upsert_document(&tx, &doc)?;
|
||||
result.issues += 1;
|
||||
for id in &chunk {
|
||||
mark_dirty(&tx, SourceType::Issue, *id)?;
|
||||
}
|
||||
tx.commit()?;
|
||||
|
||||
offset += issues.len();
|
||||
// Report progress here if using indicatif
|
||||
last_id = *chunk.last().unwrap();
|
||||
}
|
||||
|
||||
// Similar chunked loops for MRs and discussions...
|
||||
// Similar keyset-paginated seeding for MRs and discussions...
|
||||
|
||||
// Final FTS rebuild in its own transaction
|
||||
let tx = conn.transaction()?;
|
||||
tx.execute(
|
||||
"INSERT INTO documents_fts(documents_fts) VALUES('rebuild')",
|
||||
// Report: seeding complete, now regenerating
|
||||
}
|
||||
|
||||
// Both modes: drain dirty_sources through the regenerator
|
||||
let regen = regenerate_dirty_documents(&conn)?;
|
||||
|
||||
if full {
|
||||
// FTS optimize after bulk operations (compacts index segments)
|
||||
conn.execute(
|
||||
"INSERT INTO documents_fts(documents_fts) VALUES('optimize')",
|
||||
[],
|
||||
)?;
|
||||
tx.commit()?;
|
||||
} else {
|
||||
// Incremental mode: process dirty_sources only
|
||||
// 1. Query dirty_sources (bounded by LIMIT)
|
||||
// 2. Regenerate only those documents
|
||||
// 3. Clear from dirty_sources after processing
|
||||
}
|
||||
|
||||
// Map regen -> GenerateDocsResult stats
|
||||
todo!()
|
||||
}
|
||||
|
||||
@@ -849,7 +864,7 @@ pub fn search_fts(
|
||||
)?;
|
||||
|
||||
let results = stmt
|
||||
.query_map([&safe_query, &limit.to_string()], |row| {
|
||||
.query_map(rusqlite::params![safe_query, limit as i64], |row| {
|
||||
Ok(FtsResult {
|
||||
document_id: row.get(0)?,
|
||||
rank: row.get(1)?,
|
||||
@@ -897,7 +912,8 @@ pub struct SearchFilters {
|
||||
pub source_type: Option<SourceType>,
|
||||
pub author: Option<String>,
|
||||
pub project_id: Option<i64>,
|
||||
pub after: Option<i64>, // ms epoch
|
||||
pub after: Option<i64>, // ms epoch (created_at >=)
|
||||
pub updated_after: Option<i64>, // ms epoch (updated_at >=)
|
||||
pub labels: Vec<String>, // AND logic
|
||||
pub path: Option<PathFilter>,
|
||||
pub limit: usize, // Default 20, max 100
|
||||
@@ -910,6 +926,7 @@ impl SearchFilters {
|
||||
|| self.author.is_some()
|
||||
|| self.project_id.is_some()
|
||||
|| self.after.is_some()
|
||||
|| self.updated_after.is_some()
|
||||
|| !self.labels.is_empty()
|
||||
|| self.path.is_some()
|
||||
}
|
||||
@@ -990,6 +1007,11 @@ pub fn apply_filters(
|
||||
params.push(Box::new(after));
|
||||
}
|
||||
|
||||
if let Some(updated_after) = filters.updated_after {
|
||||
conditions.push("d.updated_at >= ?".into());
|
||||
params.push(Box::new(updated_after));
|
||||
}
|
||||
|
||||
// Labels: AND logic - all labels must be present
|
||||
for label in &filters.labels {
|
||||
conditions.push(
|
||||
@@ -1064,6 +1086,7 @@ pub fn apply_filters(
|
||||
| `--author` | `author_username` | Exact match |
|
||||
| `--project` | `project_id` | Resolve path to ID |
|
||||
| `--after` | `created_at` | `>= date` (ms epoch) |
|
||||
| `--updated-after` | `updated_at` | `>= date` (ms epoch), common triage filter |
|
||||
| `--label` | `document_labels` | JOIN, multiple = AND |
|
||||
| `--path` | `document_paths` | JOIN, trailing `/` = prefix |
|
||||
| `--limit` | N/A | Default 20, max 100 |
|
||||
@@ -1072,6 +1095,7 @@ pub fn apply_filters(
|
||||
- [ ] Each filter correctly restricts results
|
||||
- [ ] Multiple `--label` flags use AND logic
|
||||
- [ ] Path prefix vs exact match works correctly
|
||||
- [ ] `--updated-after` filters on updated_at (not created_at)
|
||||
- [ ] Filters compose (all applied together)
|
||||
- [ ] Ranking order preserved after filtering (ORDER BY position)
|
||||
- [ ] Limit clamped to valid range [1, 100]
|
||||
@@ -1080,7 +1104,7 @@ pub fn apply_filters(
|
||||
|
||||
---
|
||||
|
||||
### 3.4 CLI: `gi search --mode=lexical`
|
||||
### 3.4 CLI: `lore search --mode=lexical`
|
||||
|
||||
**File:** `src/cli/commands/search.rs`
|
||||
|
||||
@@ -1141,9 +1165,49 @@ pub fn run_search(
|
||||
explain: bool,
|
||||
) -> Result<SearchResponse> {
|
||||
// 1. Parse query and filters
|
||||
// 2. Execute search based on mode
|
||||
// 3. Apply post-retrieval filters
|
||||
// 4. Format and return results
|
||||
// 2. Execute search based on mode -> ranked doc_ids (+ explain ranks)
|
||||
// 3. Apply post-retrieval filters preserving ranking order
|
||||
// 4. HYDRATE in one DB round-trip (see hydration query below):
|
||||
// - documents fields (title, url, created_at, updated_at, content_text)
|
||||
// - project_path via JOIN projects
|
||||
// - labels aggregated via json_group_array
|
||||
// - paths aggregated via json_group_array (optional)
|
||||
// 5. Attach snippet:
|
||||
// - prefer FTS snippet when doc hit FTS
|
||||
// - fallback: truncated content_text via generate_fallback_snippet()
|
||||
// 6. For --mode=semantic with 0% embedding coverage:
|
||||
// return early with actionable error message (distinct from "Ollama down")
|
||||
todo!()
|
||||
}
|
||||
|
||||
/// Hydration query: fetch all display fields for ranked doc IDs in a single round-trip.
|
||||
///
|
||||
/// Uses json_each(?) to preserve ranking order from the search pipeline.
|
||||
/// Aggregates labels and paths inline to avoid N+1 queries.
|
||||
///
|
||||
/// ```sql
|
||||
/// SELECT d.id, d.source_type, d.title, d.url, d.author_username,
|
||||
/// d.created_at, d.updated_at, d.content_text,
|
||||
/// p.path AS project_path,
|
||||
/// (SELECT json_group_array(dl.label_name)
|
||||
/// FROM document_labels dl WHERE dl.document_id = d.id) AS labels,
|
||||
/// (SELECT json_group_array(dp.path)
|
||||
/// FROM document_paths dp WHERE dp.document_id = d.id) AS paths
|
||||
/// FROM json_each(?) AS j
|
||||
/// JOIN documents d ON d.id = j.value
|
||||
/// JOIN projects p ON p.id = d.project_id
|
||||
/// ORDER BY j.key
|
||||
/// ```
|
||||
///
|
||||
/// This single query replaces what would otherwise be:
|
||||
/// - 1 query per document for metadata
|
||||
/// - 1 query per document for labels
|
||||
/// - 1 query per document for paths
|
||||
/// For 20 results, that's 60 queries reduced to 1.
|
||||
fn hydrate_results(
|
||||
conn: &Connection,
|
||||
doc_ids: &[i64],
|
||||
) -> Result<Vec<HydratedDocument>> {
|
||||
todo!()
|
||||
}
|
||||
|
||||
@@ -1240,6 +1304,10 @@ pub struct SearchArgs {
|
||||
#[arg(long)]
|
||||
after: Option<String>,
|
||||
|
||||
/// Filter by updated date (recently active items)
|
||||
#[arg(long)]
|
||||
updated_after: Option<String>,
|
||||
|
||||
/// Filter by label (can specify multiple)
|
||||
#[arg(long, action = clap::ArgAction::Append)]
|
||||
label: Vec<String>,
|
||||
@@ -1266,12 +1334,15 @@ pub struct SearchArgs {
|
||||
|
||||
**Acceptance Criteria:**
|
||||
- [ ] Works without Ollama running
|
||||
- [ ] All filters functional
|
||||
- [ ] All filters functional (including `--updated-after`)
|
||||
- [ ] Human-readable output with snippets
|
||||
- [ ] Semantic-only results get fallback snippets from content_text
|
||||
- [ ] Results hydrated in single DB round-trip (no N+1 queries)
|
||||
- [ ] JSON output matches schema
|
||||
- [ ] Empty results show helpful message
|
||||
- [ ] "No data indexed" message if documents table empty
|
||||
- [ ] `--mode=semantic` with 0% embedding coverage returns actionable error
|
||||
(distinct from "Ollama unavailable" — tells user to run `lore embed` first)
|
||||
- [ ] `--fts-mode=safe` (default) preserves prefix `*` while escaping special chars
|
||||
- [ ] `--fts-mode=raw` passes FTS5 MATCH syntax through unchanged
|
||||
|
||||
@@ -1535,7 +1606,7 @@ impl GiError {
|
||||
// ... existing mappings ...
|
||||
Self::OllamaUnavailable { .. } => Some("Start Ollama: ollama serve"),
|
||||
Self::OllamaModelNotFound { model } => Some("Pull the model: ollama pull nomic-embed-text"),
|
||||
Self::EmbeddingFailed { .. } => Some("Check Ollama logs or retry with 'gi embed --retry-failed'"),
|
||||
Self::EmbeddingFailed { .. } => Some("Check Ollama logs or retry with 'lore embed --retry-failed'"),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1558,6 +1629,7 @@ use crate::embedding::OllamaClient;
|
||||
const BATCH_SIZE: usize = 32;
|
||||
|
||||
/// SQLite page size for paging through pending documents.
|
||||
/// Uses keyset paging (id > last_id) to avoid rescanning previously-processed rows.
|
||||
const DB_PAGE_SIZE: usize = 500;
|
||||
|
||||
/// Expected embedding dimensions for nomic-embed-text model.
|
||||
@@ -1584,11 +1656,16 @@ pub struct EmbedResult {
|
||||
/// Embed documents that need embedding.
|
||||
///
|
||||
/// Process:
|
||||
/// 1. Query dirty_sources ordered by queued_at
|
||||
/// 2. For each: regenerate document, compute new hash
|
||||
/// 3. ALWAYS upsert document (labels/paths may change even if content_hash unchanged)
|
||||
/// 4. Track whether content_hash changed (for stats)
|
||||
/// 5. Delete from dirty_sources (or record error on failure)
|
||||
/// 1. Select documents needing embeddings:
|
||||
/// - Pending: missing embedding_metadata row OR content_hash mismatch
|
||||
/// - RetryFailed: embedding_metadata.last_error IS NOT NULL
|
||||
/// 2. Page through candidates using keyset pagination (id > last_id)
|
||||
/// to avoid rescanning already-processed rows
|
||||
/// 3. Batch texts -> Ollama `/api/embed` with concurrent HTTP requests
|
||||
/// 4. Write embeddings + embedding_metadata in per-batch transactions
|
||||
/// 5. Failed batches record `last_error` in embedding_metadata
|
||||
/// (excluded from Pending selection; retried via RetryFailed)
|
||||
/// 6. Progress reported as (embedded + failed) vs total_pending
|
||||
pub async fn embed_documents(
|
||||
conn: &Connection,
|
||||
client: &OllamaClient,
|
||||
@@ -1605,9 +1682,11 @@ pub async fn embed_documents(
|
||||
return Ok(result);
|
||||
}
|
||||
|
||||
// Page through pending documents to avoid loading all into memory
|
||||
// Page through pending documents using keyset pagination to avoid
|
||||
// both memory pressure and OFFSET performance degradation.
|
||||
let mut last_id: i64 = 0;
|
||||
loop {
|
||||
let pending = find_pending_documents(conn, DB_PAGE_SIZE, selection)?;
|
||||
let pending = find_pending_documents(conn, DB_PAGE_SIZE, last_id, selection)?;
|
||||
if pending.is_empty() {
|
||||
break;
|
||||
}
|
||||
@@ -1640,6 +1719,11 @@ pub async fn embed_documents(
|
||||
collect_writes(conn, &meta, res, &mut result)?;
|
||||
}
|
||||
|
||||
// Advance keyset cursor for next page
|
||||
if let Some(last) = pending.last() {
|
||||
last_id = last.id;
|
||||
}
|
||||
|
||||
if let Some(ref cb) = progress_callback {
|
||||
cb(result.embedded + result.failed, total_pending);
|
||||
}
|
||||
@@ -1718,14 +1802,16 @@ fn count_pending_documents(conn: &Connection, selection: EmbedSelection) -> Resu
|
||||
Ok(count)
|
||||
}
|
||||
|
||||
/// Find pending documents for embedding.
|
||||
/// Find pending documents for embedding using keyset pagination.
|
||||
///
|
||||
/// IMPORTANT: Uses deterministic ORDER BY d.id to ensure consistent
|
||||
/// paging behavior. Without ordering, SQLite may return rows in
|
||||
/// different orders across calls, causing missed or duplicate documents.
|
||||
/// IMPORTANT: Uses keyset pagination (d.id > last_id) instead of OFFSET.
|
||||
/// OFFSET degrades O(n²) on large result sets because SQLite must scan
|
||||
/// and discard all rows before the offset. Keyset pagination is O(1) per page
|
||||
/// since the index seek goes directly to the starting row.
|
||||
fn find_pending_documents(
|
||||
conn: &Connection,
|
||||
limit: usize,
|
||||
last_id: i64,
|
||||
selection: EmbedSelection,
|
||||
) -> Result<Vec<PendingDocument>> {
|
||||
let sql = match selection {
|
||||
@@ -1733,8 +1819,9 @@ fn find_pending_documents(
|
||||
"SELECT d.id, d.content_text, d.content_hash
|
||||
FROM documents d
|
||||
LEFT JOIN embedding_metadata em ON d.id = em.document_id
|
||||
WHERE em.document_id IS NULL
|
||||
OR em.content_hash != d.content_hash
|
||||
WHERE (em.document_id IS NULL
|
||||
OR em.content_hash != d.content_hash)
|
||||
AND d.id > ?
|
||||
ORDER BY d.id
|
||||
LIMIT ?",
|
||||
EmbedSelection::RetryFailed =>
|
||||
@@ -1742,13 +1829,14 @@ fn find_pending_documents(
|
||||
FROM documents d
|
||||
JOIN embedding_metadata em ON d.id = em.document_id
|
||||
WHERE em.last_error IS NOT NULL
|
||||
AND d.id > ?
|
||||
ORDER BY d.id
|
||||
LIMIT ?",
|
||||
};
|
||||
let mut stmt = conn.prepare(sql)?;
|
||||
|
||||
let docs = stmt
|
||||
.query_map([limit], |row| {
|
||||
.query_map(rusqlite::params![last_id, limit as i64], |row| {
|
||||
Ok(PendingDocument {
|
||||
id: row.get(0)?,
|
||||
content: row.get(1)?,
|
||||
@@ -1827,7 +1915,7 @@ fn record_embedding_error(
|
||||
|
||||
---
|
||||
|
||||
### 4.5 CLI: `gi embed`
|
||||
### 4.5 CLI: `lore embed`
|
||||
|
||||
**File:** `src/cli/commands/embed.rs`
|
||||
|
||||
@@ -1928,7 +2016,7 @@ pub struct EmbedArgs {
|
||||
|
||||
---
|
||||
|
||||
### 4.6 CLI: `gi stats`
|
||||
### 4.6 CLI: `lore stats`
|
||||
|
||||
**File:** `src/cli/commands/stats.rs`
|
||||
|
||||
@@ -2034,7 +2122,13 @@ pub struct RepairResult {
|
||||
/// Fixes:
|
||||
/// - Deletes orphaned embeddings (embedding_metadata rows with no matching document)
|
||||
/// - Clears stale embedding_metadata (hash mismatch) so they get re-embedded
|
||||
/// - Repopulates FTS for documents missing from documents_fts
|
||||
/// - Rebuilds FTS index from scratch (correct-by-construction)
|
||||
///
|
||||
/// NOTE: FTS repair uses `rebuild` rather than partial row insertion.
|
||||
/// With `content='documents'` (external-content FTS), partial repopulation
|
||||
/// via INSERT of missing rows is fragile — if the external content table
|
||||
/// and FTS content diverge in any way, partial fixes can leave the index
|
||||
/// in an inconsistent state. A full rebuild is slower but guaranteed correct.
|
||||
pub fn run_repair(config: &Config) -> Result<RepairResult> {
|
||||
let conn = open_db(config)?;
|
||||
|
||||
@@ -2061,19 +2155,19 @@ pub fn run_repair(config: &Config) -> Result<RepairResult> {
|
||||
[],
|
||||
)?;
|
||||
|
||||
// Repopulate FTS for missing documents
|
||||
let fts_repopulated = conn.execute(
|
||||
"INSERT INTO documents_fts(rowid, title, content_text)
|
||||
SELECT id, COALESCE(title, ''), content_text
|
||||
FROM documents
|
||||
WHERE id NOT IN (SELECT rowid FROM documents_fts)",
|
||||
// Rebuild FTS index from scratch — correct-by-construction.
|
||||
// This re-reads all rows from the external content table (documents)
|
||||
// and rebuilds the index. Slower than partial fix but guaranteed consistent.
|
||||
conn.execute(
|
||||
"INSERT INTO documents_fts(documents_fts) VALUES('rebuild')",
|
||||
[],
|
||||
)?;
|
||||
let fts_rebuilt = 1; // rebuild is all-or-nothing
|
||||
|
||||
Ok(RepairResult {
|
||||
orphaned_embeddings_deleted: orphaned_deleted,
|
||||
stale_embeddings_cleared: stale_cleared,
|
||||
missing_fts_repopulated: fts_repopulated,
|
||||
missing_fts_repopulated: fts_rebuilt,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -2772,7 +2866,7 @@ pub fn record_fetch_error(
|
||||
current_attempt: i64,
|
||||
) -> Result<()> {
|
||||
let now = now_ms();
|
||||
let next_attempt = compute_next_attempt_at(now, current_attempt + 1);
|
||||
let next_attempt = crate::core::backoff::compute_next_attempt_at(now, current_attempt + 1);
|
||||
|
||||
conn.execute(
|
||||
"UPDATE pending_discussion_fetches
|
||||
@@ -2786,14 +2880,44 @@ pub fn record_fetch_error(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// NOTE: Backoff computation uses the shared utility in `src/core/backoff.rs`.
|
||||
// See Phase 6.X below for the shared implementation.
|
||||
```
|
||||
|
||||
**Acceptance Criteria:**
|
||||
- [ ] Updated entities queued for discussion fetch
|
||||
- [ ] Success removes from queue
|
||||
- [ ] Failure increments attempt_count and sets next_attempt_at
|
||||
- [ ] Processing bounded per run (max 100)
|
||||
- [ ] Exponential backoff uses `next_attempt_at` (index-friendly, no overflow)
|
||||
- [ ] Backoff computed with jitter to prevent thundering herd
|
||||
|
||||
---
|
||||
|
||||
### 6.X Shared Backoff Utility
|
||||
|
||||
**File:** `src/core/backoff.rs`
|
||||
|
||||
Single implementation of exponential backoff with jitter, used by both
|
||||
`dirty_sources` and `pending_discussion_fetches` queues. Living in `src/core/`
|
||||
because it's a cross-cutting concern used by multiple modules.
|
||||
|
||||
```rust
|
||||
use rand::Rng;
|
||||
|
||||
/// Compute next_attempt_at with exponential backoff and jitter.
|
||||
///
|
||||
/// Formula: now + min(3600000, 1000 * 2^attempt_count) * (0.9 to 1.1)
|
||||
/// - Capped at 1 hour to prevent runaway delays
|
||||
/// - ±10% jitter prevents synchronized retries after outages
|
||||
///
|
||||
/// Used by:
|
||||
/// - `dirty_sources` retry scheduling (document regeneration failures)
|
||||
/// - `pending_discussion_fetches` retry scheduling (API fetch failures)
|
||||
///
|
||||
/// Having one implementation prevents subtle divergence between queues
|
||||
/// (e.g., different caps or jitter ranges).
|
||||
pub fn compute_next_attempt_at(now: i64, attempt_count: i64) -> i64 {
|
||||
use rand::Rng;
|
||||
|
||||
// Cap attempt_count to prevent overflow (2^30 > 1 hour anyway)
|
||||
let capped_attempts = attempt_count.min(30) as u32;
|
||||
let base_delay_ms = 1000_i64.saturating_mul(1 << capped_attempts);
|
||||
@@ -2807,13 +2931,16 @@ pub fn compute_next_attempt_at(now: i64, attempt_count: i64) -> i64 {
|
||||
}
|
||||
```
|
||||
|
||||
**Update `src/core/mod.rs`:**
|
||||
```rust
|
||||
pub mod backoff; // Add to existing modules
|
||||
```
|
||||
|
||||
**Acceptance Criteria:**
|
||||
- [ ] Updated entities queued for discussion fetch
|
||||
- [ ] Success removes from queue
|
||||
- [ ] Failure increments attempt_count and sets next_attempt_at
|
||||
- [ ] Processing bounded per run (max 100)
|
||||
- [ ] Exponential backoff uses `next_attempt_at` (index-friendly, no overflow)
|
||||
- [ ] Backoff computed with jitter to prevent thundering herd
|
||||
- [ ] Single implementation shared by both queue retry paths
|
||||
- [ ] Cap at 1 hour prevents runaway delays
|
||||
- [ ] Jitter prevents thundering herd after outage recovery
|
||||
- [ ] Unit tests verify backoff curve and cap behavior
|
||||
|
||||
---
|
||||
|
||||
@@ -2917,19 +3044,36 @@ fn delete_document(
|
||||
}
|
||||
|
||||
/// Record a regeneration error on a dirty source for retry.
|
||||
///
|
||||
/// IMPORTANT: Sets `next_attempt_at` using the shared backoff utility.
|
||||
/// Without this, failed items would retry every run (hot-loop), defeating
|
||||
/// the backoff design documented in the schema.
|
||||
fn record_dirty_error(
|
||||
conn: &Connection,
|
||||
source_type: SourceType,
|
||||
source_id: i64,
|
||||
error: &str,
|
||||
) -> Result<()> {
|
||||
let now = now_ms();
|
||||
|
||||
// Read current attempt_count from DB to compute backoff
|
||||
let attempt_count: i64 = conn.query_row(
|
||||
"SELECT attempt_count FROM dirty_sources WHERE source_type = ? AND source_id = ?",
|
||||
rusqlite::params![source_type.as_str(), source_id],
|
||||
|row| row.get(0),
|
||||
)?;
|
||||
|
||||
// Use shared backoff utility (same as pending_discussion_fetches)
|
||||
let next_attempt_at = crate::core::backoff::compute_next_attempt_at(now, attempt_count + 1);
|
||||
|
||||
conn.execute(
|
||||
"UPDATE dirty_sources
|
||||
SET attempt_count = attempt_count + 1,
|
||||
last_attempt_at = ?,
|
||||
last_error = ?
|
||||
last_error = ?,
|
||||
next_attempt_at = ?
|
||||
WHERE source_type = ? AND source_id = ?",
|
||||
rusqlite::params![now_ms(), error, source_type.as_str(), source_id],
|
||||
rusqlite::params![now, error, next_attempt_at, source_type.as_str(), source_id],
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -3080,7 +3224,7 @@ fn get_document_id(
|
||||
|
||||
---
|
||||
|
||||
### 6.4 CLI: `gi sync`
|
||||
### 6.4 CLI: `lore sync`
|
||||
|
||||
**File:** `src/cli/commands/sync.rs`
|
||||
|
||||
@@ -3198,7 +3342,8 @@ pub struct SyncArgs {
|
||||
| FTS query sanitization | `src/search/fts.rs` (mod tests) | `to_fts_query()` edge cases: `-`, `"`, `:`, `*`, `C++` |
|
||||
| SourceType parsing | `src/documents/extractor.rs` (mod tests) | `parse()` accepts aliases: `mr`, `mrs`, `issue`, etc. |
|
||||
| SearchFilters | `src/search/filters.rs` (mod tests) | `has_any_filter()`, `clamp_limit()` |
|
||||
| Backoff logic | `src/ingestion/dirty_tracker.rs` (mod tests) | Exponential backoff query timing |
|
||||
| Backoff logic | `src/core/backoff.rs` (mod tests) | Shared exponential backoff curve, cap, jitter |
|
||||
| Hydration | `src/cli/commands/search.rs` (mod tests) | Single round-trip, label/path aggregation |
|
||||
|
||||
### Integration Tests
|
||||
|
||||
@@ -3232,25 +3377,29 @@ Each query must have at least one expected URL in top 10 results.
|
||||
|
||||
| Command | Expected | Pass Criteria |
|
||||
|---------|----------|---------------|
|
||||
| `gi generate-docs` | Progress, count | Completes, count > 0 |
|
||||
| `gi generate-docs` (re-run) | 0 regenerated | Hash comparison works |
|
||||
| `gi embed` | Progress, count | Completes, count matches docs |
|
||||
| `gi embed` (re-run) | 0 embedded | Skips unchanged |
|
||||
| `gi embed --retry-failed` | Processes failed | Only failed docs processed |
|
||||
| `gi stats` | Coverage stats | Shows 100% after embed |
|
||||
| `gi stats` | Queue depths | Shows dirty_sources and pending_discussion_fetches counts |
|
||||
| `gi search "auth" --mode=lexical` | Results | Works without Ollama |
|
||||
| `gi search "auth"` | Hybrid results | Vector + FTS combined |
|
||||
| `gi search "auth"` (Ollama down) | FTS results + warning | Graceful degradation, warning in response |
|
||||
| `gi search "auth" --explain` | Rank breakdown | Shows vector/FTS/RRF |
|
||||
| `gi search "auth" --type=mr` | Filtered results | Only MRs |
|
||||
| `gi search "auth" --type=mrs` | Filtered results | Alias works |
|
||||
| `gi search "auth" --label=bug` | Filtered results | Only labeled docs |
|
||||
| `gi search "-DWITH_SSL"` | Results | Leading dash doesn't cause FTS error |
|
||||
| `gi search 'C++'` | Results | Special chars in query work |
|
||||
| `gi search "nonexistent123"` | No results | Graceful empty state |
|
||||
| `gi sync` | Full pipeline | All steps complete |
|
||||
| `gi sync --no-embed` | Skip embedding | Docs generated, not embedded |
|
||||
| `lore generate-docs` | Progress, count | Completes, count > 0 |
|
||||
| `lore generate-docs` (re-run) | 0 regenerated | Hash comparison works |
|
||||
| `lore embed` | Progress, count | Completes, count matches docs |
|
||||
| `lore embed` (re-run) | 0 embedded | Skips unchanged |
|
||||
| `lore embed --retry-failed` | Processes failed | Only failed docs processed |
|
||||
| `lore stats` | Coverage stats | Shows 100% after embed |
|
||||
| `lore stats` | Queue depths | Shows dirty_sources and pending_discussion_fetches counts |
|
||||
| `lore search "auth" --mode=lexical` | Results | Works without Ollama |
|
||||
| `lore search "auth"` | Hybrid results | Vector + FTS combined |
|
||||
| `lore search "auth"` (Ollama down) | FTS results + warning | Graceful degradation, warning in response |
|
||||
| `lore search "auth" --explain` | Rank breakdown | Shows vector/FTS/RRF |
|
||||
| `lore search "auth" --type=mr` | Filtered results | Only MRs |
|
||||
| `lore search "auth" --type=mrs` | Filtered results | Alias works |
|
||||
| `lore search "auth" --label=bug` | Filtered results | Only labeled docs |
|
||||
| `lore search "-DWITH_SSL"` | Results | Leading dash doesn't cause FTS error |
|
||||
| `lore search 'C++'` | Results | Special chars in query work |
|
||||
| `lore search "auth" --updated-after 2024-01-01` | Filtered results | Only recently updated docs |
|
||||
| `lore search "nonexistent123"` | No results | Graceful empty state |
|
||||
| `lore search "auth" --mode=semantic` (no embeddings) | Actionable error | Tells user to run `lore embed` first |
|
||||
| `lore sync` | Full pipeline | All steps complete |
|
||||
| `lore sync --no-embed` | Skip embedding | Docs generated, not embedded |
|
||||
| `lore generate-docs --full` | Progress, count | Keyset pagination completes without OFFSET degradation |
|
||||
| `lore stats --check --repair` | Repair results | FTS rebuilt, orphans cleaned |
|
||||
|
||||
---
|
||||
|
||||
@@ -3274,43 +3423,59 @@ Each query must have at least one expected URL in top 10 results.
|
||||
|
||||
## Success Criteria
|
||||
|
||||
Checkpoint 3 is complete when:
|
||||
Checkpoint 3 is complete when all three gates pass:
|
||||
|
||||
### Gate A: Lexical MVP
|
||||
|
||||
1. **Lexical search works without Ollama**
|
||||
- `gi search "query" --mode=lexical` returns relevant results
|
||||
- All filters functional
|
||||
- `lore search "query" --mode=lexical` returns relevant results
|
||||
- All filters functional (including `--updated-after`)
|
||||
- FTS5 syntax errors prevented by query sanitization
|
||||
- Special characters in queries work correctly (`-DWITH_SSL`, `C++`)
|
||||
- Search results hydrated in single DB round-trip (no N+1)
|
||||
|
||||
2. **Semantic search works with Ollama**
|
||||
- `gi embed` completes successfully
|
||||
- `gi search "query"` returns semantically relevant results
|
||||
2. **Document generation is correct**
|
||||
- Full and incremental modes use the same regenerator codepath
|
||||
- `--full` uses keyset pagination (no OFFSET degradation)
|
||||
- FTS triggers use COALESCE for NULL-safe operation
|
||||
|
||||
### Gate B: Hybrid MVP
|
||||
|
||||
3. **Semantic search works with Ollama**
|
||||
- `lore embed` completes successfully
|
||||
- `lore search "query"` returns semantically relevant results
|
||||
- `--explain` shows ranking breakdown
|
||||
- `--mode=semantic` with 0% embedding coverage returns actionable error
|
||||
|
||||
3. **Hybrid search combines both**
|
||||
4. **Hybrid search combines both**
|
||||
- Documents appearing in both retrievers rank higher
|
||||
- Graceful degradation when Ollama unavailable (falls back to FTS)
|
||||
- Transient embed failures don't fail the entire search
|
||||
- Warning message included in response on degradation
|
||||
- Embedding pipeline uses keyset pagination for consistent paging
|
||||
|
||||
4. **Incremental sync is efficient**
|
||||
- `gi sync` only processes changed entities
|
||||
### Gate C: Sync MVP
|
||||
|
||||
5. **Incremental sync is efficient**
|
||||
- `lore sync` only processes changed entities
|
||||
- Re-embedding only happens for changed documents
|
||||
- Progress visible during long syncs
|
||||
- Queue backoff prevents hot-loop retries on persistent failures
|
||||
- Queue backoff actually prevents hot-loop retries (both queues set `next_attempt_at`)
|
||||
- Shared backoff utility ensures consistent behavior across queues
|
||||
|
||||
5. **Data integrity maintained**
|
||||
6. **Data integrity maintained**
|
||||
- All counts match between tables
|
||||
- No orphaned records
|
||||
- Hashes consistent
|
||||
- `get_existing_hash()` properly distinguishes "not found" from DB errors
|
||||
- `--repair` uses FTS `rebuild` for correct-by-construction repair
|
||||
|
||||
6. **Observability**
|
||||
- `gi stats` shows queue depths and failed item counts
|
||||
7. **Observability**
|
||||
- `lore stats` shows queue depths and failed item counts
|
||||
- Failed items visible for operator intervention
|
||||
- Deterministic ordering ensures consistent paging
|
||||
|
||||
7. **Tests pass**
|
||||
- Unit tests for core algorithms (including FTS sanitization, backoff)
|
||||
8. **Tests pass**
|
||||
- Unit tests for core algorithms (including FTS sanitization, shared backoff, hydration)
|
||||
- Integration tests for pipelines
|
||||
- Golden queries return expected results
|
||||
|
||||
@@ -17,6 +17,7 @@ use crate::ingestion::{
|
||||
};
|
||||
|
||||
/// Result of ingest command for display.
|
||||
#[derive(Default)]
|
||||
pub struct IngestResult {
|
||||
pub resource_type: String,
|
||||
pub projects_synced: usize,
|
||||
@@ -130,24 +131,7 @@ pub async fn run_ingest(
|
||||
|
||||
let mut total = IngestResult {
|
||||
resource_type: resource_type.to_string(),
|
||||
projects_synced: 0,
|
||||
// Issue fields
|
||||
issues_fetched: 0,
|
||||
issues_upserted: 0,
|
||||
issues_synced_discussions: 0,
|
||||
issues_skipped_discussion_sync: 0,
|
||||
// MR fields
|
||||
mrs_fetched: 0,
|
||||
mrs_upserted: 0,
|
||||
mrs_synced_discussions: 0,
|
||||
mrs_skipped_discussion_sync: 0,
|
||||
assignees_linked: 0,
|
||||
reviewers_linked: 0,
|
||||
diffnotes_count: 0,
|
||||
// Shared fields
|
||||
labels_created: 0,
|
||||
discussions_fetched: 0,
|
||||
notes_upserted: 0,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let type_label = if resource_type == "issues" {
|
||||
|
||||
@@ -232,8 +232,11 @@ fn query_issues(conn: &Connection, filters: &ListFilters) -> Result<ListResult>
|
||||
let mut params: Vec<Box<dyn rusqlite::ToSql>> = Vec::new();
|
||||
|
||||
if let Some(project) = filters.project {
|
||||
where_clauses.push("p.path_with_namespace LIKE ?");
|
||||
params.push(Box::new(format!("%{project}%")));
|
||||
// Exact match or suffix match after '/' to avoid partial matches
|
||||
// e.g. "foo" matches "group/foo" but NOT "group/foobar"
|
||||
where_clauses.push("(p.path_with_namespace = ? OR p.path_with_namespace LIKE ?)");
|
||||
params.push(Box::new(project.to_string()));
|
||||
params.push(Box::new(format!("%/{project}")));
|
||||
}
|
||||
|
||||
if let Some(state) = filters.state
|
||||
@@ -337,11 +340,11 @@ fn query_issues(conn: &Connection, filters: &ListFilters) -> Result<ListResult>
|
||||
i.updated_at,
|
||||
i.web_url,
|
||||
p.path_with_namespace,
|
||||
(SELECT GROUP_CONCAT(l.name, ',')
|
||||
(SELECT GROUP_CONCAT(l.name, X'1F')
|
||||
FROM issue_labels il
|
||||
JOIN labels l ON il.label_id = l.id
|
||||
WHERE il.issue_id = i.id) AS labels_csv,
|
||||
(SELECT GROUP_CONCAT(ia.username, ',')
|
||||
(SELECT GROUP_CONCAT(ia.username, X'1F')
|
||||
FROM issue_assignees ia
|
||||
WHERE ia.issue_id = i.id) AS assignees_csv,
|
||||
COALESCE(d.total, 0) AS discussion_count,
|
||||
@@ -369,12 +372,12 @@ fn query_issues(conn: &Connection, filters: &ListFilters) -> Result<ListResult>
|
||||
.query_map(param_refs.as_slice(), |row| {
|
||||
let labels_csv: Option<String> = row.get(8)?;
|
||||
let labels = labels_csv
|
||||
.map(|s| s.split(',').map(String::from).collect())
|
||||
.map(|s| s.split('\x1F').map(String::from).collect())
|
||||
.unwrap_or_default();
|
||||
|
||||
let assignees_csv: Option<String> = row.get(9)?;
|
||||
let assignees = assignees_csv
|
||||
.map(|s| s.split(',').map(String::from).collect())
|
||||
.map(|s| s.split('\x1F').map(String::from).collect())
|
||||
.unwrap_or_default();
|
||||
|
||||
Ok(IssueListRow {
|
||||
@@ -416,8 +419,11 @@ fn query_mrs(conn: &Connection, filters: &MrListFilters) -> Result<MrListResult>
|
||||
let mut params: Vec<Box<dyn rusqlite::ToSql>> = Vec::new();
|
||||
|
||||
if let Some(project) = filters.project {
|
||||
where_clauses.push("p.path_with_namespace LIKE ?");
|
||||
params.push(Box::new(format!("%{project}%")));
|
||||
// Exact match or suffix match after '/' to avoid partial matches
|
||||
// e.g. "foo" matches "group/foo" but NOT "group/foobar"
|
||||
where_clauses.push("(p.path_with_namespace = ? OR p.path_with_namespace LIKE ?)");
|
||||
params.push(Box::new(project.to_string()));
|
||||
params.push(Box::new(format!("%/{project}")));
|
||||
}
|
||||
|
||||
if let Some(state) = filters.state
|
||||
@@ -536,14 +542,14 @@ fn query_mrs(conn: &Connection, filters: &MrListFilters) -> Result<MrListResult>
|
||||
m.updated_at,
|
||||
m.web_url,
|
||||
p.path_with_namespace,
|
||||
(SELECT GROUP_CONCAT(l.name, ',')
|
||||
(SELECT GROUP_CONCAT(l.name, X'1F')
|
||||
FROM mr_labels ml
|
||||
JOIN labels l ON ml.label_id = l.id
|
||||
WHERE ml.merge_request_id = m.id) AS labels_csv,
|
||||
(SELECT GROUP_CONCAT(ma.username, ',')
|
||||
(SELECT GROUP_CONCAT(ma.username, X'1F')
|
||||
FROM mr_assignees ma
|
||||
WHERE ma.merge_request_id = m.id) AS assignees_csv,
|
||||
(SELECT GROUP_CONCAT(mr.username, ',')
|
||||
(SELECT GROUP_CONCAT(mr.username, X'1F')
|
||||
FROM mr_reviewers mr
|
||||
WHERE mr.merge_request_id = m.id) AS reviewers_csv,
|
||||
COALESCE(d.total, 0) AS discussion_count,
|
||||
@@ -571,17 +577,17 @@ fn query_mrs(conn: &Connection, filters: &MrListFilters) -> Result<MrListResult>
|
||||
.query_map(param_refs.as_slice(), |row| {
|
||||
let labels_csv: Option<String> = row.get(11)?;
|
||||
let labels = labels_csv
|
||||
.map(|s| s.split(',').map(String::from).collect())
|
||||
.map(|s| s.split('\x1F').map(String::from).collect())
|
||||
.unwrap_or_default();
|
||||
|
||||
let assignees_csv: Option<String> = row.get(12)?;
|
||||
let assignees = assignees_csv
|
||||
.map(|s| s.split(',').map(String::from).collect())
|
||||
.map(|s| s.split('\x1F').map(String::from).collect())
|
||||
.unwrap_or_default();
|
||||
|
||||
let reviewers_csv: Option<String> = row.get(13)?;
|
||||
let reviewers = reviewers_csv
|
||||
.map(|s| s.split(',').map(String::from).collect())
|
||||
.map(|s| s.split('\x1F').map(String::from).collect())
|
||||
.unwrap_or_default();
|
||||
|
||||
let draft_int: i64 = row.get(3)?;
|
||||
@@ -615,6 +621,10 @@ fn format_relative_time(ms_epoch: i64) -> String {
|
||||
let now = now_ms();
|
||||
let diff = now - ms_epoch;
|
||||
|
||||
if diff < 0 {
|
||||
return "in the future".to_string();
|
||||
}
|
||||
|
||||
match diff {
|
||||
d if d < 60_000 => "just now".to_string(),
|
||||
d if d < 3_600_000 => format!("{} min ago", d / 60_000),
|
||||
|
||||
@@ -150,8 +150,12 @@ fn find_issue(conn: &Connection, iid: i64, project_filter: Option<&str>) -> Resu
|
||||
i.created_at, i.updated_at, i.web_url, p.path_with_namespace
|
||||
FROM issues i
|
||||
JOIN projects p ON i.project_id = p.id
|
||||
WHERE i.iid = ? AND p.path_with_namespace LIKE ?",
|
||||
vec![Box::new(iid), Box::new(format!("%{}%", project))],
|
||||
WHERE i.iid = ? AND (p.path_with_namespace = ? OR p.path_with_namespace LIKE ?)",
|
||||
vec![
|
||||
Box::new(iid),
|
||||
Box::new(project.to_string()),
|
||||
Box::new(format!("%/{}", project)),
|
||||
],
|
||||
),
|
||||
None => (
|
||||
"SELECT i.id, i.iid, i.title, i.description, i.state, i.author_username,
|
||||
@@ -336,8 +340,12 @@ fn find_mr(conn: &Connection, iid: i64, project_filter: Option<&str>) -> Result<
|
||||
m.web_url, p.path_with_namespace
|
||||
FROM merge_requests m
|
||||
JOIN projects p ON m.project_id = p.id
|
||||
WHERE m.iid = ? AND p.path_with_namespace LIKE ?",
|
||||
vec![Box::new(iid), Box::new(format!("%{}%", project))],
|
||||
WHERE m.iid = ? AND (p.path_with_namespace = ? OR p.path_with_namespace LIKE ?)",
|
||||
vec![
|
||||
Box::new(iid),
|
||||
Box::new(project.to_string()),
|
||||
Box::new(format!("%/{}", project)),
|
||||
],
|
||||
),
|
||||
None => (
|
||||
"SELECT m.id, m.iid, m.title, m.description, m.state, m.draft,
|
||||
|
||||
301
src/cli/mod.rs
301
src/cli/mod.rs
@@ -11,13 +11,17 @@ use std::io::IsTerminal;
|
||||
#[command(version, about, long_about = None)]
|
||||
pub struct Cli {
|
||||
/// Path to config file
|
||||
#[arg(short, long, global = true)]
|
||||
#[arg(short = 'c', long, global = true)]
|
||||
pub config: Option<String>,
|
||||
|
||||
/// Machine-readable JSON output (auto-enabled when piped)
|
||||
#[arg(long, global = true, env = "LORE_ROBOT")]
|
||||
pub robot: bool,
|
||||
|
||||
/// JSON output (global shorthand)
|
||||
#[arg(short = 'J', long = "json", global = true)]
|
||||
pub json: bool,
|
||||
|
||||
#[command(subcommand)]
|
||||
pub command: Commands,
|
||||
}
|
||||
@@ -25,17 +29,41 @@ pub struct Cli {
|
||||
impl Cli {
|
||||
/// Check if robot mode is active (explicit flag, env var, or non-TTY stdout)
|
||||
pub fn is_robot_mode(&self) -> bool {
|
||||
self.robot || !std::io::stdout().is_terminal()
|
||||
self.robot || self.json || !std::io::stdout().is_terminal()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Subcommand)]
|
||||
#[allow(clippy::large_enum_variant)]
|
||||
pub enum Commands {
|
||||
/// List or show issues
|
||||
Issues(IssuesArgs),
|
||||
|
||||
/// List or show merge requests
|
||||
Mrs(MrsArgs),
|
||||
|
||||
/// Ingest data from GitLab
|
||||
Ingest(IngestArgs),
|
||||
|
||||
/// Count entities in local database
|
||||
Count(CountArgs),
|
||||
|
||||
/// Show sync state
|
||||
Status,
|
||||
|
||||
/// Verify GitLab authentication
|
||||
Auth,
|
||||
|
||||
/// Check environment health
|
||||
Doctor,
|
||||
|
||||
/// Show version information
|
||||
Version,
|
||||
|
||||
/// Initialize configuration and database
|
||||
Init {
|
||||
/// Skip overwrite confirmation
|
||||
#[arg(long)]
|
||||
#[arg(short = 'f', long)]
|
||||
force: bool,
|
||||
|
||||
/// Fail if prompts would be shown
|
||||
@@ -43,149 +71,67 @@ pub enum Commands {
|
||||
non_interactive: bool,
|
||||
},
|
||||
|
||||
/// Verify GitLab authentication
|
||||
AuthTest,
|
||||
|
||||
/// Check environment health
|
||||
Doctor {
|
||||
/// Output as JSON
|
||||
#[arg(long)]
|
||||
json: bool,
|
||||
},
|
||||
|
||||
/// Show version information
|
||||
Version,
|
||||
|
||||
/// Create timestamped database backup
|
||||
Backup,
|
||||
|
||||
/// Delete database and reset all state
|
||||
Reset {
|
||||
/// Skip confirmation prompt
|
||||
#[arg(long)]
|
||||
confirm: bool,
|
||||
#[arg(short = 'y', long)]
|
||||
yes: bool,
|
||||
},
|
||||
|
||||
/// Run pending database migrations
|
||||
Migrate,
|
||||
|
||||
/// Show sync state
|
||||
SyncStatus,
|
||||
|
||||
/// Ingest data from GitLab
|
||||
Ingest {
|
||||
/// Resource type to ingest
|
||||
#[arg(long, value_parser = ["issues", "mrs"])]
|
||||
r#type: String,
|
||||
|
||||
/// Filter to single project
|
||||
#[arg(long)]
|
||||
project: Option<String>,
|
||||
|
||||
/// Override stale sync lock
|
||||
#[arg(long)]
|
||||
force: bool,
|
||||
|
||||
/// Full re-sync: reset cursors and fetch all data from scratch
|
||||
#[arg(long)]
|
||||
full: bool,
|
||||
},
|
||||
|
||||
/// List issues or MRs from local database
|
||||
// --- Hidden backward-compat aliases ---
|
||||
/// List issues or MRs (deprecated: use 'lore issues' or 'lore mrs')
|
||||
#[command(hide = true)]
|
||||
List {
|
||||
/// Entity type to list
|
||||
#[arg(value_parser = ["issues", "mrs"])]
|
||||
entity: String,
|
||||
|
||||
/// Maximum results
|
||||
#[arg(long, default_value = "50")]
|
||||
limit: usize,
|
||||
|
||||
/// Filter by project path
|
||||
#[arg(long)]
|
||||
project: Option<String>,
|
||||
|
||||
/// Filter by state (opened|closed|all for issues; opened|merged|closed|locked|all for MRs)
|
||||
#[arg(long)]
|
||||
state: Option<String>,
|
||||
|
||||
/// Filter by author username
|
||||
#[arg(long)]
|
||||
author: Option<String>,
|
||||
|
||||
/// Filter by assignee username
|
||||
#[arg(long)]
|
||||
assignee: Option<String>,
|
||||
|
||||
/// Filter by label (repeatable, AND logic)
|
||||
#[arg(long)]
|
||||
label: Option<Vec<String>>,
|
||||
|
||||
/// Filter by milestone title (issues only)
|
||||
#[arg(long)]
|
||||
milestone: Option<String>,
|
||||
|
||||
/// Filter by time (7d, 2w, 1m, or YYYY-MM-DD)
|
||||
#[arg(long)]
|
||||
since: Option<String>,
|
||||
|
||||
/// Filter by due date (before this date, YYYY-MM-DD) (issues only)
|
||||
#[arg(long)]
|
||||
due_before: Option<String>,
|
||||
|
||||
/// Show only issues with a due date (issues only)
|
||||
#[arg(long)]
|
||||
has_due_date: bool,
|
||||
|
||||
/// Sort field
|
||||
#[arg(long, value_parser = ["updated", "created", "iid"], default_value = "updated")]
|
||||
sort: String,
|
||||
|
||||
/// Sort order
|
||||
#[arg(long, value_parser = ["desc", "asc"], default_value = "desc")]
|
||||
order: String,
|
||||
|
||||
/// Open first matching item in browser
|
||||
#[arg(long)]
|
||||
open: bool,
|
||||
|
||||
/// Output as JSON
|
||||
#[arg(long)]
|
||||
json: bool,
|
||||
|
||||
/// Show only draft MRs (MRs only)
|
||||
#[arg(long, conflicts_with = "no_draft")]
|
||||
draft: bool,
|
||||
|
||||
/// Exclude draft MRs (MRs only)
|
||||
#[arg(long, conflicts_with = "draft")]
|
||||
no_draft: bool,
|
||||
|
||||
/// Filter by reviewer username (MRs only)
|
||||
#[arg(long)]
|
||||
reviewer: Option<String>,
|
||||
|
||||
/// Filter by target branch (MRs only)
|
||||
#[arg(long)]
|
||||
target_branch: Option<String>,
|
||||
|
||||
/// Filter by source branch (MRs only)
|
||||
#[arg(long)]
|
||||
source_branch: Option<String>,
|
||||
},
|
||||
|
||||
/// Count entities in local database
|
||||
Count {
|
||||
/// Entity type to count
|
||||
#[arg(value_parser = ["issues", "mrs", "discussions", "notes"])]
|
||||
entity: String,
|
||||
|
||||
/// Filter by noteable type (for discussions/notes)
|
||||
#[arg(long, value_parser = ["issue", "mr"])]
|
||||
r#type: Option<String>,
|
||||
},
|
||||
|
||||
/// Show detailed entity information
|
||||
/// Show detailed entity information (deprecated: use 'lore issues <IID>' or 'lore mrs <IID>')
|
||||
#[command(hide = true)]
|
||||
Show {
|
||||
/// Entity type to show
|
||||
#[arg(value_parser = ["issue", "mr"])]
|
||||
@@ -194,12 +140,173 @@ pub enum Commands {
|
||||
/// Entity IID
|
||||
iid: i64,
|
||||
|
||||
/// Filter by project path (required if iid is ambiguous)
|
||||
#[arg(long)]
|
||||
project: Option<String>,
|
||||
|
||||
/// Output as JSON
|
||||
#[arg(long)]
|
||||
json: bool,
|
||||
},
|
||||
|
||||
/// Verify GitLab authentication (deprecated: use 'lore auth')
|
||||
#[command(hide = true, name = "auth-test")]
|
||||
AuthTest,
|
||||
|
||||
/// Show sync state (deprecated: use 'lore status')
|
||||
#[command(hide = true, name = "sync-status")]
|
||||
SyncStatus,
|
||||
}
|
||||
|
||||
/// Arguments for `lore issues [IID]`
|
||||
#[derive(Parser)]
|
||||
pub struct IssuesArgs {
|
||||
/// Issue IID (omit to list, provide to show details)
|
||||
pub iid: Option<i64>,
|
||||
|
||||
/// Maximum results
|
||||
#[arg(short = 'n', long = "limit", default_value = "50")]
|
||||
pub limit: usize,
|
||||
|
||||
/// Filter by state (opened, closed, all)
|
||||
#[arg(short = 's', long)]
|
||||
pub state: Option<String>,
|
||||
|
||||
/// Filter by project path
|
||||
#[arg(short = 'p', long)]
|
||||
pub project: Option<String>,
|
||||
|
||||
/// Filter by author username
|
||||
#[arg(short = 'a', long)]
|
||||
pub author: Option<String>,
|
||||
|
||||
/// Filter by assignee username
|
||||
#[arg(short = 'A', long)]
|
||||
pub assignee: Option<String>,
|
||||
|
||||
/// Filter by label (repeatable, AND logic)
|
||||
#[arg(short = 'l', long)]
|
||||
pub label: Option<Vec<String>>,
|
||||
|
||||
/// Filter by milestone title
|
||||
#[arg(short = 'm', long)]
|
||||
pub milestone: Option<String>,
|
||||
|
||||
/// Filter by time (7d, 2w, 1m, or YYYY-MM-DD)
|
||||
#[arg(long)]
|
||||
pub since: Option<String>,
|
||||
|
||||
/// Filter by due date (before this date, YYYY-MM-DD)
|
||||
#[arg(long = "due-before")]
|
||||
pub due_before: Option<String>,
|
||||
|
||||
/// Show only issues with a due date
|
||||
#[arg(long = "has-due")]
|
||||
pub has_due: bool,
|
||||
|
||||
/// Sort field (updated, created, iid)
|
||||
#[arg(long, value_parser = ["updated", "created", "iid"], default_value = "updated")]
|
||||
pub sort: String,
|
||||
|
||||
/// Sort ascending (default: descending)
|
||||
#[arg(long)]
|
||||
pub asc: bool,
|
||||
|
||||
/// Open first matching item in browser
|
||||
#[arg(short = 'o', long)]
|
||||
pub open: bool,
|
||||
}
|
||||
|
||||
/// Arguments for `lore mrs [IID]`
|
||||
#[derive(Parser)]
|
||||
pub struct MrsArgs {
|
||||
/// MR IID (omit to list, provide to show details)
|
||||
pub iid: Option<i64>,
|
||||
|
||||
/// Maximum results
|
||||
#[arg(short = 'n', long = "limit", default_value = "50")]
|
||||
pub limit: usize,
|
||||
|
||||
/// Filter by state (opened, merged, closed, locked, all)
|
||||
#[arg(short = 's', long)]
|
||||
pub state: Option<String>,
|
||||
|
||||
/// Filter by project path
|
||||
#[arg(short = 'p', long)]
|
||||
pub project: Option<String>,
|
||||
|
||||
/// Filter by author username
|
||||
#[arg(short = 'a', long)]
|
||||
pub author: Option<String>,
|
||||
|
||||
/// Filter by assignee username
|
||||
#[arg(short = 'A', long)]
|
||||
pub assignee: Option<String>,
|
||||
|
||||
/// Filter by reviewer username
|
||||
#[arg(short = 'r', long)]
|
||||
pub reviewer: Option<String>,
|
||||
|
||||
/// Filter by label (repeatable, AND logic)
|
||||
#[arg(short = 'l', long)]
|
||||
pub label: Option<Vec<String>>,
|
||||
|
||||
/// Filter by time (7d, 2w, 1m, or YYYY-MM-DD)
|
||||
#[arg(long)]
|
||||
pub since: Option<String>,
|
||||
|
||||
/// Show only draft MRs
|
||||
#[arg(short = 'd', long, conflicts_with = "no_draft")]
|
||||
pub draft: bool,
|
||||
|
||||
/// Exclude draft MRs
|
||||
#[arg(short = 'D', long = "no-draft", conflicts_with = "draft")]
|
||||
pub no_draft: bool,
|
||||
|
||||
/// Filter by target branch
|
||||
#[arg(long)]
|
||||
pub target: Option<String>,
|
||||
|
||||
/// Filter by source branch
|
||||
#[arg(long)]
|
||||
pub source: Option<String>,
|
||||
|
||||
/// Sort field (updated, created, iid)
|
||||
#[arg(long, value_parser = ["updated", "created", "iid"], default_value = "updated")]
|
||||
pub sort: String,
|
||||
|
||||
/// Sort ascending (default: descending)
|
||||
#[arg(long)]
|
||||
pub asc: bool,
|
||||
|
||||
/// Open first matching item in browser
|
||||
#[arg(short = 'o', long)]
|
||||
pub open: bool,
|
||||
}
|
||||
|
||||
/// Arguments for `lore ingest [ENTITY]`
|
||||
#[derive(Parser)]
|
||||
pub struct IngestArgs {
|
||||
/// Entity to ingest (issues, mrs). Omit to ingest everything.
|
||||
#[arg(value_parser = ["issues", "mrs"])]
|
||||
pub entity: Option<String>,
|
||||
|
||||
/// Filter to single project
|
||||
#[arg(short = 'p', long)]
|
||||
pub project: Option<String>,
|
||||
|
||||
/// Override stale sync lock
|
||||
#[arg(short = 'f', long)]
|
||||
pub force: bool,
|
||||
|
||||
/// Full re-sync: reset cursors and fetch all data from scratch
|
||||
#[arg(long)]
|
||||
pub full: bool,
|
||||
}
|
||||
|
||||
/// Arguments for `lore count <ENTITY>`
|
||||
#[derive(Parser)]
|
||||
pub struct CountArgs {
|
||||
/// Entity type to count (issues, mrs, discussions, notes)
|
||||
#[arg(value_parser = ["issues", "mrs", "discussions", "notes"])]
|
||||
pub entity: String,
|
||||
|
||||
/// Parent type filter: issue or mr (for discussions/notes)
|
||||
#[arg(short = 'f', long = "for", value_parser = ["issue", "mr"])]
|
||||
pub for_entity: Option<String>,
|
||||
}
|
||||
|
||||
@@ -150,27 +150,49 @@ impl GiError {
|
||||
Self::Io(_) => ErrorCode::IoError,
|
||||
Self::Transform(_) => ErrorCode::TransformError,
|
||||
Self::NotFound(_) => ErrorCode::GitLabNotFound,
|
||||
Self::Ambiguous(_) => ErrorCode::InternalError,
|
||||
Self::Ambiguous(_) => ErrorCode::GitLabNotFound,
|
||||
Self::Other(_) => ErrorCode::InternalError,
|
||||
}
|
||||
}
|
||||
|
||||
/// Get a suggestion for how to fix this error.
|
||||
/// Get a suggestion for how to fix this error, including inline examples.
|
||||
pub fn suggestion(&self) -> Option<&'static str> {
|
||||
match self {
|
||||
Self::ConfigNotFound { .. } => Some("Run 'lore init' to create configuration"),
|
||||
Self::ConfigInvalid { .. } => Some("Check config file syntax or run 'lore init' to recreate"),
|
||||
Self::GitLabAuthFailed => Some("Verify token has read_api scope and is not expired"),
|
||||
Self::GitLabNotFound { .. } => Some("Check the resource path exists and you have access"),
|
||||
Self::ConfigNotFound { .. } => Some(
|
||||
"Run 'lore init' to set up your GitLab connection.\n\n Expected: ~/.config/lore/config.json",
|
||||
),
|
||||
Self::ConfigInvalid { .. } => Some(
|
||||
"Check config file syntax or run 'lore init' to recreate.\n\n Example:\n lore init\n lore init --force",
|
||||
),
|
||||
Self::GitLabAuthFailed => Some(
|
||||
"Verify token has read_api scope and is not expired.\n\n Example:\n export GITLAB_TOKEN=glpat-xxxxxxxxxxxx\n lore auth",
|
||||
),
|
||||
Self::GitLabNotFound { .. } => Some(
|
||||
"Check the resource path exists and you have access.\n\n Example:\n lore issues -p group/project\n lore mrs -p group/project",
|
||||
),
|
||||
Self::GitLabRateLimited { .. } => Some("Wait and retry, or reduce request frequency"),
|
||||
Self::GitLabNetworkError { .. } => Some("Check network connection and GitLab URL"),
|
||||
Self::DatabaseLocked { .. } => Some("Wait for other sync to complete or use --force"),
|
||||
Self::MigrationFailed { .. } => Some("Check database file permissions or reset with 'lore reset'"),
|
||||
Self::TokenNotSet { .. } => Some("Export the token environment variable"),
|
||||
Self::Database(_) => Some("Check database file permissions or reset with 'lore reset'"),
|
||||
Self::GitLabNetworkError { .. } => Some(
|
||||
"Check network connection and GitLab URL.\n\n Example:\n lore doctor\n lore auth",
|
||||
),
|
||||
Self::DatabaseLocked { .. } => Some(
|
||||
"Wait for other sync to complete or use --force.\n\n Example:\n lore ingest --force\n lore ingest issues --force",
|
||||
),
|
||||
Self::MigrationFailed { .. } => Some(
|
||||
"Check database file permissions or reset with 'lore reset'.\n\n Example:\n lore migrate\n lore reset --yes",
|
||||
),
|
||||
Self::TokenNotSet { .. } => Some(
|
||||
"Export the token to your shell:\n\n export GITLAB_TOKEN=glpat-xxxxxxxxxxxx\n\n Your token needs the read_api scope.",
|
||||
),
|
||||
Self::Database(_) => Some(
|
||||
"Check database file permissions or reset with 'lore reset'.\n\n Example:\n lore doctor\n lore reset --yes",
|
||||
),
|
||||
Self::Http(_) => Some("Check network connection"),
|
||||
Self::NotFound(_) => Some("Verify the entity exists using 'lore list'"),
|
||||
Self::Ambiguous(_) => Some("Use --project flag to disambiguate"),
|
||||
Self::NotFound(_) => Some(
|
||||
"Verify the entity exists.\n\n Example:\n lore issues\n lore mrs",
|
||||
),
|
||||
Self::Ambiguous(_) => Some(
|
||||
"Use -p to choose a specific project.\n\n Example:\n lore issues 42 -p group/project-a\n lore mrs 99 -p group/project-b",
|
||||
),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -59,6 +59,22 @@ pub fn parse_since(input: &str) -> Option<i64> {
|
||||
iso_to_ms(input)
|
||||
}
|
||||
|
||||
/// Convert ISO 8601 timestamp to milliseconds with strict error handling.
|
||||
/// Returns Err with a descriptive message if the timestamp is invalid.
|
||||
pub fn iso_to_ms_strict(iso_string: &str) -> Result<i64, String> {
|
||||
DateTime::parse_from_rfc3339(iso_string)
|
||||
.map(|dt| dt.timestamp_millis())
|
||||
.map_err(|_| format!("Invalid timestamp: {}", iso_string))
|
||||
}
|
||||
|
||||
/// Convert optional ISO 8601 timestamp to optional milliseconds (strict).
|
||||
pub fn iso_to_ms_opt_strict(iso_string: &Option<String>) -> Result<Option<i64>, String> {
|
||||
match iso_string {
|
||||
Some(s) => iso_to_ms_strict(s).map(Some),
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
/// Format milliseconds epoch to human-readable full datetime.
|
||||
pub fn format_full_datetime(ms: i64) -> String {
|
||||
DateTime::from_timestamp_millis(ms)
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
//! Discussion and note transformers: convert GitLab discussions to local schema.
|
||||
|
||||
use chrono::DateTime;
|
||||
|
||||
use crate::core::time::now_ms;
|
||||
use crate::core::time::{iso_to_ms, iso_to_ms_strict, now_ms};
|
||||
use crate::gitlab::types::{GitLabDiscussion, GitLabNote};
|
||||
|
||||
/// Reference to the parent noteable (Issue or MergeRequest).
|
||||
@@ -60,16 +58,9 @@ pub struct NormalizedNote {
|
||||
pub position_head_sha: Option<String>, // Head commit SHA for diff
|
||||
}
|
||||
|
||||
/// Parse ISO 8601 timestamp to milliseconds, returning None on failure.
|
||||
fn parse_timestamp_opt(ts: &str) -> Option<i64> {
|
||||
DateTime::parse_from_rfc3339(ts)
|
||||
.ok()
|
||||
.map(|dt| dt.timestamp_millis())
|
||||
}
|
||||
|
||||
/// Parse ISO 8601 timestamp to milliseconds, defaulting to 0 on failure.
|
||||
fn parse_timestamp(ts: &str) -> i64 {
|
||||
parse_timestamp_opt(ts).unwrap_or(0)
|
||||
iso_to_ms(ts).unwrap_or(0)
|
||||
}
|
||||
|
||||
/// Transform a GitLab discussion into normalized schema.
|
||||
@@ -90,7 +81,7 @@ pub fn transform_discussion(
|
||||
let note_timestamps: Vec<i64> = gitlab_discussion
|
||||
.notes
|
||||
.iter()
|
||||
.filter_map(|n| parse_timestamp_opt(&n.created_at))
|
||||
.filter_map(|n| iso_to_ms(&n.created_at))
|
||||
.collect();
|
||||
|
||||
let first_note_at = note_timestamps.iter().min().copied();
|
||||
@@ -191,7 +182,7 @@ fn transform_single_note(
|
||||
resolved_at: note
|
||||
.resolved_at
|
||||
.as_ref()
|
||||
.and_then(|ts| parse_timestamp_opt(ts)),
|
||||
.and_then(|ts| iso_to_ms(ts)),
|
||||
position_old_path,
|
||||
position_new_path,
|
||||
position_old_line,
|
||||
@@ -244,13 +235,6 @@ fn extract_position_fields(
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse ISO 8601 timestamp to milliseconds with strict error handling.
|
||||
/// Returns Err with the invalid timestamp in the error message.
|
||||
fn parse_timestamp_strict(ts: &str) -> Result<i64, String> {
|
||||
DateTime::parse_from_rfc3339(ts)
|
||||
.map(|dt| dt.timestamp_millis())
|
||||
.map_err(|_| format!("Invalid timestamp: {}", ts))
|
||||
}
|
||||
|
||||
/// Transform notes from a GitLab discussion with strict timestamp parsing.
|
||||
/// Returns Err if any timestamp is invalid - no silent fallback to 0.
|
||||
@@ -275,10 +259,10 @@ fn transform_single_note_strict(
|
||||
now: i64,
|
||||
) -> Result<NormalizedNote, String> {
|
||||
// Parse timestamps with strict error handling
|
||||
let created_at = parse_timestamp_strict(¬e.created_at)?;
|
||||
let updated_at = parse_timestamp_strict(¬e.updated_at)?;
|
||||
let created_at = iso_to_ms_strict(¬e.created_at)?;
|
||||
let updated_at = iso_to_ms_strict(¬e.updated_at)?;
|
||||
let resolved_at = match ¬e.resolved_at {
|
||||
Some(ts) => Some(parse_timestamp_strict(ts)?),
|
||||
Some(ts) => Some(iso_to_ms_strict(ts)?),
|
||||
None => None,
|
||||
};
|
||||
|
||||
|
||||
@@ -1,33 +1,8 @@
|
||||
//! Merge request transformer: converts GitLabMergeRequest to local schema.
|
||||
|
||||
use chrono::DateTime;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use crate::core::time::{iso_to_ms_opt_strict, iso_to_ms_strict, now_ms};
|
||||
use crate::gitlab::types::GitLabMergeRequest;
|
||||
|
||||
/// Get current time in milliseconds since Unix epoch.
|
||||
fn now_ms() -> i64 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.expect("Time went backwards")
|
||||
.as_millis() as i64
|
||||
}
|
||||
|
||||
/// Parse ISO 8601 timestamp to milliseconds since Unix epoch.
|
||||
fn iso_to_ms(ts: &str) -> Result<i64, String> {
|
||||
DateTime::parse_from_rfc3339(ts)
|
||||
.map(|dt| dt.timestamp_millis())
|
||||
.map_err(|e| format!("Failed to parse timestamp '{}': {}", ts, e))
|
||||
}
|
||||
|
||||
/// Parse optional ISO 8601 timestamp to optional milliseconds since Unix epoch.
|
||||
fn iso_to_ms_opt(ts: &Option<String>) -> Result<Option<i64>, String> {
|
||||
match ts {
|
||||
Some(s) => iso_to_ms(s).map(Some),
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
/// Local schema representation of a merge request row.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct NormalizedMergeRequest {
|
||||
@@ -77,12 +52,12 @@ pub fn transform_merge_request(
|
||||
local_project_id: i64,
|
||||
) -> Result<MergeRequestWithMetadata, String> {
|
||||
// Parse required timestamps
|
||||
let created_at = iso_to_ms(&gitlab_mr.created_at)?;
|
||||
let updated_at = iso_to_ms(&gitlab_mr.updated_at)?;
|
||||
let created_at = iso_to_ms_strict(&gitlab_mr.created_at)?;
|
||||
let updated_at = iso_to_ms_strict(&gitlab_mr.updated_at)?;
|
||||
|
||||
// Parse optional timestamps
|
||||
let merged_at = iso_to_ms_opt(&gitlab_mr.merged_at)?;
|
||||
let closed_at = iso_to_ms_opt(&gitlab_mr.closed_at)?;
|
||||
let merged_at = iso_to_ms_opt_strict(&gitlab_mr.merged_at)?;
|
||||
let closed_at = iso_to_ms_opt_strict(&gitlab_mr.closed_at)?;
|
||||
|
||||
// Draft: prefer draft, fallback to work_in_progress
|
||||
let is_draft = gitlab_mr.draft || gitlab_mr.work_in_progress;
|
||||
|
||||
@@ -89,16 +89,10 @@ async fn ingest_discussions_for_issue(
|
||||
|
||||
// Track discussions we've seen for stale removal
|
||||
let mut seen_discussion_ids: Vec<String> = Vec::new();
|
||||
// Track if we've started receiving data (to distinguish empty result from failure)
|
||||
let mut received_first_response = false;
|
||||
// Track if any error occurred during pagination
|
||||
let mut pagination_error: Option<crate::core::error::GiError> = None;
|
||||
|
||||
while let Some(disc_result) = discussions_stream.next().await {
|
||||
// Mark that we've received at least one response from the API
|
||||
if !received_first_response {
|
||||
received_first_response = true;
|
||||
}
|
||||
|
||||
// Handle errors - record but don't delete stale data
|
||||
let gitlab_discussion = match disc_result {
|
||||
@@ -139,8 +133,6 @@ async fn ingest_discussions_for_issue(
|
||||
let tx = conn.unchecked_transaction()?;
|
||||
|
||||
upsert_discussion(&tx, &normalized, payload_id)?;
|
||||
result.discussions_upserted += 1;
|
||||
seen_discussion_ids.push(normalized.gitlab_discussion_id.clone());
|
||||
|
||||
// Get local discussion ID
|
||||
let local_discussion_id: i64 = tx.query_row(
|
||||
@@ -151,6 +143,7 @@ async fn ingest_discussions_for_issue(
|
||||
|
||||
// Transform and store notes
|
||||
let notes = transform_notes(&gitlab_discussion, local_project_id);
|
||||
let notes_count = notes.len();
|
||||
|
||||
// Delete existing notes for this discussion (full refresh)
|
||||
tx.execute(
|
||||
@@ -178,26 +171,19 @@ async fn ingest_discussions_for_issue(
|
||||
)?;
|
||||
|
||||
insert_note(&tx, local_discussion_id, ¬e, note_payload_id)?;
|
||||
result.notes_upserted += 1;
|
||||
}
|
||||
|
||||
tx.commit()?;
|
||||
|
||||
// Increment counters AFTER successful commit to keep metrics honest
|
||||
result.discussions_upserted += 1;
|
||||
result.notes_upserted += notes_count;
|
||||
seen_discussion_ids.push(normalized.gitlab_discussion_id.clone());
|
||||
}
|
||||
|
||||
// Only remove stale discussions if pagination completed without errors
|
||||
// AND we actually received a response (empty or not)
|
||||
if pagination_error.is_none() && received_first_response {
|
||||
let removed = remove_stale_discussions(conn, issue.local_issue_id, &seen_discussion_ids)?;
|
||||
result.stale_discussions_removed = removed;
|
||||
|
||||
// Update discussions_synced_for_updated_at on the issue
|
||||
update_issue_sync_timestamp(conn, issue.local_issue_id, issue.updated_at)?;
|
||||
} else if pagination_error.is_none()
|
||||
&& !received_first_response
|
||||
&& seen_discussion_ids.is_empty()
|
||||
{
|
||||
// Stream was empty but no error - issue genuinely has no discussions
|
||||
// This is safe to remove stale discussions (if any exist from before)
|
||||
// Only remove stale discussions and advance watermark if pagination completed
|
||||
// without errors. Safe for both empty results and populated results.
|
||||
if pagination_error.is_none() {
|
||||
let removed = remove_stale_discussions(conn, issue.local_issue_id, &seen_discussion_ids)?;
|
||||
result.stale_discussions_removed = removed;
|
||||
|
||||
@@ -208,7 +194,6 @@ async fn ingest_discussions_for_issue(
|
||||
discussions_seen = seen_discussion_ids.len(),
|
||||
"Skipping stale removal due to pagination error"
|
||||
);
|
||||
// Return the error to signal incomplete sync
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
|
||||
@@ -155,14 +155,13 @@ pub fn write_prefetched_mr_discussions(
|
||||
|
||||
// Write each discussion
|
||||
for disc in &prefetched.discussions {
|
||||
result.discussions_fetched += 1;
|
||||
|
||||
// Count DiffNotes
|
||||
result.diffnotes_count += disc
|
||||
// Count DiffNotes upfront (independent of transaction)
|
||||
let diffnotes_in_disc = disc
|
||||
.notes
|
||||
.iter()
|
||||
.filter(|n| n.position_new_path.is_some() || n.position_old_path.is_some())
|
||||
.count();
|
||||
let notes_in_disc = disc.notes.len();
|
||||
|
||||
// Start transaction
|
||||
let tx = conn.unchecked_transaction()?;
|
||||
@@ -182,7 +181,6 @@ pub fn write_prefetched_mr_discussions(
|
||||
|
||||
// Upsert discussion
|
||||
upsert_discussion(&tx, &disc.normalized, run_seen_at, payload_id)?;
|
||||
result.discussions_upserted += 1;
|
||||
|
||||
// Get local discussion ID
|
||||
let local_discussion_id: i64 = tx.query_row(
|
||||
@@ -219,10 +217,15 @@ pub fn write_prefetched_mr_discussions(
|
||||
};
|
||||
|
||||
upsert_note(&tx, local_discussion_id, note, run_seen_at, note_payload_id)?;
|
||||
result.notes_upserted += 1;
|
||||
}
|
||||
|
||||
tx.commit()?;
|
||||
|
||||
// Increment counters AFTER successful commit to keep metrics honest
|
||||
result.discussions_fetched += 1;
|
||||
result.discussions_upserted += 1;
|
||||
result.notes_upserted += notes_in_disc;
|
||||
result.diffnotes_count += diffnotes_in_disc;
|
||||
}
|
||||
|
||||
// Only sweep stale data and advance watermark on full success
|
||||
@@ -343,8 +346,6 @@ async fn ingest_discussions_for_mr(
|
||||
break;
|
||||
}
|
||||
};
|
||||
result.discussions_fetched += 1;
|
||||
|
||||
// CRITICAL: Parse notes BEFORE any destructive DB operations
|
||||
let notes = match transform_notes_with_diff_position(&gitlab_discussion, local_project_id) {
|
||||
Ok(notes) => notes,
|
||||
@@ -361,11 +362,12 @@ async fn ingest_discussions_for_mr(
|
||||
}
|
||||
};
|
||||
|
||||
// Count DiffNotes
|
||||
result.diffnotes_count += notes
|
||||
// Count DiffNotes upfront (independent of transaction)
|
||||
let diffnotes_in_disc = notes
|
||||
.iter()
|
||||
.filter(|n| n.position_new_path.is_some() || n.position_old_path.is_some())
|
||||
.count();
|
||||
let notes_count = notes.len();
|
||||
|
||||
// Transform discussion
|
||||
let normalized_discussion =
|
||||
@@ -389,7 +391,6 @@ async fn ingest_discussions_for_mr(
|
||||
|
||||
// Upsert discussion with run_seen_at
|
||||
upsert_discussion(&tx, &normalized_discussion, run_seen_at, payload_id)?;
|
||||
result.discussions_upserted += 1;
|
||||
|
||||
// Get local discussion ID
|
||||
let local_discussion_id: i64 = tx.query_row(
|
||||
@@ -433,10 +434,15 @@ async fn ingest_discussions_for_mr(
|
||||
};
|
||||
|
||||
upsert_note(&tx, local_discussion_id, note, run_seen_at, note_payload_id)?;
|
||||
result.notes_upserted += 1;
|
||||
}
|
||||
|
||||
tx.commit()?;
|
||||
|
||||
// Increment counters AFTER successful commit to keep metrics honest
|
||||
result.discussions_fetched += 1;
|
||||
result.discussions_upserted += 1;
|
||||
result.notes_upserted += notes_count;
|
||||
result.diffnotes_count += diffnotes_in_disc;
|
||||
}
|
||||
|
||||
// Only sweep stale data and advance watermark on full success
|
||||
|
||||
648
src/main.rs
648
src/main.rs
@@ -18,7 +18,7 @@ use lore::cli::commands::{
|
||||
run_doctor, run_ingest, run_init, run_list_issues, run_list_mrs, run_show_issue, run_show_mr,
|
||||
run_sync_status,
|
||||
};
|
||||
use lore::cli::{Cli, Commands};
|
||||
use lore::cli::{Cli, Commands, CountArgs, IngestArgs, IssuesArgs, MrsArgs};
|
||||
use lore::core::db::{create_connection, get_schema_version, run_migrations};
|
||||
use lore::core::error::{GiError, RobotErrorOutput};
|
||||
use lore::core::paths::get_config_path;
|
||||
@@ -47,33 +47,25 @@ async fn main() {
|
||||
let robot_mode = cli.is_robot_mode();
|
||||
|
||||
let result = match cli.command {
|
||||
Commands::Issues(args) => handle_issues(cli.config.as_deref(), args, robot_mode).await,
|
||||
Commands::Mrs(args) => handle_mrs(cli.config.as_deref(), args, robot_mode).await,
|
||||
Commands::Ingest(args) => handle_ingest(cli.config.as_deref(), args, robot_mode).await,
|
||||
Commands::Count(args) => {
|
||||
handle_count(cli.config.as_deref(), args, robot_mode).await
|
||||
}
|
||||
Commands::Status => handle_sync_status_cmd(cli.config.as_deref(), robot_mode).await,
|
||||
Commands::Auth => handle_auth_test(cli.config.as_deref(), robot_mode).await,
|
||||
Commands::Doctor => handle_doctor(cli.config.as_deref(), robot_mode).await,
|
||||
Commands::Version => handle_version(robot_mode),
|
||||
Commands::Init {
|
||||
force,
|
||||
non_interactive,
|
||||
} => handle_init(cli.config.as_deref(), force, non_interactive, robot_mode).await,
|
||||
Commands::AuthTest => handle_auth_test(cli.config.as_deref(), robot_mode).await,
|
||||
Commands::Doctor { json } => handle_doctor(cli.config.as_deref(), json || robot_mode).await,
|
||||
Commands::Version => handle_version(robot_mode),
|
||||
Commands::Backup => handle_backup(robot_mode),
|
||||
Commands::Reset { confirm: _ } => handle_reset(robot_mode),
|
||||
Commands::Reset { yes: _ } => handle_reset(robot_mode),
|
||||
Commands::Migrate => handle_migrate(cli.config.as_deref(), robot_mode).await,
|
||||
Commands::SyncStatus => handle_sync_status(cli.config.as_deref(), robot_mode).await,
|
||||
Commands::Ingest {
|
||||
r#type,
|
||||
project,
|
||||
force,
|
||||
full,
|
||||
} => {
|
||||
handle_ingest(
|
||||
cli.config.as_deref(),
|
||||
&r#type,
|
||||
project.as_deref(),
|
||||
force,
|
||||
full,
|
||||
robot_mode,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
// --- Backward-compat: deprecated aliases ---
|
||||
Commands::List {
|
||||
entity,
|
||||
limit,
|
||||
@@ -89,14 +81,17 @@ async fn main() {
|
||||
sort,
|
||||
order,
|
||||
open,
|
||||
json,
|
||||
draft,
|
||||
no_draft,
|
||||
reviewer,
|
||||
target_branch,
|
||||
source_branch,
|
||||
} => {
|
||||
handle_list(
|
||||
eprintln!(
|
||||
"{}",
|
||||
style("warning: 'lore list' is deprecated, use 'lore issues' or 'lore mrs'").yellow()
|
||||
);
|
||||
handle_list_compat(
|
||||
cli.config.as_deref(),
|
||||
&entity,
|
||||
limit,
|
||||
@@ -112,7 +107,7 @@ async fn main() {
|
||||
&sort,
|
||||
&order,
|
||||
open,
|
||||
json || robot_mode,
|
||||
robot_mode,
|
||||
draft,
|
||||
no_draft,
|
||||
reviewer.as_deref(),
|
||||
@@ -121,24 +116,42 @@ async fn main() {
|
||||
)
|
||||
.await
|
||||
}
|
||||
Commands::Count { entity, r#type } => {
|
||||
handle_count(cli.config.as_deref(), &entity, r#type.as_deref(), robot_mode).await
|
||||
}
|
||||
Commands::Show {
|
||||
entity,
|
||||
iid,
|
||||
project,
|
||||
json,
|
||||
} => {
|
||||
handle_show(
|
||||
eprintln!(
|
||||
"{}",
|
||||
style(format!(
|
||||
"warning: 'lore show' is deprecated, use 'lore {}s {}'",
|
||||
entity, iid
|
||||
))
|
||||
.yellow()
|
||||
);
|
||||
handle_show_compat(
|
||||
cli.config.as_deref(),
|
||||
&entity,
|
||||
iid,
|
||||
project.as_deref(),
|
||||
json || robot_mode,
|
||||
robot_mode,
|
||||
)
|
||||
.await
|
||||
}
|
||||
Commands::AuthTest => {
|
||||
eprintln!(
|
||||
"{}",
|
||||
style("warning: 'lore auth-test' is deprecated, use 'lore auth'").yellow()
|
||||
);
|
||||
handle_auth_test(cli.config.as_deref(), robot_mode).await
|
||||
}
|
||||
Commands::SyncStatus => {
|
||||
eprintln!(
|
||||
"{}",
|
||||
style("warning: 'lore sync-status' is deprecated, use 'lore status'").yellow()
|
||||
);
|
||||
handle_sync_status_cmd(cli.config.as_deref(), robot_mode).await
|
||||
}
|
||||
};
|
||||
|
||||
if let Err(e) = result {
|
||||
@@ -207,6 +220,259 @@ fn handle_error(e: Box<dyn std::error::Error>, robot_mode: bool) -> ! {
|
||||
std::process::exit(1);
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Primary command handlers
|
||||
// ============================================================================
|
||||
|
||||
async fn handle_issues(
|
||||
config_override: Option<&str>,
|
||||
args: IssuesArgs,
|
||||
robot_mode: bool,
|
||||
) -> Result<(), Box<dyn std::error::Error>> {
|
||||
let config = Config::load(config_override)?;
|
||||
let order = if args.asc { "asc" } else { "desc" };
|
||||
|
||||
if let Some(iid) = args.iid {
|
||||
// Show mode
|
||||
let result = run_show_issue(&config, iid, args.project.as_deref())?;
|
||||
if robot_mode {
|
||||
print_show_issue_json(&result);
|
||||
} else {
|
||||
print_show_issue(&result);
|
||||
}
|
||||
} else {
|
||||
// List mode
|
||||
let filters = ListFilters {
|
||||
limit: args.limit,
|
||||
project: args.project.as_deref(),
|
||||
state: args.state.as_deref(),
|
||||
author: args.author.as_deref(),
|
||||
assignee: args.assignee.as_deref(),
|
||||
labels: args.label.as_deref(),
|
||||
milestone: args.milestone.as_deref(),
|
||||
since: args.since.as_deref(),
|
||||
due_before: args.due_before.as_deref(),
|
||||
has_due_date: args.has_due,
|
||||
sort: &args.sort,
|
||||
order,
|
||||
};
|
||||
|
||||
let result = run_list_issues(&config, filters)?;
|
||||
|
||||
if args.open {
|
||||
open_issue_in_browser(&result);
|
||||
} else if robot_mode {
|
||||
print_list_issues_json(&result);
|
||||
} else {
|
||||
print_list_issues(&result);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn handle_mrs(
|
||||
config_override: Option<&str>,
|
||||
args: MrsArgs,
|
||||
robot_mode: bool,
|
||||
) -> Result<(), Box<dyn std::error::Error>> {
|
||||
let config = Config::load(config_override)?;
|
||||
let order = if args.asc { "asc" } else { "desc" };
|
||||
|
||||
if let Some(iid) = args.iid {
|
||||
// Show mode
|
||||
let result = run_show_mr(&config, iid, args.project.as_deref())?;
|
||||
if robot_mode {
|
||||
print_show_mr_json(&result);
|
||||
} else {
|
||||
print_show_mr(&result);
|
||||
}
|
||||
} else {
|
||||
// List mode
|
||||
let filters = MrListFilters {
|
||||
limit: args.limit,
|
||||
project: args.project.as_deref(),
|
||||
state: args.state.as_deref(),
|
||||
author: args.author.as_deref(),
|
||||
assignee: args.assignee.as_deref(),
|
||||
reviewer: args.reviewer.as_deref(),
|
||||
labels: args.label.as_deref(),
|
||||
since: args.since.as_deref(),
|
||||
draft: args.draft,
|
||||
no_draft: args.no_draft,
|
||||
target_branch: args.target.as_deref(),
|
||||
source_branch: args.source.as_deref(),
|
||||
sort: &args.sort,
|
||||
order,
|
||||
};
|
||||
|
||||
let result = run_list_mrs(&config, filters)?;
|
||||
|
||||
if args.open {
|
||||
open_mr_in_browser(&result);
|
||||
} else if robot_mode {
|
||||
print_list_mrs_json(&result);
|
||||
} else {
|
||||
print_list_mrs(&result);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn handle_ingest(
|
||||
config_override: Option<&str>,
|
||||
args: IngestArgs,
|
||||
robot_mode: bool,
|
||||
) -> Result<(), Box<dyn std::error::Error>> {
|
||||
let config = Config::load(config_override)?;
|
||||
|
||||
match args.entity.as_deref() {
|
||||
Some(resource_type) => {
|
||||
// Single entity ingest
|
||||
let result = run_ingest(
|
||||
&config,
|
||||
resource_type,
|
||||
args.project.as_deref(),
|
||||
args.force,
|
||||
args.full,
|
||||
robot_mode,
|
||||
)
|
||||
.await?;
|
||||
|
||||
if robot_mode {
|
||||
print_ingest_summary_json(&result);
|
||||
} else {
|
||||
print_ingest_summary(&result);
|
||||
}
|
||||
}
|
||||
None => {
|
||||
// Ingest everything: issues then MRs
|
||||
if !robot_mode {
|
||||
println!(
|
||||
"{}",
|
||||
style("Ingesting all content (issues + merge requests)...").blue()
|
||||
);
|
||||
println!();
|
||||
}
|
||||
|
||||
let issues_result = run_ingest(
|
||||
&config,
|
||||
"issues",
|
||||
args.project.as_deref(),
|
||||
args.force,
|
||||
args.full,
|
||||
robot_mode,
|
||||
)
|
||||
.await?;
|
||||
|
||||
let mrs_result = run_ingest(
|
||||
&config,
|
||||
"mrs",
|
||||
args.project.as_deref(),
|
||||
args.force,
|
||||
args.full,
|
||||
robot_mode,
|
||||
)
|
||||
.await?;
|
||||
|
||||
if robot_mode {
|
||||
print_combined_ingest_json(&issues_result, &mrs_result);
|
||||
} else {
|
||||
print_ingest_summary(&issues_result);
|
||||
print_ingest_summary(&mrs_result);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// JSON output for combined ingest (issues + mrs).
|
||||
#[derive(Serialize)]
|
||||
struct CombinedIngestOutput {
|
||||
ok: bool,
|
||||
data: CombinedIngestData,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct CombinedIngestData {
|
||||
resource_type: String,
|
||||
issues: CombinedIngestEntityStats,
|
||||
merge_requests: CombinedIngestEntityStats,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct CombinedIngestEntityStats {
|
||||
projects_synced: usize,
|
||||
fetched: usize,
|
||||
upserted: usize,
|
||||
labels_created: usize,
|
||||
discussions_fetched: usize,
|
||||
notes_upserted: usize,
|
||||
}
|
||||
|
||||
fn print_combined_ingest_json(
|
||||
issues: &lore::cli::commands::ingest::IngestResult,
|
||||
mrs: &lore::cli::commands::ingest::IngestResult,
|
||||
) {
|
||||
let output = CombinedIngestOutput {
|
||||
ok: true,
|
||||
data: CombinedIngestData {
|
||||
resource_type: "all".to_string(),
|
||||
issues: CombinedIngestEntityStats {
|
||||
projects_synced: issues.projects_synced,
|
||||
fetched: issues.issues_fetched,
|
||||
upserted: issues.issues_upserted,
|
||||
labels_created: issues.labels_created,
|
||||
discussions_fetched: issues.discussions_fetched,
|
||||
notes_upserted: issues.notes_upserted,
|
||||
},
|
||||
merge_requests: CombinedIngestEntityStats {
|
||||
projects_synced: mrs.projects_synced,
|
||||
fetched: mrs.mrs_fetched,
|
||||
upserted: mrs.mrs_upserted,
|
||||
labels_created: mrs.labels_created,
|
||||
discussions_fetched: mrs.discussions_fetched,
|
||||
notes_upserted: mrs.notes_upserted,
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
println!("{}", serde_json::to_string(&output).unwrap());
|
||||
}
|
||||
|
||||
async fn handle_count(
|
||||
config_override: Option<&str>,
|
||||
args: CountArgs,
|
||||
robot_mode: bool,
|
||||
) -> Result<(), Box<dyn std::error::Error>> {
|
||||
let config = Config::load(config_override)?;
|
||||
|
||||
let result = run_count(&config, &args.entity, args.for_entity.as_deref())?;
|
||||
if robot_mode {
|
||||
print_count_json(&result);
|
||||
} else {
|
||||
print_count(&result);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn handle_sync_status_cmd(
|
||||
config_override: Option<&str>,
|
||||
robot_mode: bool,
|
||||
) -> Result<(), Box<dyn std::error::Error>> {
|
||||
let config = Config::load(config_override)?;
|
||||
|
||||
let result = run_sync_status(&config)?;
|
||||
if robot_mode {
|
||||
print_sync_status_json(&result);
|
||||
} else {
|
||||
print_sync_status(&result);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn handle_init(
|
||||
config_override: Option<&str>,
|
||||
force: bool,
|
||||
@@ -389,11 +655,11 @@ async fn handle_auth_test(
|
||||
|
||||
async fn handle_doctor(
|
||||
config_override: Option<&str>,
|
||||
json: bool,
|
||||
robot_mode: bool,
|
||||
) -> Result<(), Box<dyn std::error::Error>> {
|
||||
let result = run_doctor(config_override).await;
|
||||
|
||||
if json {
|
||||
if robot_mode {
|
||||
println!("{}", serde_json::to_string_pretty(&result)?);
|
||||
} else {
|
||||
print_doctor_results(&result);
|
||||
@@ -406,191 +672,6 @@ async fn handle_doctor(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn handle_ingest(
|
||||
config_override: Option<&str>,
|
||||
resource_type: &str,
|
||||
project_filter: Option<&str>,
|
||||
force: bool,
|
||||
full: bool,
|
||||
robot_mode: bool,
|
||||
) -> Result<(), Box<dyn std::error::Error>> {
|
||||
let config = Config::load(config_override)?;
|
||||
|
||||
match run_ingest(&config, resource_type, project_filter, force, full, robot_mode).await {
|
||||
Ok(result) => {
|
||||
if robot_mode {
|
||||
print_ingest_summary_json(&result);
|
||||
} else {
|
||||
print_ingest_summary(&result);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
Err(e) => {
|
||||
eprintln!("{}", style(format!("Error: {e}")).red());
|
||||
std::process::exit(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn handle_list(
|
||||
config_override: Option<&str>,
|
||||
entity: &str,
|
||||
limit: usize,
|
||||
project_filter: Option<&str>,
|
||||
state_filter: Option<&str>,
|
||||
author_filter: Option<&str>,
|
||||
assignee_filter: Option<&str>,
|
||||
label_filter: Option<&[String]>,
|
||||
milestone_filter: Option<&str>,
|
||||
since_filter: Option<&str>,
|
||||
due_before_filter: Option<&str>,
|
||||
has_due_date: bool,
|
||||
sort: &str,
|
||||
order: &str,
|
||||
open_browser: bool,
|
||||
json_output: bool,
|
||||
draft: bool,
|
||||
no_draft: bool,
|
||||
reviewer_filter: Option<&str>,
|
||||
target_branch_filter: Option<&str>,
|
||||
source_branch_filter: Option<&str>,
|
||||
) -> Result<(), Box<dyn std::error::Error>> {
|
||||
let config = Config::load(config_override)?;
|
||||
|
||||
match entity {
|
||||
"issues" => {
|
||||
let filters = ListFilters {
|
||||
limit,
|
||||
project: project_filter,
|
||||
state: state_filter,
|
||||
author: author_filter,
|
||||
assignee: assignee_filter,
|
||||
labels: label_filter,
|
||||
milestone: milestone_filter,
|
||||
since: since_filter,
|
||||
due_before: due_before_filter,
|
||||
has_due_date,
|
||||
sort,
|
||||
order,
|
||||
};
|
||||
|
||||
let result = run_list_issues(&config, filters)?;
|
||||
|
||||
if open_browser {
|
||||
open_issue_in_browser(&result);
|
||||
} else if json_output {
|
||||
print_list_issues_json(&result);
|
||||
} else {
|
||||
print_list_issues(&result);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
"mrs" => {
|
||||
let filters = MrListFilters {
|
||||
limit,
|
||||
project: project_filter,
|
||||
state: state_filter,
|
||||
author: author_filter,
|
||||
assignee: assignee_filter,
|
||||
reviewer: reviewer_filter,
|
||||
labels: label_filter,
|
||||
since: since_filter,
|
||||
draft,
|
||||
no_draft,
|
||||
target_branch: target_branch_filter,
|
||||
source_branch: source_branch_filter,
|
||||
sort,
|
||||
order,
|
||||
};
|
||||
|
||||
let result = run_list_mrs(&config, filters)?;
|
||||
|
||||
if open_browser {
|
||||
open_mr_in_browser(&result);
|
||||
} else if json_output {
|
||||
print_list_mrs_json(&result);
|
||||
} else {
|
||||
print_list_mrs(&result);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
_ => {
|
||||
eprintln!("{}", style(format!("Unknown entity: {entity}")).red());
|
||||
std::process::exit(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn handle_count(
|
||||
config_override: Option<&str>,
|
||||
entity: &str,
|
||||
type_filter: Option<&str>,
|
||||
robot_mode: bool,
|
||||
) -> Result<(), Box<dyn std::error::Error>> {
|
||||
let config = Config::load(config_override)?;
|
||||
|
||||
let result = run_count(&config, entity, type_filter)?;
|
||||
if robot_mode {
|
||||
print_count_json(&result);
|
||||
} else {
|
||||
print_count(&result);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn handle_sync_status(
|
||||
config_override: Option<&str>,
|
||||
robot_mode: bool,
|
||||
) -> Result<(), Box<dyn std::error::Error>> {
|
||||
let config = Config::load(config_override)?;
|
||||
|
||||
let result = run_sync_status(&config)?;
|
||||
if robot_mode {
|
||||
print_sync_status_json(&result);
|
||||
} else {
|
||||
print_sync_status(&result);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn handle_show(
|
||||
config_override: Option<&str>,
|
||||
entity: &str,
|
||||
iid: i64,
|
||||
project_filter: Option<&str>,
|
||||
json: bool,
|
||||
) -> Result<(), Box<dyn std::error::Error>> {
|
||||
let config = Config::load(config_override)?;
|
||||
|
||||
match entity {
|
||||
"issue" => {
|
||||
let result = run_show_issue(&config, iid, project_filter)?;
|
||||
if json {
|
||||
print_show_issue_json(&result);
|
||||
} else {
|
||||
print_show_issue(&result);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
"mr" => {
|
||||
let result = run_show_mr(&config, iid, project_filter)?;
|
||||
if json {
|
||||
print_show_mr_json(&result);
|
||||
} else {
|
||||
print_show_mr(&result);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
_ => {
|
||||
eprintln!("{}", style(format!("Unknown entity: {entity}")).red());
|
||||
std::process::exit(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// JSON output for version command.
|
||||
#[derive(Serialize)]
|
||||
struct VersionOutput {
|
||||
@@ -758,3 +839,134 @@ async fn handle_migrate(
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Backward-compat handlers (deprecated, delegate to new handlers)
|
||||
// ============================================================================
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn handle_list_compat(
|
||||
config_override: Option<&str>,
|
||||
entity: &str,
|
||||
limit: usize,
|
||||
project_filter: Option<&str>,
|
||||
state_filter: Option<&str>,
|
||||
author_filter: Option<&str>,
|
||||
assignee_filter: Option<&str>,
|
||||
label_filter: Option<&[String]>,
|
||||
milestone_filter: Option<&str>,
|
||||
since_filter: Option<&str>,
|
||||
due_before_filter: Option<&str>,
|
||||
has_due_date: bool,
|
||||
sort: &str,
|
||||
order: &str,
|
||||
open_browser: bool,
|
||||
json_output: bool,
|
||||
draft: bool,
|
||||
no_draft: bool,
|
||||
reviewer_filter: Option<&str>,
|
||||
target_branch_filter: Option<&str>,
|
||||
source_branch_filter: Option<&str>,
|
||||
) -> Result<(), Box<dyn std::error::Error>> {
|
||||
let config = Config::load(config_override)?;
|
||||
|
||||
match entity {
|
||||
"issues" => {
|
||||
let filters = ListFilters {
|
||||
limit,
|
||||
project: project_filter,
|
||||
state: state_filter,
|
||||
author: author_filter,
|
||||
assignee: assignee_filter,
|
||||
labels: label_filter,
|
||||
milestone: milestone_filter,
|
||||
since: since_filter,
|
||||
due_before: due_before_filter,
|
||||
has_due_date,
|
||||
sort,
|
||||
order,
|
||||
};
|
||||
|
||||
let result = run_list_issues(&config, filters)?;
|
||||
|
||||
if open_browser {
|
||||
open_issue_in_browser(&result);
|
||||
} else if json_output {
|
||||
print_list_issues_json(&result);
|
||||
} else {
|
||||
print_list_issues(&result);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
"mrs" => {
|
||||
let filters = MrListFilters {
|
||||
limit,
|
||||
project: project_filter,
|
||||
state: state_filter,
|
||||
author: author_filter,
|
||||
assignee: assignee_filter,
|
||||
reviewer: reviewer_filter,
|
||||
labels: label_filter,
|
||||
since: since_filter,
|
||||
draft,
|
||||
no_draft,
|
||||
target_branch: target_branch_filter,
|
||||
source_branch: source_branch_filter,
|
||||
sort,
|
||||
order,
|
||||
};
|
||||
|
||||
let result = run_list_mrs(&config, filters)?;
|
||||
|
||||
if open_browser {
|
||||
open_mr_in_browser(&result);
|
||||
} else if json_output {
|
||||
print_list_mrs_json(&result);
|
||||
} else {
|
||||
print_list_mrs(&result);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
_ => {
|
||||
eprintln!("{}", style(format!("Unknown entity: {entity}")).red());
|
||||
std::process::exit(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn handle_show_compat(
|
||||
config_override: Option<&str>,
|
||||
entity: &str,
|
||||
iid: i64,
|
||||
project_filter: Option<&str>,
|
||||
json: bool,
|
||||
) -> Result<(), Box<dyn std::error::Error>> {
|
||||
let config = Config::load(config_override)?;
|
||||
|
||||
match entity {
|
||||
"issue" => {
|
||||
let result = run_show_issue(&config, iid, project_filter)?;
|
||||
if json {
|
||||
print_show_issue_json(&result);
|
||||
} else {
|
||||
print_show_issue(&result);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
"mr" => {
|
||||
let result = run_show_mr(&config, iid, project_filter)?;
|
||||
if json {
|
||||
print_show_mr_json(&result);
|
||||
} else {
|
||||
print_show_mr(&result);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
_ => {
|
||||
eprintln!("{}", style(format!("Unknown entity: {entity}")).red());
|
||||
std::process::exit(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user