Compare commits
6 Commits
55b895a2eb
...
f4dba386c9
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f4dba386c9 | ||
|
|
856aad1641 | ||
|
|
8fe5feda7e | ||
|
|
753ff46bb4 | ||
|
|
d3a05cfb87 | ||
|
|
390f8a9288 |
@@ -1,7 +1,5 @@
|
|||||||
# Checkpoint 3: Search & Sync MVP
|
# Checkpoint 3: Search & Sync MVP
|
||||||
|
|
||||||
> **Note:** The project was renamed from "gitlab-inbox" to "gitlore" and the CLI from "gi" to "lore". References to "gi" in this document should be read as "lore".
|
|
||||||
|
|
||||||
> **Status:** Planning
|
> **Status:** Planning
|
||||||
> **Prerequisite:** Checkpoints 0, 1, 2 complete (issues, MRs, discussions ingested)
|
> **Prerequisite:** Checkpoints 0, 1, 2 complete (issues, MRs, discussions ingested)
|
||||||
> **Goal:** Deliver working semantic + lexical hybrid search with efficient incremental sync
|
> **Goal:** Deliver working semantic + lexical hybrid search with efficient incremental sync
|
||||||
@@ -17,13 +15,27 @@ All code integrates with existing `gitlore` infrastructure:
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## Executive Summary
|
## Executive Summary (Gated Milestones)
|
||||||
|
|
||||||
|
This checkpoint ships in three gates to reduce integration risk. Each gate is independently verifiable and shippable:
|
||||||
|
|
||||||
|
**Gate A (Lexical MVP):** documents + FTS + filters + `lore search --mode=lexical` + `lore stats`
|
||||||
|
**Gate B (Hybrid MVP):** embeddings + vector + RRF fusion + graceful degradation
|
||||||
|
**Gate C (Sync MVP):** `lore sync` orchestration + queues/backoff + integrity check/repair
|
||||||
|
|
||||||
**Deliverables:**
|
**Deliverables:**
|
||||||
|
|
||||||
|
**Gate A**
|
||||||
1. Document generation from issues/MRs/discussions with FTS5 indexing
|
1. Document generation from issues/MRs/discussions with FTS5 indexing
|
||||||
2. Ollama-powered embedding pipeline with sqlite-vec storage
|
2. Lexical search + filters + snippets + `lore stats`
|
||||||
3. Hybrid search (RRF-ranked vector + lexical) with rich filtering
|
|
||||||
4. Orchestrated `gi sync` command with incremental re-embedding
|
**Gate B**
|
||||||
|
3. Ollama-powered embedding pipeline with sqlite-vec storage
|
||||||
|
4. Hybrid search (RRF-ranked vector + lexical) with rich filtering + graceful degradation
|
||||||
|
|
||||||
|
**Gate C**
|
||||||
|
5. Orchestrated `lore sync` command with incremental doc regen + re-embedding
|
||||||
|
6. Integrity checks + repair paths for FTS/embeddings consistency
|
||||||
|
|
||||||
**Key Design Decisions:**
|
**Key Design Decisions:**
|
||||||
- Documents are the search unit (not raw entities)
|
- Documents are the search unit (not raw entities)
|
||||||
@@ -144,15 +156,19 @@ CREATE VIRTUAL TABLE documents_fts USING fts5(
|
|||||||
prefix='2 3 4'
|
prefix='2 3 4'
|
||||||
);
|
);
|
||||||
|
|
||||||
-- Keep FTS in sync via triggers
|
-- Keep FTS in sync via triggers.
|
||||||
|
-- IMPORTANT: COALESCE(title, '') ensures FTS5 external-content table never
|
||||||
|
-- receives NULL values, which can cause inconsistencies with delete operations.
|
||||||
|
-- FTS5 delete requires exact match of original values; NULL != NULL in SQL,
|
||||||
|
-- so a NULL title on insert would make the delete trigger fail silently.
|
||||||
CREATE TRIGGER documents_ai AFTER INSERT ON documents BEGIN
|
CREATE TRIGGER documents_ai AFTER INSERT ON documents BEGIN
|
||||||
INSERT INTO documents_fts(rowid, title, content_text)
|
INSERT INTO documents_fts(rowid, title, content_text)
|
||||||
VALUES (new.id, new.title, new.content_text);
|
VALUES (new.id, COALESCE(new.title, ''), new.content_text);
|
||||||
END;
|
END;
|
||||||
|
|
||||||
CREATE TRIGGER documents_ad AFTER DELETE ON documents BEGIN
|
CREATE TRIGGER documents_ad AFTER DELETE ON documents BEGIN
|
||||||
INSERT INTO documents_fts(documents_fts, rowid, title, content_text)
|
INSERT INTO documents_fts(documents_fts, rowid, title, content_text)
|
||||||
VALUES('delete', old.id, old.title, old.content_text);
|
VALUES('delete', old.id, COALESCE(old.title, ''), old.content_text);
|
||||||
END;
|
END;
|
||||||
|
|
||||||
-- Only rebuild FTS when searchable text actually changes (not metadata-only updates)
|
-- Only rebuild FTS when searchable text actually changes (not metadata-only updates)
|
||||||
@@ -160,9 +176,9 @@ CREATE TRIGGER documents_au AFTER UPDATE ON documents
|
|||||||
WHEN old.title IS NOT new.title OR old.content_text != new.content_text
|
WHEN old.title IS NOT new.title OR old.content_text != new.content_text
|
||||||
BEGIN
|
BEGIN
|
||||||
INSERT INTO documents_fts(documents_fts, rowid, title, content_text)
|
INSERT INTO documents_fts(documents_fts, rowid, title, content_text)
|
||||||
VALUES('delete', old.id, old.title, old.content_text);
|
VALUES('delete', old.id, COALESCE(old.title, ''), old.content_text);
|
||||||
INSERT INTO documents_fts(rowid, title, content_text)
|
INSERT INTO documents_fts(rowid, title, content_text)
|
||||||
VALUES (new.id, new.title, new.content_text);
|
VALUES (new.id, COALESCE(new.title, ''), new.content_text);
|
||||||
END;
|
END;
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -490,7 +506,7 @@ pub struct NoteContent {
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
### 2.4 CLI: `gi generate-docs` (Incremental by Default)
|
### 2.4 CLI: `lore generate-docs` (Incremental by Default)
|
||||||
|
|
||||||
**File:** `src/cli/commands/generate_docs.rs`
|
**File:** `src/cli/commands/generate_docs.rs`
|
||||||
|
|
||||||
@@ -518,84 +534,83 @@ pub struct GenerateDocsResult {
|
|||||||
pub skipped: usize, // Unchanged documents
|
pub skipped: usize, // Unchanged documents
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Chunk size for --full mode transactions.
|
/// Chunk size for --full mode dirty queue seeding.
|
||||||
/// Balances throughput against WAL file growth and memory pressure.
|
/// Balances throughput against WAL file growth and memory pressure.
|
||||||
const FULL_MODE_CHUNK_SIZE: usize = 2000;
|
const FULL_MODE_CHUNK_SIZE: usize = 2000;
|
||||||
|
|
||||||
/// Run document generation (incremental by default).
|
/// Run document generation (incremental by default).
|
||||||
///
|
///
|
||||||
|
/// IMPORTANT: Both modes use the same regenerator codepath to avoid
|
||||||
|
/// logic divergence in label/path hashing, deletion semantics, and
|
||||||
|
/// write-optimization behavior. The only difference is how dirty_sources
|
||||||
|
/// gets populated.
|
||||||
|
///
|
||||||
/// Incremental mode (default):
|
/// Incremental mode (default):
|
||||||
/// - Processes only items in dirty_sources queue
|
/// - Processes only items already in dirty_sources queue
|
||||||
/// - Fast for routine syncs
|
/// - Fast for routine syncs
|
||||||
///
|
///
|
||||||
/// Full mode (--full):
|
/// Full mode (--full):
|
||||||
/// - Regenerates ALL documents from scratch
|
/// - Seeds dirty_sources with ALL source entities in chunks
|
||||||
/// - Uses chunked transactions (2k docs/tx) to bound WAL growth
|
/// - Drains through the same regenerator pipeline
|
||||||
|
/// - Uses keyset pagination (WHERE id > last_id) to avoid OFFSET degradation
|
||||||
|
/// - Final FTS optimize after all chunks complete
|
||||||
/// - Use when schema changes or after migration
|
/// - Use when schema changes or after migration
|
||||||
pub fn run_generate_docs(
|
pub fn run_generate_docs(
|
||||||
config: &Config,
|
config: &Config,
|
||||||
full: bool,
|
full: bool,
|
||||||
project_filter: Option<&str>,
|
project_filter: Option<&str>,
|
||||||
) -> Result<GenerateDocsResult> {
|
) -> Result<GenerateDocsResult> {
|
||||||
|
let conn = open_db(config)?;
|
||||||
|
|
||||||
if full {
|
if full {
|
||||||
// Full mode: regenerate everything using chunked transactions
|
// Full mode: seed dirty_sources with all source entities, then drain.
|
||||||
|
// Uses keyset pagination to avoid O(n²) OFFSET degradation on large tables.
|
||||||
//
|
//
|
||||||
// Using chunked transactions instead of a single giant transaction:
|
// Seeding is chunked to bound WAL growth:
|
||||||
// - Bounds WAL file growth (single 50k-doc tx could balloon WAL)
|
// 1. For each source type (issues, MRs, discussions):
|
||||||
// - Reduces memory pressure from statement caches
|
// a. Query next chunk WHERE id > last_id ORDER BY id LIMIT chunk_size
|
||||||
// - Allows progress reporting between chunks
|
// b. INSERT OR IGNORE each into dirty_sources
|
||||||
// - Crash partway through leaves partial but consistent state
|
// c. Advance last_id = chunk.last().id
|
||||||
|
// d. Loop until chunk is empty
|
||||||
|
// 2. Drain dirty_sources through regenerator (same as incremental)
|
||||||
|
// 3. Final FTS optimize (not full rebuild — triggers handle consistency)
|
||||||
//
|
//
|
||||||
// Steps per chunk:
|
// Benefits of unified codepath:
|
||||||
// 1. BEGIN IMMEDIATE transaction
|
// - No divergence in label/path hash behavior
|
||||||
// 2. Query next batch of sources (issues/MRs/discussions)
|
// - No divergence in deletion semantics
|
||||||
// 3. For each: generate document, compute hash
|
// - No divergence in write-optimization logic (labels_hash, paths_hash)
|
||||||
// 4. Upsert into `documents` table (FTS triggers auto-fire)
|
// - FTS triggers fire identically in both modes
|
||||||
// 5. Populate `document_labels` and `document_paths`
|
|
||||||
// 6. COMMIT
|
|
||||||
// 7. Report progress, loop to next chunk
|
|
||||||
//
|
|
||||||
// After all chunks:
|
|
||||||
// 8. Single final transaction for FTS rebuild:
|
|
||||||
// INSERT INTO documents_fts(documents_fts) VALUES('rebuild')
|
|
||||||
//
|
|
||||||
// Example implementation:
|
|
||||||
let conn = open_db(config)?;
|
|
||||||
let mut result = GenerateDocsResult::default();
|
|
||||||
let mut offset = 0;
|
|
||||||
|
|
||||||
|
// Seed issues
|
||||||
|
let mut last_id: i64 = 0;
|
||||||
loop {
|
loop {
|
||||||
// Process issues in chunks
|
let chunk = query_issue_ids_after(&conn, project_filter, FULL_MODE_CHUNK_SIZE, last_id)?;
|
||||||
let issues: Vec<Issue> = query_issues(&conn, project_filter, FULL_MODE_CHUNK_SIZE, offset)?;
|
if chunk.is_empty() { break; }
|
||||||
if issues.is_empty() { break; }
|
|
||||||
|
|
||||||
let tx = conn.transaction()?;
|
let tx = conn.transaction()?;
|
||||||
for issue in &issues {
|
for id in &chunk {
|
||||||
let doc = generate_issue_document(issue)?;
|
mark_dirty(&tx, SourceType::Issue, *id)?;
|
||||||
upsert_document(&tx, &doc)?;
|
|
||||||
result.issues += 1;
|
|
||||||
}
|
}
|
||||||
tx.commit()?;
|
tx.commit()?;
|
||||||
|
last_id = *chunk.last().unwrap();
|
||||||
offset += issues.len();
|
|
||||||
// Report progress here if using indicatif
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Similar chunked loops for MRs and discussions...
|
// Similar keyset-paginated seeding for MRs and discussions...
|
||||||
|
|
||||||
// Final FTS rebuild in its own transaction
|
// Report: seeding complete, now regenerating
|
||||||
let tx = conn.transaction()?;
|
}
|
||||||
tx.execute(
|
|
||||||
"INSERT INTO documents_fts(documents_fts) VALUES('rebuild')",
|
// Both modes: drain dirty_sources through the regenerator
|
||||||
|
let regen = regenerate_dirty_documents(&conn)?;
|
||||||
|
|
||||||
|
if full {
|
||||||
|
// FTS optimize after bulk operations (compacts index segments)
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO documents_fts(documents_fts) VALUES('optimize')",
|
||||||
[],
|
[],
|
||||||
)?;
|
)?;
|
||||||
tx.commit()?;
|
|
||||||
} else {
|
|
||||||
// Incremental mode: process dirty_sources only
|
|
||||||
// 1. Query dirty_sources (bounded by LIMIT)
|
|
||||||
// 2. Regenerate only those documents
|
|
||||||
// 3. Clear from dirty_sources after processing
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Map regen -> GenerateDocsResult stats
|
||||||
todo!()
|
todo!()
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -849,7 +864,7 @@ pub fn search_fts(
|
|||||||
)?;
|
)?;
|
||||||
|
|
||||||
let results = stmt
|
let results = stmt
|
||||||
.query_map([&safe_query, &limit.to_string()], |row| {
|
.query_map(rusqlite::params![safe_query, limit as i64], |row| {
|
||||||
Ok(FtsResult {
|
Ok(FtsResult {
|
||||||
document_id: row.get(0)?,
|
document_id: row.get(0)?,
|
||||||
rank: row.get(1)?,
|
rank: row.get(1)?,
|
||||||
@@ -897,10 +912,11 @@ pub struct SearchFilters {
|
|||||||
pub source_type: Option<SourceType>,
|
pub source_type: Option<SourceType>,
|
||||||
pub author: Option<String>,
|
pub author: Option<String>,
|
||||||
pub project_id: Option<i64>,
|
pub project_id: Option<i64>,
|
||||||
pub after: Option<i64>, // ms epoch
|
pub after: Option<i64>, // ms epoch (created_at >=)
|
||||||
pub labels: Vec<String>, // AND logic
|
pub updated_after: Option<i64>, // ms epoch (updated_at >=)
|
||||||
|
pub labels: Vec<String>, // AND logic
|
||||||
pub path: Option<PathFilter>,
|
pub path: Option<PathFilter>,
|
||||||
pub limit: usize, // Default 20, max 100
|
pub limit: usize, // Default 20, max 100
|
||||||
}
|
}
|
||||||
|
|
||||||
impl SearchFilters {
|
impl SearchFilters {
|
||||||
@@ -910,6 +926,7 @@ impl SearchFilters {
|
|||||||
|| self.author.is_some()
|
|| self.author.is_some()
|
||||||
|| self.project_id.is_some()
|
|| self.project_id.is_some()
|
||||||
|| self.after.is_some()
|
|| self.after.is_some()
|
||||||
|
|| self.updated_after.is_some()
|
||||||
|| !self.labels.is_empty()
|
|| !self.labels.is_empty()
|
||||||
|| self.path.is_some()
|
|| self.path.is_some()
|
||||||
}
|
}
|
||||||
@@ -990,6 +1007,11 @@ pub fn apply_filters(
|
|||||||
params.push(Box::new(after));
|
params.push(Box::new(after));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if let Some(updated_after) = filters.updated_after {
|
||||||
|
conditions.push("d.updated_at >= ?".into());
|
||||||
|
params.push(Box::new(updated_after));
|
||||||
|
}
|
||||||
|
|
||||||
// Labels: AND logic - all labels must be present
|
// Labels: AND logic - all labels must be present
|
||||||
for label in &filters.labels {
|
for label in &filters.labels {
|
||||||
conditions.push(
|
conditions.push(
|
||||||
@@ -1064,6 +1086,7 @@ pub fn apply_filters(
|
|||||||
| `--author` | `author_username` | Exact match |
|
| `--author` | `author_username` | Exact match |
|
||||||
| `--project` | `project_id` | Resolve path to ID |
|
| `--project` | `project_id` | Resolve path to ID |
|
||||||
| `--after` | `created_at` | `>= date` (ms epoch) |
|
| `--after` | `created_at` | `>= date` (ms epoch) |
|
||||||
|
| `--updated-after` | `updated_at` | `>= date` (ms epoch), common triage filter |
|
||||||
| `--label` | `document_labels` | JOIN, multiple = AND |
|
| `--label` | `document_labels` | JOIN, multiple = AND |
|
||||||
| `--path` | `document_paths` | JOIN, trailing `/` = prefix |
|
| `--path` | `document_paths` | JOIN, trailing `/` = prefix |
|
||||||
| `--limit` | N/A | Default 20, max 100 |
|
| `--limit` | N/A | Default 20, max 100 |
|
||||||
@@ -1072,6 +1095,7 @@ pub fn apply_filters(
|
|||||||
- [ ] Each filter correctly restricts results
|
- [ ] Each filter correctly restricts results
|
||||||
- [ ] Multiple `--label` flags use AND logic
|
- [ ] Multiple `--label` flags use AND logic
|
||||||
- [ ] Path prefix vs exact match works correctly
|
- [ ] Path prefix vs exact match works correctly
|
||||||
|
- [ ] `--updated-after` filters on updated_at (not created_at)
|
||||||
- [ ] Filters compose (all applied together)
|
- [ ] Filters compose (all applied together)
|
||||||
- [ ] Ranking order preserved after filtering (ORDER BY position)
|
- [ ] Ranking order preserved after filtering (ORDER BY position)
|
||||||
- [ ] Limit clamped to valid range [1, 100]
|
- [ ] Limit clamped to valid range [1, 100]
|
||||||
@@ -1080,7 +1104,7 @@ pub fn apply_filters(
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
### 3.4 CLI: `gi search --mode=lexical`
|
### 3.4 CLI: `lore search --mode=lexical`
|
||||||
|
|
||||||
**File:** `src/cli/commands/search.rs`
|
**File:** `src/cli/commands/search.rs`
|
||||||
|
|
||||||
@@ -1141,9 +1165,49 @@ pub fn run_search(
|
|||||||
explain: bool,
|
explain: bool,
|
||||||
) -> Result<SearchResponse> {
|
) -> Result<SearchResponse> {
|
||||||
// 1. Parse query and filters
|
// 1. Parse query and filters
|
||||||
// 2. Execute search based on mode
|
// 2. Execute search based on mode -> ranked doc_ids (+ explain ranks)
|
||||||
// 3. Apply post-retrieval filters
|
// 3. Apply post-retrieval filters preserving ranking order
|
||||||
// 4. Format and return results
|
// 4. HYDRATE in one DB round-trip (see hydration query below):
|
||||||
|
// - documents fields (title, url, created_at, updated_at, content_text)
|
||||||
|
// - project_path via JOIN projects
|
||||||
|
// - labels aggregated via json_group_array
|
||||||
|
// - paths aggregated via json_group_array (optional)
|
||||||
|
// 5. Attach snippet:
|
||||||
|
// - prefer FTS snippet when doc hit FTS
|
||||||
|
// - fallback: truncated content_text via generate_fallback_snippet()
|
||||||
|
// 6. For --mode=semantic with 0% embedding coverage:
|
||||||
|
// return early with actionable error message (distinct from "Ollama down")
|
||||||
|
todo!()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Hydration query: fetch all display fields for ranked doc IDs in a single round-trip.
|
||||||
|
///
|
||||||
|
/// Uses json_each(?) to preserve ranking order from the search pipeline.
|
||||||
|
/// Aggregates labels and paths inline to avoid N+1 queries.
|
||||||
|
///
|
||||||
|
/// ```sql
|
||||||
|
/// SELECT d.id, d.source_type, d.title, d.url, d.author_username,
|
||||||
|
/// d.created_at, d.updated_at, d.content_text,
|
||||||
|
/// p.path AS project_path,
|
||||||
|
/// (SELECT json_group_array(dl.label_name)
|
||||||
|
/// FROM document_labels dl WHERE dl.document_id = d.id) AS labels,
|
||||||
|
/// (SELECT json_group_array(dp.path)
|
||||||
|
/// FROM document_paths dp WHERE dp.document_id = d.id) AS paths
|
||||||
|
/// FROM json_each(?) AS j
|
||||||
|
/// JOIN documents d ON d.id = j.value
|
||||||
|
/// JOIN projects p ON p.id = d.project_id
|
||||||
|
/// ORDER BY j.key
|
||||||
|
/// ```
|
||||||
|
///
|
||||||
|
/// This single query replaces what would otherwise be:
|
||||||
|
/// - 1 query per document for metadata
|
||||||
|
/// - 1 query per document for labels
|
||||||
|
/// - 1 query per document for paths
|
||||||
|
/// For 20 results, that's 60 queries reduced to 1.
|
||||||
|
fn hydrate_results(
|
||||||
|
conn: &Connection,
|
||||||
|
doc_ids: &[i64],
|
||||||
|
) -> Result<Vec<HydratedDocument>> {
|
||||||
todo!()
|
todo!()
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1240,6 +1304,10 @@ pub struct SearchArgs {
|
|||||||
#[arg(long)]
|
#[arg(long)]
|
||||||
after: Option<String>,
|
after: Option<String>,
|
||||||
|
|
||||||
|
/// Filter by updated date (recently active items)
|
||||||
|
#[arg(long)]
|
||||||
|
updated_after: Option<String>,
|
||||||
|
|
||||||
/// Filter by label (can specify multiple)
|
/// Filter by label (can specify multiple)
|
||||||
#[arg(long, action = clap::ArgAction::Append)]
|
#[arg(long, action = clap::ArgAction::Append)]
|
||||||
label: Vec<String>,
|
label: Vec<String>,
|
||||||
@@ -1266,12 +1334,15 @@ pub struct SearchArgs {
|
|||||||
|
|
||||||
**Acceptance Criteria:**
|
**Acceptance Criteria:**
|
||||||
- [ ] Works without Ollama running
|
- [ ] Works without Ollama running
|
||||||
- [ ] All filters functional
|
- [ ] All filters functional (including `--updated-after`)
|
||||||
- [ ] Human-readable output with snippets
|
- [ ] Human-readable output with snippets
|
||||||
- [ ] Semantic-only results get fallback snippets from content_text
|
- [ ] Semantic-only results get fallback snippets from content_text
|
||||||
|
- [ ] Results hydrated in single DB round-trip (no N+1 queries)
|
||||||
- [ ] JSON output matches schema
|
- [ ] JSON output matches schema
|
||||||
- [ ] Empty results show helpful message
|
- [ ] Empty results show helpful message
|
||||||
- [ ] "No data indexed" message if documents table empty
|
- [ ] "No data indexed" message if documents table empty
|
||||||
|
- [ ] `--mode=semantic` with 0% embedding coverage returns actionable error
|
||||||
|
(distinct from "Ollama unavailable" — tells user to run `lore embed` first)
|
||||||
- [ ] `--fts-mode=safe` (default) preserves prefix `*` while escaping special chars
|
- [ ] `--fts-mode=safe` (default) preserves prefix `*` while escaping special chars
|
||||||
- [ ] `--fts-mode=raw` passes FTS5 MATCH syntax through unchanged
|
- [ ] `--fts-mode=raw` passes FTS5 MATCH syntax through unchanged
|
||||||
|
|
||||||
@@ -1535,7 +1606,7 @@ impl GiError {
|
|||||||
// ... existing mappings ...
|
// ... existing mappings ...
|
||||||
Self::OllamaUnavailable { .. } => Some("Start Ollama: ollama serve"),
|
Self::OllamaUnavailable { .. } => Some("Start Ollama: ollama serve"),
|
||||||
Self::OllamaModelNotFound { model } => Some("Pull the model: ollama pull nomic-embed-text"),
|
Self::OllamaModelNotFound { model } => Some("Pull the model: ollama pull nomic-embed-text"),
|
||||||
Self::EmbeddingFailed { .. } => Some("Check Ollama logs or retry with 'gi embed --retry-failed'"),
|
Self::EmbeddingFailed { .. } => Some("Check Ollama logs or retry with 'lore embed --retry-failed'"),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1558,6 +1629,7 @@ use crate::embedding::OllamaClient;
|
|||||||
const BATCH_SIZE: usize = 32;
|
const BATCH_SIZE: usize = 32;
|
||||||
|
|
||||||
/// SQLite page size for paging through pending documents.
|
/// SQLite page size for paging through pending documents.
|
||||||
|
/// Uses keyset paging (id > last_id) to avoid rescanning previously-processed rows.
|
||||||
const DB_PAGE_SIZE: usize = 500;
|
const DB_PAGE_SIZE: usize = 500;
|
||||||
|
|
||||||
/// Expected embedding dimensions for nomic-embed-text model.
|
/// Expected embedding dimensions for nomic-embed-text model.
|
||||||
@@ -1584,11 +1656,16 @@ pub struct EmbedResult {
|
|||||||
/// Embed documents that need embedding.
|
/// Embed documents that need embedding.
|
||||||
///
|
///
|
||||||
/// Process:
|
/// Process:
|
||||||
/// 1. Query dirty_sources ordered by queued_at
|
/// 1. Select documents needing embeddings:
|
||||||
/// 2. For each: regenerate document, compute new hash
|
/// - Pending: missing embedding_metadata row OR content_hash mismatch
|
||||||
/// 3. ALWAYS upsert document (labels/paths may change even if content_hash unchanged)
|
/// - RetryFailed: embedding_metadata.last_error IS NOT NULL
|
||||||
/// 4. Track whether content_hash changed (for stats)
|
/// 2. Page through candidates using keyset pagination (id > last_id)
|
||||||
/// 5. Delete from dirty_sources (or record error on failure)
|
/// to avoid rescanning already-processed rows
|
||||||
|
/// 3. Batch texts -> Ollama `/api/embed` with concurrent HTTP requests
|
||||||
|
/// 4. Write embeddings + embedding_metadata in per-batch transactions
|
||||||
|
/// 5. Failed batches record `last_error` in embedding_metadata
|
||||||
|
/// (excluded from Pending selection; retried via RetryFailed)
|
||||||
|
/// 6. Progress reported as (embedded + failed) vs total_pending
|
||||||
pub async fn embed_documents(
|
pub async fn embed_documents(
|
||||||
conn: &Connection,
|
conn: &Connection,
|
||||||
client: &OllamaClient,
|
client: &OllamaClient,
|
||||||
@@ -1605,9 +1682,11 @@ pub async fn embed_documents(
|
|||||||
return Ok(result);
|
return Ok(result);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Page through pending documents to avoid loading all into memory
|
// Page through pending documents using keyset pagination to avoid
|
||||||
|
// both memory pressure and OFFSET performance degradation.
|
||||||
|
let mut last_id: i64 = 0;
|
||||||
loop {
|
loop {
|
||||||
let pending = find_pending_documents(conn, DB_PAGE_SIZE, selection)?;
|
let pending = find_pending_documents(conn, DB_PAGE_SIZE, last_id, selection)?;
|
||||||
if pending.is_empty() {
|
if pending.is_empty() {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
@@ -1640,6 +1719,11 @@ pub async fn embed_documents(
|
|||||||
collect_writes(conn, &meta, res, &mut result)?;
|
collect_writes(conn, &meta, res, &mut result)?;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Advance keyset cursor for next page
|
||||||
|
if let Some(last) = pending.last() {
|
||||||
|
last_id = last.id;
|
||||||
|
}
|
||||||
|
|
||||||
if let Some(ref cb) = progress_callback {
|
if let Some(ref cb) = progress_callback {
|
||||||
cb(result.embedded + result.failed, total_pending);
|
cb(result.embedded + result.failed, total_pending);
|
||||||
}
|
}
|
||||||
@@ -1718,14 +1802,16 @@ fn count_pending_documents(conn: &Connection, selection: EmbedSelection) -> Resu
|
|||||||
Ok(count)
|
Ok(count)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Find pending documents for embedding.
|
/// Find pending documents for embedding using keyset pagination.
|
||||||
///
|
///
|
||||||
/// IMPORTANT: Uses deterministic ORDER BY d.id to ensure consistent
|
/// IMPORTANT: Uses keyset pagination (d.id > last_id) instead of OFFSET.
|
||||||
/// paging behavior. Without ordering, SQLite may return rows in
|
/// OFFSET degrades O(n²) on large result sets because SQLite must scan
|
||||||
/// different orders across calls, causing missed or duplicate documents.
|
/// and discard all rows before the offset. Keyset pagination is O(1) per page
|
||||||
|
/// since the index seek goes directly to the starting row.
|
||||||
fn find_pending_documents(
|
fn find_pending_documents(
|
||||||
conn: &Connection,
|
conn: &Connection,
|
||||||
limit: usize,
|
limit: usize,
|
||||||
|
last_id: i64,
|
||||||
selection: EmbedSelection,
|
selection: EmbedSelection,
|
||||||
) -> Result<Vec<PendingDocument>> {
|
) -> Result<Vec<PendingDocument>> {
|
||||||
let sql = match selection {
|
let sql = match selection {
|
||||||
@@ -1733,8 +1819,9 @@ fn find_pending_documents(
|
|||||||
"SELECT d.id, d.content_text, d.content_hash
|
"SELECT d.id, d.content_text, d.content_hash
|
||||||
FROM documents d
|
FROM documents d
|
||||||
LEFT JOIN embedding_metadata em ON d.id = em.document_id
|
LEFT JOIN embedding_metadata em ON d.id = em.document_id
|
||||||
WHERE em.document_id IS NULL
|
WHERE (em.document_id IS NULL
|
||||||
OR em.content_hash != d.content_hash
|
OR em.content_hash != d.content_hash)
|
||||||
|
AND d.id > ?
|
||||||
ORDER BY d.id
|
ORDER BY d.id
|
||||||
LIMIT ?",
|
LIMIT ?",
|
||||||
EmbedSelection::RetryFailed =>
|
EmbedSelection::RetryFailed =>
|
||||||
@@ -1742,13 +1829,14 @@ fn find_pending_documents(
|
|||||||
FROM documents d
|
FROM documents d
|
||||||
JOIN embedding_metadata em ON d.id = em.document_id
|
JOIN embedding_metadata em ON d.id = em.document_id
|
||||||
WHERE em.last_error IS NOT NULL
|
WHERE em.last_error IS NOT NULL
|
||||||
|
AND d.id > ?
|
||||||
ORDER BY d.id
|
ORDER BY d.id
|
||||||
LIMIT ?",
|
LIMIT ?",
|
||||||
};
|
};
|
||||||
let mut stmt = conn.prepare(sql)?;
|
let mut stmt = conn.prepare(sql)?;
|
||||||
|
|
||||||
let docs = stmt
|
let docs = stmt
|
||||||
.query_map([limit], |row| {
|
.query_map(rusqlite::params![last_id, limit as i64], |row| {
|
||||||
Ok(PendingDocument {
|
Ok(PendingDocument {
|
||||||
id: row.get(0)?,
|
id: row.get(0)?,
|
||||||
content: row.get(1)?,
|
content: row.get(1)?,
|
||||||
@@ -1827,7 +1915,7 @@ fn record_embedding_error(
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
### 4.5 CLI: `gi embed`
|
### 4.5 CLI: `lore embed`
|
||||||
|
|
||||||
**File:** `src/cli/commands/embed.rs`
|
**File:** `src/cli/commands/embed.rs`
|
||||||
|
|
||||||
@@ -1928,7 +2016,7 @@ pub struct EmbedArgs {
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
### 4.6 CLI: `gi stats`
|
### 4.6 CLI: `lore stats`
|
||||||
|
|
||||||
**File:** `src/cli/commands/stats.rs`
|
**File:** `src/cli/commands/stats.rs`
|
||||||
|
|
||||||
@@ -2034,7 +2122,13 @@ pub struct RepairResult {
|
|||||||
/// Fixes:
|
/// Fixes:
|
||||||
/// - Deletes orphaned embeddings (embedding_metadata rows with no matching document)
|
/// - Deletes orphaned embeddings (embedding_metadata rows with no matching document)
|
||||||
/// - Clears stale embedding_metadata (hash mismatch) so they get re-embedded
|
/// - Clears stale embedding_metadata (hash mismatch) so they get re-embedded
|
||||||
/// - Repopulates FTS for documents missing from documents_fts
|
/// - Rebuilds FTS index from scratch (correct-by-construction)
|
||||||
|
///
|
||||||
|
/// NOTE: FTS repair uses `rebuild` rather than partial row insertion.
|
||||||
|
/// With `content='documents'` (external-content FTS), partial repopulation
|
||||||
|
/// via INSERT of missing rows is fragile — if the external content table
|
||||||
|
/// and FTS content diverge in any way, partial fixes can leave the index
|
||||||
|
/// in an inconsistent state. A full rebuild is slower but guaranteed correct.
|
||||||
pub fn run_repair(config: &Config) -> Result<RepairResult> {
|
pub fn run_repair(config: &Config) -> Result<RepairResult> {
|
||||||
let conn = open_db(config)?;
|
let conn = open_db(config)?;
|
||||||
|
|
||||||
@@ -2061,19 +2155,19 @@ pub fn run_repair(config: &Config) -> Result<RepairResult> {
|
|||||||
[],
|
[],
|
||||||
)?;
|
)?;
|
||||||
|
|
||||||
// Repopulate FTS for missing documents
|
// Rebuild FTS index from scratch — correct-by-construction.
|
||||||
let fts_repopulated = conn.execute(
|
// This re-reads all rows from the external content table (documents)
|
||||||
"INSERT INTO documents_fts(rowid, title, content_text)
|
// and rebuilds the index. Slower than partial fix but guaranteed consistent.
|
||||||
SELECT id, COALESCE(title, ''), content_text
|
conn.execute(
|
||||||
FROM documents
|
"INSERT INTO documents_fts(documents_fts) VALUES('rebuild')",
|
||||||
WHERE id NOT IN (SELECT rowid FROM documents_fts)",
|
|
||||||
[],
|
[],
|
||||||
)?;
|
)?;
|
||||||
|
let fts_rebuilt = 1; // rebuild is all-or-nothing
|
||||||
|
|
||||||
Ok(RepairResult {
|
Ok(RepairResult {
|
||||||
orphaned_embeddings_deleted: orphaned_deleted,
|
orphaned_embeddings_deleted: orphaned_deleted,
|
||||||
stale_embeddings_cleared: stale_cleared,
|
stale_embeddings_cleared: stale_cleared,
|
||||||
missing_fts_repopulated: fts_repopulated,
|
missing_fts_repopulated: fts_rebuilt,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2772,7 +2866,7 @@ pub fn record_fetch_error(
|
|||||||
current_attempt: i64,
|
current_attempt: i64,
|
||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
let now = now_ms();
|
let now = now_ms();
|
||||||
let next_attempt = compute_next_attempt_at(now, current_attempt + 1);
|
let next_attempt = crate::core::backoff::compute_next_attempt_at(now, current_attempt + 1);
|
||||||
|
|
||||||
conn.execute(
|
conn.execute(
|
||||||
"UPDATE pending_discussion_fetches
|
"UPDATE pending_discussion_fetches
|
||||||
@@ -2786,14 +2880,44 @@ pub fn record_fetch_error(
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// NOTE: Backoff computation uses the shared utility in `src/core/backoff.rs`.
|
||||||
|
// See Phase 6.X below for the shared implementation.
|
||||||
|
```
|
||||||
|
|
||||||
|
**Acceptance Criteria:**
|
||||||
|
- [ ] Updated entities queued for discussion fetch
|
||||||
|
- [ ] Success removes from queue
|
||||||
|
- [ ] Failure increments attempt_count and sets next_attempt_at
|
||||||
|
- [ ] Processing bounded per run (max 100)
|
||||||
|
- [ ] Exponential backoff uses `next_attempt_at` (index-friendly, no overflow)
|
||||||
|
- [ ] Backoff computed with jitter to prevent thundering herd
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 6.X Shared Backoff Utility
|
||||||
|
|
||||||
|
**File:** `src/core/backoff.rs`
|
||||||
|
|
||||||
|
Single implementation of exponential backoff with jitter, used by both
|
||||||
|
`dirty_sources` and `pending_discussion_fetches` queues. Living in `src/core/`
|
||||||
|
because it's a cross-cutting concern used by multiple modules.
|
||||||
|
|
||||||
|
```rust
|
||||||
|
use rand::Rng;
|
||||||
|
|
||||||
/// Compute next_attempt_at with exponential backoff and jitter.
|
/// Compute next_attempt_at with exponential backoff and jitter.
|
||||||
///
|
///
|
||||||
/// Formula: now + min(3600000, 1000 * 2^attempt_count) * (0.9 to 1.1)
|
/// Formula: now + min(3600000, 1000 * 2^attempt_count) * (0.9 to 1.1)
|
||||||
/// - Capped at 1 hour to prevent runaway delays
|
/// - Capped at 1 hour to prevent runaway delays
|
||||||
/// - ±10% jitter prevents synchronized retries after outages
|
/// - ±10% jitter prevents synchronized retries after outages
|
||||||
|
///
|
||||||
|
/// Used by:
|
||||||
|
/// - `dirty_sources` retry scheduling (document regeneration failures)
|
||||||
|
/// - `pending_discussion_fetches` retry scheduling (API fetch failures)
|
||||||
|
///
|
||||||
|
/// Having one implementation prevents subtle divergence between queues
|
||||||
|
/// (e.g., different caps or jitter ranges).
|
||||||
pub fn compute_next_attempt_at(now: i64, attempt_count: i64) -> i64 {
|
pub fn compute_next_attempt_at(now: i64, attempt_count: i64) -> i64 {
|
||||||
use rand::Rng;
|
|
||||||
|
|
||||||
// Cap attempt_count to prevent overflow (2^30 > 1 hour anyway)
|
// Cap attempt_count to prevent overflow (2^30 > 1 hour anyway)
|
||||||
let capped_attempts = attempt_count.min(30) as u32;
|
let capped_attempts = attempt_count.min(30) as u32;
|
||||||
let base_delay_ms = 1000_i64.saturating_mul(1 << capped_attempts);
|
let base_delay_ms = 1000_i64.saturating_mul(1 << capped_attempts);
|
||||||
@@ -2807,13 +2931,16 @@ pub fn compute_next_attempt_at(now: i64, attempt_count: i64) -> i64 {
|
|||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
|
**Update `src/core/mod.rs`:**
|
||||||
|
```rust
|
||||||
|
pub mod backoff; // Add to existing modules
|
||||||
|
```
|
||||||
|
|
||||||
**Acceptance Criteria:**
|
**Acceptance Criteria:**
|
||||||
- [ ] Updated entities queued for discussion fetch
|
- [ ] Single implementation shared by both queue retry paths
|
||||||
- [ ] Success removes from queue
|
- [ ] Cap at 1 hour prevents runaway delays
|
||||||
- [ ] Failure increments attempt_count and sets next_attempt_at
|
- [ ] Jitter prevents thundering herd after outage recovery
|
||||||
- [ ] Processing bounded per run (max 100)
|
- [ ] Unit tests verify backoff curve and cap behavior
|
||||||
- [ ] Exponential backoff uses `next_attempt_at` (index-friendly, no overflow)
|
|
||||||
- [ ] Backoff computed with jitter to prevent thundering herd
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -2917,19 +3044,36 @@ fn delete_document(
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Record a regeneration error on a dirty source for retry.
|
/// Record a regeneration error on a dirty source for retry.
|
||||||
|
///
|
||||||
|
/// IMPORTANT: Sets `next_attempt_at` using the shared backoff utility.
|
||||||
|
/// Without this, failed items would retry every run (hot-loop), defeating
|
||||||
|
/// the backoff design documented in the schema.
|
||||||
fn record_dirty_error(
|
fn record_dirty_error(
|
||||||
conn: &Connection,
|
conn: &Connection,
|
||||||
source_type: SourceType,
|
source_type: SourceType,
|
||||||
source_id: i64,
|
source_id: i64,
|
||||||
error: &str,
|
error: &str,
|
||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
|
let now = now_ms();
|
||||||
|
|
||||||
|
// Read current attempt_count from DB to compute backoff
|
||||||
|
let attempt_count: i64 = conn.query_row(
|
||||||
|
"SELECT attempt_count FROM dirty_sources WHERE source_type = ? AND source_id = ?",
|
||||||
|
rusqlite::params![source_type.as_str(), source_id],
|
||||||
|
|row| row.get(0),
|
||||||
|
)?;
|
||||||
|
|
||||||
|
// Use shared backoff utility (same as pending_discussion_fetches)
|
||||||
|
let next_attempt_at = crate::core::backoff::compute_next_attempt_at(now, attempt_count + 1);
|
||||||
|
|
||||||
conn.execute(
|
conn.execute(
|
||||||
"UPDATE dirty_sources
|
"UPDATE dirty_sources
|
||||||
SET attempt_count = attempt_count + 1,
|
SET attempt_count = attempt_count + 1,
|
||||||
last_attempt_at = ?,
|
last_attempt_at = ?,
|
||||||
last_error = ?
|
last_error = ?,
|
||||||
|
next_attempt_at = ?
|
||||||
WHERE source_type = ? AND source_id = ?",
|
WHERE source_type = ? AND source_id = ?",
|
||||||
rusqlite::params![now_ms(), error, source_type.as_str(), source_id],
|
rusqlite::params![now, error, next_attempt_at, source_type.as_str(), source_id],
|
||||||
)?;
|
)?;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
@@ -3080,7 +3224,7 @@ fn get_document_id(
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
### 6.4 CLI: `gi sync`
|
### 6.4 CLI: `lore sync`
|
||||||
|
|
||||||
**File:** `src/cli/commands/sync.rs`
|
**File:** `src/cli/commands/sync.rs`
|
||||||
|
|
||||||
@@ -3198,7 +3342,8 @@ pub struct SyncArgs {
|
|||||||
| FTS query sanitization | `src/search/fts.rs` (mod tests) | `to_fts_query()` edge cases: `-`, `"`, `:`, `*`, `C++` |
|
| FTS query sanitization | `src/search/fts.rs` (mod tests) | `to_fts_query()` edge cases: `-`, `"`, `:`, `*`, `C++` |
|
||||||
| SourceType parsing | `src/documents/extractor.rs` (mod tests) | `parse()` accepts aliases: `mr`, `mrs`, `issue`, etc. |
|
| SourceType parsing | `src/documents/extractor.rs` (mod tests) | `parse()` accepts aliases: `mr`, `mrs`, `issue`, etc. |
|
||||||
| SearchFilters | `src/search/filters.rs` (mod tests) | `has_any_filter()`, `clamp_limit()` |
|
| SearchFilters | `src/search/filters.rs` (mod tests) | `has_any_filter()`, `clamp_limit()` |
|
||||||
| Backoff logic | `src/ingestion/dirty_tracker.rs` (mod tests) | Exponential backoff query timing |
|
| Backoff logic | `src/core/backoff.rs` (mod tests) | Shared exponential backoff curve, cap, jitter |
|
||||||
|
| Hydration | `src/cli/commands/search.rs` (mod tests) | Single round-trip, label/path aggregation |
|
||||||
|
|
||||||
### Integration Tests
|
### Integration Tests
|
||||||
|
|
||||||
@@ -3232,25 +3377,29 @@ Each query must have at least one expected URL in top 10 results.
|
|||||||
|
|
||||||
| Command | Expected | Pass Criteria |
|
| Command | Expected | Pass Criteria |
|
||||||
|---------|----------|---------------|
|
|---------|----------|---------------|
|
||||||
| `gi generate-docs` | Progress, count | Completes, count > 0 |
|
| `lore generate-docs` | Progress, count | Completes, count > 0 |
|
||||||
| `gi generate-docs` (re-run) | 0 regenerated | Hash comparison works |
|
| `lore generate-docs` (re-run) | 0 regenerated | Hash comparison works |
|
||||||
| `gi embed` | Progress, count | Completes, count matches docs |
|
| `lore embed` | Progress, count | Completes, count matches docs |
|
||||||
| `gi embed` (re-run) | 0 embedded | Skips unchanged |
|
| `lore embed` (re-run) | 0 embedded | Skips unchanged |
|
||||||
| `gi embed --retry-failed` | Processes failed | Only failed docs processed |
|
| `lore embed --retry-failed` | Processes failed | Only failed docs processed |
|
||||||
| `gi stats` | Coverage stats | Shows 100% after embed |
|
| `lore stats` | Coverage stats | Shows 100% after embed |
|
||||||
| `gi stats` | Queue depths | Shows dirty_sources and pending_discussion_fetches counts |
|
| `lore stats` | Queue depths | Shows dirty_sources and pending_discussion_fetches counts |
|
||||||
| `gi search "auth" --mode=lexical` | Results | Works without Ollama |
|
| `lore search "auth" --mode=lexical` | Results | Works without Ollama |
|
||||||
| `gi search "auth"` | Hybrid results | Vector + FTS combined |
|
| `lore search "auth"` | Hybrid results | Vector + FTS combined |
|
||||||
| `gi search "auth"` (Ollama down) | FTS results + warning | Graceful degradation, warning in response |
|
| `lore search "auth"` (Ollama down) | FTS results + warning | Graceful degradation, warning in response |
|
||||||
| `gi search "auth" --explain` | Rank breakdown | Shows vector/FTS/RRF |
|
| `lore search "auth" --explain` | Rank breakdown | Shows vector/FTS/RRF |
|
||||||
| `gi search "auth" --type=mr` | Filtered results | Only MRs |
|
| `lore search "auth" --type=mr` | Filtered results | Only MRs |
|
||||||
| `gi search "auth" --type=mrs` | Filtered results | Alias works |
|
| `lore search "auth" --type=mrs` | Filtered results | Alias works |
|
||||||
| `gi search "auth" --label=bug` | Filtered results | Only labeled docs |
|
| `lore search "auth" --label=bug` | Filtered results | Only labeled docs |
|
||||||
| `gi search "-DWITH_SSL"` | Results | Leading dash doesn't cause FTS error |
|
| `lore search "-DWITH_SSL"` | Results | Leading dash doesn't cause FTS error |
|
||||||
| `gi search 'C++'` | Results | Special chars in query work |
|
| `lore search 'C++'` | Results | Special chars in query work |
|
||||||
| `gi search "nonexistent123"` | No results | Graceful empty state |
|
| `lore search "auth" --updated-after 2024-01-01` | Filtered results | Only recently updated docs |
|
||||||
| `gi sync` | Full pipeline | All steps complete |
|
| `lore search "nonexistent123"` | No results | Graceful empty state |
|
||||||
| `gi sync --no-embed` | Skip embedding | Docs generated, not embedded |
|
| `lore search "auth" --mode=semantic` (no embeddings) | Actionable error | Tells user to run `lore embed` first |
|
||||||
|
| `lore sync` | Full pipeline | All steps complete |
|
||||||
|
| `lore sync --no-embed` | Skip embedding | Docs generated, not embedded |
|
||||||
|
| `lore generate-docs --full` | Progress, count | Keyset pagination completes without OFFSET degradation |
|
||||||
|
| `lore stats --check --repair` | Repair results | FTS rebuilt, orphans cleaned |
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -3274,43 +3423,59 @@ Each query must have at least one expected URL in top 10 results.
|
|||||||
|
|
||||||
## Success Criteria
|
## Success Criteria
|
||||||
|
|
||||||
Checkpoint 3 is complete when:
|
Checkpoint 3 is complete when all three gates pass:
|
||||||
|
|
||||||
|
### Gate A: Lexical MVP
|
||||||
|
|
||||||
1. **Lexical search works without Ollama**
|
1. **Lexical search works without Ollama**
|
||||||
- `gi search "query" --mode=lexical` returns relevant results
|
- `lore search "query" --mode=lexical` returns relevant results
|
||||||
- All filters functional
|
- All filters functional (including `--updated-after`)
|
||||||
- FTS5 syntax errors prevented by query sanitization
|
- FTS5 syntax errors prevented by query sanitization
|
||||||
- Special characters in queries work correctly (`-DWITH_SSL`, `C++`)
|
- Special characters in queries work correctly (`-DWITH_SSL`, `C++`)
|
||||||
|
- Search results hydrated in single DB round-trip (no N+1)
|
||||||
|
|
||||||
2. **Semantic search works with Ollama**
|
2. **Document generation is correct**
|
||||||
- `gi embed` completes successfully
|
- Full and incremental modes use the same regenerator codepath
|
||||||
- `gi search "query"` returns semantically relevant results
|
- `--full` uses keyset pagination (no OFFSET degradation)
|
||||||
|
- FTS triggers use COALESCE for NULL-safe operation
|
||||||
|
|
||||||
|
### Gate B: Hybrid MVP
|
||||||
|
|
||||||
|
3. **Semantic search works with Ollama**
|
||||||
|
- `lore embed` completes successfully
|
||||||
|
- `lore search "query"` returns semantically relevant results
|
||||||
- `--explain` shows ranking breakdown
|
- `--explain` shows ranking breakdown
|
||||||
|
- `--mode=semantic` with 0% embedding coverage returns actionable error
|
||||||
|
|
||||||
3. **Hybrid search combines both**
|
4. **Hybrid search combines both**
|
||||||
- Documents appearing in both retrievers rank higher
|
- Documents appearing in both retrievers rank higher
|
||||||
- Graceful degradation when Ollama unavailable (falls back to FTS)
|
- Graceful degradation when Ollama unavailable (falls back to FTS)
|
||||||
- Transient embed failures don't fail the entire search
|
- Transient embed failures don't fail the entire search
|
||||||
- Warning message included in response on degradation
|
- Warning message included in response on degradation
|
||||||
|
- Embedding pipeline uses keyset pagination for consistent paging
|
||||||
|
|
||||||
4. **Incremental sync is efficient**
|
### Gate C: Sync MVP
|
||||||
- `gi sync` only processes changed entities
|
|
||||||
|
5. **Incremental sync is efficient**
|
||||||
|
- `lore sync` only processes changed entities
|
||||||
- Re-embedding only happens for changed documents
|
- Re-embedding only happens for changed documents
|
||||||
- Progress visible during long syncs
|
- Progress visible during long syncs
|
||||||
- Queue backoff prevents hot-loop retries on persistent failures
|
- Queue backoff actually prevents hot-loop retries (both queues set `next_attempt_at`)
|
||||||
|
- Shared backoff utility ensures consistent behavior across queues
|
||||||
|
|
||||||
5. **Data integrity maintained**
|
6. **Data integrity maintained**
|
||||||
- All counts match between tables
|
- All counts match between tables
|
||||||
- No orphaned records
|
- No orphaned records
|
||||||
- Hashes consistent
|
- Hashes consistent
|
||||||
- `get_existing_hash()` properly distinguishes "not found" from DB errors
|
- `get_existing_hash()` properly distinguishes "not found" from DB errors
|
||||||
|
- `--repair` uses FTS `rebuild` for correct-by-construction repair
|
||||||
|
|
||||||
6. **Observability**
|
7. **Observability**
|
||||||
- `gi stats` shows queue depths and failed item counts
|
- `lore stats` shows queue depths and failed item counts
|
||||||
- Failed items visible for operator intervention
|
- Failed items visible for operator intervention
|
||||||
- Deterministic ordering ensures consistent paging
|
- Deterministic ordering ensures consistent paging
|
||||||
|
|
||||||
7. **Tests pass**
|
8. **Tests pass**
|
||||||
- Unit tests for core algorithms (including FTS sanitization, backoff)
|
- Unit tests for core algorithms (including FTS sanitization, shared backoff, hydration)
|
||||||
- Integration tests for pipelines
|
- Integration tests for pipelines
|
||||||
- Golden queries return expected results
|
- Golden queries return expected results
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ use crate::ingestion::{
|
|||||||
};
|
};
|
||||||
|
|
||||||
/// Result of ingest command for display.
|
/// Result of ingest command for display.
|
||||||
|
#[derive(Default)]
|
||||||
pub struct IngestResult {
|
pub struct IngestResult {
|
||||||
pub resource_type: String,
|
pub resource_type: String,
|
||||||
pub projects_synced: usize,
|
pub projects_synced: usize,
|
||||||
@@ -130,24 +131,7 @@ pub async fn run_ingest(
|
|||||||
|
|
||||||
let mut total = IngestResult {
|
let mut total = IngestResult {
|
||||||
resource_type: resource_type.to_string(),
|
resource_type: resource_type.to_string(),
|
||||||
projects_synced: 0,
|
..Default::default()
|
||||||
// Issue fields
|
|
||||||
issues_fetched: 0,
|
|
||||||
issues_upserted: 0,
|
|
||||||
issues_synced_discussions: 0,
|
|
||||||
issues_skipped_discussion_sync: 0,
|
|
||||||
// MR fields
|
|
||||||
mrs_fetched: 0,
|
|
||||||
mrs_upserted: 0,
|
|
||||||
mrs_synced_discussions: 0,
|
|
||||||
mrs_skipped_discussion_sync: 0,
|
|
||||||
assignees_linked: 0,
|
|
||||||
reviewers_linked: 0,
|
|
||||||
diffnotes_count: 0,
|
|
||||||
// Shared fields
|
|
||||||
labels_created: 0,
|
|
||||||
discussions_fetched: 0,
|
|
||||||
notes_upserted: 0,
|
|
||||||
};
|
};
|
||||||
|
|
||||||
let type_label = if resource_type == "issues" {
|
let type_label = if resource_type == "issues" {
|
||||||
|
|||||||
@@ -232,8 +232,11 @@ fn query_issues(conn: &Connection, filters: &ListFilters) -> Result<ListResult>
|
|||||||
let mut params: Vec<Box<dyn rusqlite::ToSql>> = Vec::new();
|
let mut params: Vec<Box<dyn rusqlite::ToSql>> = Vec::new();
|
||||||
|
|
||||||
if let Some(project) = filters.project {
|
if let Some(project) = filters.project {
|
||||||
where_clauses.push("p.path_with_namespace LIKE ?");
|
// Exact match or suffix match after '/' to avoid partial matches
|
||||||
params.push(Box::new(format!("%{project}%")));
|
// e.g. "foo" matches "group/foo" but NOT "group/foobar"
|
||||||
|
where_clauses.push("(p.path_with_namespace = ? OR p.path_with_namespace LIKE ?)");
|
||||||
|
params.push(Box::new(project.to_string()));
|
||||||
|
params.push(Box::new(format!("%/{project}")));
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Some(state) = filters.state
|
if let Some(state) = filters.state
|
||||||
@@ -337,11 +340,11 @@ fn query_issues(conn: &Connection, filters: &ListFilters) -> Result<ListResult>
|
|||||||
i.updated_at,
|
i.updated_at,
|
||||||
i.web_url,
|
i.web_url,
|
||||||
p.path_with_namespace,
|
p.path_with_namespace,
|
||||||
(SELECT GROUP_CONCAT(l.name, ',')
|
(SELECT GROUP_CONCAT(l.name, X'1F')
|
||||||
FROM issue_labels il
|
FROM issue_labels il
|
||||||
JOIN labels l ON il.label_id = l.id
|
JOIN labels l ON il.label_id = l.id
|
||||||
WHERE il.issue_id = i.id) AS labels_csv,
|
WHERE il.issue_id = i.id) AS labels_csv,
|
||||||
(SELECT GROUP_CONCAT(ia.username, ',')
|
(SELECT GROUP_CONCAT(ia.username, X'1F')
|
||||||
FROM issue_assignees ia
|
FROM issue_assignees ia
|
||||||
WHERE ia.issue_id = i.id) AS assignees_csv,
|
WHERE ia.issue_id = i.id) AS assignees_csv,
|
||||||
COALESCE(d.total, 0) AS discussion_count,
|
COALESCE(d.total, 0) AS discussion_count,
|
||||||
@@ -369,12 +372,12 @@ fn query_issues(conn: &Connection, filters: &ListFilters) -> Result<ListResult>
|
|||||||
.query_map(param_refs.as_slice(), |row| {
|
.query_map(param_refs.as_slice(), |row| {
|
||||||
let labels_csv: Option<String> = row.get(8)?;
|
let labels_csv: Option<String> = row.get(8)?;
|
||||||
let labels = labels_csv
|
let labels = labels_csv
|
||||||
.map(|s| s.split(',').map(String::from).collect())
|
.map(|s| s.split('\x1F').map(String::from).collect())
|
||||||
.unwrap_or_default();
|
.unwrap_or_default();
|
||||||
|
|
||||||
let assignees_csv: Option<String> = row.get(9)?;
|
let assignees_csv: Option<String> = row.get(9)?;
|
||||||
let assignees = assignees_csv
|
let assignees = assignees_csv
|
||||||
.map(|s| s.split(',').map(String::from).collect())
|
.map(|s| s.split('\x1F').map(String::from).collect())
|
||||||
.unwrap_or_default();
|
.unwrap_or_default();
|
||||||
|
|
||||||
Ok(IssueListRow {
|
Ok(IssueListRow {
|
||||||
@@ -416,8 +419,11 @@ fn query_mrs(conn: &Connection, filters: &MrListFilters) -> Result<MrListResult>
|
|||||||
let mut params: Vec<Box<dyn rusqlite::ToSql>> = Vec::new();
|
let mut params: Vec<Box<dyn rusqlite::ToSql>> = Vec::new();
|
||||||
|
|
||||||
if let Some(project) = filters.project {
|
if let Some(project) = filters.project {
|
||||||
where_clauses.push("p.path_with_namespace LIKE ?");
|
// Exact match or suffix match after '/' to avoid partial matches
|
||||||
params.push(Box::new(format!("%{project}%")));
|
// e.g. "foo" matches "group/foo" but NOT "group/foobar"
|
||||||
|
where_clauses.push("(p.path_with_namespace = ? OR p.path_with_namespace LIKE ?)");
|
||||||
|
params.push(Box::new(project.to_string()));
|
||||||
|
params.push(Box::new(format!("%/{project}")));
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Some(state) = filters.state
|
if let Some(state) = filters.state
|
||||||
@@ -536,14 +542,14 @@ fn query_mrs(conn: &Connection, filters: &MrListFilters) -> Result<MrListResult>
|
|||||||
m.updated_at,
|
m.updated_at,
|
||||||
m.web_url,
|
m.web_url,
|
||||||
p.path_with_namespace,
|
p.path_with_namespace,
|
||||||
(SELECT GROUP_CONCAT(l.name, ',')
|
(SELECT GROUP_CONCAT(l.name, X'1F')
|
||||||
FROM mr_labels ml
|
FROM mr_labels ml
|
||||||
JOIN labels l ON ml.label_id = l.id
|
JOIN labels l ON ml.label_id = l.id
|
||||||
WHERE ml.merge_request_id = m.id) AS labels_csv,
|
WHERE ml.merge_request_id = m.id) AS labels_csv,
|
||||||
(SELECT GROUP_CONCAT(ma.username, ',')
|
(SELECT GROUP_CONCAT(ma.username, X'1F')
|
||||||
FROM mr_assignees ma
|
FROM mr_assignees ma
|
||||||
WHERE ma.merge_request_id = m.id) AS assignees_csv,
|
WHERE ma.merge_request_id = m.id) AS assignees_csv,
|
||||||
(SELECT GROUP_CONCAT(mr.username, ',')
|
(SELECT GROUP_CONCAT(mr.username, X'1F')
|
||||||
FROM mr_reviewers mr
|
FROM mr_reviewers mr
|
||||||
WHERE mr.merge_request_id = m.id) AS reviewers_csv,
|
WHERE mr.merge_request_id = m.id) AS reviewers_csv,
|
||||||
COALESCE(d.total, 0) AS discussion_count,
|
COALESCE(d.total, 0) AS discussion_count,
|
||||||
@@ -571,17 +577,17 @@ fn query_mrs(conn: &Connection, filters: &MrListFilters) -> Result<MrListResult>
|
|||||||
.query_map(param_refs.as_slice(), |row| {
|
.query_map(param_refs.as_slice(), |row| {
|
||||||
let labels_csv: Option<String> = row.get(11)?;
|
let labels_csv: Option<String> = row.get(11)?;
|
||||||
let labels = labels_csv
|
let labels = labels_csv
|
||||||
.map(|s| s.split(',').map(String::from).collect())
|
.map(|s| s.split('\x1F').map(String::from).collect())
|
||||||
.unwrap_or_default();
|
.unwrap_or_default();
|
||||||
|
|
||||||
let assignees_csv: Option<String> = row.get(12)?;
|
let assignees_csv: Option<String> = row.get(12)?;
|
||||||
let assignees = assignees_csv
|
let assignees = assignees_csv
|
||||||
.map(|s| s.split(',').map(String::from).collect())
|
.map(|s| s.split('\x1F').map(String::from).collect())
|
||||||
.unwrap_or_default();
|
.unwrap_or_default();
|
||||||
|
|
||||||
let reviewers_csv: Option<String> = row.get(13)?;
|
let reviewers_csv: Option<String> = row.get(13)?;
|
||||||
let reviewers = reviewers_csv
|
let reviewers = reviewers_csv
|
||||||
.map(|s| s.split(',').map(String::from).collect())
|
.map(|s| s.split('\x1F').map(String::from).collect())
|
||||||
.unwrap_or_default();
|
.unwrap_or_default();
|
||||||
|
|
||||||
let draft_int: i64 = row.get(3)?;
|
let draft_int: i64 = row.get(3)?;
|
||||||
@@ -615,6 +621,10 @@ fn format_relative_time(ms_epoch: i64) -> String {
|
|||||||
let now = now_ms();
|
let now = now_ms();
|
||||||
let diff = now - ms_epoch;
|
let diff = now - ms_epoch;
|
||||||
|
|
||||||
|
if diff < 0 {
|
||||||
|
return "in the future".to_string();
|
||||||
|
}
|
||||||
|
|
||||||
match diff {
|
match diff {
|
||||||
d if d < 60_000 => "just now".to_string(),
|
d if d < 60_000 => "just now".to_string(),
|
||||||
d if d < 3_600_000 => format!("{} min ago", d / 60_000),
|
d if d < 3_600_000 => format!("{} min ago", d / 60_000),
|
||||||
|
|||||||
@@ -150,8 +150,12 @@ fn find_issue(conn: &Connection, iid: i64, project_filter: Option<&str>) -> Resu
|
|||||||
i.created_at, i.updated_at, i.web_url, p.path_with_namespace
|
i.created_at, i.updated_at, i.web_url, p.path_with_namespace
|
||||||
FROM issues i
|
FROM issues i
|
||||||
JOIN projects p ON i.project_id = p.id
|
JOIN projects p ON i.project_id = p.id
|
||||||
WHERE i.iid = ? AND p.path_with_namespace LIKE ?",
|
WHERE i.iid = ? AND (p.path_with_namespace = ? OR p.path_with_namespace LIKE ?)",
|
||||||
vec![Box::new(iid), Box::new(format!("%{}%", project))],
|
vec![
|
||||||
|
Box::new(iid),
|
||||||
|
Box::new(project.to_string()),
|
||||||
|
Box::new(format!("%/{}", project)),
|
||||||
|
],
|
||||||
),
|
),
|
||||||
None => (
|
None => (
|
||||||
"SELECT i.id, i.iid, i.title, i.description, i.state, i.author_username,
|
"SELECT i.id, i.iid, i.title, i.description, i.state, i.author_username,
|
||||||
@@ -336,8 +340,12 @@ fn find_mr(conn: &Connection, iid: i64, project_filter: Option<&str>) -> Result<
|
|||||||
m.web_url, p.path_with_namespace
|
m.web_url, p.path_with_namespace
|
||||||
FROM merge_requests m
|
FROM merge_requests m
|
||||||
JOIN projects p ON m.project_id = p.id
|
JOIN projects p ON m.project_id = p.id
|
||||||
WHERE m.iid = ? AND p.path_with_namespace LIKE ?",
|
WHERE m.iid = ? AND (p.path_with_namespace = ? OR p.path_with_namespace LIKE ?)",
|
||||||
vec![Box::new(iid), Box::new(format!("%{}%", project))],
|
vec![
|
||||||
|
Box::new(iid),
|
||||||
|
Box::new(project.to_string()),
|
||||||
|
Box::new(format!("%/{}", project)),
|
||||||
|
],
|
||||||
),
|
),
|
||||||
None => (
|
None => (
|
||||||
"SELECT m.id, m.iid, m.title, m.description, m.state, m.draft,
|
"SELECT m.id, m.iid, m.title, m.description, m.state, m.draft,
|
||||||
|
|||||||
301
src/cli/mod.rs
301
src/cli/mod.rs
@@ -11,13 +11,17 @@ use std::io::IsTerminal;
|
|||||||
#[command(version, about, long_about = None)]
|
#[command(version, about, long_about = None)]
|
||||||
pub struct Cli {
|
pub struct Cli {
|
||||||
/// Path to config file
|
/// Path to config file
|
||||||
#[arg(short, long, global = true)]
|
#[arg(short = 'c', long, global = true)]
|
||||||
pub config: Option<String>,
|
pub config: Option<String>,
|
||||||
|
|
||||||
/// Machine-readable JSON output (auto-enabled when piped)
|
/// Machine-readable JSON output (auto-enabled when piped)
|
||||||
#[arg(long, global = true, env = "LORE_ROBOT")]
|
#[arg(long, global = true, env = "LORE_ROBOT")]
|
||||||
pub robot: bool,
|
pub robot: bool,
|
||||||
|
|
||||||
|
/// JSON output (global shorthand)
|
||||||
|
#[arg(short = 'J', long = "json", global = true)]
|
||||||
|
pub json: bool,
|
||||||
|
|
||||||
#[command(subcommand)]
|
#[command(subcommand)]
|
||||||
pub command: Commands,
|
pub command: Commands,
|
||||||
}
|
}
|
||||||
@@ -25,17 +29,41 @@ pub struct Cli {
|
|||||||
impl Cli {
|
impl Cli {
|
||||||
/// Check if robot mode is active (explicit flag, env var, or non-TTY stdout)
|
/// Check if robot mode is active (explicit flag, env var, or non-TTY stdout)
|
||||||
pub fn is_robot_mode(&self) -> bool {
|
pub fn is_robot_mode(&self) -> bool {
|
||||||
self.robot || !std::io::stdout().is_terminal()
|
self.robot || self.json || !std::io::stdout().is_terminal()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Subcommand)]
|
#[derive(Subcommand)]
|
||||||
#[allow(clippy::large_enum_variant)]
|
#[allow(clippy::large_enum_variant)]
|
||||||
pub enum Commands {
|
pub enum Commands {
|
||||||
|
/// List or show issues
|
||||||
|
Issues(IssuesArgs),
|
||||||
|
|
||||||
|
/// List or show merge requests
|
||||||
|
Mrs(MrsArgs),
|
||||||
|
|
||||||
|
/// Ingest data from GitLab
|
||||||
|
Ingest(IngestArgs),
|
||||||
|
|
||||||
|
/// Count entities in local database
|
||||||
|
Count(CountArgs),
|
||||||
|
|
||||||
|
/// Show sync state
|
||||||
|
Status,
|
||||||
|
|
||||||
|
/// Verify GitLab authentication
|
||||||
|
Auth,
|
||||||
|
|
||||||
|
/// Check environment health
|
||||||
|
Doctor,
|
||||||
|
|
||||||
|
/// Show version information
|
||||||
|
Version,
|
||||||
|
|
||||||
/// Initialize configuration and database
|
/// Initialize configuration and database
|
||||||
Init {
|
Init {
|
||||||
/// Skip overwrite confirmation
|
/// Skip overwrite confirmation
|
||||||
#[arg(long)]
|
#[arg(short = 'f', long)]
|
||||||
force: bool,
|
force: bool,
|
||||||
|
|
||||||
/// Fail if prompts would be shown
|
/// Fail if prompts would be shown
|
||||||
@@ -43,149 +71,67 @@ pub enum Commands {
|
|||||||
non_interactive: bool,
|
non_interactive: bool,
|
||||||
},
|
},
|
||||||
|
|
||||||
/// Verify GitLab authentication
|
|
||||||
AuthTest,
|
|
||||||
|
|
||||||
/// Check environment health
|
|
||||||
Doctor {
|
|
||||||
/// Output as JSON
|
|
||||||
#[arg(long)]
|
|
||||||
json: bool,
|
|
||||||
},
|
|
||||||
|
|
||||||
/// Show version information
|
|
||||||
Version,
|
|
||||||
|
|
||||||
/// Create timestamped database backup
|
/// Create timestamped database backup
|
||||||
Backup,
|
Backup,
|
||||||
|
|
||||||
/// Delete database and reset all state
|
/// Delete database and reset all state
|
||||||
Reset {
|
Reset {
|
||||||
/// Skip confirmation prompt
|
/// Skip confirmation prompt
|
||||||
#[arg(long)]
|
#[arg(short = 'y', long)]
|
||||||
confirm: bool,
|
yes: bool,
|
||||||
},
|
},
|
||||||
|
|
||||||
/// Run pending database migrations
|
/// Run pending database migrations
|
||||||
Migrate,
|
Migrate,
|
||||||
|
|
||||||
/// Show sync state
|
// --- Hidden backward-compat aliases ---
|
||||||
SyncStatus,
|
/// List issues or MRs (deprecated: use 'lore issues' or 'lore mrs')
|
||||||
|
#[command(hide = true)]
|
||||||
/// Ingest data from GitLab
|
|
||||||
Ingest {
|
|
||||||
/// Resource type to ingest
|
|
||||||
#[arg(long, value_parser = ["issues", "mrs"])]
|
|
||||||
r#type: String,
|
|
||||||
|
|
||||||
/// Filter to single project
|
|
||||||
#[arg(long)]
|
|
||||||
project: Option<String>,
|
|
||||||
|
|
||||||
/// Override stale sync lock
|
|
||||||
#[arg(long)]
|
|
||||||
force: bool,
|
|
||||||
|
|
||||||
/// Full re-sync: reset cursors and fetch all data from scratch
|
|
||||||
#[arg(long)]
|
|
||||||
full: bool,
|
|
||||||
},
|
|
||||||
|
|
||||||
/// List issues or MRs from local database
|
|
||||||
List {
|
List {
|
||||||
/// Entity type to list
|
/// Entity type to list
|
||||||
#[arg(value_parser = ["issues", "mrs"])]
|
#[arg(value_parser = ["issues", "mrs"])]
|
||||||
entity: String,
|
entity: String,
|
||||||
|
|
||||||
/// Maximum results
|
|
||||||
#[arg(long, default_value = "50")]
|
#[arg(long, default_value = "50")]
|
||||||
limit: usize,
|
limit: usize,
|
||||||
|
|
||||||
/// Filter by project path
|
|
||||||
#[arg(long)]
|
#[arg(long)]
|
||||||
project: Option<String>,
|
project: Option<String>,
|
||||||
|
|
||||||
/// Filter by state (opened|closed|all for issues; opened|merged|closed|locked|all for MRs)
|
|
||||||
#[arg(long)]
|
#[arg(long)]
|
||||||
state: Option<String>,
|
state: Option<String>,
|
||||||
|
|
||||||
/// Filter by author username
|
|
||||||
#[arg(long)]
|
#[arg(long)]
|
||||||
author: Option<String>,
|
author: Option<String>,
|
||||||
|
|
||||||
/// Filter by assignee username
|
|
||||||
#[arg(long)]
|
#[arg(long)]
|
||||||
assignee: Option<String>,
|
assignee: Option<String>,
|
||||||
|
|
||||||
/// Filter by label (repeatable, AND logic)
|
|
||||||
#[arg(long)]
|
#[arg(long)]
|
||||||
label: Option<Vec<String>>,
|
label: Option<Vec<String>>,
|
||||||
|
|
||||||
/// Filter by milestone title (issues only)
|
|
||||||
#[arg(long)]
|
#[arg(long)]
|
||||||
milestone: Option<String>,
|
milestone: Option<String>,
|
||||||
|
|
||||||
/// Filter by time (7d, 2w, 1m, or YYYY-MM-DD)
|
|
||||||
#[arg(long)]
|
#[arg(long)]
|
||||||
since: Option<String>,
|
since: Option<String>,
|
||||||
|
|
||||||
/// Filter by due date (before this date, YYYY-MM-DD) (issues only)
|
|
||||||
#[arg(long)]
|
#[arg(long)]
|
||||||
due_before: Option<String>,
|
due_before: Option<String>,
|
||||||
|
|
||||||
/// Show only issues with a due date (issues only)
|
|
||||||
#[arg(long)]
|
#[arg(long)]
|
||||||
has_due_date: bool,
|
has_due_date: bool,
|
||||||
|
|
||||||
/// Sort field
|
|
||||||
#[arg(long, value_parser = ["updated", "created", "iid"], default_value = "updated")]
|
#[arg(long, value_parser = ["updated", "created", "iid"], default_value = "updated")]
|
||||||
sort: String,
|
sort: String,
|
||||||
|
|
||||||
/// Sort order
|
|
||||||
#[arg(long, value_parser = ["desc", "asc"], default_value = "desc")]
|
#[arg(long, value_parser = ["desc", "asc"], default_value = "desc")]
|
||||||
order: String,
|
order: String,
|
||||||
|
|
||||||
/// Open first matching item in browser
|
|
||||||
#[arg(long)]
|
#[arg(long)]
|
||||||
open: bool,
|
open: bool,
|
||||||
|
|
||||||
/// Output as JSON
|
|
||||||
#[arg(long)]
|
|
||||||
json: bool,
|
|
||||||
|
|
||||||
/// Show only draft MRs (MRs only)
|
|
||||||
#[arg(long, conflicts_with = "no_draft")]
|
#[arg(long, conflicts_with = "no_draft")]
|
||||||
draft: bool,
|
draft: bool,
|
||||||
|
|
||||||
/// Exclude draft MRs (MRs only)
|
|
||||||
#[arg(long, conflicts_with = "draft")]
|
#[arg(long, conflicts_with = "draft")]
|
||||||
no_draft: bool,
|
no_draft: bool,
|
||||||
|
|
||||||
/// Filter by reviewer username (MRs only)
|
|
||||||
#[arg(long)]
|
#[arg(long)]
|
||||||
reviewer: Option<String>,
|
reviewer: Option<String>,
|
||||||
|
|
||||||
/// Filter by target branch (MRs only)
|
|
||||||
#[arg(long)]
|
#[arg(long)]
|
||||||
target_branch: Option<String>,
|
target_branch: Option<String>,
|
||||||
|
|
||||||
/// Filter by source branch (MRs only)
|
|
||||||
#[arg(long)]
|
#[arg(long)]
|
||||||
source_branch: Option<String>,
|
source_branch: Option<String>,
|
||||||
},
|
},
|
||||||
|
|
||||||
/// Count entities in local database
|
/// Show detailed entity information (deprecated: use 'lore issues <IID>' or 'lore mrs <IID>')
|
||||||
Count {
|
#[command(hide = true)]
|
||||||
/// Entity type to count
|
|
||||||
#[arg(value_parser = ["issues", "mrs", "discussions", "notes"])]
|
|
||||||
entity: String,
|
|
||||||
|
|
||||||
/// Filter by noteable type (for discussions/notes)
|
|
||||||
#[arg(long, value_parser = ["issue", "mr"])]
|
|
||||||
r#type: Option<String>,
|
|
||||||
},
|
|
||||||
|
|
||||||
/// Show detailed entity information
|
|
||||||
Show {
|
Show {
|
||||||
/// Entity type to show
|
/// Entity type to show
|
||||||
#[arg(value_parser = ["issue", "mr"])]
|
#[arg(value_parser = ["issue", "mr"])]
|
||||||
@@ -194,12 +140,173 @@ pub enum Commands {
|
|||||||
/// Entity IID
|
/// Entity IID
|
||||||
iid: i64,
|
iid: i64,
|
||||||
|
|
||||||
/// Filter by project path (required if iid is ambiguous)
|
|
||||||
#[arg(long)]
|
#[arg(long)]
|
||||||
project: Option<String>,
|
project: Option<String>,
|
||||||
|
|
||||||
/// Output as JSON
|
|
||||||
#[arg(long)]
|
|
||||||
json: bool,
|
|
||||||
},
|
},
|
||||||
|
|
||||||
|
/// Verify GitLab authentication (deprecated: use 'lore auth')
|
||||||
|
#[command(hide = true, name = "auth-test")]
|
||||||
|
AuthTest,
|
||||||
|
|
||||||
|
/// Show sync state (deprecated: use 'lore status')
|
||||||
|
#[command(hide = true, name = "sync-status")]
|
||||||
|
SyncStatus,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Arguments for `lore issues [IID]`
|
||||||
|
#[derive(Parser)]
|
||||||
|
pub struct IssuesArgs {
|
||||||
|
/// Issue IID (omit to list, provide to show details)
|
||||||
|
pub iid: Option<i64>,
|
||||||
|
|
||||||
|
/// Maximum results
|
||||||
|
#[arg(short = 'n', long = "limit", default_value = "50")]
|
||||||
|
pub limit: usize,
|
||||||
|
|
||||||
|
/// Filter by state (opened, closed, all)
|
||||||
|
#[arg(short = 's', long)]
|
||||||
|
pub state: Option<String>,
|
||||||
|
|
||||||
|
/// Filter by project path
|
||||||
|
#[arg(short = 'p', long)]
|
||||||
|
pub project: Option<String>,
|
||||||
|
|
||||||
|
/// Filter by author username
|
||||||
|
#[arg(short = 'a', long)]
|
||||||
|
pub author: Option<String>,
|
||||||
|
|
||||||
|
/// Filter by assignee username
|
||||||
|
#[arg(short = 'A', long)]
|
||||||
|
pub assignee: Option<String>,
|
||||||
|
|
||||||
|
/// Filter by label (repeatable, AND logic)
|
||||||
|
#[arg(short = 'l', long)]
|
||||||
|
pub label: Option<Vec<String>>,
|
||||||
|
|
||||||
|
/// Filter by milestone title
|
||||||
|
#[arg(short = 'm', long)]
|
||||||
|
pub milestone: Option<String>,
|
||||||
|
|
||||||
|
/// Filter by time (7d, 2w, 1m, or YYYY-MM-DD)
|
||||||
|
#[arg(long)]
|
||||||
|
pub since: Option<String>,
|
||||||
|
|
||||||
|
/// Filter by due date (before this date, YYYY-MM-DD)
|
||||||
|
#[arg(long = "due-before")]
|
||||||
|
pub due_before: Option<String>,
|
||||||
|
|
||||||
|
/// Show only issues with a due date
|
||||||
|
#[arg(long = "has-due")]
|
||||||
|
pub has_due: bool,
|
||||||
|
|
||||||
|
/// Sort field (updated, created, iid)
|
||||||
|
#[arg(long, value_parser = ["updated", "created", "iid"], default_value = "updated")]
|
||||||
|
pub sort: String,
|
||||||
|
|
||||||
|
/// Sort ascending (default: descending)
|
||||||
|
#[arg(long)]
|
||||||
|
pub asc: bool,
|
||||||
|
|
||||||
|
/// Open first matching item in browser
|
||||||
|
#[arg(short = 'o', long)]
|
||||||
|
pub open: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Arguments for `lore mrs [IID]`
|
||||||
|
#[derive(Parser)]
|
||||||
|
pub struct MrsArgs {
|
||||||
|
/// MR IID (omit to list, provide to show details)
|
||||||
|
pub iid: Option<i64>,
|
||||||
|
|
||||||
|
/// Maximum results
|
||||||
|
#[arg(short = 'n', long = "limit", default_value = "50")]
|
||||||
|
pub limit: usize,
|
||||||
|
|
||||||
|
/// Filter by state (opened, merged, closed, locked, all)
|
||||||
|
#[arg(short = 's', long)]
|
||||||
|
pub state: Option<String>,
|
||||||
|
|
||||||
|
/// Filter by project path
|
||||||
|
#[arg(short = 'p', long)]
|
||||||
|
pub project: Option<String>,
|
||||||
|
|
||||||
|
/// Filter by author username
|
||||||
|
#[arg(short = 'a', long)]
|
||||||
|
pub author: Option<String>,
|
||||||
|
|
||||||
|
/// Filter by assignee username
|
||||||
|
#[arg(short = 'A', long)]
|
||||||
|
pub assignee: Option<String>,
|
||||||
|
|
||||||
|
/// Filter by reviewer username
|
||||||
|
#[arg(short = 'r', long)]
|
||||||
|
pub reviewer: Option<String>,
|
||||||
|
|
||||||
|
/// Filter by label (repeatable, AND logic)
|
||||||
|
#[arg(short = 'l', long)]
|
||||||
|
pub label: Option<Vec<String>>,
|
||||||
|
|
||||||
|
/// Filter by time (7d, 2w, 1m, or YYYY-MM-DD)
|
||||||
|
#[arg(long)]
|
||||||
|
pub since: Option<String>,
|
||||||
|
|
||||||
|
/// Show only draft MRs
|
||||||
|
#[arg(short = 'd', long, conflicts_with = "no_draft")]
|
||||||
|
pub draft: bool,
|
||||||
|
|
||||||
|
/// Exclude draft MRs
|
||||||
|
#[arg(short = 'D', long = "no-draft", conflicts_with = "draft")]
|
||||||
|
pub no_draft: bool,
|
||||||
|
|
||||||
|
/// Filter by target branch
|
||||||
|
#[arg(long)]
|
||||||
|
pub target: Option<String>,
|
||||||
|
|
||||||
|
/// Filter by source branch
|
||||||
|
#[arg(long)]
|
||||||
|
pub source: Option<String>,
|
||||||
|
|
||||||
|
/// Sort field (updated, created, iid)
|
||||||
|
#[arg(long, value_parser = ["updated", "created", "iid"], default_value = "updated")]
|
||||||
|
pub sort: String,
|
||||||
|
|
||||||
|
/// Sort ascending (default: descending)
|
||||||
|
#[arg(long)]
|
||||||
|
pub asc: bool,
|
||||||
|
|
||||||
|
/// Open first matching item in browser
|
||||||
|
#[arg(short = 'o', long)]
|
||||||
|
pub open: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Arguments for `lore ingest [ENTITY]`
|
||||||
|
#[derive(Parser)]
|
||||||
|
pub struct IngestArgs {
|
||||||
|
/// Entity to ingest (issues, mrs). Omit to ingest everything.
|
||||||
|
#[arg(value_parser = ["issues", "mrs"])]
|
||||||
|
pub entity: Option<String>,
|
||||||
|
|
||||||
|
/// Filter to single project
|
||||||
|
#[arg(short = 'p', long)]
|
||||||
|
pub project: Option<String>,
|
||||||
|
|
||||||
|
/// Override stale sync lock
|
||||||
|
#[arg(short = 'f', long)]
|
||||||
|
pub force: bool,
|
||||||
|
|
||||||
|
/// Full re-sync: reset cursors and fetch all data from scratch
|
||||||
|
#[arg(long)]
|
||||||
|
pub full: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Arguments for `lore count <ENTITY>`
|
||||||
|
#[derive(Parser)]
|
||||||
|
pub struct CountArgs {
|
||||||
|
/// Entity type to count (issues, mrs, discussions, notes)
|
||||||
|
#[arg(value_parser = ["issues", "mrs", "discussions", "notes"])]
|
||||||
|
pub entity: String,
|
||||||
|
|
||||||
|
/// Parent type filter: issue or mr (for discussions/notes)
|
||||||
|
#[arg(short = 'f', long = "for", value_parser = ["issue", "mr"])]
|
||||||
|
pub for_entity: Option<String>,
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -150,27 +150,49 @@ impl GiError {
|
|||||||
Self::Io(_) => ErrorCode::IoError,
|
Self::Io(_) => ErrorCode::IoError,
|
||||||
Self::Transform(_) => ErrorCode::TransformError,
|
Self::Transform(_) => ErrorCode::TransformError,
|
||||||
Self::NotFound(_) => ErrorCode::GitLabNotFound,
|
Self::NotFound(_) => ErrorCode::GitLabNotFound,
|
||||||
Self::Ambiguous(_) => ErrorCode::InternalError,
|
Self::Ambiguous(_) => ErrorCode::GitLabNotFound,
|
||||||
Self::Other(_) => ErrorCode::InternalError,
|
Self::Other(_) => ErrorCode::InternalError,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Get a suggestion for how to fix this error.
|
/// Get a suggestion for how to fix this error, including inline examples.
|
||||||
pub fn suggestion(&self) -> Option<&'static str> {
|
pub fn suggestion(&self) -> Option<&'static str> {
|
||||||
match self {
|
match self {
|
||||||
Self::ConfigNotFound { .. } => Some("Run 'lore init' to create configuration"),
|
Self::ConfigNotFound { .. } => Some(
|
||||||
Self::ConfigInvalid { .. } => Some("Check config file syntax or run 'lore init' to recreate"),
|
"Run 'lore init' to set up your GitLab connection.\n\n Expected: ~/.config/lore/config.json",
|
||||||
Self::GitLabAuthFailed => Some("Verify token has read_api scope and is not expired"),
|
),
|
||||||
Self::GitLabNotFound { .. } => Some("Check the resource path exists and you have access"),
|
Self::ConfigInvalid { .. } => Some(
|
||||||
|
"Check config file syntax or run 'lore init' to recreate.\n\n Example:\n lore init\n lore init --force",
|
||||||
|
),
|
||||||
|
Self::GitLabAuthFailed => Some(
|
||||||
|
"Verify token has read_api scope and is not expired.\n\n Example:\n export GITLAB_TOKEN=glpat-xxxxxxxxxxxx\n lore auth",
|
||||||
|
),
|
||||||
|
Self::GitLabNotFound { .. } => Some(
|
||||||
|
"Check the resource path exists and you have access.\n\n Example:\n lore issues -p group/project\n lore mrs -p group/project",
|
||||||
|
),
|
||||||
Self::GitLabRateLimited { .. } => Some("Wait and retry, or reduce request frequency"),
|
Self::GitLabRateLimited { .. } => Some("Wait and retry, or reduce request frequency"),
|
||||||
Self::GitLabNetworkError { .. } => Some("Check network connection and GitLab URL"),
|
Self::GitLabNetworkError { .. } => Some(
|
||||||
Self::DatabaseLocked { .. } => Some("Wait for other sync to complete or use --force"),
|
"Check network connection and GitLab URL.\n\n Example:\n lore doctor\n lore auth",
|
||||||
Self::MigrationFailed { .. } => Some("Check database file permissions or reset with 'lore reset'"),
|
),
|
||||||
Self::TokenNotSet { .. } => Some("Export the token environment variable"),
|
Self::DatabaseLocked { .. } => Some(
|
||||||
Self::Database(_) => Some("Check database file permissions or reset with 'lore reset'"),
|
"Wait for other sync to complete or use --force.\n\n Example:\n lore ingest --force\n lore ingest issues --force",
|
||||||
|
),
|
||||||
|
Self::MigrationFailed { .. } => Some(
|
||||||
|
"Check database file permissions or reset with 'lore reset'.\n\n Example:\n lore migrate\n lore reset --yes",
|
||||||
|
),
|
||||||
|
Self::TokenNotSet { .. } => Some(
|
||||||
|
"Export the token to your shell:\n\n export GITLAB_TOKEN=glpat-xxxxxxxxxxxx\n\n Your token needs the read_api scope.",
|
||||||
|
),
|
||||||
|
Self::Database(_) => Some(
|
||||||
|
"Check database file permissions or reset with 'lore reset'.\n\n Example:\n lore doctor\n lore reset --yes",
|
||||||
|
),
|
||||||
Self::Http(_) => Some("Check network connection"),
|
Self::Http(_) => Some("Check network connection"),
|
||||||
Self::NotFound(_) => Some("Verify the entity exists using 'lore list'"),
|
Self::NotFound(_) => Some(
|
||||||
Self::Ambiguous(_) => Some("Use --project flag to disambiguate"),
|
"Verify the entity exists.\n\n Example:\n lore issues\n lore mrs",
|
||||||
|
),
|
||||||
|
Self::Ambiguous(_) => Some(
|
||||||
|
"Use -p to choose a specific project.\n\n Example:\n lore issues 42 -p group/project-a\n lore mrs 99 -p group/project-b",
|
||||||
|
),
|
||||||
_ => None,
|
_ => None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -59,6 +59,22 @@ pub fn parse_since(input: &str) -> Option<i64> {
|
|||||||
iso_to_ms(input)
|
iso_to_ms(input)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Convert ISO 8601 timestamp to milliseconds with strict error handling.
|
||||||
|
/// Returns Err with a descriptive message if the timestamp is invalid.
|
||||||
|
pub fn iso_to_ms_strict(iso_string: &str) -> Result<i64, String> {
|
||||||
|
DateTime::parse_from_rfc3339(iso_string)
|
||||||
|
.map(|dt| dt.timestamp_millis())
|
||||||
|
.map_err(|_| format!("Invalid timestamp: {}", iso_string))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Convert optional ISO 8601 timestamp to optional milliseconds (strict).
|
||||||
|
pub fn iso_to_ms_opt_strict(iso_string: &Option<String>) -> Result<Option<i64>, String> {
|
||||||
|
match iso_string {
|
||||||
|
Some(s) => iso_to_ms_strict(s).map(Some),
|
||||||
|
None => Ok(None),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Format milliseconds epoch to human-readable full datetime.
|
/// Format milliseconds epoch to human-readable full datetime.
|
||||||
pub fn format_full_datetime(ms: i64) -> String {
|
pub fn format_full_datetime(ms: i64) -> String {
|
||||||
DateTime::from_timestamp_millis(ms)
|
DateTime::from_timestamp_millis(ms)
|
||||||
|
|||||||
@@ -1,8 +1,6 @@
|
|||||||
//! Discussion and note transformers: convert GitLab discussions to local schema.
|
//! Discussion and note transformers: convert GitLab discussions to local schema.
|
||||||
|
|
||||||
use chrono::DateTime;
|
use crate::core::time::{iso_to_ms, iso_to_ms_strict, now_ms};
|
||||||
|
|
||||||
use crate::core::time::now_ms;
|
|
||||||
use crate::gitlab::types::{GitLabDiscussion, GitLabNote};
|
use crate::gitlab::types::{GitLabDiscussion, GitLabNote};
|
||||||
|
|
||||||
/// Reference to the parent noteable (Issue or MergeRequest).
|
/// Reference to the parent noteable (Issue or MergeRequest).
|
||||||
@@ -60,16 +58,9 @@ pub struct NormalizedNote {
|
|||||||
pub position_head_sha: Option<String>, // Head commit SHA for diff
|
pub position_head_sha: Option<String>, // Head commit SHA for diff
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Parse ISO 8601 timestamp to milliseconds, returning None on failure.
|
|
||||||
fn parse_timestamp_opt(ts: &str) -> Option<i64> {
|
|
||||||
DateTime::parse_from_rfc3339(ts)
|
|
||||||
.ok()
|
|
||||||
.map(|dt| dt.timestamp_millis())
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Parse ISO 8601 timestamp to milliseconds, defaulting to 0 on failure.
|
/// Parse ISO 8601 timestamp to milliseconds, defaulting to 0 on failure.
|
||||||
fn parse_timestamp(ts: &str) -> i64 {
|
fn parse_timestamp(ts: &str) -> i64 {
|
||||||
parse_timestamp_opt(ts).unwrap_or(0)
|
iso_to_ms(ts).unwrap_or(0)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Transform a GitLab discussion into normalized schema.
|
/// Transform a GitLab discussion into normalized schema.
|
||||||
@@ -90,7 +81,7 @@ pub fn transform_discussion(
|
|||||||
let note_timestamps: Vec<i64> = gitlab_discussion
|
let note_timestamps: Vec<i64> = gitlab_discussion
|
||||||
.notes
|
.notes
|
||||||
.iter()
|
.iter()
|
||||||
.filter_map(|n| parse_timestamp_opt(&n.created_at))
|
.filter_map(|n| iso_to_ms(&n.created_at))
|
||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
let first_note_at = note_timestamps.iter().min().copied();
|
let first_note_at = note_timestamps.iter().min().copied();
|
||||||
@@ -191,7 +182,7 @@ fn transform_single_note(
|
|||||||
resolved_at: note
|
resolved_at: note
|
||||||
.resolved_at
|
.resolved_at
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.and_then(|ts| parse_timestamp_opt(ts)),
|
.and_then(|ts| iso_to_ms(ts)),
|
||||||
position_old_path,
|
position_old_path,
|
||||||
position_new_path,
|
position_new_path,
|
||||||
position_old_line,
|
position_old_line,
|
||||||
@@ -244,13 +235,6 @@ fn extract_position_fields(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Parse ISO 8601 timestamp to milliseconds with strict error handling.
|
|
||||||
/// Returns Err with the invalid timestamp in the error message.
|
|
||||||
fn parse_timestamp_strict(ts: &str) -> Result<i64, String> {
|
|
||||||
DateTime::parse_from_rfc3339(ts)
|
|
||||||
.map(|dt| dt.timestamp_millis())
|
|
||||||
.map_err(|_| format!("Invalid timestamp: {}", ts))
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Transform notes from a GitLab discussion with strict timestamp parsing.
|
/// Transform notes from a GitLab discussion with strict timestamp parsing.
|
||||||
/// Returns Err if any timestamp is invalid - no silent fallback to 0.
|
/// Returns Err if any timestamp is invalid - no silent fallback to 0.
|
||||||
@@ -275,10 +259,10 @@ fn transform_single_note_strict(
|
|||||||
now: i64,
|
now: i64,
|
||||||
) -> Result<NormalizedNote, String> {
|
) -> Result<NormalizedNote, String> {
|
||||||
// Parse timestamps with strict error handling
|
// Parse timestamps with strict error handling
|
||||||
let created_at = parse_timestamp_strict(¬e.created_at)?;
|
let created_at = iso_to_ms_strict(¬e.created_at)?;
|
||||||
let updated_at = parse_timestamp_strict(¬e.updated_at)?;
|
let updated_at = iso_to_ms_strict(¬e.updated_at)?;
|
||||||
let resolved_at = match ¬e.resolved_at {
|
let resolved_at = match ¬e.resolved_at {
|
||||||
Some(ts) => Some(parse_timestamp_strict(ts)?),
|
Some(ts) => Some(iso_to_ms_strict(ts)?),
|
||||||
None => None,
|
None => None,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
@@ -1,33 +1,8 @@
|
|||||||
//! Merge request transformer: converts GitLabMergeRequest to local schema.
|
//! Merge request transformer: converts GitLabMergeRequest to local schema.
|
||||||
|
|
||||||
use chrono::DateTime;
|
use crate::core::time::{iso_to_ms_opt_strict, iso_to_ms_strict, now_ms};
|
||||||
use std::time::{SystemTime, UNIX_EPOCH};
|
|
||||||
|
|
||||||
use crate::gitlab::types::GitLabMergeRequest;
|
use crate::gitlab::types::GitLabMergeRequest;
|
||||||
|
|
||||||
/// Get current time in milliseconds since Unix epoch.
|
|
||||||
fn now_ms() -> i64 {
|
|
||||||
SystemTime::now()
|
|
||||||
.duration_since(UNIX_EPOCH)
|
|
||||||
.expect("Time went backwards")
|
|
||||||
.as_millis() as i64
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Parse ISO 8601 timestamp to milliseconds since Unix epoch.
|
|
||||||
fn iso_to_ms(ts: &str) -> Result<i64, String> {
|
|
||||||
DateTime::parse_from_rfc3339(ts)
|
|
||||||
.map(|dt| dt.timestamp_millis())
|
|
||||||
.map_err(|e| format!("Failed to parse timestamp '{}': {}", ts, e))
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Parse optional ISO 8601 timestamp to optional milliseconds since Unix epoch.
|
|
||||||
fn iso_to_ms_opt(ts: &Option<String>) -> Result<Option<i64>, String> {
|
|
||||||
match ts {
|
|
||||||
Some(s) => iso_to_ms(s).map(Some),
|
|
||||||
None => Ok(None),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Local schema representation of a merge request row.
|
/// Local schema representation of a merge request row.
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
pub struct NormalizedMergeRequest {
|
pub struct NormalizedMergeRequest {
|
||||||
@@ -77,12 +52,12 @@ pub fn transform_merge_request(
|
|||||||
local_project_id: i64,
|
local_project_id: i64,
|
||||||
) -> Result<MergeRequestWithMetadata, String> {
|
) -> Result<MergeRequestWithMetadata, String> {
|
||||||
// Parse required timestamps
|
// Parse required timestamps
|
||||||
let created_at = iso_to_ms(&gitlab_mr.created_at)?;
|
let created_at = iso_to_ms_strict(&gitlab_mr.created_at)?;
|
||||||
let updated_at = iso_to_ms(&gitlab_mr.updated_at)?;
|
let updated_at = iso_to_ms_strict(&gitlab_mr.updated_at)?;
|
||||||
|
|
||||||
// Parse optional timestamps
|
// Parse optional timestamps
|
||||||
let merged_at = iso_to_ms_opt(&gitlab_mr.merged_at)?;
|
let merged_at = iso_to_ms_opt_strict(&gitlab_mr.merged_at)?;
|
||||||
let closed_at = iso_to_ms_opt(&gitlab_mr.closed_at)?;
|
let closed_at = iso_to_ms_opt_strict(&gitlab_mr.closed_at)?;
|
||||||
|
|
||||||
// Draft: prefer draft, fallback to work_in_progress
|
// Draft: prefer draft, fallback to work_in_progress
|
||||||
let is_draft = gitlab_mr.draft || gitlab_mr.work_in_progress;
|
let is_draft = gitlab_mr.draft || gitlab_mr.work_in_progress;
|
||||||
|
|||||||
@@ -89,16 +89,10 @@ async fn ingest_discussions_for_issue(
|
|||||||
|
|
||||||
// Track discussions we've seen for stale removal
|
// Track discussions we've seen for stale removal
|
||||||
let mut seen_discussion_ids: Vec<String> = Vec::new();
|
let mut seen_discussion_ids: Vec<String> = Vec::new();
|
||||||
// Track if we've started receiving data (to distinguish empty result from failure)
|
|
||||||
let mut received_first_response = false;
|
|
||||||
// Track if any error occurred during pagination
|
// Track if any error occurred during pagination
|
||||||
let mut pagination_error: Option<crate::core::error::GiError> = None;
|
let mut pagination_error: Option<crate::core::error::GiError> = None;
|
||||||
|
|
||||||
while let Some(disc_result) = discussions_stream.next().await {
|
while let Some(disc_result) = discussions_stream.next().await {
|
||||||
// Mark that we've received at least one response from the API
|
|
||||||
if !received_first_response {
|
|
||||||
received_first_response = true;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Handle errors - record but don't delete stale data
|
// Handle errors - record but don't delete stale data
|
||||||
let gitlab_discussion = match disc_result {
|
let gitlab_discussion = match disc_result {
|
||||||
@@ -139,8 +133,6 @@ async fn ingest_discussions_for_issue(
|
|||||||
let tx = conn.unchecked_transaction()?;
|
let tx = conn.unchecked_transaction()?;
|
||||||
|
|
||||||
upsert_discussion(&tx, &normalized, payload_id)?;
|
upsert_discussion(&tx, &normalized, payload_id)?;
|
||||||
result.discussions_upserted += 1;
|
|
||||||
seen_discussion_ids.push(normalized.gitlab_discussion_id.clone());
|
|
||||||
|
|
||||||
// Get local discussion ID
|
// Get local discussion ID
|
||||||
let local_discussion_id: i64 = tx.query_row(
|
let local_discussion_id: i64 = tx.query_row(
|
||||||
@@ -151,6 +143,7 @@ async fn ingest_discussions_for_issue(
|
|||||||
|
|
||||||
// Transform and store notes
|
// Transform and store notes
|
||||||
let notes = transform_notes(&gitlab_discussion, local_project_id);
|
let notes = transform_notes(&gitlab_discussion, local_project_id);
|
||||||
|
let notes_count = notes.len();
|
||||||
|
|
||||||
// Delete existing notes for this discussion (full refresh)
|
// Delete existing notes for this discussion (full refresh)
|
||||||
tx.execute(
|
tx.execute(
|
||||||
@@ -178,26 +171,19 @@ async fn ingest_discussions_for_issue(
|
|||||||
)?;
|
)?;
|
||||||
|
|
||||||
insert_note(&tx, local_discussion_id, ¬e, note_payload_id)?;
|
insert_note(&tx, local_discussion_id, ¬e, note_payload_id)?;
|
||||||
result.notes_upserted += 1;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
tx.commit()?;
|
tx.commit()?;
|
||||||
|
|
||||||
|
// Increment counters AFTER successful commit to keep metrics honest
|
||||||
|
result.discussions_upserted += 1;
|
||||||
|
result.notes_upserted += notes_count;
|
||||||
|
seen_discussion_ids.push(normalized.gitlab_discussion_id.clone());
|
||||||
}
|
}
|
||||||
|
|
||||||
// Only remove stale discussions if pagination completed without errors
|
// Only remove stale discussions and advance watermark if pagination completed
|
||||||
// AND we actually received a response (empty or not)
|
// without errors. Safe for both empty results and populated results.
|
||||||
if pagination_error.is_none() && received_first_response {
|
if pagination_error.is_none() {
|
||||||
let removed = remove_stale_discussions(conn, issue.local_issue_id, &seen_discussion_ids)?;
|
|
||||||
result.stale_discussions_removed = removed;
|
|
||||||
|
|
||||||
// Update discussions_synced_for_updated_at on the issue
|
|
||||||
update_issue_sync_timestamp(conn, issue.local_issue_id, issue.updated_at)?;
|
|
||||||
} else if pagination_error.is_none()
|
|
||||||
&& !received_first_response
|
|
||||||
&& seen_discussion_ids.is_empty()
|
|
||||||
{
|
|
||||||
// Stream was empty but no error - issue genuinely has no discussions
|
|
||||||
// This is safe to remove stale discussions (if any exist from before)
|
|
||||||
let removed = remove_stale_discussions(conn, issue.local_issue_id, &seen_discussion_ids)?;
|
let removed = remove_stale_discussions(conn, issue.local_issue_id, &seen_discussion_ids)?;
|
||||||
result.stale_discussions_removed = removed;
|
result.stale_discussions_removed = removed;
|
||||||
|
|
||||||
@@ -208,7 +194,6 @@ async fn ingest_discussions_for_issue(
|
|||||||
discussions_seen = seen_discussion_ids.len(),
|
discussions_seen = seen_discussion_ids.len(),
|
||||||
"Skipping stale removal due to pagination error"
|
"Skipping stale removal due to pagination error"
|
||||||
);
|
);
|
||||||
// Return the error to signal incomplete sync
|
|
||||||
return Err(err);
|
return Err(err);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -155,14 +155,13 @@ pub fn write_prefetched_mr_discussions(
|
|||||||
|
|
||||||
// Write each discussion
|
// Write each discussion
|
||||||
for disc in &prefetched.discussions {
|
for disc in &prefetched.discussions {
|
||||||
result.discussions_fetched += 1;
|
// Count DiffNotes upfront (independent of transaction)
|
||||||
|
let diffnotes_in_disc = disc
|
||||||
// Count DiffNotes
|
|
||||||
result.diffnotes_count += disc
|
|
||||||
.notes
|
.notes
|
||||||
.iter()
|
.iter()
|
||||||
.filter(|n| n.position_new_path.is_some() || n.position_old_path.is_some())
|
.filter(|n| n.position_new_path.is_some() || n.position_old_path.is_some())
|
||||||
.count();
|
.count();
|
||||||
|
let notes_in_disc = disc.notes.len();
|
||||||
|
|
||||||
// Start transaction
|
// Start transaction
|
||||||
let tx = conn.unchecked_transaction()?;
|
let tx = conn.unchecked_transaction()?;
|
||||||
@@ -182,7 +181,6 @@ pub fn write_prefetched_mr_discussions(
|
|||||||
|
|
||||||
// Upsert discussion
|
// Upsert discussion
|
||||||
upsert_discussion(&tx, &disc.normalized, run_seen_at, payload_id)?;
|
upsert_discussion(&tx, &disc.normalized, run_seen_at, payload_id)?;
|
||||||
result.discussions_upserted += 1;
|
|
||||||
|
|
||||||
// Get local discussion ID
|
// Get local discussion ID
|
||||||
let local_discussion_id: i64 = tx.query_row(
|
let local_discussion_id: i64 = tx.query_row(
|
||||||
@@ -219,10 +217,15 @@ pub fn write_prefetched_mr_discussions(
|
|||||||
};
|
};
|
||||||
|
|
||||||
upsert_note(&tx, local_discussion_id, note, run_seen_at, note_payload_id)?;
|
upsert_note(&tx, local_discussion_id, note, run_seen_at, note_payload_id)?;
|
||||||
result.notes_upserted += 1;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
tx.commit()?;
|
tx.commit()?;
|
||||||
|
|
||||||
|
// Increment counters AFTER successful commit to keep metrics honest
|
||||||
|
result.discussions_fetched += 1;
|
||||||
|
result.discussions_upserted += 1;
|
||||||
|
result.notes_upserted += notes_in_disc;
|
||||||
|
result.diffnotes_count += diffnotes_in_disc;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Only sweep stale data and advance watermark on full success
|
// Only sweep stale data and advance watermark on full success
|
||||||
@@ -343,8 +346,6 @@ async fn ingest_discussions_for_mr(
|
|||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
result.discussions_fetched += 1;
|
|
||||||
|
|
||||||
// CRITICAL: Parse notes BEFORE any destructive DB operations
|
// CRITICAL: Parse notes BEFORE any destructive DB operations
|
||||||
let notes = match transform_notes_with_diff_position(&gitlab_discussion, local_project_id) {
|
let notes = match transform_notes_with_diff_position(&gitlab_discussion, local_project_id) {
|
||||||
Ok(notes) => notes,
|
Ok(notes) => notes,
|
||||||
@@ -361,11 +362,12 @@ async fn ingest_discussions_for_mr(
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
// Count DiffNotes
|
// Count DiffNotes upfront (independent of transaction)
|
||||||
result.diffnotes_count += notes
|
let diffnotes_in_disc = notes
|
||||||
.iter()
|
.iter()
|
||||||
.filter(|n| n.position_new_path.is_some() || n.position_old_path.is_some())
|
.filter(|n| n.position_new_path.is_some() || n.position_old_path.is_some())
|
||||||
.count();
|
.count();
|
||||||
|
let notes_count = notes.len();
|
||||||
|
|
||||||
// Transform discussion
|
// Transform discussion
|
||||||
let normalized_discussion =
|
let normalized_discussion =
|
||||||
@@ -389,7 +391,6 @@ async fn ingest_discussions_for_mr(
|
|||||||
|
|
||||||
// Upsert discussion with run_seen_at
|
// Upsert discussion with run_seen_at
|
||||||
upsert_discussion(&tx, &normalized_discussion, run_seen_at, payload_id)?;
|
upsert_discussion(&tx, &normalized_discussion, run_seen_at, payload_id)?;
|
||||||
result.discussions_upserted += 1;
|
|
||||||
|
|
||||||
// Get local discussion ID
|
// Get local discussion ID
|
||||||
let local_discussion_id: i64 = tx.query_row(
|
let local_discussion_id: i64 = tx.query_row(
|
||||||
@@ -433,10 +434,15 @@ async fn ingest_discussions_for_mr(
|
|||||||
};
|
};
|
||||||
|
|
||||||
upsert_note(&tx, local_discussion_id, note, run_seen_at, note_payload_id)?;
|
upsert_note(&tx, local_discussion_id, note, run_seen_at, note_payload_id)?;
|
||||||
result.notes_upserted += 1;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
tx.commit()?;
|
tx.commit()?;
|
||||||
|
|
||||||
|
// Increment counters AFTER successful commit to keep metrics honest
|
||||||
|
result.discussions_fetched += 1;
|
||||||
|
result.discussions_upserted += 1;
|
||||||
|
result.notes_upserted += notes_count;
|
||||||
|
result.diffnotes_count += diffnotes_in_disc;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Only sweep stale data and advance watermark on full success
|
// Only sweep stale data and advance watermark on full success
|
||||||
|
|||||||
648
src/main.rs
648
src/main.rs
@@ -18,7 +18,7 @@ use lore::cli::commands::{
|
|||||||
run_doctor, run_ingest, run_init, run_list_issues, run_list_mrs, run_show_issue, run_show_mr,
|
run_doctor, run_ingest, run_init, run_list_issues, run_list_mrs, run_show_issue, run_show_mr,
|
||||||
run_sync_status,
|
run_sync_status,
|
||||||
};
|
};
|
||||||
use lore::cli::{Cli, Commands};
|
use lore::cli::{Cli, Commands, CountArgs, IngestArgs, IssuesArgs, MrsArgs};
|
||||||
use lore::core::db::{create_connection, get_schema_version, run_migrations};
|
use lore::core::db::{create_connection, get_schema_version, run_migrations};
|
||||||
use lore::core::error::{GiError, RobotErrorOutput};
|
use lore::core::error::{GiError, RobotErrorOutput};
|
||||||
use lore::core::paths::get_config_path;
|
use lore::core::paths::get_config_path;
|
||||||
@@ -47,33 +47,25 @@ async fn main() {
|
|||||||
let robot_mode = cli.is_robot_mode();
|
let robot_mode = cli.is_robot_mode();
|
||||||
|
|
||||||
let result = match cli.command {
|
let result = match cli.command {
|
||||||
|
Commands::Issues(args) => handle_issues(cli.config.as_deref(), args, robot_mode).await,
|
||||||
|
Commands::Mrs(args) => handle_mrs(cli.config.as_deref(), args, robot_mode).await,
|
||||||
|
Commands::Ingest(args) => handle_ingest(cli.config.as_deref(), args, robot_mode).await,
|
||||||
|
Commands::Count(args) => {
|
||||||
|
handle_count(cli.config.as_deref(), args, robot_mode).await
|
||||||
|
}
|
||||||
|
Commands::Status => handle_sync_status_cmd(cli.config.as_deref(), robot_mode).await,
|
||||||
|
Commands::Auth => handle_auth_test(cli.config.as_deref(), robot_mode).await,
|
||||||
|
Commands::Doctor => handle_doctor(cli.config.as_deref(), robot_mode).await,
|
||||||
|
Commands::Version => handle_version(robot_mode),
|
||||||
Commands::Init {
|
Commands::Init {
|
||||||
force,
|
force,
|
||||||
non_interactive,
|
non_interactive,
|
||||||
} => handle_init(cli.config.as_deref(), force, non_interactive, robot_mode).await,
|
} => handle_init(cli.config.as_deref(), force, non_interactive, robot_mode).await,
|
||||||
Commands::AuthTest => handle_auth_test(cli.config.as_deref(), robot_mode).await,
|
|
||||||
Commands::Doctor { json } => handle_doctor(cli.config.as_deref(), json || robot_mode).await,
|
|
||||||
Commands::Version => handle_version(robot_mode),
|
|
||||||
Commands::Backup => handle_backup(robot_mode),
|
Commands::Backup => handle_backup(robot_mode),
|
||||||
Commands::Reset { confirm: _ } => handle_reset(robot_mode),
|
Commands::Reset { yes: _ } => handle_reset(robot_mode),
|
||||||
Commands::Migrate => handle_migrate(cli.config.as_deref(), robot_mode).await,
|
Commands::Migrate => handle_migrate(cli.config.as_deref(), robot_mode).await,
|
||||||
Commands::SyncStatus => handle_sync_status(cli.config.as_deref(), robot_mode).await,
|
|
||||||
Commands::Ingest {
|
// --- Backward-compat: deprecated aliases ---
|
||||||
r#type,
|
|
||||||
project,
|
|
||||||
force,
|
|
||||||
full,
|
|
||||||
} => {
|
|
||||||
handle_ingest(
|
|
||||||
cli.config.as_deref(),
|
|
||||||
&r#type,
|
|
||||||
project.as_deref(),
|
|
||||||
force,
|
|
||||||
full,
|
|
||||||
robot_mode,
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
}
|
|
||||||
Commands::List {
|
Commands::List {
|
||||||
entity,
|
entity,
|
||||||
limit,
|
limit,
|
||||||
@@ -89,14 +81,17 @@ async fn main() {
|
|||||||
sort,
|
sort,
|
||||||
order,
|
order,
|
||||||
open,
|
open,
|
||||||
json,
|
|
||||||
draft,
|
draft,
|
||||||
no_draft,
|
no_draft,
|
||||||
reviewer,
|
reviewer,
|
||||||
target_branch,
|
target_branch,
|
||||||
source_branch,
|
source_branch,
|
||||||
} => {
|
} => {
|
||||||
handle_list(
|
eprintln!(
|
||||||
|
"{}",
|
||||||
|
style("warning: 'lore list' is deprecated, use 'lore issues' or 'lore mrs'").yellow()
|
||||||
|
);
|
||||||
|
handle_list_compat(
|
||||||
cli.config.as_deref(),
|
cli.config.as_deref(),
|
||||||
&entity,
|
&entity,
|
||||||
limit,
|
limit,
|
||||||
@@ -112,7 +107,7 @@ async fn main() {
|
|||||||
&sort,
|
&sort,
|
||||||
&order,
|
&order,
|
||||||
open,
|
open,
|
||||||
json || robot_mode,
|
robot_mode,
|
||||||
draft,
|
draft,
|
||||||
no_draft,
|
no_draft,
|
||||||
reviewer.as_deref(),
|
reviewer.as_deref(),
|
||||||
@@ -121,24 +116,42 @@ async fn main() {
|
|||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
Commands::Count { entity, r#type } => {
|
|
||||||
handle_count(cli.config.as_deref(), &entity, r#type.as_deref(), robot_mode).await
|
|
||||||
}
|
|
||||||
Commands::Show {
|
Commands::Show {
|
||||||
entity,
|
entity,
|
||||||
iid,
|
iid,
|
||||||
project,
|
project,
|
||||||
json,
|
|
||||||
} => {
|
} => {
|
||||||
handle_show(
|
eprintln!(
|
||||||
|
"{}",
|
||||||
|
style(format!(
|
||||||
|
"warning: 'lore show' is deprecated, use 'lore {}s {}'",
|
||||||
|
entity, iid
|
||||||
|
))
|
||||||
|
.yellow()
|
||||||
|
);
|
||||||
|
handle_show_compat(
|
||||||
cli.config.as_deref(),
|
cli.config.as_deref(),
|
||||||
&entity,
|
&entity,
|
||||||
iid,
|
iid,
|
||||||
project.as_deref(),
|
project.as_deref(),
|
||||||
json || robot_mode,
|
robot_mode,
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
|
Commands::AuthTest => {
|
||||||
|
eprintln!(
|
||||||
|
"{}",
|
||||||
|
style("warning: 'lore auth-test' is deprecated, use 'lore auth'").yellow()
|
||||||
|
);
|
||||||
|
handle_auth_test(cli.config.as_deref(), robot_mode).await
|
||||||
|
}
|
||||||
|
Commands::SyncStatus => {
|
||||||
|
eprintln!(
|
||||||
|
"{}",
|
||||||
|
style("warning: 'lore sync-status' is deprecated, use 'lore status'").yellow()
|
||||||
|
);
|
||||||
|
handle_sync_status_cmd(cli.config.as_deref(), robot_mode).await
|
||||||
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
if let Err(e) = result {
|
if let Err(e) = result {
|
||||||
@@ -207,6 +220,259 @@ fn handle_error(e: Box<dyn std::error::Error>, robot_mode: bool) -> ! {
|
|||||||
std::process::exit(1);
|
std::process::exit(1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ============================================================================
|
||||||
|
// Primary command handlers
|
||||||
|
// ============================================================================
|
||||||
|
|
||||||
|
async fn handle_issues(
|
||||||
|
config_override: Option<&str>,
|
||||||
|
args: IssuesArgs,
|
||||||
|
robot_mode: bool,
|
||||||
|
) -> Result<(), Box<dyn std::error::Error>> {
|
||||||
|
let config = Config::load(config_override)?;
|
||||||
|
let order = if args.asc { "asc" } else { "desc" };
|
||||||
|
|
||||||
|
if let Some(iid) = args.iid {
|
||||||
|
// Show mode
|
||||||
|
let result = run_show_issue(&config, iid, args.project.as_deref())?;
|
||||||
|
if robot_mode {
|
||||||
|
print_show_issue_json(&result);
|
||||||
|
} else {
|
||||||
|
print_show_issue(&result);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// List mode
|
||||||
|
let filters = ListFilters {
|
||||||
|
limit: args.limit,
|
||||||
|
project: args.project.as_deref(),
|
||||||
|
state: args.state.as_deref(),
|
||||||
|
author: args.author.as_deref(),
|
||||||
|
assignee: args.assignee.as_deref(),
|
||||||
|
labels: args.label.as_deref(),
|
||||||
|
milestone: args.milestone.as_deref(),
|
||||||
|
since: args.since.as_deref(),
|
||||||
|
due_before: args.due_before.as_deref(),
|
||||||
|
has_due_date: args.has_due,
|
||||||
|
sort: &args.sort,
|
||||||
|
order,
|
||||||
|
};
|
||||||
|
|
||||||
|
let result = run_list_issues(&config, filters)?;
|
||||||
|
|
||||||
|
if args.open {
|
||||||
|
open_issue_in_browser(&result);
|
||||||
|
} else if robot_mode {
|
||||||
|
print_list_issues_json(&result);
|
||||||
|
} else {
|
||||||
|
print_list_issues(&result);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn handle_mrs(
|
||||||
|
config_override: Option<&str>,
|
||||||
|
args: MrsArgs,
|
||||||
|
robot_mode: bool,
|
||||||
|
) -> Result<(), Box<dyn std::error::Error>> {
|
||||||
|
let config = Config::load(config_override)?;
|
||||||
|
let order = if args.asc { "asc" } else { "desc" };
|
||||||
|
|
||||||
|
if let Some(iid) = args.iid {
|
||||||
|
// Show mode
|
||||||
|
let result = run_show_mr(&config, iid, args.project.as_deref())?;
|
||||||
|
if robot_mode {
|
||||||
|
print_show_mr_json(&result);
|
||||||
|
} else {
|
||||||
|
print_show_mr(&result);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// List mode
|
||||||
|
let filters = MrListFilters {
|
||||||
|
limit: args.limit,
|
||||||
|
project: args.project.as_deref(),
|
||||||
|
state: args.state.as_deref(),
|
||||||
|
author: args.author.as_deref(),
|
||||||
|
assignee: args.assignee.as_deref(),
|
||||||
|
reviewer: args.reviewer.as_deref(),
|
||||||
|
labels: args.label.as_deref(),
|
||||||
|
since: args.since.as_deref(),
|
||||||
|
draft: args.draft,
|
||||||
|
no_draft: args.no_draft,
|
||||||
|
target_branch: args.target.as_deref(),
|
||||||
|
source_branch: args.source.as_deref(),
|
||||||
|
sort: &args.sort,
|
||||||
|
order,
|
||||||
|
};
|
||||||
|
|
||||||
|
let result = run_list_mrs(&config, filters)?;
|
||||||
|
|
||||||
|
if args.open {
|
||||||
|
open_mr_in_browser(&result);
|
||||||
|
} else if robot_mode {
|
||||||
|
print_list_mrs_json(&result);
|
||||||
|
} else {
|
||||||
|
print_list_mrs(&result);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn handle_ingest(
|
||||||
|
config_override: Option<&str>,
|
||||||
|
args: IngestArgs,
|
||||||
|
robot_mode: bool,
|
||||||
|
) -> Result<(), Box<dyn std::error::Error>> {
|
||||||
|
let config = Config::load(config_override)?;
|
||||||
|
|
||||||
|
match args.entity.as_deref() {
|
||||||
|
Some(resource_type) => {
|
||||||
|
// Single entity ingest
|
||||||
|
let result = run_ingest(
|
||||||
|
&config,
|
||||||
|
resource_type,
|
||||||
|
args.project.as_deref(),
|
||||||
|
args.force,
|
||||||
|
args.full,
|
||||||
|
robot_mode,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
if robot_mode {
|
||||||
|
print_ingest_summary_json(&result);
|
||||||
|
} else {
|
||||||
|
print_ingest_summary(&result);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
None => {
|
||||||
|
// Ingest everything: issues then MRs
|
||||||
|
if !robot_mode {
|
||||||
|
println!(
|
||||||
|
"{}",
|
||||||
|
style("Ingesting all content (issues + merge requests)...").blue()
|
||||||
|
);
|
||||||
|
println!();
|
||||||
|
}
|
||||||
|
|
||||||
|
let issues_result = run_ingest(
|
||||||
|
&config,
|
||||||
|
"issues",
|
||||||
|
args.project.as_deref(),
|
||||||
|
args.force,
|
||||||
|
args.full,
|
||||||
|
robot_mode,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
let mrs_result = run_ingest(
|
||||||
|
&config,
|
||||||
|
"mrs",
|
||||||
|
args.project.as_deref(),
|
||||||
|
args.force,
|
||||||
|
args.full,
|
||||||
|
robot_mode,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
if robot_mode {
|
||||||
|
print_combined_ingest_json(&issues_result, &mrs_result);
|
||||||
|
} else {
|
||||||
|
print_ingest_summary(&issues_result);
|
||||||
|
print_ingest_summary(&mrs_result);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// JSON output for combined ingest (issues + mrs).
|
||||||
|
#[derive(Serialize)]
|
||||||
|
struct CombinedIngestOutput {
|
||||||
|
ok: bool,
|
||||||
|
data: CombinedIngestData,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
struct CombinedIngestData {
|
||||||
|
resource_type: String,
|
||||||
|
issues: CombinedIngestEntityStats,
|
||||||
|
merge_requests: CombinedIngestEntityStats,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
struct CombinedIngestEntityStats {
|
||||||
|
projects_synced: usize,
|
||||||
|
fetched: usize,
|
||||||
|
upserted: usize,
|
||||||
|
labels_created: usize,
|
||||||
|
discussions_fetched: usize,
|
||||||
|
notes_upserted: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn print_combined_ingest_json(
|
||||||
|
issues: &lore::cli::commands::ingest::IngestResult,
|
||||||
|
mrs: &lore::cli::commands::ingest::IngestResult,
|
||||||
|
) {
|
||||||
|
let output = CombinedIngestOutput {
|
||||||
|
ok: true,
|
||||||
|
data: CombinedIngestData {
|
||||||
|
resource_type: "all".to_string(),
|
||||||
|
issues: CombinedIngestEntityStats {
|
||||||
|
projects_synced: issues.projects_synced,
|
||||||
|
fetched: issues.issues_fetched,
|
||||||
|
upserted: issues.issues_upserted,
|
||||||
|
labels_created: issues.labels_created,
|
||||||
|
discussions_fetched: issues.discussions_fetched,
|
||||||
|
notes_upserted: issues.notes_upserted,
|
||||||
|
},
|
||||||
|
merge_requests: CombinedIngestEntityStats {
|
||||||
|
projects_synced: mrs.projects_synced,
|
||||||
|
fetched: mrs.mrs_fetched,
|
||||||
|
upserted: mrs.mrs_upserted,
|
||||||
|
labels_created: mrs.labels_created,
|
||||||
|
discussions_fetched: mrs.discussions_fetched,
|
||||||
|
notes_upserted: mrs.notes_upserted,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
println!("{}", serde_json::to_string(&output).unwrap());
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn handle_count(
|
||||||
|
config_override: Option<&str>,
|
||||||
|
args: CountArgs,
|
||||||
|
robot_mode: bool,
|
||||||
|
) -> Result<(), Box<dyn std::error::Error>> {
|
||||||
|
let config = Config::load(config_override)?;
|
||||||
|
|
||||||
|
let result = run_count(&config, &args.entity, args.for_entity.as_deref())?;
|
||||||
|
if robot_mode {
|
||||||
|
print_count_json(&result);
|
||||||
|
} else {
|
||||||
|
print_count(&result);
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn handle_sync_status_cmd(
|
||||||
|
config_override: Option<&str>,
|
||||||
|
robot_mode: bool,
|
||||||
|
) -> Result<(), Box<dyn std::error::Error>> {
|
||||||
|
let config = Config::load(config_override)?;
|
||||||
|
|
||||||
|
let result = run_sync_status(&config)?;
|
||||||
|
if robot_mode {
|
||||||
|
print_sync_status_json(&result);
|
||||||
|
} else {
|
||||||
|
print_sync_status(&result);
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
async fn handle_init(
|
async fn handle_init(
|
||||||
config_override: Option<&str>,
|
config_override: Option<&str>,
|
||||||
force: bool,
|
force: bool,
|
||||||
@@ -389,11 +655,11 @@ async fn handle_auth_test(
|
|||||||
|
|
||||||
async fn handle_doctor(
|
async fn handle_doctor(
|
||||||
config_override: Option<&str>,
|
config_override: Option<&str>,
|
||||||
json: bool,
|
robot_mode: bool,
|
||||||
) -> Result<(), Box<dyn std::error::Error>> {
|
) -> Result<(), Box<dyn std::error::Error>> {
|
||||||
let result = run_doctor(config_override).await;
|
let result = run_doctor(config_override).await;
|
||||||
|
|
||||||
if json {
|
if robot_mode {
|
||||||
println!("{}", serde_json::to_string_pretty(&result)?);
|
println!("{}", serde_json::to_string_pretty(&result)?);
|
||||||
} else {
|
} else {
|
||||||
print_doctor_results(&result);
|
print_doctor_results(&result);
|
||||||
@@ -406,191 +672,6 @@ async fn handle_doctor(
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn handle_ingest(
|
|
||||||
config_override: Option<&str>,
|
|
||||||
resource_type: &str,
|
|
||||||
project_filter: Option<&str>,
|
|
||||||
force: bool,
|
|
||||||
full: bool,
|
|
||||||
robot_mode: bool,
|
|
||||||
) -> Result<(), Box<dyn std::error::Error>> {
|
|
||||||
let config = Config::load(config_override)?;
|
|
||||||
|
|
||||||
match run_ingest(&config, resource_type, project_filter, force, full, robot_mode).await {
|
|
||||||
Ok(result) => {
|
|
||||||
if robot_mode {
|
|
||||||
print_ingest_summary_json(&result);
|
|
||||||
} else {
|
|
||||||
print_ingest_summary(&result);
|
|
||||||
}
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
Err(e) => {
|
|
||||||
eprintln!("{}", style(format!("Error: {e}")).red());
|
|
||||||
std::process::exit(1);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[allow(clippy::too_many_arguments)]
|
|
||||||
async fn handle_list(
|
|
||||||
config_override: Option<&str>,
|
|
||||||
entity: &str,
|
|
||||||
limit: usize,
|
|
||||||
project_filter: Option<&str>,
|
|
||||||
state_filter: Option<&str>,
|
|
||||||
author_filter: Option<&str>,
|
|
||||||
assignee_filter: Option<&str>,
|
|
||||||
label_filter: Option<&[String]>,
|
|
||||||
milestone_filter: Option<&str>,
|
|
||||||
since_filter: Option<&str>,
|
|
||||||
due_before_filter: Option<&str>,
|
|
||||||
has_due_date: bool,
|
|
||||||
sort: &str,
|
|
||||||
order: &str,
|
|
||||||
open_browser: bool,
|
|
||||||
json_output: bool,
|
|
||||||
draft: bool,
|
|
||||||
no_draft: bool,
|
|
||||||
reviewer_filter: Option<&str>,
|
|
||||||
target_branch_filter: Option<&str>,
|
|
||||||
source_branch_filter: Option<&str>,
|
|
||||||
) -> Result<(), Box<dyn std::error::Error>> {
|
|
||||||
let config = Config::load(config_override)?;
|
|
||||||
|
|
||||||
match entity {
|
|
||||||
"issues" => {
|
|
||||||
let filters = ListFilters {
|
|
||||||
limit,
|
|
||||||
project: project_filter,
|
|
||||||
state: state_filter,
|
|
||||||
author: author_filter,
|
|
||||||
assignee: assignee_filter,
|
|
||||||
labels: label_filter,
|
|
||||||
milestone: milestone_filter,
|
|
||||||
since: since_filter,
|
|
||||||
due_before: due_before_filter,
|
|
||||||
has_due_date,
|
|
||||||
sort,
|
|
||||||
order,
|
|
||||||
};
|
|
||||||
|
|
||||||
let result = run_list_issues(&config, filters)?;
|
|
||||||
|
|
||||||
if open_browser {
|
|
||||||
open_issue_in_browser(&result);
|
|
||||||
} else if json_output {
|
|
||||||
print_list_issues_json(&result);
|
|
||||||
} else {
|
|
||||||
print_list_issues(&result);
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
"mrs" => {
|
|
||||||
let filters = MrListFilters {
|
|
||||||
limit,
|
|
||||||
project: project_filter,
|
|
||||||
state: state_filter,
|
|
||||||
author: author_filter,
|
|
||||||
assignee: assignee_filter,
|
|
||||||
reviewer: reviewer_filter,
|
|
||||||
labels: label_filter,
|
|
||||||
since: since_filter,
|
|
||||||
draft,
|
|
||||||
no_draft,
|
|
||||||
target_branch: target_branch_filter,
|
|
||||||
source_branch: source_branch_filter,
|
|
||||||
sort,
|
|
||||||
order,
|
|
||||||
};
|
|
||||||
|
|
||||||
let result = run_list_mrs(&config, filters)?;
|
|
||||||
|
|
||||||
if open_browser {
|
|
||||||
open_mr_in_browser(&result);
|
|
||||||
} else if json_output {
|
|
||||||
print_list_mrs_json(&result);
|
|
||||||
} else {
|
|
||||||
print_list_mrs(&result);
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
_ => {
|
|
||||||
eprintln!("{}", style(format!("Unknown entity: {entity}")).red());
|
|
||||||
std::process::exit(1);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn handle_count(
|
|
||||||
config_override: Option<&str>,
|
|
||||||
entity: &str,
|
|
||||||
type_filter: Option<&str>,
|
|
||||||
robot_mode: bool,
|
|
||||||
) -> Result<(), Box<dyn std::error::Error>> {
|
|
||||||
let config = Config::load(config_override)?;
|
|
||||||
|
|
||||||
let result = run_count(&config, entity, type_filter)?;
|
|
||||||
if robot_mode {
|
|
||||||
print_count_json(&result);
|
|
||||||
} else {
|
|
||||||
print_count(&result);
|
|
||||||
}
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn handle_sync_status(
|
|
||||||
config_override: Option<&str>,
|
|
||||||
robot_mode: bool,
|
|
||||||
) -> Result<(), Box<dyn std::error::Error>> {
|
|
||||||
let config = Config::load(config_override)?;
|
|
||||||
|
|
||||||
let result = run_sync_status(&config)?;
|
|
||||||
if robot_mode {
|
|
||||||
print_sync_status_json(&result);
|
|
||||||
} else {
|
|
||||||
print_sync_status(&result);
|
|
||||||
}
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn handle_show(
|
|
||||||
config_override: Option<&str>,
|
|
||||||
entity: &str,
|
|
||||||
iid: i64,
|
|
||||||
project_filter: Option<&str>,
|
|
||||||
json: bool,
|
|
||||||
) -> Result<(), Box<dyn std::error::Error>> {
|
|
||||||
let config = Config::load(config_override)?;
|
|
||||||
|
|
||||||
match entity {
|
|
||||||
"issue" => {
|
|
||||||
let result = run_show_issue(&config, iid, project_filter)?;
|
|
||||||
if json {
|
|
||||||
print_show_issue_json(&result);
|
|
||||||
} else {
|
|
||||||
print_show_issue(&result);
|
|
||||||
}
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
"mr" => {
|
|
||||||
let result = run_show_mr(&config, iid, project_filter)?;
|
|
||||||
if json {
|
|
||||||
print_show_mr_json(&result);
|
|
||||||
} else {
|
|
||||||
print_show_mr(&result);
|
|
||||||
}
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
_ => {
|
|
||||||
eprintln!("{}", style(format!("Unknown entity: {entity}")).red());
|
|
||||||
std::process::exit(1);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// JSON output for version command.
|
/// JSON output for version command.
|
||||||
#[derive(Serialize)]
|
#[derive(Serialize)]
|
||||||
struct VersionOutput {
|
struct VersionOutput {
|
||||||
@@ -758,3 +839,134 @@ async fn handle_migrate(
|
|||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ============================================================================
|
||||||
|
// Backward-compat handlers (deprecated, delegate to new handlers)
|
||||||
|
// ============================================================================
|
||||||
|
|
||||||
|
#[allow(clippy::too_many_arguments)]
|
||||||
|
async fn handle_list_compat(
|
||||||
|
config_override: Option<&str>,
|
||||||
|
entity: &str,
|
||||||
|
limit: usize,
|
||||||
|
project_filter: Option<&str>,
|
||||||
|
state_filter: Option<&str>,
|
||||||
|
author_filter: Option<&str>,
|
||||||
|
assignee_filter: Option<&str>,
|
||||||
|
label_filter: Option<&[String]>,
|
||||||
|
milestone_filter: Option<&str>,
|
||||||
|
since_filter: Option<&str>,
|
||||||
|
due_before_filter: Option<&str>,
|
||||||
|
has_due_date: bool,
|
||||||
|
sort: &str,
|
||||||
|
order: &str,
|
||||||
|
open_browser: bool,
|
||||||
|
json_output: bool,
|
||||||
|
draft: bool,
|
||||||
|
no_draft: bool,
|
||||||
|
reviewer_filter: Option<&str>,
|
||||||
|
target_branch_filter: Option<&str>,
|
||||||
|
source_branch_filter: Option<&str>,
|
||||||
|
) -> Result<(), Box<dyn std::error::Error>> {
|
||||||
|
let config = Config::load(config_override)?;
|
||||||
|
|
||||||
|
match entity {
|
||||||
|
"issues" => {
|
||||||
|
let filters = ListFilters {
|
||||||
|
limit,
|
||||||
|
project: project_filter,
|
||||||
|
state: state_filter,
|
||||||
|
author: author_filter,
|
||||||
|
assignee: assignee_filter,
|
||||||
|
labels: label_filter,
|
||||||
|
milestone: milestone_filter,
|
||||||
|
since: since_filter,
|
||||||
|
due_before: due_before_filter,
|
||||||
|
has_due_date,
|
||||||
|
sort,
|
||||||
|
order,
|
||||||
|
};
|
||||||
|
|
||||||
|
let result = run_list_issues(&config, filters)?;
|
||||||
|
|
||||||
|
if open_browser {
|
||||||
|
open_issue_in_browser(&result);
|
||||||
|
} else if json_output {
|
||||||
|
print_list_issues_json(&result);
|
||||||
|
} else {
|
||||||
|
print_list_issues(&result);
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
"mrs" => {
|
||||||
|
let filters = MrListFilters {
|
||||||
|
limit,
|
||||||
|
project: project_filter,
|
||||||
|
state: state_filter,
|
||||||
|
author: author_filter,
|
||||||
|
assignee: assignee_filter,
|
||||||
|
reviewer: reviewer_filter,
|
||||||
|
labels: label_filter,
|
||||||
|
since: since_filter,
|
||||||
|
draft,
|
||||||
|
no_draft,
|
||||||
|
target_branch: target_branch_filter,
|
||||||
|
source_branch: source_branch_filter,
|
||||||
|
sort,
|
||||||
|
order,
|
||||||
|
};
|
||||||
|
|
||||||
|
let result = run_list_mrs(&config, filters)?;
|
||||||
|
|
||||||
|
if open_browser {
|
||||||
|
open_mr_in_browser(&result);
|
||||||
|
} else if json_output {
|
||||||
|
print_list_mrs_json(&result);
|
||||||
|
} else {
|
||||||
|
print_list_mrs(&result);
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
_ => {
|
||||||
|
eprintln!("{}", style(format!("Unknown entity: {entity}")).red());
|
||||||
|
std::process::exit(1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn handle_show_compat(
|
||||||
|
config_override: Option<&str>,
|
||||||
|
entity: &str,
|
||||||
|
iid: i64,
|
||||||
|
project_filter: Option<&str>,
|
||||||
|
json: bool,
|
||||||
|
) -> Result<(), Box<dyn std::error::Error>> {
|
||||||
|
let config = Config::load(config_override)?;
|
||||||
|
|
||||||
|
match entity {
|
||||||
|
"issue" => {
|
||||||
|
let result = run_show_issue(&config, iid, project_filter)?;
|
||||||
|
if json {
|
||||||
|
print_show_issue_json(&result);
|
||||||
|
} else {
|
||||||
|
print_show_issue(&result);
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
"mr" => {
|
||||||
|
let result = run_show_mr(&config, iid, project_filter)?;
|
||||||
|
if json {
|
||||||
|
print_show_mr_json(&result);
|
||||||
|
} else {
|
||||||
|
print_show_mr(&result);
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
_ => {
|
||||||
|
eprintln!("{}", style(format!("Unknown entity: {entity}")).red());
|
||||||
|
std::process::exit(1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user