feat(core): Implement infrastructure layer for CLI operations

Establishes foundational modules that all other components depend on.

src/core/config.rs - Configuration management:
- JSON-based config file with Zod-like validation via serde
- GitLab settings: base URL, token environment variable
- Project list with paths to track
- Sync settings: backfill days, stale lock timeout, cursor rewind
- Storage settings: database path, payload compression toggle
- XDG-compliant config path resolution via dirs crate
- Loads GITLAB_TOKEN from configured environment variable

src/core/db.rs - Database connection and migrations:
- Opens or creates SQLite database with WAL mode for concurrency
- Embeds migration SQL as const strings (001-005)
- Runs migrations idempotently with checksum verification
- Provides thread-safe connection management

src/core/error.rs - Unified error handling:
- GiError enum with variants for all failure modes
- Config, Database, GitLab, Ingestion, Lock, IO, Parse errors
- thiserror derive for automatic Display/Error impls
- Result type alias for ergonomic error propagation

src/core/lock.rs - Distributed sync locking:
- File-based locks to prevent concurrent syncs
- Stale lock detection with configurable timeout
- Force override for recovery scenarios
- Lock file contains PID and timestamp for debugging

src/core/paths.rs - Path resolution:
- XDG Base Directory Specification compliance
- Config: ~/.config/gi/config.json
- Data: ~/.local/share/gi/gi.db
- Creates parent directories on first access

src/core/payloads.rs - Raw payload storage:
- Optional gzip compression for storage efficiency
- SHA-256 content addressing for deduplication
- Type-prefixed keys (issue:, discussion:, note:)
- Batch insert with UPSERT for idempotent ingestion

src/core/time.rs - Timestamp utilities:
- Relative time parsing (7d, 2w, 1m) for --since flag
- ISO 8601 date parsing for absolute dates
- Human-friendly relative time formatting

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Taylor Eernisse
2026-01-26 11:28:07 -05:00
parent d15f457a58
commit 7aaa51f645
8 changed files with 1244 additions and 0 deletions

215
src/core/payloads.rs Normal file
View File

@@ -0,0 +1,215 @@
//! Raw payload storage with optional compression and deduplication.
use flate2::Compression;
use flate2::read::GzDecoder;
use flate2::write::GzEncoder;
use rusqlite::Connection;
use sha2::{Digest, Sha256};
use std::io::{Read, Write};
use super::error::Result;
use super::time::now_ms;
/// Options for storing a payload.
pub struct StorePayloadOptions<'a> {
pub project_id: Option<i64>,
pub resource_type: &'a str, // 'project' | 'issue' | 'mr' | 'note' | 'discussion'
pub gitlab_id: &'a str, // TEXT because discussion IDs are strings
pub payload: &'a serde_json::Value,
pub compress: bool,
}
/// Store a raw API payload with optional compression and deduplication.
/// Returns the row ID (either new or existing if duplicate).
pub fn store_payload(conn: &Connection, options: StorePayloadOptions) -> Result<i64> {
// 1. JSON stringify the payload
let json_bytes = serde_json::to_vec(options.payload)?;
// 2. SHA-256 hash the JSON bytes (pre-compression)
let mut hasher = Sha256::new();
hasher.update(&json_bytes);
let payload_hash = format!("{:x}", hasher.finalize());
// 3. Check for duplicate by (project_id, resource_type, gitlab_id, payload_hash)
let existing: Option<i64> = conn
.query_row(
"SELECT id FROM raw_payloads
WHERE project_id IS ? AND resource_type = ? AND gitlab_id = ? AND payload_hash = ?",
(
options.project_id,
options.resource_type,
options.gitlab_id,
&payload_hash,
),
|row| row.get(0),
)
.ok();
// 4. If duplicate, return existing ID
if let Some(id) = existing {
return Ok(id);
}
// 5. Compress if requested
let (encoding, payload_bytes) = if options.compress {
let mut encoder = GzEncoder::new(Vec::new(), Compression::default());
encoder.write_all(&json_bytes)?;
("gzip", encoder.finish()?)
} else {
("identity", json_bytes)
};
// 6. INSERT with content_encoding
conn.execute(
"INSERT INTO raw_payloads
(source, project_id, resource_type, gitlab_id, fetched_at, content_encoding, payload_hash, payload)
VALUES ('gitlab', ?, ?, ?, ?, ?, ?, ?)",
(
options.project_id,
options.resource_type,
options.gitlab_id,
now_ms(),
encoding,
&payload_hash,
&payload_bytes,
),
)?;
Ok(conn.last_insert_rowid())
}
/// Read a raw payload by ID, decompressing if necessary.
/// Returns None if not found.
pub fn read_payload(conn: &Connection, id: i64) -> Result<Option<serde_json::Value>> {
let row: Option<(String, Vec<u8>)> = conn
.query_row(
"SELECT content_encoding, payload FROM raw_payloads WHERE id = ?",
[id],
|row| Ok((row.get(0)?, row.get(1)?)),
)
.ok();
let Some((encoding, payload_bytes)) = row else {
return Ok(None);
};
// Decompress if needed
let json_bytes = if encoding == "gzip" {
let mut decoder = GzDecoder::new(&payload_bytes[..]);
let mut decompressed = Vec::new();
decoder.read_to_end(&mut decompressed)?;
decompressed
} else {
payload_bytes
};
let value: serde_json::Value = serde_json::from_slice(&json_bytes)?;
Ok(Some(value))
}
#[cfg(test)]
mod tests {
use super::*;
use crate::core::db::create_connection;
use tempfile::tempdir;
fn setup_test_db() -> Connection {
let dir = tempdir().unwrap();
let db_path = dir.path().join("test.db");
let conn = create_connection(&db_path).unwrap();
// Create minimal schema for testing
conn.execute_batch(
"CREATE TABLE raw_payloads (
id INTEGER PRIMARY KEY,
source TEXT NOT NULL,
project_id INTEGER,
resource_type TEXT NOT NULL,
gitlab_id TEXT NOT NULL,
fetched_at INTEGER NOT NULL,
content_encoding TEXT NOT NULL DEFAULT 'identity',
payload_hash TEXT NOT NULL,
payload BLOB NOT NULL
);
CREATE UNIQUE INDEX uq_raw_payloads_dedupe
ON raw_payloads(project_id, resource_type, gitlab_id, payload_hash);",
)
.unwrap();
conn
}
#[test]
fn test_store_and_read_payload() {
let conn = setup_test_db();
let payload = serde_json::json!({"title": "Test Issue", "id": 123});
let id = store_payload(
&conn,
StorePayloadOptions {
project_id: Some(1),
resource_type: "issue",
gitlab_id: "123",
payload: &payload,
compress: false,
},
)
.unwrap();
let result = read_payload(&conn, id).unwrap().unwrap();
assert_eq!(result["title"], "Test Issue");
}
#[test]
fn test_compression_roundtrip() {
let conn = setup_test_db();
let payload = serde_json::json!({"data": "x".repeat(1000)});
let id = store_payload(
&conn,
StorePayloadOptions {
project_id: Some(1),
resource_type: "issue",
gitlab_id: "456",
payload: &payload,
compress: true,
},
)
.unwrap();
let result = read_payload(&conn, id).unwrap().unwrap();
assert_eq!(result["data"], "x".repeat(1000));
}
#[test]
fn test_deduplication() {
let conn = setup_test_db();
let payload = serde_json::json!({"id": 789});
let id1 = store_payload(
&conn,
StorePayloadOptions {
project_id: Some(1),
resource_type: "issue",
gitlab_id: "789",
payload: &payload,
compress: false,
},
)
.unwrap();
let id2 = store_payload(
&conn,
StorePayloadOptions {
project_id: Some(1),
resource_type: "issue",
gitlab_id: "789",
payload: &payload,
compress: false,
},
)
.unwrap();
assert_eq!(id1, id2); // Same payload returns same ID
}
}