feat(core): Implement infrastructure layer for CLI operations
Establishes foundational modules that all other components depend on. src/core/config.rs - Configuration management: - JSON-based config file with Zod-like validation via serde - GitLab settings: base URL, token environment variable - Project list with paths to track - Sync settings: backfill days, stale lock timeout, cursor rewind - Storage settings: database path, payload compression toggle - XDG-compliant config path resolution via dirs crate - Loads GITLAB_TOKEN from configured environment variable src/core/db.rs - Database connection and migrations: - Opens or creates SQLite database with WAL mode for concurrency - Embeds migration SQL as const strings (001-005) - Runs migrations idempotently with checksum verification - Provides thread-safe connection management src/core/error.rs - Unified error handling: - GiError enum with variants for all failure modes - Config, Database, GitLab, Ingestion, Lock, IO, Parse errors - thiserror derive for automatic Display/Error impls - Result type alias for ergonomic error propagation src/core/lock.rs - Distributed sync locking: - File-based locks to prevent concurrent syncs - Stale lock detection with configurable timeout - Force override for recovery scenarios - Lock file contains PID and timestamp for debugging src/core/paths.rs - Path resolution: - XDG Base Directory Specification compliance - Config: ~/.config/gi/config.json - Data: ~/.local/share/gi/gi.db - Creates parent directories on first access src/core/payloads.rs - Raw payload storage: - Optional gzip compression for storage efficiency - SHA-256 content addressing for deduplication - Type-prefixed keys (issue:, discussion:, note:) - Batch insert with UPSERT for idempotent ingestion src/core/time.rs - Timestamp utilities: - Relative time parsing (7d, 2w, 1m) for --since flag - ISO 8601 date parsing for absolute dates - Human-friendly relative time formatting Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
215
src/core/payloads.rs
Normal file
215
src/core/payloads.rs
Normal file
@@ -0,0 +1,215 @@
|
||||
//! Raw payload storage with optional compression and deduplication.
|
||||
|
||||
use flate2::Compression;
|
||||
use flate2::read::GzDecoder;
|
||||
use flate2::write::GzEncoder;
|
||||
use rusqlite::Connection;
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::io::{Read, Write};
|
||||
|
||||
use super::error::Result;
|
||||
use super::time::now_ms;
|
||||
|
||||
/// Options for storing a payload.
|
||||
pub struct StorePayloadOptions<'a> {
|
||||
pub project_id: Option<i64>,
|
||||
pub resource_type: &'a str, // 'project' | 'issue' | 'mr' | 'note' | 'discussion'
|
||||
pub gitlab_id: &'a str, // TEXT because discussion IDs are strings
|
||||
pub payload: &'a serde_json::Value,
|
||||
pub compress: bool,
|
||||
}
|
||||
|
||||
/// Store a raw API payload with optional compression and deduplication.
|
||||
/// Returns the row ID (either new or existing if duplicate).
|
||||
pub fn store_payload(conn: &Connection, options: StorePayloadOptions) -> Result<i64> {
|
||||
// 1. JSON stringify the payload
|
||||
let json_bytes = serde_json::to_vec(options.payload)?;
|
||||
|
||||
// 2. SHA-256 hash the JSON bytes (pre-compression)
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(&json_bytes);
|
||||
let payload_hash = format!("{:x}", hasher.finalize());
|
||||
|
||||
// 3. Check for duplicate by (project_id, resource_type, gitlab_id, payload_hash)
|
||||
let existing: Option<i64> = conn
|
||||
.query_row(
|
||||
"SELECT id FROM raw_payloads
|
||||
WHERE project_id IS ? AND resource_type = ? AND gitlab_id = ? AND payload_hash = ?",
|
||||
(
|
||||
options.project_id,
|
||||
options.resource_type,
|
||||
options.gitlab_id,
|
||||
&payload_hash,
|
||||
),
|
||||
|row| row.get(0),
|
||||
)
|
||||
.ok();
|
||||
|
||||
// 4. If duplicate, return existing ID
|
||||
if let Some(id) = existing {
|
||||
return Ok(id);
|
||||
}
|
||||
|
||||
// 5. Compress if requested
|
||||
let (encoding, payload_bytes) = if options.compress {
|
||||
let mut encoder = GzEncoder::new(Vec::new(), Compression::default());
|
||||
encoder.write_all(&json_bytes)?;
|
||||
("gzip", encoder.finish()?)
|
||||
} else {
|
||||
("identity", json_bytes)
|
||||
};
|
||||
|
||||
// 6. INSERT with content_encoding
|
||||
conn.execute(
|
||||
"INSERT INTO raw_payloads
|
||||
(source, project_id, resource_type, gitlab_id, fetched_at, content_encoding, payload_hash, payload)
|
||||
VALUES ('gitlab', ?, ?, ?, ?, ?, ?, ?)",
|
||||
(
|
||||
options.project_id,
|
||||
options.resource_type,
|
||||
options.gitlab_id,
|
||||
now_ms(),
|
||||
encoding,
|
||||
&payload_hash,
|
||||
&payload_bytes,
|
||||
),
|
||||
)?;
|
||||
|
||||
Ok(conn.last_insert_rowid())
|
||||
}
|
||||
|
||||
/// Read a raw payload by ID, decompressing if necessary.
|
||||
/// Returns None if not found.
|
||||
pub fn read_payload(conn: &Connection, id: i64) -> Result<Option<serde_json::Value>> {
|
||||
let row: Option<(String, Vec<u8>)> = conn
|
||||
.query_row(
|
||||
"SELECT content_encoding, payload FROM raw_payloads WHERE id = ?",
|
||||
[id],
|
||||
|row| Ok((row.get(0)?, row.get(1)?)),
|
||||
)
|
||||
.ok();
|
||||
|
||||
let Some((encoding, payload_bytes)) = row else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
// Decompress if needed
|
||||
let json_bytes = if encoding == "gzip" {
|
||||
let mut decoder = GzDecoder::new(&payload_bytes[..]);
|
||||
let mut decompressed = Vec::new();
|
||||
decoder.read_to_end(&mut decompressed)?;
|
||||
decompressed
|
||||
} else {
|
||||
payload_bytes
|
||||
};
|
||||
|
||||
let value: serde_json::Value = serde_json::from_slice(&json_bytes)?;
|
||||
Ok(Some(value))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::core::db::create_connection;
|
||||
use tempfile::tempdir;
|
||||
|
||||
fn setup_test_db() -> Connection {
|
||||
let dir = tempdir().unwrap();
|
||||
let db_path = dir.path().join("test.db");
|
||||
let conn = create_connection(&db_path).unwrap();
|
||||
|
||||
// Create minimal schema for testing
|
||||
conn.execute_batch(
|
||||
"CREATE TABLE raw_payloads (
|
||||
id INTEGER PRIMARY KEY,
|
||||
source TEXT NOT NULL,
|
||||
project_id INTEGER,
|
||||
resource_type TEXT NOT NULL,
|
||||
gitlab_id TEXT NOT NULL,
|
||||
fetched_at INTEGER NOT NULL,
|
||||
content_encoding TEXT NOT NULL DEFAULT 'identity',
|
||||
payload_hash TEXT NOT NULL,
|
||||
payload BLOB NOT NULL
|
||||
);
|
||||
CREATE UNIQUE INDEX uq_raw_payloads_dedupe
|
||||
ON raw_payloads(project_id, resource_type, gitlab_id, payload_hash);",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
conn
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_store_and_read_payload() {
|
||||
let conn = setup_test_db();
|
||||
let payload = serde_json::json!({"title": "Test Issue", "id": 123});
|
||||
|
||||
let id = store_payload(
|
||||
&conn,
|
||||
StorePayloadOptions {
|
||||
project_id: Some(1),
|
||||
resource_type: "issue",
|
||||
gitlab_id: "123",
|
||||
payload: &payload,
|
||||
compress: false,
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let result = read_payload(&conn, id).unwrap().unwrap();
|
||||
assert_eq!(result["title"], "Test Issue");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_compression_roundtrip() {
|
||||
let conn = setup_test_db();
|
||||
let payload = serde_json::json!({"data": "x".repeat(1000)});
|
||||
|
||||
let id = store_payload(
|
||||
&conn,
|
||||
StorePayloadOptions {
|
||||
project_id: Some(1),
|
||||
resource_type: "issue",
|
||||
gitlab_id: "456",
|
||||
payload: &payload,
|
||||
compress: true,
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let result = read_payload(&conn, id).unwrap().unwrap();
|
||||
assert_eq!(result["data"], "x".repeat(1000));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_deduplication() {
|
||||
let conn = setup_test_db();
|
||||
let payload = serde_json::json!({"id": 789});
|
||||
|
||||
let id1 = store_payload(
|
||||
&conn,
|
||||
StorePayloadOptions {
|
||||
project_id: Some(1),
|
||||
resource_type: "issue",
|
||||
gitlab_id: "789",
|
||||
payload: &payload,
|
||||
compress: false,
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let id2 = store_payload(
|
||||
&conn,
|
||||
StorePayloadOptions {
|
||||
project_id: Some(1),
|
||||
resource_type: "issue",
|
||||
gitlab_id: "789",
|
||||
payload: &payload,
|
||||
compress: false,
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(id1, id2); // Same payload returns same ID
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user