theirongolddev/gitlore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(events): Implement Gate 1 resource events infrastructure

Browse Source 
Add complete infrastructure for ingesting GitLab Resource Events
(state, label, milestone) into local SQLite tables. This enables
temporal queries (timeline, file-history, trace) in later gates.

- Add migration 011: resource_state/label/milestone_events tables,
  entity_references table, pending_dependent_fetches queue
- Add 6 serde types for GitLab Resource Events API responses
- Add fetchResourceEvents config flag with --no-events CLI override
- Add fetch_all_pages<T> generic paginator and 6 API endpoint methods
- Add DB upsert functions with savepoint atomicity (events_db.rs)
- Add dependent fetch queue with exponential backoff (dependent_queue.rs)
- Add 'lore count events' command with human table and robot JSON output
- Extend 'lore stats --check' with event FK integrity and queue health
- Add 8 unit tests for resource event type deserialization

Closes: bd-hu3, bd-2e8, bd-2fm, bd-sqw, bd-1uc, bd-tir, bd-3sh, bd-1m8

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Taylor Eernisse
2026-02-03 11:23:44 -05:00
parent 549a0646d7
commit 98907ac666
18 changed files with 1227 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

128
migrations/011_resource_events.sql Normal file
View File

@@ -0,0 +1,128 @@
-- Migration 011: Resource event tables, entity references, and dependent fetch queue
-- Powers temporal queries (timeline, file-history, trace) via GitLab Resource Events APIs.
-- State change events (opened/closed/reopened/merged/locked)
CREATE TABLE resource_state_events (
id INTEGER PRIMARY KEY,
gitlab_id INTEGER NOT NULL,
project_id INTEGER NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
issue_id INTEGER REFERENCES issues(id) ON DELETE CASCADE,
merge_request_id INTEGER REFERENCES merge_requests(id) ON DELETE CASCADE,
state TEXT NOT NULL,
actor_gitlab_id INTEGER,
actor_username TEXT,
created_at INTEGER NOT NULL, -- ms epoch UTC
source_commit TEXT,
source_merge_request_iid INTEGER, -- iid from source_merge_request ref
CHECK (
(issue_id IS NOT NULL AND merge_request_id IS NULL) OR
(issue_id IS NULL AND merge_request_id IS NOT NULL)
)
);
CREATE UNIQUE INDEX uq_state_events_gitlab ON resource_state_events(gitlab_id, project_id);
CREATE INDEX idx_state_events_issue ON resource_state_events(issue_id) WHERE issue_id IS NOT NULL;
CREATE INDEX idx_state_events_mr ON resource_state_events(merge_request_id) WHERE merge_request_id IS NOT NULL;
CREATE INDEX idx_state_events_created ON resource_state_events(created_at);
-- Label change events (add/remove)
CREATE TABLE resource_label_events (
id INTEGER PRIMARY KEY,
gitlab_id INTEGER NOT NULL,
project_id INTEGER NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
issue_id INTEGER REFERENCES issues(id) ON DELETE CASCADE,
merge_request_id INTEGER REFERENCES merge_requests(id) ON DELETE CASCADE,
action TEXT NOT NULL CHECK (action IN ('add', 'remove')),
label_name TEXT NOT NULL,
actor_gitlab_id INTEGER,
actor_username TEXT,
created_at INTEGER NOT NULL, -- ms epoch UTC
CHECK (
(issue_id IS NOT NULL AND merge_request_id IS NULL) OR
(issue_id IS NULL AND merge_request_id IS NOT NULL)
)
);
CREATE UNIQUE INDEX uq_label_events_gitlab ON resource_label_events(gitlab_id, project_id);
CREATE INDEX idx_label_events_issue ON resource_label_events(issue_id) WHERE issue_id IS NOT NULL;
CREATE INDEX idx_label_events_mr ON resource_label_events(merge_request_id) WHERE merge_request_id IS NOT NULL;
CREATE INDEX idx_label_events_created ON resource_label_events(created_at);
-- Milestone change events (add/remove)
CREATE TABLE resource_milestone_events (
id INTEGER PRIMARY KEY,
gitlab_id INTEGER NOT NULL,
project_id INTEGER NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
issue_id INTEGER REFERENCES issues(id) ON DELETE CASCADE,
merge_request_id INTEGER REFERENCES merge_requests(id) ON DELETE CASCADE,
action TEXT NOT NULL CHECK (action IN ('add', 'remove')),
milestone_title TEXT NOT NULL,
milestone_id INTEGER,
actor_gitlab_id INTEGER,
actor_username TEXT,
created_at INTEGER NOT NULL, -- ms epoch UTC
CHECK (
(issue_id IS NOT NULL AND merge_request_id IS NULL) OR
(issue_id IS NULL AND merge_request_id IS NOT NULL)
)
);
CREATE UNIQUE INDEX uq_milestone_events_gitlab ON resource_milestone_events(gitlab_id, project_id);
CREATE INDEX idx_milestone_events_issue ON resource_milestone_events(issue_id) WHERE issue_id IS NOT NULL;
CREATE INDEX idx_milestone_events_mr ON resource_milestone_events(merge_request_id) WHERE merge_request_id IS NOT NULL;
CREATE INDEX idx_milestone_events_created ON resource_milestone_events(created_at);
-- Cross-reference table (Gate 2): source/target entity pairs
CREATE TABLE entity_references (
id INTEGER PRIMARY KEY,
project_id INTEGER NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
source_entity_type TEXT NOT NULL CHECK (source_entity_type IN ('issue', 'merge_request')),
source_entity_id INTEGER NOT NULL, -- local DB id
target_entity_type TEXT NOT NULL CHECK (target_entity_type IN ('issue', 'merge_request')),
target_entity_id INTEGER, -- local DB id (NULL if unresolved)
target_project_path TEXT, -- for unresolved cross-project refs
target_entity_iid INTEGER, -- for unresolved refs
reference_type TEXT NOT NULL CHECK (reference_type IN ('closes', 'mentioned', 'related')),
source_method TEXT NOT NULL CHECK (source_method IN ('api', 'note_parse', 'description_parse')),
created_at INTEGER NOT NULL -- ms epoch UTC
);
CREATE UNIQUE INDEX uq_entity_refs ON entity_references(
project_id,
source_entity_type,
source_entity_id,
target_entity_type,
COALESCE(target_entity_id, -1),
COALESCE(target_project_path, ''),
COALESCE(target_entity_iid, -1),
reference_type,
source_method
);
CREATE INDEX idx_entity_refs_source ON entity_references(source_entity_type, source_entity_id);
CREATE INDEX idx_entity_refs_target ON entity_references(target_entity_id) WHERE target_entity_id IS NOT NULL;
CREATE INDEX idx_entity_refs_unresolved ON entity_references(target_project_path, target_entity_iid) WHERE target_entity_id IS NULL;
-- Generic dependent fetch queue (resource_events, mr_closes_issues, mr_diffs)
CREATE TABLE pending_dependent_fetches (
id INTEGER PRIMARY KEY,
project_id INTEGER NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
entity_type TEXT NOT NULL CHECK (entity_type IN ('issue', 'merge_request')),
entity_iid INTEGER NOT NULL,
entity_local_id INTEGER NOT NULL,
job_type TEXT NOT NULL CHECK (job_type IN ('resource_events', 'mr_closes_issues', 'mr_diffs')),
payload_json TEXT, -- optional extra data for the job
enqueued_at INTEGER NOT NULL, -- ms epoch UTC
locked_at INTEGER, -- ms epoch UTC (NULL = available)
attempts INTEGER NOT NULL DEFAULT 0,
next_retry_at INTEGER, -- ms epoch UTC (NULL = no backoff)
last_error TEXT
);
CREATE UNIQUE INDEX uq_pending_fetches ON pending_dependent_fetches(project_id, entity_type, entity_iid, job_type);
CREATE INDEX idx_pending_fetches_claimable ON pending_dependent_fetches(job_type, locked_at) WHERE locked_at IS NULL;
CREATE INDEX idx_pending_fetches_retryable ON pending_dependent_fetches(next_retry_at) WHERE locked_at IS NULL AND next_retry_at IS NOT NULL;
-- Update schema version
INSERT INTO schema_version (version, applied_at, description)
VALUES (11, strftime('%s', 'now') * 1000, 'Resource events, entity references, and dependent fetch queue');