From ce5cd9c95d26447057244be2dd2489b3efe42cdc Mon Sep 17 00:00:00 2001 From: Taylor Eernisse Date: Tue, 3 Feb 2026 12:06:43 -0500 Subject: [PATCH] feat(schema): Add migration 011 for resource events, entity references, and dependent fetch queue Introduces five new tables that power temporal queries (timeline, file-history, trace) via GitLab Resource Events APIs: - resource_state_events: State transitions (opened/closed/reopened/merged/locked) with actor tracking, source commit, and source MR references - resource_label_events: Label add/remove history per entity - resource_milestone_events: Milestone assignment changes per entity - entity_references: Cross-reference table (Gate 2 prep) linking source/target entity pairs with reference type and discovery method - pending_dependent_fetches: Generic job queue for resource_events, mr_closes_issues, and mr_diffs with exponential backoff retry All event tables enforce entity exclusivity via CHECK constraints (exactly one of issue_id or merge_request_id must be non-NULL). Deduplication handled via UNIQUE indexes on (gitlab_id, project_id). FK cascades ensure cleanup when parent entities are removed. The dependent fetch queue uses a UNIQUE constraint on (project_id, entity_type, entity_iid, job_type) for idempotent enqueue, with partial indexes optimizing claim and retry queries. Registered as migration 011 in the embedded MIGRATIONS array in db.rs. Co-Authored-By: Claude Opus 4.5 --- migrations/011_resource_events.sql | 128 +++++++++++++++++++++++++++++ src/core/db.rs | 4 + 2 files changed, 132 insertions(+) create mode 100644 migrations/011_resource_events.sql diff --git a/migrations/011_resource_events.sql b/migrations/011_resource_events.sql new file mode 100644 index 0000000..43a277b --- /dev/null +++ b/migrations/011_resource_events.sql @@ -0,0 +1,128 @@ +-- Migration 011: Resource event tables, entity references, and dependent fetch queue +-- Powers temporal queries (timeline, file-history, trace) via GitLab Resource Events APIs. + +-- State change events (opened/closed/reopened/merged/locked) +CREATE TABLE resource_state_events ( + id INTEGER PRIMARY KEY, + gitlab_id INTEGER NOT NULL, + project_id INTEGER NOT NULL REFERENCES projects(id) ON DELETE CASCADE, + issue_id INTEGER REFERENCES issues(id) ON DELETE CASCADE, + merge_request_id INTEGER REFERENCES merge_requests(id) ON DELETE CASCADE, + state TEXT NOT NULL, + actor_gitlab_id INTEGER, + actor_username TEXT, + created_at INTEGER NOT NULL, -- ms epoch UTC + source_commit TEXT, + source_merge_request_iid INTEGER, -- iid from source_merge_request ref + CHECK ( + (issue_id IS NOT NULL AND merge_request_id IS NULL) OR + (issue_id IS NULL AND merge_request_id IS NOT NULL) + ) +); + +CREATE UNIQUE INDEX uq_state_events_gitlab ON resource_state_events(gitlab_id, project_id); +CREATE INDEX idx_state_events_issue ON resource_state_events(issue_id) WHERE issue_id IS NOT NULL; +CREATE INDEX idx_state_events_mr ON resource_state_events(merge_request_id) WHERE merge_request_id IS NOT NULL; +CREATE INDEX idx_state_events_created ON resource_state_events(created_at); + +-- Label change events (add/remove) +CREATE TABLE resource_label_events ( + id INTEGER PRIMARY KEY, + gitlab_id INTEGER NOT NULL, + project_id INTEGER NOT NULL REFERENCES projects(id) ON DELETE CASCADE, + issue_id INTEGER REFERENCES issues(id) ON DELETE CASCADE, + merge_request_id INTEGER REFERENCES merge_requests(id) ON DELETE CASCADE, + action TEXT NOT NULL CHECK (action IN ('add', 'remove')), + label_name TEXT NOT NULL, + actor_gitlab_id INTEGER, + actor_username TEXT, + created_at INTEGER NOT NULL, -- ms epoch UTC + CHECK ( + (issue_id IS NOT NULL AND merge_request_id IS NULL) OR + (issue_id IS NULL AND merge_request_id IS NOT NULL) + ) +); + +CREATE UNIQUE INDEX uq_label_events_gitlab ON resource_label_events(gitlab_id, project_id); +CREATE INDEX idx_label_events_issue ON resource_label_events(issue_id) WHERE issue_id IS NOT NULL; +CREATE INDEX idx_label_events_mr ON resource_label_events(merge_request_id) WHERE merge_request_id IS NOT NULL; +CREATE INDEX idx_label_events_created ON resource_label_events(created_at); + +-- Milestone change events (add/remove) +CREATE TABLE resource_milestone_events ( + id INTEGER PRIMARY KEY, + gitlab_id INTEGER NOT NULL, + project_id INTEGER NOT NULL REFERENCES projects(id) ON DELETE CASCADE, + issue_id INTEGER REFERENCES issues(id) ON DELETE CASCADE, + merge_request_id INTEGER REFERENCES merge_requests(id) ON DELETE CASCADE, + action TEXT NOT NULL CHECK (action IN ('add', 'remove')), + milestone_title TEXT NOT NULL, + milestone_id INTEGER, + actor_gitlab_id INTEGER, + actor_username TEXT, + created_at INTEGER NOT NULL, -- ms epoch UTC + CHECK ( + (issue_id IS NOT NULL AND merge_request_id IS NULL) OR + (issue_id IS NULL AND merge_request_id IS NOT NULL) + ) +); + +CREATE UNIQUE INDEX uq_milestone_events_gitlab ON resource_milestone_events(gitlab_id, project_id); +CREATE INDEX idx_milestone_events_issue ON resource_milestone_events(issue_id) WHERE issue_id IS NOT NULL; +CREATE INDEX idx_milestone_events_mr ON resource_milestone_events(merge_request_id) WHERE merge_request_id IS NOT NULL; +CREATE INDEX idx_milestone_events_created ON resource_milestone_events(created_at); + +-- Cross-reference table (Gate 2): source/target entity pairs +CREATE TABLE entity_references ( + id INTEGER PRIMARY KEY, + project_id INTEGER NOT NULL REFERENCES projects(id) ON DELETE CASCADE, + source_entity_type TEXT NOT NULL CHECK (source_entity_type IN ('issue', 'merge_request')), + source_entity_id INTEGER NOT NULL, -- local DB id + target_entity_type TEXT NOT NULL CHECK (target_entity_type IN ('issue', 'merge_request')), + target_entity_id INTEGER, -- local DB id (NULL if unresolved) + target_project_path TEXT, -- for unresolved cross-project refs + target_entity_iid INTEGER, -- for unresolved refs + reference_type TEXT NOT NULL CHECK (reference_type IN ('closes', 'mentioned', 'related')), + source_method TEXT NOT NULL CHECK (source_method IN ('api', 'note_parse', 'description_parse')), + created_at INTEGER NOT NULL -- ms epoch UTC +); + +CREATE UNIQUE INDEX uq_entity_refs ON entity_references( + project_id, + source_entity_type, + source_entity_id, + target_entity_type, + COALESCE(target_entity_id, -1), + COALESCE(target_project_path, ''), + COALESCE(target_entity_iid, -1), + reference_type, + source_method +); + +CREATE INDEX idx_entity_refs_source ON entity_references(source_entity_type, source_entity_id); +CREATE INDEX idx_entity_refs_target ON entity_references(target_entity_id) WHERE target_entity_id IS NOT NULL; +CREATE INDEX idx_entity_refs_unresolved ON entity_references(target_project_path, target_entity_iid) WHERE target_entity_id IS NULL; + +-- Generic dependent fetch queue (resource_events, mr_closes_issues, mr_diffs) +CREATE TABLE pending_dependent_fetches ( + id INTEGER PRIMARY KEY, + project_id INTEGER NOT NULL REFERENCES projects(id) ON DELETE CASCADE, + entity_type TEXT NOT NULL CHECK (entity_type IN ('issue', 'merge_request')), + entity_iid INTEGER NOT NULL, + entity_local_id INTEGER NOT NULL, + job_type TEXT NOT NULL CHECK (job_type IN ('resource_events', 'mr_closes_issues', 'mr_diffs')), + payload_json TEXT, -- optional extra data for the job + enqueued_at INTEGER NOT NULL, -- ms epoch UTC + locked_at INTEGER, -- ms epoch UTC (NULL = available) + attempts INTEGER NOT NULL DEFAULT 0, + next_retry_at INTEGER, -- ms epoch UTC (NULL = no backoff) + last_error TEXT +); + +CREATE UNIQUE INDEX uq_pending_fetches ON pending_dependent_fetches(project_id, entity_type, entity_iid, job_type); +CREATE INDEX idx_pending_fetches_claimable ON pending_dependent_fetches(job_type, locked_at) WHERE locked_at IS NULL; +CREATE INDEX idx_pending_fetches_retryable ON pending_dependent_fetches(next_retry_at) WHERE locked_at IS NULL AND next_retry_at IS NOT NULL; + +-- Update schema version +INSERT INTO schema_version (version, applied_at, description) +VALUES (11, strftime('%s', 'now') * 1000, 'Resource events, entity references, and dependent fetch queue'); diff --git a/src/core/db.rs b/src/core/db.rs index ad39bdf..2c6fc3c 100644 --- a/src/core/db.rs +++ b/src/core/db.rs @@ -47,6 +47,10 @@ const MIGRATIONS: &[(&str, &str)] = &[ "010", include_str!("../../migrations/010_chunk_config.sql"), ), + ( + "011", + include_str!("../../migrations/011_resource_events.sql"), + ), ]; /// Create a database connection with production-grade pragmas.