feat(db): Add SQLite database migrations for GitLab data model
Implements a comprehensive relational schema for storing GitLab data with full audit trail and raw payload preservation. Migration 001_initial.sql establishes core metadata tables: - projects: Tracked GitLab projects with paths and namespace - sync_watermarks: Cursor-based incremental sync state per project - schema_migrations: Migration tracking with checksums for integrity Migration 002_issues.sql creates the issues data model: - issues: Core issue data with timestamps, author, state, counts - labels: Project-specific label definitions with colors/descriptions - issue_labels: Many-to-many junction for issue-label relationships - milestones: Project milestones with state and due dates - discussions: Threaded discussions linked to issues/MRs - notes: Individual notes within discussions with full metadata - raw_payloads: Compressed original API responses keyed by entity Migration 003_indexes.sql adds performance indexes: - Covering indexes for common query patterns (state, updated_at) - Composite indexes for filtered queries (project + state) Migration 004_discussions_payload.sql extends discussions: - Adds raw_payload column for discussion-level API preservation - Enables debugging and data recovery from original responses Migration 005_assignees_milestone_duedate.sql completes the model: - issue_assignees: Many-to-many for multiple assignees per issue - Adds milestone_id, due_date columns to issues table - Indexes for assignee and milestone filtering Schema supports both incremental sync and full historical queries. Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
105
migrations/002_issues.sql
Normal file
105
migrations/002_issues.sql
Normal file
@@ -0,0 +1,105 @@
|
||||
-- Migration 002: Issue Ingestion Tables
|
||||
-- Applies on top of 001_initial.sql
|
||||
|
||||
-- Issues table
|
||||
CREATE TABLE issues (
|
||||
id INTEGER PRIMARY KEY,
|
||||
gitlab_id INTEGER UNIQUE NOT NULL,
|
||||
project_id INTEGER NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
||||
iid INTEGER NOT NULL,
|
||||
title TEXT,
|
||||
description TEXT,
|
||||
state TEXT NOT NULL CHECK (state IN ('opened', 'closed')),
|
||||
author_username TEXT,
|
||||
created_at INTEGER NOT NULL, -- ms epoch UTC
|
||||
updated_at INTEGER NOT NULL, -- ms epoch UTC
|
||||
last_seen_at INTEGER NOT NULL, -- updated on every upsert
|
||||
discussions_synced_for_updated_at INTEGER, -- watermark for dependent sync
|
||||
web_url TEXT,
|
||||
raw_payload_id INTEGER REFERENCES raw_payloads(id)
|
||||
);
|
||||
|
||||
CREATE INDEX idx_issues_project_updated ON issues(project_id, updated_at);
|
||||
CREATE INDEX idx_issues_author ON issues(author_username);
|
||||
CREATE UNIQUE INDEX uq_issues_project_iid ON issues(project_id, iid);
|
||||
|
||||
-- Labels table (name-only for CP1)
|
||||
CREATE TABLE labels (
|
||||
id INTEGER PRIMARY KEY,
|
||||
gitlab_id INTEGER, -- optional, for future Labels API
|
||||
project_id INTEGER NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
||||
name TEXT NOT NULL,
|
||||
color TEXT,
|
||||
description TEXT
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX uq_labels_project_name ON labels(project_id, name);
|
||||
CREATE INDEX idx_labels_name ON labels(name);
|
||||
|
||||
-- Issue-label junction (DELETE before INSERT for stale removal)
|
||||
CREATE TABLE issue_labels (
|
||||
issue_id INTEGER NOT NULL REFERENCES issues(id) ON DELETE CASCADE,
|
||||
label_id INTEGER NOT NULL REFERENCES labels(id) ON DELETE CASCADE,
|
||||
PRIMARY KEY(issue_id, label_id)
|
||||
);
|
||||
|
||||
CREATE INDEX idx_issue_labels_label ON issue_labels(label_id);
|
||||
|
||||
-- Discussion threads for issues (MR discussions added in CP2)
|
||||
CREATE TABLE discussions (
|
||||
id INTEGER PRIMARY KEY,
|
||||
gitlab_discussion_id TEXT NOT NULL, -- GitLab string ID (e.g., "6a9c1750b37d...")
|
||||
project_id INTEGER NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
||||
issue_id INTEGER REFERENCES issues(id) ON DELETE CASCADE,
|
||||
merge_request_id INTEGER, -- FK added in CP2 via ALTER TABLE
|
||||
noteable_type TEXT NOT NULL CHECK (noteable_type IN ('Issue', 'MergeRequest')),
|
||||
individual_note INTEGER NOT NULL DEFAULT 0, -- 0=threaded, 1=standalone
|
||||
first_note_at INTEGER, -- min(note.created_at) for ordering
|
||||
last_note_at INTEGER, -- max(note.created_at) for "recently active"
|
||||
last_seen_at INTEGER NOT NULL, -- updated on every upsert
|
||||
resolvable INTEGER NOT NULL DEFAULT 0, -- MR discussions can be resolved
|
||||
resolved INTEGER NOT NULL DEFAULT 0,
|
||||
CHECK (
|
||||
(noteable_type = 'Issue' AND issue_id IS NOT NULL AND merge_request_id IS NULL) OR
|
||||
(noteable_type = 'MergeRequest' AND merge_request_id IS NOT NULL AND issue_id IS NULL)
|
||||
)
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX uq_discussions_project_discussion_id ON discussions(project_id, gitlab_discussion_id);
|
||||
CREATE INDEX idx_discussions_issue ON discussions(issue_id);
|
||||
CREATE INDEX idx_discussions_mr ON discussions(merge_request_id);
|
||||
CREATE INDEX idx_discussions_last_note ON discussions(last_note_at);
|
||||
|
||||
-- Notes belong to discussions
|
||||
CREATE TABLE notes (
|
||||
id INTEGER PRIMARY KEY,
|
||||
gitlab_id INTEGER UNIQUE NOT NULL,
|
||||
discussion_id INTEGER NOT NULL REFERENCES discussions(id) ON DELETE CASCADE,
|
||||
project_id INTEGER NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
||||
note_type TEXT, -- 'DiscussionNote' | 'DiffNote' | null
|
||||
is_system INTEGER NOT NULL DEFAULT 0, -- 1 for system-generated notes
|
||||
author_username TEXT,
|
||||
body TEXT,
|
||||
created_at INTEGER NOT NULL, -- ms epoch
|
||||
updated_at INTEGER NOT NULL, -- ms epoch
|
||||
last_seen_at INTEGER NOT NULL, -- updated on every upsert
|
||||
position INTEGER, -- 0-indexed array order from API
|
||||
resolvable INTEGER NOT NULL DEFAULT 0,
|
||||
resolved INTEGER NOT NULL DEFAULT 0,
|
||||
resolved_by TEXT,
|
||||
resolved_at INTEGER,
|
||||
-- DiffNote position metadata (populated for MR DiffNotes in CP2)
|
||||
position_old_path TEXT,
|
||||
position_new_path TEXT,
|
||||
position_old_line INTEGER,
|
||||
position_new_line INTEGER,
|
||||
raw_payload_id INTEGER REFERENCES raw_payloads(id)
|
||||
);
|
||||
|
||||
CREATE INDEX idx_notes_discussion ON notes(discussion_id);
|
||||
CREATE INDEX idx_notes_author ON notes(author_username);
|
||||
CREATE INDEX idx_notes_system ON notes(is_system);
|
||||
|
||||
-- Update schema version
|
||||
INSERT INTO schema_version (version, applied_at, description)
|
||||
VALUES (2, strftime('%s', 'now') * 1000, 'Issue ingestion tables');
|
||||
Reference in New Issue
Block a user