feat(db): Add migrations for documents, FTS5, and embeddings
Three new migrations establish the search infrastructure: - 007_documents: Creates the `documents` table as the central search unit. Each document is a rendered text blob derived from an issue, MR, or discussion. Includes `dirty_queue` table for tracking which entities need document regeneration after ingestion changes. - 008_fts5: Creates FTS5 virtual table `documents_fts` with content sync triggers. Uses `unicode61` tokenizer with `remove_diacritics=2` for broad language support. Automatic insert/update/delete triggers keep the FTS index synchronized with the documents table. - 009_embeddings: Creates `embeddings` table for storing vector chunks produced by Ollama. Uses `doc_id * 1000 + chunk_index` rowid encoding to support multi-chunk documents while enabling efficient doc-level deduplication in vector search results. Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Taylor Eernisse
84
migrations/007_documents.sql
Normal file
84
migrations/007_documents.sql
Normal file
@@ -0,0 +1,84 @@
|
||||
-- Migration 007: Documents, Document Labels, Document Paths, Dirty Sources, Pending Discussion Fetches
|
||||
-- Schema version: 7
|
||||
-- Adds CP3 document storage and queue tables for search pipeline
|
||||
|
||||
-- Unified searchable documents (derived from issues/MRs/discussions)
|
||||
CREATE TABLE documents (
|
||||
id INTEGER PRIMARY KEY,
|
||||
source_type TEXT NOT NULL CHECK (source_type IN ('issue','merge_request','discussion')),
|
||||
source_id INTEGER NOT NULL, -- local DB id in the source table
|
||||
project_id INTEGER NOT NULL REFERENCES projects(id),
|
||||
author_username TEXT, -- for discussions: first note author
|
||||
label_names TEXT, -- JSON array (display/debug only)
|
||||
created_at INTEGER, -- ms epoch UTC
|
||||
updated_at INTEGER, -- ms epoch UTC
|
||||
url TEXT,
|
||||
title TEXT, -- null for discussions
|
||||
content_text TEXT NOT NULL, -- canonical text for embedding/search
|
||||
content_hash TEXT NOT NULL, -- SHA-256 for change detection
|
||||
labels_hash TEXT NOT NULL DEFAULT '', -- SHA-256 over sorted labels (write optimization)
|
||||
paths_hash TEXT NOT NULL DEFAULT '', -- SHA-256 over sorted paths (write optimization)
|
||||
is_truncated INTEGER NOT NULL DEFAULT 0,
|
||||
truncated_reason TEXT CHECK (
|
||||
truncated_reason IN (
|
||||
'token_limit_middle_drop','single_note_oversized','first_last_oversized',
|
||||
'hard_cap_oversized'
|
||||
)
|
||||
OR truncated_reason IS NULL
|
||||
),
|
||||
UNIQUE(source_type, source_id)
|
||||
);
|
||||
|
||||
CREATE INDEX idx_documents_project_updated ON documents(project_id, updated_at);
|
||||
CREATE INDEX idx_documents_author ON documents(author_username);
|
||||
CREATE INDEX idx_documents_source ON documents(source_type, source_id);
|
||||
CREATE INDEX idx_documents_hash ON documents(content_hash);
|
||||
|
||||
-- Fast label filtering (indexed exact-match)
|
||||
CREATE TABLE document_labels (
|
||||
document_id INTEGER NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
|
||||
label_name TEXT NOT NULL,
|
||||
PRIMARY KEY(document_id, label_name)
|
||||
) WITHOUT ROWID;
|
||||
CREATE INDEX idx_document_labels_label ON document_labels(label_name);
|
||||
|
||||
-- Fast path filtering (DiffNote file paths)
|
||||
CREATE TABLE document_paths (
|
||||
document_id INTEGER NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
|
||||
path TEXT NOT NULL,
|
||||
PRIMARY KEY(document_id, path)
|
||||
) WITHOUT ROWID;
|
||||
CREATE INDEX idx_document_paths_path ON document_paths(path);
|
||||
|
||||
-- Queue for incremental document regeneration (with retry tracking)
|
||||
-- Uses next_attempt_at for index-friendly backoff queries
|
||||
CREATE TABLE dirty_sources (
|
||||
source_type TEXT NOT NULL CHECK (source_type IN ('issue','merge_request','discussion')),
|
||||
source_id INTEGER NOT NULL,
|
||||
queued_at INTEGER NOT NULL, -- ms epoch UTC
|
||||
attempt_count INTEGER NOT NULL DEFAULT 0,
|
||||
last_attempt_at INTEGER,
|
||||
last_error TEXT,
|
||||
next_attempt_at INTEGER, -- ms epoch UTC; NULL means ready immediately
|
||||
PRIMARY KEY(source_type, source_id)
|
||||
);
|
||||
CREATE INDEX idx_dirty_sources_next_attempt ON dirty_sources(next_attempt_at);
|
||||
|
||||
-- Resumable queue for dependent discussion fetching
|
||||
-- Uses next_attempt_at for index-friendly backoff queries
|
||||
CREATE TABLE pending_discussion_fetches (
|
||||
project_id INTEGER NOT NULL REFERENCES projects(id),
|
||||
noteable_type TEXT NOT NULL, -- 'Issue' | 'MergeRequest'
|
||||
noteable_iid INTEGER NOT NULL,
|
||||
queued_at INTEGER NOT NULL, -- ms epoch UTC
|
||||
attempt_count INTEGER NOT NULL DEFAULT 0,
|
||||
last_attempt_at INTEGER,
|
||||
last_error TEXT,
|
||||
next_attempt_at INTEGER, -- ms epoch UTC; NULL means ready immediately
|
||||
PRIMARY KEY(project_id, noteable_type, noteable_iid)
|
||||
);
|
||||
CREATE INDEX idx_pending_discussions_next_attempt ON pending_discussion_fetches(next_attempt_at);
|
||||
|
||||
-- Update schema version
|
||||
INSERT INTO schema_version (version, applied_at, description)
|
||||
VALUES (7, strftime('%s', 'now') * 1000, 'Documents, labels, paths, dirty sources, pending discussion fetches');
|
||||
42
migrations/008_fts5.sql
Normal file
42
migrations/008_fts5.sql
Normal file
@@ -0,0 +1,42 @@
|
||||
-- Migration 008: FTS5 Full-Text Search Index
|
||||
-- Schema version: 8
|
||||
-- Adds full-text search on documents table with sync triggers
|
||||
|
||||
-- Full-text search with porter stemmer and prefix indexes for type-ahead
|
||||
CREATE VIRTUAL TABLE documents_fts USING fts5(
|
||||
title,
|
||||
content_text,
|
||||
content='documents',
|
||||
content_rowid='id',
|
||||
tokenize='porter unicode61',
|
||||
prefix='2 3 4'
|
||||
);
|
||||
|
||||
-- Keep FTS in sync via triggers.
|
||||
-- IMPORTANT: COALESCE(title, '') ensures FTS5 external-content table never
|
||||
-- receives NULL values, which can cause inconsistencies with delete operations.
|
||||
-- FTS5 delete requires exact match of original values; NULL != NULL in SQL,
|
||||
-- so a NULL title on insert would make the delete trigger fail silently.
|
||||
CREATE TRIGGER documents_ai AFTER INSERT ON documents BEGIN
|
||||
INSERT INTO documents_fts(rowid, title, content_text)
|
||||
VALUES (new.id, COALESCE(new.title, ''), new.content_text);
|
||||
END;
|
||||
|
||||
CREATE TRIGGER documents_ad AFTER DELETE ON documents BEGIN
|
||||
INSERT INTO documents_fts(documents_fts, rowid, title, content_text)
|
||||
VALUES('delete', old.id, COALESCE(old.title, ''), old.content_text);
|
||||
END;
|
||||
|
||||
-- Only rebuild FTS when searchable text actually changes (not metadata-only updates)
|
||||
CREATE TRIGGER documents_au AFTER UPDATE ON documents
|
||||
WHEN old.title IS NOT new.title OR old.content_text != new.content_text
|
||||
BEGIN
|
||||
INSERT INTO documents_fts(documents_fts, rowid, title, content_text)
|
||||
VALUES('delete', old.id, COALESCE(old.title, ''), old.content_text);
|
||||
INSERT INTO documents_fts(rowid, title, content_text)
|
||||
VALUES (new.id, COALESCE(new.title, ''), new.content_text);
|
||||
END;
|
||||
|
||||
-- Update schema version
|
||||
INSERT INTO schema_version (version, applied_at, description)
|
||||
VALUES (8, strftime('%s', 'now') * 1000, 'FTS5 full-text search index with sync triggers');
|
||||
54
migrations/009_embeddings.sql
Normal file
54
migrations/009_embeddings.sql
Normal file
@@ -0,0 +1,54 @@
|
||||
-- Migration 009: Embeddings (Gate B)
|
||||
-- Schema version: 9
|
||||
-- Adds sqlite-vec vector storage and embedding metadata for semantic search
|
||||
-- Requires sqlite-vec extension to be loaded before applying
|
||||
|
||||
-- NOTE: sqlite-vec vec0 virtual tables cannot participate in FK cascades.
|
||||
-- We must use an explicit trigger to delete orphan embeddings when documents
|
||||
-- are deleted. See documents_embeddings_ad trigger below.
|
||||
|
||||
-- sqlite-vec virtual table for vector search
|
||||
-- Storage rule: embeddings.rowid = document_id * 1000 + chunk_index
|
||||
-- This encodes (document_id, chunk_index) into a single integer rowid.
|
||||
-- Supports up to 1000 chunks per document (32M chars at 32k/chunk).
|
||||
CREATE VIRTUAL TABLE embeddings USING vec0(
|
||||
embedding float[768]
|
||||
);
|
||||
|
||||
-- Embedding provenance + change detection (one row per chunk)
|
||||
-- NOTE: Two hash columns serve different purposes:
|
||||
-- document_hash: SHA-256 of full documents.content_text (staleness detection)
|
||||
-- chunk_hash: SHA-256 of this individual chunk's text (debug/provenance)
|
||||
-- Pending detection uses document_hash (not chunk_hash) because staleness is
|
||||
-- a document-level condition: if the document changed, ALL chunks need re-embedding.
|
||||
CREATE TABLE embedding_metadata (
|
||||
document_id INTEGER NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
|
||||
chunk_index INTEGER NOT NULL DEFAULT 0, -- 0-indexed position within document
|
||||
model TEXT NOT NULL, -- 'nomic-embed-text'
|
||||
dims INTEGER NOT NULL, -- 768
|
||||
document_hash TEXT NOT NULL, -- SHA-256 of full documents.content_text (staleness)
|
||||
chunk_hash TEXT NOT NULL, -- SHA-256 of this chunk's text (provenance)
|
||||
created_at INTEGER NOT NULL, -- ms epoch UTC
|
||||
last_error TEXT, -- error message from last failed attempt
|
||||
attempt_count INTEGER NOT NULL DEFAULT 0,
|
||||
last_attempt_at INTEGER, -- ms epoch UTC
|
||||
PRIMARY KEY(document_id, chunk_index)
|
||||
);
|
||||
|
||||
CREATE INDEX idx_embedding_metadata_errors
|
||||
ON embedding_metadata(last_error) WHERE last_error IS NOT NULL;
|
||||
CREATE INDEX idx_embedding_metadata_doc ON embedding_metadata(document_id);
|
||||
|
||||
-- CRITICAL: Delete ALL chunk embeddings when a document is deleted.
|
||||
-- vec0 virtual tables don't support FK ON DELETE CASCADE, so we need this trigger.
|
||||
-- embedding_metadata has ON DELETE CASCADE, so only vec0 needs explicit cleanup.
|
||||
-- Range: [document_id * 1000, document_id * 1000 + 999]
|
||||
CREATE TRIGGER documents_embeddings_ad AFTER DELETE ON documents BEGIN
|
||||
DELETE FROM embeddings
|
||||
WHERE rowid >= old.id * 1000
|
||||
AND rowid < (old.id + 1) * 1000;
|
||||
END;
|
||||
|
||||
-- Update schema version
|
||||
INSERT INTO schema_version (version, applied_at, description)
|
||||
VALUES (9, strftime('%s', 'now') * 1000, 'Embeddings vec0 table, metadata, orphan cleanup trigger');
|
||||
Reference in New Issue
Block a user