afce0bd3e5
Switch from external Gemini API (3072 dims, $0.15/1M tokens) to local Ollama mxbai-embed-large (1024 dims, free) for cost savings and HNSW index support. Changes: - Updated embeddings.ts: model 'mxbai-embed-large', API URL fixed - Updated migration 015: vector(1024) with HNSW index - Regenerated 268 tool_docs embeddings with new model Benefits: - Free embeddings (no API costs) - HNSW index enabled (1024 < 2000 dim limit) - Fast similarity search (O(log n) vs O(n)) - No external API dependency Trade-offs: - 5% quality loss (MTEB 64.68 vs ~70 Gemini) - Uses local compute (1.2GB RAM, <1s per embedding) Task: CF-251 Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
38 lines
1.7 KiB
SQL
38 lines
1.7 KiB
SQL
-- Migration 015: Tool Documentation
|
|
-- Creates tool_docs table for queryable tool documentation with semantic search
|
|
-- Dependencies: 001_base_schema.sql (pgvector extension)
|
|
|
|
-- Tool documentation table
|
|
CREATE TABLE IF NOT EXISTS tool_docs (
|
|
id SERIAL PRIMARY KEY,
|
|
tool_name TEXT NOT NULL,
|
|
category TEXT NOT NULL CHECK (category IN ('mcp', 'cli', 'script', 'internal', 'deprecated')),
|
|
title TEXT NOT NULL,
|
|
description TEXT NOT NULL,
|
|
usage_example TEXT,
|
|
parameters JSONB, -- Structured parameter definitions
|
|
notes TEXT, -- Additional notes, gotchas, tips
|
|
tags TEXT[], -- Searchable tags (e.g., ['backup', 'database', 'postgresql'])
|
|
source_file TEXT, -- Original source file (TOOLS.md, script path, etc.)
|
|
embedding vector(1024), -- mxbai-embed-large embedding (1024 dimensions)
|
|
created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
|
|
updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
|
|
);
|
|
|
|
-- Indexes for fast lookups
|
|
CREATE INDEX IF NOT EXISTS idx_tool_docs_name ON tool_docs(tool_name);
|
|
CREATE INDEX IF NOT EXISTS idx_tool_docs_category ON tool_docs(category);
|
|
CREATE INDEX IF NOT EXISTS idx_tool_docs_tags ON tool_docs USING gin(tags);
|
|
|
|
-- HNSW index for fast semantic similarity search (O(log n) vs O(n) sequential scan)
|
|
-- mxbai-embed-large (1024 dims) < 2000 dim limit, so HNSW index works
|
|
CREATE INDEX IF NOT EXISTS idx_tool_docs_embedding ON tool_docs USING hnsw (embedding vector_cosine_ops);
|
|
|
|
-- Full-text search on title + description
|
|
CREATE INDEX IF NOT EXISTS idx_tool_docs_fts ON tool_docs
|
|
USING gin(to_tsvector('english', title || ' ' || description || ' ' || COALESCE(notes, '')));
|
|
|
|
-- Record migration
|
|
INSERT INTO schema_migrations (version) VALUES ('015_tool_docs')
|
|
ON CONFLICT (version) DO NOTHING;
|