Add tool documentation MCP tools for queryable TOOLS.md replacement

- Created tool_docs table with pgvector support (migration 015)
- Added 5 MCP tools: tool_doc_add, tool_doc_search, tool_doc_get, tool_doc_list, tool_doc_export
- Imported 268 tools from TOOLS.md to database
- Enables semantic search for tool documentation (when embeddings available)
- Reduces context pollution (TOOLS.md is 11,769 lines, now queryable)

Task: CF-249

Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>

migrations/015_tool_docs.sql

@@ -0,0 +1,36 @@
+-- Migration 015: Tool Documentation
+-- Creates tool_docs table for queryable tool documentation with semantic search
+-- Dependencies: 001_base_schema.sql (pgvector extension)
+
+-- Tool documentation table
+CREATE TABLE IF NOT EXISTS tool_docs (
+    id SERIAL PRIMARY KEY,
+    tool_name TEXT NOT NULL,
+    category TEXT NOT NULL CHECK (category IN ('mcp', 'cli', 'script', 'internal', 'deprecated')),
+    title TEXT NOT NULL,
+    description TEXT NOT NULL,
+    usage_example TEXT,
+    parameters JSONB,  -- Structured parameter definitions
+    notes TEXT,  -- Additional notes, gotchas, tips
+    tags TEXT[],  -- Searchable tags (e.g., ['backup', 'database', 'postgresql'])
+    source_file TEXT,  -- Original source file (TOOLS.md, script path, etc.)
+    embedding vector(1024),  -- Semantic search embedding
+    created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
+    updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
+);
+
+-- Indexes for fast lookups
+CREATE INDEX IF NOT EXISTS idx_tool_docs_name ON tool_docs(tool_name);
+CREATE INDEX IF NOT EXISTS idx_tool_docs_category ON tool_docs(category);
+CREATE INDEX IF NOT EXISTS idx_tool_docs_tags ON tool_docs USING gin(tags);
+
+-- HNSW index for semantic similarity search
+CREATE INDEX IF NOT EXISTS idx_tool_docs_embedding ON tool_docs USING hnsw (embedding vector_cosine_ops);
+
+-- Full-text search on title + description
+CREATE INDEX IF NOT EXISTS idx_tool_docs_fts ON tool_docs
+    USING gin(to_tsvector('english', title || ' ' || description || ' ' || COALESCE(notes, '')));
+
+-- Record migration
+INSERT INTO schema_migrations (version) VALUES ('015_tool_docs')
+ON CONFLICT (version) DO NOTHING;

src/index.ts

@@ -38,6 +38,7 @@ import {
   componentGraph,
 } from './tools/impact.js';
 import { memoryAdd, memorySearch, memoryList, memoryContext } from './tools/memories.js';
+import { toolDocAdd, toolDocSearch, toolDocGet, toolDocList, toolDocExport } from './tools/tool-docs.js';
 import {
   sessionStart,
   sessionUpdate,
@@ -374,6 +375,44 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
         result = await memoryContext(a.project, a.task_description);
         break;
 
+      // Tool Documentation
+      case 'tool_doc_add':
+        result = await toolDocAdd({
+          tool_name: a.tool_name,
+          category: a.category,
+          title: a.title,
+          description: a.description,
+          usage_example: a.usage_example,
+          parameters: a.parameters,
+          notes: a.notes,
+          tags: a.tags,
+          source_file: a.source_file,
+        });
+        break;
+      case 'tool_doc_search':
+        result = await toolDocSearch({
+          query: a.query,
+          category: a.category,
+          tags: a.tags,
+          limit: a.limit,
+        });
+        break;
+      case 'tool_doc_get':
+        result = await toolDocGet({
+          tool_name: a.tool_name,
+        });
+        break;
+      case 'tool_doc_list':
+        result = await toolDocList({
+          category: a.category,
+          tag: a.tag,
+          limit: a.limit,
+        });
+        break;
+      case 'tool_doc_export':
+        result = await toolDocExport();
+        break;
+
       // Sessions
       case 'session_start':
         result = await sessionStart({

src/tools/index.ts

@@ -644,6 +644,72 @@ export const toolDefinitions = [
     },
   },
 
+  // Tool Documentation Tools
+  {
+    name: 'tool_doc_add',
+    description: 'Add a new tool documentation entry',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        tool_name: { type: 'string', description: 'Tool or command name' },
+        category: { type: 'string', enum: ['mcp', 'cli', 'script', 'internal', 'deprecated'], description: 'Tool category' },
+        title: { type: 'string', description: 'Short descriptive title' },
+        description: { type: 'string', description: 'Detailed description of what the tool does' },
+        usage_example: { type: 'string', description: 'Usage example (optional)' },
+        parameters: { type: 'object', description: 'Parameter definitions (optional)' },
+        notes: { type: 'string', description: 'Additional notes, gotchas, tips (optional)' },
+        tags: { type: 'array', items: { type: 'string' }, description: 'Searchable tags (optional)' },
+        source_file: { type: 'string', description: 'Original source file (optional)' },
+      },
+      required: ['tool_name', 'category', 'title', 'description'],
+    },
+  },
+  {
+    name: 'tool_doc_search',
+    description: 'Search tool documentation semantically',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        query: { type: 'string', description: 'Search query' },
+        category: { type: 'string', enum: ['mcp', 'cli', 'script', 'internal', 'deprecated'], description: 'Filter by category (optional)' },
+        tags: { type: 'array', items: { type: 'string' }, description: 'Filter by tags (optional)' },
+        limit: { type: 'number', description: 'Max results (default: 5)' },
+      },
+      required: ['query'],
+    },
+  },
+  {
+    name: 'tool_doc_get',
+    description: 'Get specific tool documentation by name',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        tool_name: { type: 'string', description: 'Tool or command name' },
+      },
+      required: ['tool_name'],
+    },
+  },
+  {
+    name: 'tool_doc_list',
+    description: 'List tool documentation entries',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        category: { type: 'string', enum: ['mcp', 'cli', 'script', 'internal', 'deprecated'], description: 'Filter by category (optional)' },
+        tag: { type: 'string', description: 'Filter by tag (optional)' },
+        limit: { type: 'number', description: 'Max results (default: 20)' },
+      },
+    },
+  },
+  {
+    name: 'tool_doc_export',
+    description: 'Export all tool documentation as markdown (for backup/migration)',
+    inputSchema: {
+      type: 'object',
+      properties: {},
+    },
+  },
+
   // Session Management Tools
   {
     name: 'session_start',

src/tools/tool-docs.ts

@@ -0,0 +1,336 @@
+// Tool documentation operations for queryable TOOLS.md replacement
+
+import { query, queryOne, execute } from '../db.js';
+import { getEmbedding, formatEmbedding } from '../embeddings.js';
+
+type ToolCategory = 'mcp' | 'cli' | 'script' | 'internal' | 'deprecated';
+
+interface ToolDoc {
+  id: number;
+  tool_name: string;
+  category: ToolCategory;
+  title: string;
+  description: string;
+  usage_example: string | null;
+  parameters: Record<string, unknown> | null;
+  notes: string | null;
+  tags: string[];
+  source_file: string | null;
+  created_at: string;
+  updated_at: string;
+}
+
+interface ToolDocAddArgs {
+  tool_name: string;
+  category: ToolCategory;
+  title: string;
+  description: string;
+  usage_example?: string;
+  parameters?: Record<string, unknown>;
+  notes?: string;
+  tags?: string[];
+  source_file?: string;
+}
+
+interface ToolDocSearchArgs {
+  query: string;
+  category?: ToolCategory;
+  tags?: string[];
+  limit?: number;
+}
+
+interface ToolDocGetArgs {
+  tool_name: string;
+}
+
+interface ToolDocListArgs {
+  category?: ToolCategory;
+  tag?: string;
+  limit?: number;
+}
+
+/**
+ * Add a new tool documentation entry
+ */
+export async function toolDocAdd(args: ToolDocAddArgs): Promise<string> {
+  const {
+    tool_name,
+    category,
+    title,
+    description,
+    usage_example,
+    parameters,
+    notes,
+    tags = [],
+    source_file
+  } = args;
+
+  // Generate embedding for semantic search
+  const embedText = `${title}. ${description}. ${tags.join(' ')}. ${notes || ''}`;
+  const embedding = await getEmbedding(embedText);
+  const embeddingValue = embedding ? formatEmbedding(embedding) : null;
+
+  if (embeddingValue) {
+    await execute(
+      `INSERT INTO tool_docs (tool_name, category, title, description, usage_example, parameters, notes, tags, source_file, embedding)
+       VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)`,
+      [
+        tool_name,
+        category,
+        title,
+        description,
+        usage_example || null,
+        parameters ? JSON.stringify(parameters) : null,
+        notes || null,
+        tags,
+        source_file || null,
+        embeddingValue
+      ]
+    );
+  } else {
+    await execute(
+      `INSERT INTO tool_docs (tool_name, category, title, description, usage_example, parameters, notes, tags, source_file)
+       VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)`,
+      [
+        tool_name,
+        category,
+        title,
+        description,
+        usage_example || null,
+        parameters ? JSON.stringify(parameters) : null,
+        notes || null,
+        tags,
+        source_file || null
+      ]
+    );
+  }
+
+  return `Added tool documentation: ${tool_name}`;
+}
+
+/**
+ * Search tool documentation semantically
+ */
+export async function toolDocSearch(args: ToolDocSearchArgs): Promise<string> {
+  const { query: searchQuery, category, tags, limit = 5 } = args;
+
+  // Generate embedding for search
+  const embedding = await getEmbedding(searchQuery);
+
+  if (!embedding) {
+    return 'Error: Could not generate embedding for search';
+  }
+
+  const embeddingStr = formatEmbedding(embedding);
+
+  let whereClause = 'WHERE embedding IS NOT NULL';
+  const params: unknown[] = [embeddingStr, limit];
+  let paramIndex = 3;
+
+  if (category) {
+    whereClause += ` AND category = $${paramIndex++}`;
+    params.splice(params.length - 1, 0, category);
+  }
+  if (tags && tags.length > 0) {
+    whereClause += ` AND tags && $${paramIndex++}`;
+    params.splice(params.length - 1, 0, tags);
+  }
+
+  const docs = await query<ToolDoc & { similarity: number }>(
+    `SELECT id, tool_name, category, title, description, usage_example, notes, tags,
+            1 - (embedding <=> $1) as similarity
+     FROM tool_docs
+     ${whereClause}
+     ORDER BY embedding <=> $1
+     LIMIT $2`,
+    params
+  );
+
+  if (docs.length === 0) {
+    return 'No relevant tool documentation found';
+  }
+
+  const lines = ['Relevant tool documentation:\n'];
+  for (const doc of docs) {
+    const sim = Math.round(doc.similarity * 100);
+    const tagStr = doc.tags.length > 0 ? ` [${doc.tags.join(', ')}]` : '';
+    lines.push(`**${doc.tool_name}** (${doc.category}, ${sim}% match)${tagStr}`);
+    lines.push(`  ${doc.description}`);
+
+    if (doc.usage_example) {
+      lines.push(`\n  Usage:\n  \`\`\`\n  ${doc.usage_example}\n  \`\`\``);
+    }
+
+    if (doc.notes) {
+      lines.push(`  _Notes: ${doc.notes}_`);
+    }
+
+    lines.push('');
+  }
+
+  return lines.join('\n');
+}
+
+/**
+ * Get specific tool documentation by name
+ */
+export async function toolDocGet(args: ToolDocGetArgs): Promise<string> {
+  const { tool_name } = args;
+
+  const doc = await queryOne<ToolDoc>(
+    `SELECT * FROM tool_docs WHERE tool_name = $1`,
+    [tool_name]
+  );
+
+  if (!doc) {
+    return `Tool documentation not found: ${tool_name}`;
+  }
+
+  const lines = [
+    `# ${doc.tool_name}`,
+    `**Category:** ${doc.category}`,
+    `**Title:** ${doc.title}`,
+    '',
+    `**Description:**`,
+    doc.description,
+    ''
+  ];
+
+  if (doc.usage_example) {
+    lines.push('**Usage:**');
+    lines.push('```');
+    lines.push(doc.usage_example);
+    lines.push('```');
+    lines.push('');
+  }
+
+  if (doc.parameters) {
+    lines.push('**Parameters:**');
+    lines.push('```json');
+    lines.push(JSON.stringify(doc.parameters, null, 2));
+    lines.push('```');
+    lines.push('');
+  }
+
+  if (doc.notes) {
+    lines.push('**Notes:**');
+    lines.push(doc.notes);
+    lines.push('');
+  }
+
+  if (doc.tags.length > 0) {
+    lines.push(`**Tags:** ${doc.tags.join(', ')}`);
+  }
+
+  if (doc.source_file) {
+    lines.push(`**Source:** ${doc.source_file}`);
+  }
+
+  return lines.join('\n');
+}
+
+/**
+ * List tool documentation entries
+ */
+export async function toolDocList(args: ToolDocListArgs): Promise<string> {
+  const { category, tag, limit = 20 } = args;
+
+  let whereClause = 'WHERE 1=1';
+  const params: unknown[] = [];
+  let paramIndex = 1;
+
+  if (category) {
+    whereClause += ` AND category = $${paramIndex++}`;
+    params.push(category);
+  }
+  if (tag) {
+    whereClause += ` AND $${paramIndex++} = ANY(tags)`;
+    params.push(tag);
+  }
+
+  params.push(limit);
+
+  const docs = await query<ToolDoc>(
+    `SELECT tool_name, category, title, tags
+     FROM tool_docs
+     ${whereClause}
+     ORDER BY category, tool_name
+     LIMIT $${paramIndex}`,
+    params
+  );
+
+  if (docs.length === 0) {
+    return `No tool documentation found${category ? ` for category ${category}` : ''}`;
+  }
+
+  const lines = [`Tool Documentation${category ? ` (${category})` : ''}:\n`];
+
+  let currentCategory = '';
+  for (const doc of docs) {
+    if (doc.category !== currentCategory) {
+      currentCategory = doc.category;
+      lines.push(`\n**${currentCategory.toUpperCase()}:**`);
+    }
+    const tagStr = doc.tags.length > 0 ? ` [${doc.tags.join(', ')}]` : '';
+    lines.push(`  • ${doc.tool_name}: ${doc.title}${tagStr}`);
+  }
+
+  return lines.join('\n');
+}
+
+/**
+ * Export all tool documentation as markdown (for backup/migration)
+ */
+export async function toolDocExport(): Promise<string> {
+  const docs = await query<ToolDoc>(
+    `SELECT * FROM tool_docs ORDER BY category, tool_name`,
+    []
+  );
+
+  if (docs.length === 0) {
+    return 'No tool documentation to export';
+  }
+
+  const lines = ['# Tool Documentation Database Export\n'];
+
+  let currentCategory = '';
+  for (const doc of docs) {
+    if (doc.category !== currentCategory) {
+      currentCategory = doc.category;
+      lines.push(`\n## ${currentCategory.toUpperCase()}\n`);
+    }
+
+    lines.push(`### ${doc.tool_name}`);
+    lines.push(`**${doc.title}**\n`);
+    lines.push(doc.description);
+    lines.push('');
+
+    if (doc.usage_example) {
+      lines.push('**Usage:**');
+      lines.push('```');
+      lines.push(doc.usage_example);
+      lines.push('```\n');
+    }
+
+    if (doc.parameters) {
+      lines.push('**Parameters:**');
+      lines.push('```json');
+      lines.push(JSON.stringify(doc.parameters, null, 2));
+      lines.push('```\n');
+    }
+
+    if (doc.notes) {
+      lines.push('**Notes:**');
+      lines.push(doc.notes);
+      lines.push('');
+    }
+
+    if (doc.tags.length > 0) {
+      lines.push(`**Tags:** ${doc.tags.join(', ')}\n`);
+    }
+
+    lines.push('---\n');
+  }
+
+  return lines.join('\n');
+}