diff --git a/migrate-cf-docs-batch.mjs b/migrate-cf-docs-batch.mjs new file mode 100755 index 0000000..c0a775a --- /dev/null +++ b/migrate-cf-docs-batch.mjs @@ -0,0 +1,255 @@ +#!/usr/bin/env node +/** + * Automated batch migration of ClaudeFramework docs to task-mcp database + * + * Migrates: + * - docs/investigations/*.md → project_archives (archive_type: investigation) + * - docs/*AUDIT*.md → project_archives (archive_type: audit) + * - docs/*.md → project_archives (archive_type: research) + * - guides/*.md → project_archives (archive_type: research) + * - delegation/*.md → project_archives (archive_type: research) + */ + +import { readFileSync, readdirSync, renameSync, mkdirSync, existsSync, statSync } from 'fs'; +import { join, basename, dirname } from 'path'; +import { homedir } from 'os'; +import dotenv from 'dotenv'; +import { archiveAdd } from './dist/tools/archives.js'; + +// Load environment +dotenv.config(); + +// Configuration +const CF_DIR = join(homedir(), 'Development/Infrastructure/ClaudeFramework'); +const BACKUP_DIR = join(CF_DIR, '.migrated-to-mcp'); + +// Exclusions +const SKIP_FILES = [ + 'README.md', + 'CHANGELOG.md', + 'LICENSE.md', + 'CONTRIBUTING.md', + 'CLAUDE.md', // Keep active instructions + 'planning.md', // Keep active planning + 'PRD.md' // Keep product requirements +]; + +/** + * Determine archive type based on file path and name + */ +function getArchiveType(filePath) { + const filename = basename(filePath); + const dirName = dirname(filePath); + + // Investigation files + if (dirName.includes('/investigations/')) { + return 'investigation'; + } + + // Audit files + if (filename.includes('AUDIT') || filename.includes('_AUDIT')) { + return 'audit'; + } + + // Research/documentation files + return 'research'; +} + +/** + * Extract title from markdown content (first H1) + */ +function extractTitle(content, filename) { + const lines = content.split('\n'); + for (const line of lines) { + if (line.startsWith('# ')) { + return line.slice(2).trim().substring(0, 500); + } + } + // Fallback to filename + return filename + .replace('.md', '') + .replace(/_/g, ' ') + .replace(/-/g, ' ') + .split(' ') + .map(word => word.charAt(0).toUpperCase() + word.slice(1).toLowerCase()) + .join(' ') + .substring(0, 500); +} + +/** + * Migrate a single file + */ +async function migrateFile(filePath) { + const filename = basename(filePath); + + try { + const content = readFileSync(filePath, 'utf-8'); + const title = extractTitle(content, filename); + const fileSize = statSync(filePath).size; + const archiveType = getArchiveType(filePath); + + console.log(`\n[${new Date().toISOString().substring(11, 19)}] Migrating: ${filePath.replace(CF_DIR, '')}`); + console.log(` Title: ${title}`); + console.log(` Size: ${Math.round(fileSize / 1024)}KB`); + console.log(` Type: ${archiveType}`); + + // Call archive_add + const result = await archiveAdd({ + project: 'CF', + archive_type: archiveType, + title, + content, + original_path: filePath, + file_size: fileSize + }); + + console.log(` ✓ Migrated successfully`); + + // Move to backup + const relativePath = filePath.replace(CF_DIR + '/', ''); + const backupPath = join(BACKUP_DIR, relativePath); + const backupDir = dirname(backupPath); + + if (!existsSync(backupDir)) { + mkdirSync(backupDir, { recursive: true }); + } + + renameSync(filePath, backupPath); + console.log(` ✓ Moved to backup`); + + return { success: true, filename, title, fileSize, archiveType }; + + } catch (error) { + console.error(` ✗ Error: ${error.message}`); + return { success: false, filename, error: error.message }; + } +} + +/** + * Find all markdown files to migrate + */ +function findMarkdownFiles() { + const files = []; + const dirs = [ + join(CF_DIR, 'docs'), + join(CF_DIR, 'guides'), + join(CF_DIR, 'delegation') + ]; + + function scanDir(dir) { + if (!existsSync(dir)) return; + + const entries = readdirSync(dir, { withFileTypes: true }); + + for (const entry of entries) { + const fullPath = join(dir, entry.name); + + if (entry.isDirectory()) { + scanDir(fullPath); + } else if (entry.isFile() && entry.name.endsWith('.md')) { + if (!SKIP_FILES.includes(entry.name)) { + files.push(fullPath); + } + } + } + } + + dirs.forEach(scanDir); + return files.sort(); +} + +/** + * Main migration function + */ +async function main() { + console.log('================================================================================'); + console.log('Batch Migration: ClaudeFramework docs → task-mcp database'); + console.log('================================================================================\n'); + + // Find all files + const filesToMigrate = findMarkdownFiles(); + + console.log(`Found ${filesToMigrate.length} markdown files to migrate\n`); + + if (filesToMigrate.length === 0) { + console.log('✓ No files to migrate!'); + return; + } + + // Show sample paths + console.log('Sample files:'); + filesToMigrate.slice(0, 5).forEach(f => { + console.log(` - ${f.replace(CF_DIR, '')}`); + }); + console.log(` ... and ${filesToMigrate.length - 5} more\n`); + + console.log('Starting migration...\n'); + + const results = { + success: [], + failed: [] + }; + + // Migrate each file + for (let i = 0; i < filesToMigrate.length; i++) { + const filePath = filesToMigrate[i]; + const result = await migrateFile(filePath); + + if (result.success) { + results.success.push(result); + } else { + results.failed.push(result); + } + + // Progress indicator + const progress = Math.round(((i + 1) / filesToMigrate.length) * 100); + console.log(` Progress: ${i + 1}/${filesToMigrate.length} (${progress}%)`); + + // Small delay to avoid overwhelming the database + await new Promise(resolve => setTimeout(resolve, 50)); + } + + // Summary + console.log('\n================================================================================'); + console.log('Migration Complete'); + console.log('================================================================================\n'); + console.log(`✓ Successfully migrated: ${results.success.length} files`); + console.log(`✗ Failed: ${results.failed.length} files`); + + if (results.failed.length > 0) { + console.log('\nFailed files:'); + results.failed.forEach(f => { + console.log(` - ${f.filename}: ${f.error}`); + }); + } + + // Group by type + const byType = {}; + results.success.forEach(r => { + byType[r.archiveType] = (byType[r.archiveType] || 0) + 1; + }); + + console.log('\nBy archive type:'); + Object.entries(byType).forEach(([type, count]) => { + console.log(` - ${type}: ${count} files`); + }); + + console.log(`\nBackup location: ${BACKUP_DIR}`); + console.log('Original files can be restored if needed.'); + + // Calculate total size + const totalSize = results.success.reduce((sum, r) => sum + (r.fileSize || 0), 0); + console.log(`\nTotal data migrated: ${Math.round(totalSize / 1024 / 1024 * 100) / 100}MB`); +} + +// Run migration +main() + .then(() => { + console.log('\n✓ Migration script completed successfully'); + process.exit(0); + }) + .catch(error => { + console.error('\n✗ Migration script failed:', error); + console.error(error.stack); + process.exit(1); + }); diff --git a/migrate-plans-batch.mjs b/migrate-plans-batch.mjs new file mode 100644 index 0000000..dd2326f --- /dev/null +++ b/migrate-plans-batch.mjs @@ -0,0 +1,168 @@ +#!/usr/bin/env node +/** + * Automated batch migration of ~/.claude/plans/*.md to task-mcp database + * + * Run from task-mcp directory: node migrate-plans-batch.mjs + */ + +import { readFileSync, readdirSync, renameSync, mkdirSync, existsSync, statSync } from 'fs'; +import { join } from 'path'; +import { homedir } from 'os'; +import dotenv from 'dotenv'; +import { archiveAdd } from './dist/tools/archives.js'; + +// Load environment +dotenv.config(); + +// Configuration +const PLANS_DIR = join(homedir(), '.claude', 'plans'); +const BACKUP_DIR = join(PLANS_DIR, '.migrated-to-mcp'); +const ALREADY_MIGRATED = [ + 'adaptive-enchanting-truffle.md', + 'adaptive-sleeping-pearl.md' +]; + +/** + * Extract title from markdown content (first H1) + */ +function extractTitle(content, filename) { + const lines = content.split('\n'); + for (const line of lines) { + if (line.startsWith('# ')) { + return line.slice(2).trim().substring(0, 500); + } + } + // Fallback to filename + return filename + .replace('.md', '') + .replace(/-/g, ' ') + .split(' ') + .map(word => word.charAt(0).toUpperCase() + word.slice(1)) + .join(' ') + .substring(0, 500); +} + +/** + * Migrate a single plan file + */ +async function migratePlan(filename) { + const filePath = join(PLANS_DIR, filename); + + try { + const content = readFileSync(filePath, 'utf-8'); + const title = extractTitle(content, filename); + const fileSize = statSync(filePath).size; + + console.log(`\n[${new Date().toISOString().substring(11, 19)}] Migrating: ${filename}`); + console.log(` Title: ${title}`); + console.log(` Size: ${Math.round(fileSize / 1024)}KB`); + + // Call archive_add + const result = await archiveAdd({ + project: 'CF', + archive_type: 'session', + title, + content, + original_path: filePath, + file_size: fileSize + }); + + console.log(` ✓ Migrated successfully`); + + // Move to backup + if (!existsSync(BACKUP_DIR)) { + mkdirSync(BACKUP_DIR, { recursive: true }); + } + + const backupPath = join(BACKUP_DIR, filename); + renameSync(filePath, backupPath); + console.log(` ✓ Moved to backup`); + + return { success: true, filename, title, fileSize }; + + } catch (error) { + console.error(` ✗ Error: ${error.message}`); + return { success: false, filename, error: error.message }; + } +} + +/** + * Main migration function + */ +async function main() { + console.log('================================================================================'); + console.log('Batch Migration: ~/.claude/plans/*.md → task-mcp database'); + console.log('================================================================================\n'); + + // Get all plan files + const allFiles = readdirSync(PLANS_DIR).filter(f => f.endsWith('.md')); + const filesToMigrate = allFiles.filter(f => !ALREADY_MIGRATED.includes(f)); + + console.log(`Total plan files: ${allFiles.length}`); + console.log(`Already migrated: ${ALREADY_MIGRATED.length}`); + console.log(`To migrate: ${filesToMigrate.length}\n`); + + if (filesToMigrate.length === 0) { + console.log('✓ All files already migrated!'); + return; + } + + console.log('Starting migration...\n'); + + const results = { + success: [], + failed: [] + }; + + // Migrate each file + for (let i = 0; i < filesToMigrate.length; i++) { + const filename = filesToMigrate[i]; + const result = await migratePlan(filename); + + if (result.success) { + results.success.push(result); + } else { + results.failed.push(result); + } + + // Progress indicator + const progress = Math.round(((i + 1) / filesToMigrate.length) * 100); + console.log(` Progress: ${i + 1}/${filesToMigrate.length} (${progress}%)`); + + // Small delay to avoid overwhelming the database + await new Promise(resolve => setTimeout(resolve, 50)); + } + + // Summary + console.log('\n================================================================================'); + console.log('Migration Complete'); + console.log('================================================================================\n'); + console.log(`✓ Successfully migrated: ${results.success.length} files`); + console.log(`✗ Failed: ${results.failed.length} files`); + + if (results.failed.length > 0) { + console.log('\nFailed files:'); + results.failed.forEach(f => { + console.log(` - ${f.filename}: ${f.error}`); + }); + } + + console.log(`\nBackup location: ${BACKUP_DIR}`); + console.log('Original files can be restored if needed.'); + + // Calculate total size + const totalSize = results.success.reduce((sum, r) => sum + (r.fileSize || 0), 0); + console.log(`\nTotal data migrated: ${Math.round(totalSize / 1024 / 1024 * 100) / 100}MB`); +} + +// Run migration +main() + .then(() => { + console.log('\n✓ Migration script completed successfully'); + process.exit(0); + }) + .catch(error => { + console.error('\n✗ Migration script failed:', error); + console.error(error.stack); + process.exit(1); + }); diff --git a/migrate-project-docs-batch.mjs b/migrate-project-docs-batch.mjs new file mode 100755 index 0000000..f81afbe --- /dev/null +++ b/migrate-project-docs-batch.mjs @@ -0,0 +1,303 @@ +#!/usr/bin/env node +/** + * Automated batch migration of project-specific documentation to task-mcp database + * + * Migrates: + * - CLAUDE_HISTORY.md → project_archives (archive_type: session) + * - planning.md → project_archives (archive_type: completed) + * - PRD.md → project_archives (archive_type: research) + * + * Excludes: + * - .framework-backup directories + * - Archived projects + * - ClaudeFramework (already migrated) + */ + +import { readFileSync, readdirSync, renameSync, mkdirSync, existsSync, statSync } from 'fs'; +import { join, basename, dirname, relative } from 'path'; +import { homedir } from 'os'; +import dotenv from 'dotenv'; +import { archiveAdd } from './dist/tools/archives.js'; + +// Load environment +dotenv.config(); + +// Configuration +const DEV_DIR = join(homedir(), 'Development'); +const EXCLUDED_PATTERNS = [ + '.framework-backup', + '/Archived/', + '/ClaudeFramework/' +]; + +/** + * Extract project key from directory path + * Examples: + * /Apps/RealEstate → RE + * /Apps/eToroGridbot → GB + * /Apps/ZorkiOS → ZK + * /Infrastructure/mcp-servers/cloudmemorymcp → CM + */ +function getProjectKey(filePath) { + const parts = filePath.split('/'); + + // Find the most specific directory name + let projectName = ''; + + if (filePath.includes('/Apps/')) { + const idx = parts.indexOf('Apps'); + projectName = parts[idx + 1] || ''; + } else if (filePath.includes('/Infrastructure/')) { + // Use the deepest directory name + projectName = parts[parts.length - 2] || ''; + } + + // Generate 2-letter key from project name + const normalized = projectName + .replace(/([A-Z])/g, ' $1') // Split camelCase + .trim() + .toUpperCase(); + + const words = normalized.split(/\s+/); + + if (words.length >= 2) { + return words[0][0] + words[1][0]; // First letter of first two words + } else if (words[0].length >= 2) { + return words[0].substring(0, 2); // First two letters + } else { + return words[0][0] + 'X'; // Fallback + } +} + +/** + * Determine archive type based on filename + */ +function getArchiveType(filename) { + if (filename === 'CLAUDE_HISTORY.md') { + return 'session'; + } else if (filename === 'planning.md') { + return 'completed'; + } else if (filename === 'PRD.md') { + return 'research'; + } + return 'research'; +} + +/** + * Extract title from markdown content (first H1) + */ +function extractTitle(content, filename, projectKey) { + const lines = content.split('\n'); + for (const line of lines) { + if (line.startsWith('# ')) { + return line.slice(2).trim().substring(0, 500); + } + } + + // Fallback to project key + filename + const baseName = filename.replace('.md', '').replace(/_/g, ' '); + return `${projectKey} - ${baseName}`.substring(0, 500); +} + +/** + * Check if file should be excluded + */ +function shouldExclude(filePath) { + return EXCLUDED_PATTERNS.some(pattern => filePath.includes(pattern)); +} + +/** + * Migrate a single file + */ +async function migrateFile(filePath) { + const filename = basename(filePath); + const projectKey = getProjectKey(filePath); + + try { + const content = readFileSync(filePath, 'utf-8'); + const title = extractTitle(content, filename, projectKey); + const fileSize = statSync(filePath).size; + const archiveType = getArchiveType(filename); + + console.log(`\n[${new Date().toISOString().substring(11, 19)}] Migrating: ${relative(DEV_DIR, filePath)}`); + console.log(` Project: ${projectKey}`); + console.log(` Title: ${title}`); + console.log(` Size: ${Math.round(fileSize / 1024)}KB`); + console.log(` Type: ${archiveType}`); + + // Call archive_add + const result = await archiveAdd({ + project: projectKey, + archive_type: archiveType, + title, + content, + original_path: filePath, + file_size: fileSize + }); + + console.log(` ✓ Migrated successfully`); + + // Move to backup + const backupDir = join(dirname(filePath), '.migrated-to-mcp'); + if (!existsSync(backupDir)) { + mkdirSync(backupDir, { recursive: true }); + } + + const backupPath = join(backupDir, filename); + renameSync(filePath, backupPath); + console.log(` ✓ Moved to backup`); + + return { success: true, filename, projectKey, title, fileSize, archiveType, filePath }; + + } catch (error) { + console.error(` ✗ Error: ${error.message}`); + return { success: false, filename, projectKey, error: error.message, filePath }; + } +} + +/** + * Find all documentation files to migrate + */ +function findDocFiles() { + const files = []; + const targetFiles = ['CLAUDE_HISTORY.md', 'planning.md', 'PRD.md']; + + function scanDir(dir) { + if (!existsSync(dir)) return; + + try { + const entries = readdirSync(dir, { withFileTypes: true }); + + for (const entry of entries) { + const fullPath = join(dir, entry.name); + + // Skip excluded paths + if (shouldExclude(fullPath)) continue; + + if (entry.isDirectory()) { + // Skip node_modules, .git, etc. + if (!entry.name.startsWith('.') && entry.name !== 'node_modules') { + scanDir(fullPath); + } + } else if (entry.isFile() && targetFiles.includes(entry.name)) { + files.push(fullPath); + } + } + } catch (error) { + // Skip directories we can't read + } + } + + scanDir(DEV_DIR); + return files.sort(); +} + +/** + * Main migration function + */ +async function main() { + console.log('================================================================================'); + console.log('Batch Migration: Project Documentation → task-mcp database'); + console.log('================================================================================\n'); + + // Find all files + const filesToMigrate = findDocFiles(); + + console.log(`Found ${filesToMigrate.length} documentation files to migrate\n`); + + if (filesToMigrate.length === 0) { + console.log('✓ No files to migrate!'); + return; + } + + // Show sample paths + console.log('Sample files:'); + filesToMigrate.slice(0, 10).forEach(f => { + console.log(` - ${relative(DEV_DIR, f)}`); + }); + if (filesToMigrate.length > 10) { + console.log(` ... and ${filesToMigrate.length - 10} more`); + } + console.log('\nStarting migration...\n'); + + const results = { + success: [], + failed: [] + }; + + // Migrate each file + for (let i = 0; i < filesToMigrate.length; i++) { + const filePath = filesToMigrate[i]; + const result = await migrateFile(filePath); + + if (result.success) { + results.success.push(result); + } else { + results.failed.push(result); + } + + // Progress indicator + const progress = Math.round(((i + 1) / filesToMigrate.length) * 100); + console.log(` Progress: ${i + 1}/${filesToMigrate.length} (${progress}%)`); + + // Small delay to avoid overwhelming the database + await new Promise(resolve => setTimeout(resolve, 50)); + } + + // Summary + console.log('\n================================================================================'); + console.log('Migration Complete'); + console.log('================================================================================\n'); + console.log(`✓ Successfully migrated: ${results.success.length} files`); + console.log(`✗ Failed: ${results.failed.length} files`); + + if (results.failed.length > 0) { + console.log('\nFailed files:'); + results.failed.forEach(f => { + console.log(` - ${relative(DEV_DIR, f.filePath)}: ${f.error}`); + }); + } + + // Group by type + const byType = {}; + results.success.forEach(r => { + byType[r.archiveType] = (byType[r.archiveType] || 0) + 1; + }); + + console.log('\nBy archive type:'); + Object.entries(byType).forEach(([type, count]) => { + console.log(` - ${type}: ${count} files`); + }); + + // Group by project + const byProject = {}; + results.success.forEach(r => { + byProject[r.projectKey] = (byProject[r.projectKey] || 0) + 1; + }); + + console.log('\nBy project:'); + Object.entries(byProject) + .sort((a, b) => b[1] - a[1]) + .forEach(([project, count]) => { + console.log(` - ${project}: ${count} files`); + }); + + console.log('\nBackup location: /.migrated-to-mcp/'); + console.log('Original files can be restored if needed.'); + + // Calculate total size + const totalSize = results.success.reduce((sum, r) => sum + (r.fileSize || 0), 0); + console.log(`\nTotal data migrated: ${Math.round(totalSize / 1024 / 1024 * 100) / 100}MB`); +} + +// Run migration +main() + .then(() => { + console.log('\n✓ Migration script completed successfully'); + process.exit(0); + }) + .catch(error => { + console.error('\n✗ Migration script failed:', error); + console.error(error.stack); + process.exit(1); + }); diff --git a/migrations/017_fix_archives_embedding_dimension.sql b/migrations/017_fix_archives_embedding_dimension.sql new file mode 100644 index 0000000..5c1396f --- /dev/null +++ b/migrations/017_fix_archives_embedding_dimension.sql @@ -0,0 +1,17 @@ +-- Migration 017: Fix project_archives embedding dimension +-- Changes vector(1536) to vector(1024) to match mxbai-embed-large model + +-- Drop existing index first +DROP INDEX IF EXISTS idx_archives_embedding; + +-- Alter the embedding column dimension +ALTER TABLE project_archives + ALTER COLUMN embedding TYPE vector(1024); + +-- Recreate index with correct dimension +CREATE INDEX idx_archives_embedding ON project_archives + USING hnsw (embedding vector_cosine_ops); + +-- Record migration +INSERT INTO schema_migrations (version) VALUES ('017_fix_archives_embedding_dimension') +ON CONFLICT (version) DO NOTHING; diff --git a/run-migration.js b/run-migration.js new file mode 100644 index 0000000..85ade2f --- /dev/null +++ b/run-migration.js @@ -0,0 +1,51 @@ +#!/usr/bin/env node +/** + * Run a specific migration file + * Usage: node run-migration.js migrations/017_fix_archives_embedding_dimension.sql + */ + +import pg from 'pg'; +import { readFileSync } from 'fs'; +import { resolve } from 'path'; + +const { Pool } = pg; + +// Configuration - Direct WireGuard connection to INFRA VM PostgreSQL +const config = { + host: process.env.POSTGRES_HOST || 'infra.agiliton.internal', + port: 5432, + database: 'agiliton', + user: 'agiliton', + password: 'QtqiwCOAUpQNF6pjzOMAREzUny2bY8V1', +}; + +async function runMigration(migrationFile) { + const pool = new Pool(config); + + try { + // Read migration SQL + const sql = readFileSync(resolve(migrationFile), 'utf-8'); + + console.log(`Running migration: ${migrationFile}`); + console.log('SQL:', sql); + + // Execute migration + await pool.query(sql); + + console.log('Migration completed successfully!'); + } catch (error) { + console.error('Migration failed:', error); + process.exit(1); + } finally { + await pool.end(); + } +} + +// Get migration file from command line +const migrationFile = process.argv[2]; +if (!migrationFile) { + console.error('Usage: node run-migration.js '); + process.exit(1); +} + +runMigration(migrationFile);