Add Phase 3 migration: project documentation

- migrate-project-docs-batch.mjs: 60 files across 23 projects
- migrate-cf-docs-batch.mjs: ClaudeFramework documentation
- migrate-plans-batch.mjs: Session plan files
- 017_fix_archives_embedding_dimension.sql: Fix vector dimension
- run-migration.js: Node.js migration runner utility

Total migrated: 332 files, 8.85MB with semantic embeddings

Fixes: CF-277
This commit is contained in:
Christian Gick
2026-01-19 14:09:03 +02:00
parent 6c497ec6c5
commit eb6b51837f
5 changed files with 794 additions and 0 deletions

255
migrate-cf-docs-batch.mjs Executable file
View File

@@ -0,0 +1,255 @@
#!/usr/bin/env node
/**
* Automated batch migration of ClaudeFramework docs to task-mcp database
*
* Migrates:
* - docs/investigations/*.md → project_archives (archive_type: investigation)
* - docs/*AUDIT*.md → project_archives (archive_type: audit)
* - docs/*.md → project_archives (archive_type: research)
* - guides/*.md → project_archives (archive_type: research)
* - delegation/*.md → project_archives (archive_type: research)
*/
import { readFileSync, readdirSync, renameSync, mkdirSync, existsSync, statSync } from 'fs';
import { join, basename, dirname } from 'path';
import { homedir } from 'os';
import dotenv from 'dotenv';
import { archiveAdd } from './dist/tools/archives.js';
// Load environment
dotenv.config();
// Configuration
const CF_DIR = join(homedir(), 'Development/Infrastructure/ClaudeFramework');
const BACKUP_DIR = join(CF_DIR, '.migrated-to-mcp');
// Exclusions
const SKIP_FILES = [
'README.md',
'CHANGELOG.md',
'LICENSE.md',
'CONTRIBUTING.md',
'CLAUDE.md', // Keep active instructions
'planning.md', // Keep active planning
'PRD.md' // Keep product requirements
];
/**
* Determine archive type based on file path and name
*/
function getArchiveType(filePath) {
const filename = basename(filePath);
const dirName = dirname(filePath);
// Investigation files
if (dirName.includes('/investigations/')) {
return 'investigation';
}
// Audit files
if (filename.includes('AUDIT') || filename.includes('_AUDIT')) {
return 'audit';
}
// Research/documentation files
return 'research';
}
/**
* Extract title from markdown content (first H1)
*/
function extractTitle(content, filename) {
const lines = content.split('\n');
for (const line of lines) {
if (line.startsWith('# ')) {
return line.slice(2).trim().substring(0, 500);
}
}
// Fallback to filename
return filename
.replace('.md', '')
.replace(/_/g, ' ')
.replace(/-/g, ' ')
.split(' ')
.map(word => word.charAt(0).toUpperCase() + word.slice(1).toLowerCase())
.join(' ')
.substring(0, 500);
}
/**
* Migrate a single file
*/
async function migrateFile(filePath) {
const filename = basename(filePath);
try {
const content = readFileSync(filePath, 'utf-8');
const title = extractTitle(content, filename);
const fileSize = statSync(filePath).size;
const archiveType = getArchiveType(filePath);
console.log(`\n[${new Date().toISOString().substring(11, 19)}] Migrating: ${filePath.replace(CF_DIR, '')}`);
console.log(` Title: ${title}`);
console.log(` Size: ${Math.round(fileSize / 1024)}KB`);
console.log(` Type: ${archiveType}`);
// Call archive_add
const result = await archiveAdd({
project: 'CF',
archive_type: archiveType,
title,
content,
original_path: filePath,
file_size: fileSize
});
console.log(` ✓ Migrated successfully`);
// Move to backup
const relativePath = filePath.replace(CF_DIR + '/', '');
const backupPath = join(BACKUP_DIR, relativePath);
const backupDir = dirname(backupPath);
if (!existsSync(backupDir)) {
mkdirSync(backupDir, { recursive: true });
}
renameSync(filePath, backupPath);
console.log(` ✓ Moved to backup`);
return { success: true, filename, title, fileSize, archiveType };
} catch (error) {
console.error(` ✗ Error: ${error.message}`);
return { success: false, filename, error: error.message };
}
}
/**
* Find all markdown files to migrate
*/
function findMarkdownFiles() {
const files = [];
const dirs = [
join(CF_DIR, 'docs'),
join(CF_DIR, 'guides'),
join(CF_DIR, 'delegation')
];
function scanDir(dir) {
if (!existsSync(dir)) return;
const entries = readdirSync(dir, { withFileTypes: true });
for (const entry of entries) {
const fullPath = join(dir, entry.name);
if (entry.isDirectory()) {
scanDir(fullPath);
} else if (entry.isFile() && entry.name.endsWith('.md')) {
if (!SKIP_FILES.includes(entry.name)) {
files.push(fullPath);
}
}
}
}
dirs.forEach(scanDir);
return files.sort();
}
/**
* Main migration function
*/
async function main() {
console.log('================================================================================');
console.log('Batch Migration: ClaudeFramework docs → task-mcp database');
console.log('================================================================================\n');
// Find all files
const filesToMigrate = findMarkdownFiles();
console.log(`Found ${filesToMigrate.length} markdown files to migrate\n`);
if (filesToMigrate.length === 0) {
console.log('✓ No files to migrate!');
return;
}
// Show sample paths
console.log('Sample files:');
filesToMigrate.slice(0, 5).forEach(f => {
console.log(` - ${f.replace(CF_DIR, '')}`);
});
console.log(` ... and ${filesToMigrate.length - 5} more\n`);
console.log('Starting migration...\n');
const results = {
success: [],
failed: []
};
// Migrate each file
for (let i = 0; i < filesToMigrate.length; i++) {
const filePath = filesToMigrate[i];
const result = await migrateFile(filePath);
if (result.success) {
results.success.push(result);
} else {
results.failed.push(result);
}
// Progress indicator
const progress = Math.round(((i + 1) / filesToMigrate.length) * 100);
console.log(` Progress: ${i + 1}/${filesToMigrate.length} (${progress}%)`);
// Small delay to avoid overwhelming the database
await new Promise(resolve => setTimeout(resolve, 50));
}
// Summary
console.log('\n================================================================================');
console.log('Migration Complete');
console.log('================================================================================\n');
console.log(`✓ Successfully migrated: ${results.success.length} files`);
console.log(`✗ Failed: ${results.failed.length} files`);
if (results.failed.length > 0) {
console.log('\nFailed files:');
results.failed.forEach(f => {
console.log(` - ${f.filename}: ${f.error}`);
});
}
// Group by type
const byType = {};
results.success.forEach(r => {
byType[r.archiveType] = (byType[r.archiveType] || 0) + 1;
});
console.log('\nBy archive type:');
Object.entries(byType).forEach(([type, count]) => {
console.log(` - ${type}: ${count} files`);
});
console.log(`\nBackup location: ${BACKUP_DIR}`);
console.log('Original files can be restored if needed.');
// Calculate total size
const totalSize = results.success.reduce((sum, r) => sum + (r.fileSize || 0), 0);
console.log(`\nTotal data migrated: ${Math.round(totalSize / 1024 / 1024 * 100) / 100}MB`);
}
// Run migration
main()
.then(() => {
console.log('\n✓ Migration script completed successfully');
process.exit(0);
})
.catch(error => {
console.error('\n✗ Migration script failed:', error);
console.error(error.stack);
process.exit(1);
});

168
migrate-plans-batch.mjs Normal file
View File

@@ -0,0 +1,168 @@
#!/usr/bin/env node
/**
* Automated batch migration of ~/.claude/plans/*.md to task-mcp database
*
* Run from task-mcp directory: node migrate-plans-batch.mjs
*/
import { readFileSync, readdirSync, renameSync, mkdirSync, existsSync, statSync } from 'fs';
import { join } from 'path';
import { homedir } from 'os';
import dotenv from 'dotenv';
import { archiveAdd } from './dist/tools/archives.js';
// Load environment
dotenv.config();
// Configuration
const PLANS_DIR = join(homedir(), '.claude', 'plans');
const BACKUP_DIR = join(PLANS_DIR, '.migrated-to-mcp');
const ALREADY_MIGRATED = [
'adaptive-enchanting-truffle.md',
'adaptive-sleeping-pearl.md'
];
/**
* Extract title from markdown content (first H1)
*/
function extractTitle(content, filename) {
const lines = content.split('\n');
for (const line of lines) {
if (line.startsWith('# ')) {
return line.slice(2).trim().substring(0, 500);
}
}
// Fallback to filename
return filename
.replace('.md', '')
.replace(/-/g, ' ')
.split(' ')
.map(word => word.charAt(0).toUpperCase() + word.slice(1))
.join(' ')
.substring(0, 500);
}
/**
* Migrate a single plan file
*/
async function migratePlan(filename) {
const filePath = join(PLANS_DIR, filename);
try {
const content = readFileSync(filePath, 'utf-8');
const title = extractTitle(content, filename);
const fileSize = statSync(filePath).size;
console.log(`\n[${new Date().toISOString().substring(11, 19)}] Migrating: ${filename}`);
console.log(` Title: ${title}`);
console.log(` Size: ${Math.round(fileSize / 1024)}KB`);
// Call archive_add
const result = await archiveAdd({
project: 'CF',
archive_type: 'session',
title,
content,
original_path: filePath,
file_size: fileSize
});
console.log(` ✓ Migrated successfully`);
// Move to backup
if (!existsSync(BACKUP_DIR)) {
mkdirSync(BACKUP_DIR, { recursive: true });
}
const backupPath = join(BACKUP_DIR, filename);
renameSync(filePath, backupPath);
console.log(` ✓ Moved to backup`);
return { success: true, filename, title, fileSize };
} catch (error) {
console.error(` ✗ Error: ${error.message}`);
return { success: false, filename, error: error.message };
}
}
/**
* Main migration function
*/
async function main() {
console.log('================================================================================');
console.log('Batch Migration: ~/.claude/plans/*.md → task-mcp database');
console.log('================================================================================\n');
// Get all plan files
const allFiles = readdirSync(PLANS_DIR).filter(f => f.endsWith('.md'));
const filesToMigrate = allFiles.filter(f => !ALREADY_MIGRATED.includes(f));
console.log(`Total plan files: ${allFiles.length}`);
console.log(`Already migrated: ${ALREADY_MIGRATED.length}`);
console.log(`To migrate: ${filesToMigrate.length}\n`);
if (filesToMigrate.length === 0) {
console.log('✓ All files already migrated!');
return;
}
console.log('Starting migration...\n');
const results = {
success: [],
failed: []
};
// Migrate each file
for (let i = 0; i < filesToMigrate.length; i++) {
const filename = filesToMigrate[i];
const result = await migratePlan(filename);
if (result.success) {
results.success.push(result);
} else {
results.failed.push(result);
}
// Progress indicator
const progress = Math.round(((i + 1) / filesToMigrate.length) * 100);
console.log(` Progress: ${i + 1}/${filesToMigrate.length} (${progress}%)`);
// Small delay to avoid overwhelming the database
await new Promise(resolve => setTimeout(resolve, 50));
}
// Summary
console.log('\n================================================================================');
console.log('Migration Complete');
console.log('================================================================================\n');
console.log(`✓ Successfully migrated: ${results.success.length} files`);
console.log(`✗ Failed: ${results.failed.length} files`);
if (results.failed.length > 0) {
console.log('\nFailed files:');
results.failed.forEach(f => {
console.log(` - ${f.filename}: ${f.error}`);
});
}
console.log(`\nBackup location: ${BACKUP_DIR}`);
console.log('Original files can be restored if needed.');
// Calculate total size
const totalSize = results.success.reduce((sum, r) => sum + (r.fileSize || 0), 0);
console.log(`\nTotal data migrated: ${Math.round(totalSize / 1024 / 1024 * 100) / 100}MB`);
}
// Run migration
main()
.then(() => {
console.log('\n✓ Migration script completed successfully');
process.exit(0);
})
.catch(error => {
console.error('\n✗ Migration script failed:', error);
console.error(error.stack);
process.exit(1);
});

303
migrate-project-docs-batch.mjs Executable file
View File

@@ -0,0 +1,303 @@
#!/usr/bin/env node
/**
* Automated batch migration of project-specific documentation to task-mcp database
*
* Migrates:
* - CLAUDE_HISTORY.md → project_archives (archive_type: session)
* - planning.md → project_archives (archive_type: completed)
* - PRD.md → project_archives (archive_type: research)
*
* Excludes:
* - .framework-backup directories
* - Archived projects
* - ClaudeFramework (already migrated)
*/
import { readFileSync, readdirSync, renameSync, mkdirSync, existsSync, statSync } from 'fs';
import { join, basename, dirname, relative } from 'path';
import { homedir } from 'os';
import dotenv from 'dotenv';
import { archiveAdd } from './dist/tools/archives.js';
// Load environment
dotenv.config();
// Configuration
const DEV_DIR = join(homedir(), 'Development');
const EXCLUDED_PATTERNS = [
'.framework-backup',
'/Archived/',
'/ClaudeFramework/'
];
/**
* Extract project key from directory path
* Examples:
* /Apps/RealEstate → RE
* /Apps/eToroGridbot → GB
* /Apps/ZorkiOS → ZK
* /Infrastructure/mcp-servers/cloudmemorymcp → CM
*/
function getProjectKey(filePath) {
const parts = filePath.split('/');
// Find the most specific directory name
let projectName = '';
if (filePath.includes('/Apps/')) {
const idx = parts.indexOf('Apps');
projectName = parts[idx + 1] || '';
} else if (filePath.includes('/Infrastructure/')) {
// Use the deepest directory name
projectName = parts[parts.length - 2] || '';
}
// Generate 2-letter key from project name
const normalized = projectName
.replace(/([A-Z])/g, ' $1') // Split camelCase
.trim()
.toUpperCase();
const words = normalized.split(/\s+/);
if (words.length >= 2) {
return words[0][0] + words[1][0]; // First letter of first two words
} else if (words[0].length >= 2) {
return words[0].substring(0, 2); // First two letters
} else {
return words[0][0] + 'X'; // Fallback
}
}
/**
* Determine archive type based on filename
*/
function getArchiveType(filename) {
if (filename === 'CLAUDE_HISTORY.md') {
return 'session';
} else if (filename === 'planning.md') {
return 'completed';
} else if (filename === 'PRD.md') {
return 'research';
}
return 'research';
}
/**
* Extract title from markdown content (first H1)
*/
function extractTitle(content, filename, projectKey) {
const lines = content.split('\n');
for (const line of lines) {
if (line.startsWith('# ')) {
return line.slice(2).trim().substring(0, 500);
}
}
// Fallback to project key + filename
const baseName = filename.replace('.md', '').replace(/_/g, ' ');
return `${projectKey} - ${baseName}`.substring(0, 500);
}
/**
* Check if file should be excluded
*/
function shouldExclude(filePath) {
return EXCLUDED_PATTERNS.some(pattern => filePath.includes(pattern));
}
/**
* Migrate a single file
*/
async function migrateFile(filePath) {
const filename = basename(filePath);
const projectKey = getProjectKey(filePath);
try {
const content = readFileSync(filePath, 'utf-8');
const title = extractTitle(content, filename, projectKey);
const fileSize = statSync(filePath).size;
const archiveType = getArchiveType(filename);
console.log(`\n[${new Date().toISOString().substring(11, 19)}] Migrating: ${relative(DEV_DIR, filePath)}`);
console.log(` Project: ${projectKey}`);
console.log(` Title: ${title}`);
console.log(` Size: ${Math.round(fileSize / 1024)}KB`);
console.log(` Type: ${archiveType}`);
// Call archive_add
const result = await archiveAdd({
project: projectKey,
archive_type: archiveType,
title,
content,
original_path: filePath,
file_size: fileSize
});
console.log(` ✓ Migrated successfully`);
// Move to backup
const backupDir = join(dirname(filePath), '.migrated-to-mcp');
if (!existsSync(backupDir)) {
mkdirSync(backupDir, { recursive: true });
}
const backupPath = join(backupDir, filename);
renameSync(filePath, backupPath);
console.log(` ✓ Moved to backup`);
return { success: true, filename, projectKey, title, fileSize, archiveType, filePath };
} catch (error) {
console.error(` ✗ Error: ${error.message}`);
return { success: false, filename, projectKey, error: error.message, filePath };
}
}
/**
* Find all documentation files to migrate
*/
function findDocFiles() {
const files = [];
const targetFiles = ['CLAUDE_HISTORY.md', 'planning.md', 'PRD.md'];
function scanDir(dir) {
if (!existsSync(dir)) return;
try {
const entries = readdirSync(dir, { withFileTypes: true });
for (const entry of entries) {
const fullPath = join(dir, entry.name);
// Skip excluded paths
if (shouldExclude(fullPath)) continue;
if (entry.isDirectory()) {
// Skip node_modules, .git, etc.
if (!entry.name.startsWith('.') && entry.name !== 'node_modules') {
scanDir(fullPath);
}
} else if (entry.isFile() && targetFiles.includes(entry.name)) {
files.push(fullPath);
}
}
} catch (error) {
// Skip directories we can't read
}
}
scanDir(DEV_DIR);
return files.sort();
}
/**
* Main migration function
*/
async function main() {
console.log('================================================================================');
console.log('Batch Migration: Project Documentation → task-mcp database');
console.log('================================================================================\n');
// Find all files
const filesToMigrate = findDocFiles();
console.log(`Found ${filesToMigrate.length} documentation files to migrate\n`);
if (filesToMigrate.length === 0) {
console.log('✓ No files to migrate!');
return;
}
// Show sample paths
console.log('Sample files:');
filesToMigrate.slice(0, 10).forEach(f => {
console.log(` - ${relative(DEV_DIR, f)}`);
});
if (filesToMigrate.length > 10) {
console.log(` ... and ${filesToMigrate.length - 10} more`);
}
console.log('\nStarting migration...\n');
const results = {
success: [],
failed: []
};
// Migrate each file
for (let i = 0; i < filesToMigrate.length; i++) {
const filePath = filesToMigrate[i];
const result = await migrateFile(filePath);
if (result.success) {
results.success.push(result);
} else {
results.failed.push(result);
}
// Progress indicator
const progress = Math.round(((i + 1) / filesToMigrate.length) * 100);
console.log(` Progress: ${i + 1}/${filesToMigrate.length} (${progress}%)`);
// Small delay to avoid overwhelming the database
await new Promise(resolve => setTimeout(resolve, 50));
}
// Summary
console.log('\n================================================================================');
console.log('Migration Complete');
console.log('================================================================================\n');
console.log(`✓ Successfully migrated: ${results.success.length} files`);
console.log(`✗ Failed: ${results.failed.length} files`);
if (results.failed.length > 0) {
console.log('\nFailed files:');
results.failed.forEach(f => {
console.log(` - ${relative(DEV_DIR, f.filePath)}: ${f.error}`);
});
}
// Group by type
const byType = {};
results.success.forEach(r => {
byType[r.archiveType] = (byType[r.archiveType] || 0) + 1;
});
console.log('\nBy archive type:');
Object.entries(byType).forEach(([type, count]) => {
console.log(` - ${type}: ${count} files`);
});
// Group by project
const byProject = {};
results.success.forEach(r => {
byProject[r.projectKey] = (byProject[r.projectKey] || 0) + 1;
});
console.log('\nBy project:');
Object.entries(byProject)
.sort((a, b) => b[1] - a[1])
.forEach(([project, count]) => {
console.log(` - ${project}: ${count} files`);
});
console.log('\nBackup location: <project-dir>/.migrated-to-mcp/');
console.log('Original files can be restored if needed.');
// Calculate total size
const totalSize = results.success.reduce((sum, r) => sum + (r.fileSize || 0), 0);
console.log(`\nTotal data migrated: ${Math.round(totalSize / 1024 / 1024 * 100) / 100}MB`);
}
// Run migration
main()
.then(() => {
console.log('\n✓ Migration script completed successfully');
process.exit(0);
})
.catch(error => {
console.error('\n✗ Migration script failed:', error);
console.error(error.stack);
process.exit(1);
});

View File

@@ -0,0 +1,17 @@
-- Migration 017: Fix project_archives embedding dimension
-- Changes vector(1536) to vector(1024) to match mxbai-embed-large model
-- Drop existing index first
DROP INDEX IF EXISTS idx_archives_embedding;
-- Alter the embedding column dimension
ALTER TABLE project_archives
ALTER COLUMN embedding TYPE vector(1024);
-- Recreate index with correct dimension
CREATE INDEX idx_archives_embedding ON project_archives
USING hnsw (embedding vector_cosine_ops);
-- Record migration
INSERT INTO schema_migrations (version) VALUES ('017_fix_archives_embedding_dimension')
ON CONFLICT (version) DO NOTHING;

51
run-migration.js Normal file
View File

@@ -0,0 +1,51 @@
#!/usr/bin/env node
/**
* Run a specific migration file
* Usage: node run-migration.js migrations/017_fix_archives_embedding_dimension.sql
*/
import pg from 'pg';
import { readFileSync } from 'fs';
import { resolve } from 'path';
const { Pool } = pg;
// Configuration - Direct WireGuard connection to INFRA VM PostgreSQL
const config = {
host: process.env.POSTGRES_HOST || 'infra.agiliton.internal',
port: 5432,
database: 'agiliton',
user: 'agiliton',
password: 'QtqiwCOAUpQNF6pjzOMAREzUny2bY8V1',
};
async function runMigration(migrationFile) {
const pool = new Pool(config);
try {
// Read migration SQL
const sql = readFileSync(resolve(migrationFile), 'utf-8');
console.log(`Running migration: ${migrationFile}`);
console.log('SQL:', sql);
// Execute migration
await pool.query(sql);
console.log('Migration completed successfully!');
} catch (error) {
console.error('Migration failed:', error);
process.exit(1);
} finally {
await pool.end();
}
}
// Get migration file from command line
const migrationFile = process.argv[2];
if (!migrationFile) {
console.error('Usage: node run-migration.js <migration-file.sql>');
process.exit(1);
}
runMigration(migrationFile);