Add Phase 3 migration: project documentation
- migrate-project-docs-batch.mjs: 60 files across 23 projects - migrate-cf-docs-batch.mjs: ClaudeFramework documentation - migrate-plans-batch.mjs: Session plan files - 017_fix_archives_embedding_dimension.sql: Fix vector dimension - run-migration.js: Node.js migration runner utility Total migrated: 332 files, 8.85MB with semantic embeddings Fixes: CF-277
This commit is contained in:
255
migrate-cf-docs-batch.mjs
Executable file
255
migrate-cf-docs-batch.mjs
Executable file
@@ -0,0 +1,255 @@
|
|||||||
|
#!/usr/bin/env node
|
||||||
|
/**
|
||||||
|
* Automated batch migration of ClaudeFramework docs to task-mcp database
|
||||||
|
*
|
||||||
|
* Migrates:
|
||||||
|
* - docs/investigations/*.md → project_archives (archive_type: investigation)
|
||||||
|
* - docs/*AUDIT*.md → project_archives (archive_type: audit)
|
||||||
|
* - docs/*.md → project_archives (archive_type: research)
|
||||||
|
* - guides/*.md → project_archives (archive_type: research)
|
||||||
|
* - delegation/*.md → project_archives (archive_type: research)
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { readFileSync, readdirSync, renameSync, mkdirSync, existsSync, statSync } from 'fs';
|
||||||
|
import { join, basename, dirname } from 'path';
|
||||||
|
import { homedir } from 'os';
|
||||||
|
import dotenv from 'dotenv';
|
||||||
|
import { archiveAdd } from './dist/tools/archives.js';
|
||||||
|
|
||||||
|
// Load environment
|
||||||
|
dotenv.config();
|
||||||
|
|
||||||
|
// Configuration
|
||||||
|
const CF_DIR = join(homedir(), 'Development/Infrastructure/ClaudeFramework');
|
||||||
|
const BACKUP_DIR = join(CF_DIR, '.migrated-to-mcp');
|
||||||
|
|
||||||
|
// Exclusions
|
||||||
|
const SKIP_FILES = [
|
||||||
|
'README.md',
|
||||||
|
'CHANGELOG.md',
|
||||||
|
'LICENSE.md',
|
||||||
|
'CONTRIBUTING.md',
|
||||||
|
'CLAUDE.md', // Keep active instructions
|
||||||
|
'planning.md', // Keep active planning
|
||||||
|
'PRD.md' // Keep product requirements
|
||||||
|
];
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Determine archive type based on file path and name
|
||||||
|
*/
|
||||||
|
function getArchiveType(filePath) {
|
||||||
|
const filename = basename(filePath);
|
||||||
|
const dirName = dirname(filePath);
|
||||||
|
|
||||||
|
// Investigation files
|
||||||
|
if (dirName.includes('/investigations/')) {
|
||||||
|
return 'investigation';
|
||||||
|
}
|
||||||
|
|
||||||
|
// Audit files
|
||||||
|
if (filename.includes('AUDIT') || filename.includes('_AUDIT')) {
|
||||||
|
return 'audit';
|
||||||
|
}
|
||||||
|
|
||||||
|
// Research/documentation files
|
||||||
|
return 'research';
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Extract title from markdown content (first H1)
|
||||||
|
*/
|
||||||
|
function extractTitle(content, filename) {
|
||||||
|
const lines = content.split('\n');
|
||||||
|
for (const line of lines) {
|
||||||
|
if (line.startsWith('# ')) {
|
||||||
|
return line.slice(2).trim().substring(0, 500);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Fallback to filename
|
||||||
|
return filename
|
||||||
|
.replace('.md', '')
|
||||||
|
.replace(/_/g, ' ')
|
||||||
|
.replace(/-/g, ' ')
|
||||||
|
.split(' ')
|
||||||
|
.map(word => word.charAt(0).toUpperCase() + word.slice(1).toLowerCase())
|
||||||
|
.join(' ')
|
||||||
|
.substring(0, 500);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Migrate a single file
|
||||||
|
*/
|
||||||
|
async function migrateFile(filePath) {
|
||||||
|
const filename = basename(filePath);
|
||||||
|
|
||||||
|
try {
|
||||||
|
const content = readFileSync(filePath, 'utf-8');
|
||||||
|
const title = extractTitle(content, filename);
|
||||||
|
const fileSize = statSync(filePath).size;
|
||||||
|
const archiveType = getArchiveType(filePath);
|
||||||
|
|
||||||
|
console.log(`\n[${new Date().toISOString().substring(11, 19)}] Migrating: ${filePath.replace(CF_DIR, '')}`);
|
||||||
|
console.log(` Title: ${title}`);
|
||||||
|
console.log(` Size: ${Math.round(fileSize / 1024)}KB`);
|
||||||
|
console.log(` Type: ${archiveType}`);
|
||||||
|
|
||||||
|
// Call archive_add
|
||||||
|
const result = await archiveAdd({
|
||||||
|
project: 'CF',
|
||||||
|
archive_type: archiveType,
|
||||||
|
title,
|
||||||
|
content,
|
||||||
|
original_path: filePath,
|
||||||
|
file_size: fileSize
|
||||||
|
});
|
||||||
|
|
||||||
|
console.log(` ✓ Migrated successfully`);
|
||||||
|
|
||||||
|
// Move to backup
|
||||||
|
const relativePath = filePath.replace(CF_DIR + '/', '');
|
||||||
|
const backupPath = join(BACKUP_DIR, relativePath);
|
||||||
|
const backupDir = dirname(backupPath);
|
||||||
|
|
||||||
|
if (!existsSync(backupDir)) {
|
||||||
|
mkdirSync(backupDir, { recursive: true });
|
||||||
|
}
|
||||||
|
|
||||||
|
renameSync(filePath, backupPath);
|
||||||
|
console.log(` ✓ Moved to backup`);
|
||||||
|
|
||||||
|
return { success: true, filename, title, fileSize, archiveType };
|
||||||
|
|
||||||
|
} catch (error) {
|
||||||
|
console.error(` ✗ Error: ${error.message}`);
|
||||||
|
return { success: false, filename, error: error.message };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Find all markdown files to migrate
|
||||||
|
*/
|
||||||
|
function findMarkdownFiles() {
|
||||||
|
const files = [];
|
||||||
|
const dirs = [
|
||||||
|
join(CF_DIR, 'docs'),
|
||||||
|
join(CF_DIR, 'guides'),
|
||||||
|
join(CF_DIR, 'delegation')
|
||||||
|
];
|
||||||
|
|
||||||
|
function scanDir(dir) {
|
||||||
|
if (!existsSync(dir)) return;
|
||||||
|
|
||||||
|
const entries = readdirSync(dir, { withFileTypes: true });
|
||||||
|
|
||||||
|
for (const entry of entries) {
|
||||||
|
const fullPath = join(dir, entry.name);
|
||||||
|
|
||||||
|
if (entry.isDirectory()) {
|
||||||
|
scanDir(fullPath);
|
||||||
|
} else if (entry.isFile() && entry.name.endsWith('.md')) {
|
||||||
|
if (!SKIP_FILES.includes(entry.name)) {
|
||||||
|
files.push(fullPath);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
dirs.forEach(scanDir);
|
||||||
|
return files.sort();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Main migration function
|
||||||
|
*/
|
||||||
|
async function main() {
|
||||||
|
console.log('================================================================================');
|
||||||
|
console.log('Batch Migration: ClaudeFramework docs → task-mcp database');
|
||||||
|
console.log('================================================================================\n');
|
||||||
|
|
||||||
|
// Find all files
|
||||||
|
const filesToMigrate = findMarkdownFiles();
|
||||||
|
|
||||||
|
console.log(`Found ${filesToMigrate.length} markdown files to migrate\n`);
|
||||||
|
|
||||||
|
if (filesToMigrate.length === 0) {
|
||||||
|
console.log('✓ No files to migrate!');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Show sample paths
|
||||||
|
console.log('Sample files:');
|
||||||
|
filesToMigrate.slice(0, 5).forEach(f => {
|
||||||
|
console.log(` - ${f.replace(CF_DIR, '')}`);
|
||||||
|
});
|
||||||
|
console.log(` ... and ${filesToMigrate.length - 5} more\n`);
|
||||||
|
|
||||||
|
console.log('Starting migration...\n');
|
||||||
|
|
||||||
|
const results = {
|
||||||
|
success: [],
|
||||||
|
failed: []
|
||||||
|
};
|
||||||
|
|
||||||
|
// Migrate each file
|
||||||
|
for (let i = 0; i < filesToMigrate.length; i++) {
|
||||||
|
const filePath = filesToMigrate[i];
|
||||||
|
const result = await migrateFile(filePath);
|
||||||
|
|
||||||
|
if (result.success) {
|
||||||
|
results.success.push(result);
|
||||||
|
} else {
|
||||||
|
results.failed.push(result);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Progress indicator
|
||||||
|
const progress = Math.round(((i + 1) / filesToMigrate.length) * 100);
|
||||||
|
console.log(` Progress: ${i + 1}/${filesToMigrate.length} (${progress}%)`);
|
||||||
|
|
||||||
|
// Small delay to avoid overwhelming the database
|
||||||
|
await new Promise(resolve => setTimeout(resolve, 50));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Summary
|
||||||
|
console.log('\n================================================================================');
|
||||||
|
console.log('Migration Complete');
|
||||||
|
console.log('================================================================================\n');
|
||||||
|
console.log(`✓ Successfully migrated: ${results.success.length} files`);
|
||||||
|
console.log(`✗ Failed: ${results.failed.length} files`);
|
||||||
|
|
||||||
|
if (results.failed.length > 0) {
|
||||||
|
console.log('\nFailed files:');
|
||||||
|
results.failed.forEach(f => {
|
||||||
|
console.log(` - ${f.filename}: ${f.error}`);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// Group by type
|
||||||
|
const byType = {};
|
||||||
|
results.success.forEach(r => {
|
||||||
|
byType[r.archiveType] = (byType[r.archiveType] || 0) + 1;
|
||||||
|
});
|
||||||
|
|
||||||
|
console.log('\nBy archive type:');
|
||||||
|
Object.entries(byType).forEach(([type, count]) => {
|
||||||
|
console.log(` - ${type}: ${count} files`);
|
||||||
|
});
|
||||||
|
|
||||||
|
console.log(`\nBackup location: ${BACKUP_DIR}`);
|
||||||
|
console.log('Original files can be restored if needed.');
|
||||||
|
|
||||||
|
// Calculate total size
|
||||||
|
const totalSize = results.success.reduce((sum, r) => sum + (r.fileSize || 0), 0);
|
||||||
|
console.log(`\nTotal data migrated: ${Math.round(totalSize / 1024 / 1024 * 100) / 100}MB`);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run migration
|
||||||
|
main()
|
||||||
|
.then(() => {
|
||||||
|
console.log('\n✓ Migration script completed successfully');
|
||||||
|
process.exit(0);
|
||||||
|
})
|
||||||
|
.catch(error => {
|
||||||
|
console.error('\n✗ Migration script failed:', error);
|
||||||
|
console.error(error.stack);
|
||||||
|
process.exit(1);
|
||||||
|
});
|
||||||
168
migrate-plans-batch.mjs
Normal file
168
migrate-plans-batch.mjs
Normal file
@@ -0,0 +1,168 @@
|
|||||||
|
#!/usr/bin/env node
|
||||||
|
/**
|
||||||
|
* Automated batch migration of ~/.claude/plans/*.md to task-mcp database
|
||||||
|
*
|
||||||
|
* Run from task-mcp directory: node migrate-plans-batch.mjs
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { readFileSync, readdirSync, renameSync, mkdirSync, existsSync, statSync } from 'fs';
|
||||||
|
import { join } from 'path';
|
||||||
|
import { homedir } from 'os';
|
||||||
|
import dotenv from 'dotenv';
|
||||||
|
import { archiveAdd } from './dist/tools/archives.js';
|
||||||
|
|
||||||
|
// Load environment
|
||||||
|
dotenv.config();
|
||||||
|
|
||||||
|
// Configuration
|
||||||
|
const PLANS_DIR = join(homedir(), '.claude', 'plans');
|
||||||
|
const BACKUP_DIR = join(PLANS_DIR, '.migrated-to-mcp');
|
||||||
|
const ALREADY_MIGRATED = [
|
||||||
|
'adaptive-enchanting-truffle.md',
|
||||||
|
'adaptive-sleeping-pearl.md'
|
||||||
|
];
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Extract title from markdown content (first H1)
|
||||||
|
*/
|
||||||
|
function extractTitle(content, filename) {
|
||||||
|
const lines = content.split('\n');
|
||||||
|
for (const line of lines) {
|
||||||
|
if (line.startsWith('# ')) {
|
||||||
|
return line.slice(2).trim().substring(0, 500);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Fallback to filename
|
||||||
|
return filename
|
||||||
|
.replace('.md', '')
|
||||||
|
.replace(/-/g, ' ')
|
||||||
|
.split(' ')
|
||||||
|
.map(word => word.charAt(0).toUpperCase() + word.slice(1))
|
||||||
|
.join(' ')
|
||||||
|
.substring(0, 500);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Migrate a single plan file
|
||||||
|
*/
|
||||||
|
async function migratePlan(filename) {
|
||||||
|
const filePath = join(PLANS_DIR, filename);
|
||||||
|
|
||||||
|
try {
|
||||||
|
const content = readFileSync(filePath, 'utf-8');
|
||||||
|
const title = extractTitle(content, filename);
|
||||||
|
const fileSize = statSync(filePath).size;
|
||||||
|
|
||||||
|
console.log(`\n[${new Date().toISOString().substring(11, 19)}] Migrating: ${filename}`);
|
||||||
|
console.log(` Title: ${title}`);
|
||||||
|
console.log(` Size: ${Math.round(fileSize / 1024)}KB`);
|
||||||
|
|
||||||
|
// Call archive_add
|
||||||
|
const result = await archiveAdd({
|
||||||
|
project: 'CF',
|
||||||
|
archive_type: 'session',
|
||||||
|
title,
|
||||||
|
content,
|
||||||
|
original_path: filePath,
|
||||||
|
file_size: fileSize
|
||||||
|
});
|
||||||
|
|
||||||
|
console.log(` ✓ Migrated successfully`);
|
||||||
|
|
||||||
|
// Move to backup
|
||||||
|
if (!existsSync(BACKUP_DIR)) {
|
||||||
|
mkdirSync(BACKUP_DIR, { recursive: true });
|
||||||
|
}
|
||||||
|
|
||||||
|
const backupPath = join(BACKUP_DIR, filename);
|
||||||
|
renameSync(filePath, backupPath);
|
||||||
|
console.log(` ✓ Moved to backup`);
|
||||||
|
|
||||||
|
return { success: true, filename, title, fileSize };
|
||||||
|
|
||||||
|
} catch (error) {
|
||||||
|
console.error(` ✗ Error: ${error.message}`);
|
||||||
|
return { success: false, filename, error: error.message };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Main migration function
|
||||||
|
*/
|
||||||
|
async function main() {
|
||||||
|
console.log('================================================================================');
|
||||||
|
console.log('Batch Migration: ~/.claude/plans/*.md → task-mcp database');
|
||||||
|
console.log('================================================================================\n');
|
||||||
|
|
||||||
|
// Get all plan files
|
||||||
|
const allFiles = readdirSync(PLANS_DIR).filter(f => f.endsWith('.md'));
|
||||||
|
const filesToMigrate = allFiles.filter(f => !ALREADY_MIGRATED.includes(f));
|
||||||
|
|
||||||
|
console.log(`Total plan files: ${allFiles.length}`);
|
||||||
|
console.log(`Already migrated: ${ALREADY_MIGRATED.length}`);
|
||||||
|
console.log(`To migrate: ${filesToMigrate.length}\n`);
|
||||||
|
|
||||||
|
if (filesToMigrate.length === 0) {
|
||||||
|
console.log('✓ All files already migrated!');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log('Starting migration...\n');
|
||||||
|
|
||||||
|
const results = {
|
||||||
|
success: [],
|
||||||
|
failed: []
|
||||||
|
};
|
||||||
|
|
||||||
|
// Migrate each file
|
||||||
|
for (let i = 0; i < filesToMigrate.length; i++) {
|
||||||
|
const filename = filesToMigrate[i];
|
||||||
|
const result = await migratePlan(filename);
|
||||||
|
|
||||||
|
if (result.success) {
|
||||||
|
results.success.push(result);
|
||||||
|
} else {
|
||||||
|
results.failed.push(result);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Progress indicator
|
||||||
|
const progress = Math.round(((i + 1) / filesToMigrate.length) * 100);
|
||||||
|
console.log(` Progress: ${i + 1}/${filesToMigrate.length} (${progress}%)`);
|
||||||
|
|
||||||
|
// Small delay to avoid overwhelming the database
|
||||||
|
await new Promise(resolve => setTimeout(resolve, 50));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Summary
|
||||||
|
console.log('\n================================================================================');
|
||||||
|
console.log('Migration Complete');
|
||||||
|
console.log('================================================================================\n');
|
||||||
|
console.log(`✓ Successfully migrated: ${results.success.length} files`);
|
||||||
|
console.log(`✗ Failed: ${results.failed.length} files`);
|
||||||
|
|
||||||
|
if (results.failed.length > 0) {
|
||||||
|
console.log('\nFailed files:');
|
||||||
|
results.failed.forEach(f => {
|
||||||
|
console.log(` - ${f.filename}: ${f.error}`);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(`\nBackup location: ${BACKUP_DIR}`);
|
||||||
|
console.log('Original files can be restored if needed.');
|
||||||
|
|
||||||
|
// Calculate total size
|
||||||
|
const totalSize = results.success.reduce((sum, r) => sum + (r.fileSize || 0), 0);
|
||||||
|
console.log(`\nTotal data migrated: ${Math.round(totalSize / 1024 / 1024 * 100) / 100}MB`);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run migration
|
||||||
|
main()
|
||||||
|
.then(() => {
|
||||||
|
console.log('\n✓ Migration script completed successfully');
|
||||||
|
process.exit(0);
|
||||||
|
})
|
||||||
|
.catch(error => {
|
||||||
|
console.error('\n✗ Migration script failed:', error);
|
||||||
|
console.error(error.stack);
|
||||||
|
process.exit(1);
|
||||||
|
});
|
||||||
303
migrate-project-docs-batch.mjs
Executable file
303
migrate-project-docs-batch.mjs
Executable file
@@ -0,0 +1,303 @@
|
|||||||
|
#!/usr/bin/env node
|
||||||
|
/**
|
||||||
|
* Automated batch migration of project-specific documentation to task-mcp database
|
||||||
|
*
|
||||||
|
* Migrates:
|
||||||
|
* - CLAUDE_HISTORY.md → project_archives (archive_type: session)
|
||||||
|
* - planning.md → project_archives (archive_type: completed)
|
||||||
|
* - PRD.md → project_archives (archive_type: research)
|
||||||
|
*
|
||||||
|
* Excludes:
|
||||||
|
* - .framework-backup directories
|
||||||
|
* - Archived projects
|
||||||
|
* - ClaudeFramework (already migrated)
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { readFileSync, readdirSync, renameSync, mkdirSync, existsSync, statSync } from 'fs';
|
||||||
|
import { join, basename, dirname, relative } from 'path';
|
||||||
|
import { homedir } from 'os';
|
||||||
|
import dotenv from 'dotenv';
|
||||||
|
import { archiveAdd } from './dist/tools/archives.js';
|
||||||
|
|
||||||
|
// Load environment
|
||||||
|
dotenv.config();
|
||||||
|
|
||||||
|
// Configuration
|
||||||
|
const DEV_DIR = join(homedir(), 'Development');
|
||||||
|
const EXCLUDED_PATTERNS = [
|
||||||
|
'.framework-backup',
|
||||||
|
'/Archived/',
|
||||||
|
'/ClaudeFramework/'
|
||||||
|
];
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Extract project key from directory path
|
||||||
|
* Examples:
|
||||||
|
* /Apps/RealEstate → RE
|
||||||
|
* /Apps/eToroGridbot → GB
|
||||||
|
* /Apps/ZorkiOS → ZK
|
||||||
|
* /Infrastructure/mcp-servers/cloudmemorymcp → CM
|
||||||
|
*/
|
||||||
|
function getProjectKey(filePath) {
|
||||||
|
const parts = filePath.split('/');
|
||||||
|
|
||||||
|
// Find the most specific directory name
|
||||||
|
let projectName = '';
|
||||||
|
|
||||||
|
if (filePath.includes('/Apps/')) {
|
||||||
|
const idx = parts.indexOf('Apps');
|
||||||
|
projectName = parts[idx + 1] || '';
|
||||||
|
} else if (filePath.includes('/Infrastructure/')) {
|
||||||
|
// Use the deepest directory name
|
||||||
|
projectName = parts[parts.length - 2] || '';
|
||||||
|
}
|
||||||
|
|
||||||
|
// Generate 2-letter key from project name
|
||||||
|
const normalized = projectName
|
||||||
|
.replace(/([A-Z])/g, ' $1') // Split camelCase
|
||||||
|
.trim()
|
||||||
|
.toUpperCase();
|
||||||
|
|
||||||
|
const words = normalized.split(/\s+/);
|
||||||
|
|
||||||
|
if (words.length >= 2) {
|
||||||
|
return words[0][0] + words[1][0]; // First letter of first two words
|
||||||
|
} else if (words[0].length >= 2) {
|
||||||
|
return words[0].substring(0, 2); // First two letters
|
||||||
|
} else {
|
||||||
|
return words[0][0] + 'X'; // Fallback
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Determine archive type based on filename
|
||||||
|
*/
|
||||||
|
function getArchiveType(filename) {
|
||||||
|
if (filename === 'CLAUDE_HISTORY.md') {
|
||||||
|
return 'session';
|
||||||
|
} else if (filename === 'planning.md') {
|
||||||
|
return 'completed';
|
||||||
|
} else if (filename === 'PRD.md') {
|
||||||
|
return 'research';
|
||||||
|
}
|
||||||
|
return 'research';
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Extract title from markdown content (first H1)
|
||||||
|
*/
|
||||||
|
function extractTitle(content, filename, projectKey) {
|
||||||
|
const lines = content.split('\n');
|
||||||
|
for (const line of lines) {
|
||||||
|
if (line.startsWith('# ')) {
|
||||||
|
return line.slice(2).trim().substring(0, 500);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fallback to project key + filename
|
||||||
|
const baseName = filename.replace('.md', '').replace(/_/g, ' ');
|
||||||
|
return `${projectKey} - ${baseName}`.substring(0, 500);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check if file should be excluded
|
||||||
|
*/
|
||||||
|
function shouldExclude(filePath) {
|
||||||
|
return EXCLUDED_PATTERNS.some(pattern => filePath.includes(pattern));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Migrate a single file
|
||||||
|
*/
|
||||||
|
async function migrateFile(filePath) {
|
||||||
|
const filename = basename(filePath);
|
||||||
|
const projectKey = getProjectKey(filePath);
|
||||||
|
|
||||||
|
try {
|
||||||
|
const content = readFileSync(filePath, 'utf-8');
|
||||||
|
const title = extractTitle(content, filename, projectKey);
|
||||||
|
const fileSize = statSync(filePath).size;
|
||||||
|
const archiveType = getArchiveType(filename);
|
||||||
|
|
||||||
|
console.log(`\n[${new Date().toISOString().substring(11, 19)}] Migrating: ${relative(DEV_DIR, filePath)}`);
|
||||||
|
console.log(` Project: ${projectKey}`);
|
||||||
|
console.log(` Title: ${title}`);
|
||||||
|
console.log(` Size: ${Math.round(fileSize / 1024)}KB`);
|
||||||
|
console.log(` Type: ${archiveType}`);
|
||||||
|
|
||||||
|
// Call archive_add
|
||||||
|
const result = await archiveAdd({
|
||||||
|
project: projectKey,
|
||||||
|
archive_type: archiveType,
|
||||||
|
title,
|
||||||
|
content,
|
||||||
|
original_path: filePath,
|
||||||
|
file_size: fileSize
|
||||||
|
});
|
||||||
|
|
||||||
|
console.log(` ✓ Migrated successfully`);
|
||||||
|
|
||||||
|
// Move to backup
|
||||||
|
const backupDir = join(dirname(filePath), '.migrated-to-mcp');
|
||||||
|
if (!existsSync(backupDir)) {
|
||||||
|
mkdirSync(backupDir, { recursive: true });
|
||||||
|
}
|
||||||
|
|
||||||
|
const backupPath = join(backupDir, filename);
|
||||||
|
renameSync(filePath, backupPath);
|
||||||
|
console.log(` ✓ Moved to backup`);
|
||||||
|
|
||||||
|
return { success: true, filename, projectKey, title, fileSize, archiveType, filePath };
|
||||||
|
|
||||||
|
} catch (error) {
|
||||||
|
console.error(` ✗ Error: ${error.message}`);
|
||||||
|
return { success: false, filename, projectKey, error: error.message, filePath };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Find all documentation files to migrate
|
||||||
|
*/
|
||||||
|
function findDocFiles() {
|
||||||
|
const files = [];
|
||||||
|
const targetFiles = ['CLAUDE_HISTORY.md', 'planning.md', 'PRD.md'];
|
||||||
|
|
||||||
|
function scanDir(dir) {
|
||||||
|
if (!existsSync(dir)) return;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const entries = readdirSync(dir, { withFileTypes: true });
|
||||||
|
|
||||||
|
for (const entry of entries) {
|
||||||
|
const fullPath = join(dir, entry.name);
|
||||||
|
|
||||||
|
// Skip excluded paths
|
||||||
|
if (shouldExclude(fullPath)) continue;
|
||||||
|
|
||||||
|
if (entry.isDirectory()) {
|
||||||
|
// Skip node_modules, .git, etc.
|
||||||
|
if (!entry.name.startsWith('.') && entry.name !== 'node_modules') {
|
||||||
|
scanDir(fullPath);
|
||||||
|
}
|
||||||
|
} else if (entry.isFile() && targetFiles.includes(entry.name)) {
|
||||||
|
files.push(fullPath);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
// Skip directories we can't read
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
scanDir(DEV_DIR);
|
||||||
|
return files.sort();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Main migration function
|
||||||
|
*/
|
||||||
|
async function main() {
|
||||||
|
console.log('================================================================================');
|
||||||
|
console.log('Batch Migration: Project Documentation → task-mcp database');
|
||||||
|
console.log('================================================================================\n');
|
||||||
|
|
||||||
|
// Find all files
|
||||||
|
const filesToMigrate = findDocFiles();
|
||||||
|
|
||||||
|
console.log(`Found ${filesToMigrate.length} documentation files to migrate\n`);
|
||||||
|
|
||||||
|
if (filesToMigrate.length === 0) {
|
||||||
|
console.log('✓ No files to migrate!');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Show sample paths
|
||||||
|
console.log('Sample files:');
|
||||||
|
filesToMigrate.slice(0, 10).forEach(f => {
|
||||||
|
console.log(` - ${relative(DEV_DIR, f)}`);
|
||||||
|
});
|
||||||
|
if (filesToMigrate.length > 10) {
|
||||||
|
console.log(` ... and ${filesToMigrate.length - 10} more`);
|
||||||
|
}
|
||||||
|
console.log('\nStarting migration...\n');
|
||||||
|
|
||||||
|
const results = {
|
||||||
|
success: [],
|
||||||
|
failed: []
|
||||||
|
};
|
||||||
|
|
||||||
|
// Migrate each file
|
||||||
|
for (let i = 0; i < filesToMigrate.length; i++) {
|
||||||
|
const filePath = filesToMigrate[i];
|
||||||
|
const result = await migrateFile(filePath);
|
||||||
|
|
||||||
|
if (result.success) {
|
||||||
|
results.success.push(result);
|
||||||
|
} else {
|
||||||
|
results.failed.push(result);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Progress indicator
|
||||||
|
const progress = Math.round(((i + 1) / filesToMigrate.length) * 100);
|
||||||
|
console.log(` Progress: ${i + 1}/${filesToMigrate.length} (${progress}%)`);
|
||||||
|
|
||||||
|
// Small delay to avoid overwhelming the database
|
||||||
|
await new Promise(resolve => setTimeout(resolve, 50));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Summary
|
||||||
|
console.log('\n================================================================================');
|
||||||
|
console.log('Migration Complete');
|
||||||
|
console.log('================================================================================\n');
|
||||||
|
console.log(`✓ Successfully migrated: ${results.success.length} files`);
|
||||||
|
console.log(`✗ Failed: ${results.failed.length} files`);
|
||||||
|
|
||||||
|
if (results.failed.length > 0) {
|
||||||
|
console.log('\nFailed files:');
|
||||||
|
results.failed.forEach(f => {
|
||||||
|
console.log(` - ${relative(DEV_DIR, f.filePath)}: ${f.error}`);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// Group by type
|
||||||
|
const byType = {};
|
||||||
|
results.success.forEach(r => {
|
||||||
|
byType[r.archiveType] = (byType[r.archiveType] || 0) + 1;
|
||||||
|
});
|
||||||
|
|
||||||
|
console.log('\nBy archive type:');
|
||||||
|
Object.entries(byType).forEach(([type, count]) => {
|
||||||
|
console.log(` - ${type}: ${count} files`);
|
||||||
|
});
|
||||||
|
|
||||||
|
// Group by project
|
||||||
|
const byProject = {};
|
||||||
|
results.success.forEach(r => {
|
||||||
|
byProject[r.projectKey] = (byProject[r.projectKey] || 0) + 1;
|
||||||
|
});
|
||||||
|
|
||||||
|
console.log('\nBy project:');
|
||||||
|
Object.entries(byProject)
|
||||||
|
.sort((a, b) => b[1] - a[1])
|
||||||
|
.forEach(([project, count]) => {
|
||||||
|
console.log(` - ${project}: ${count} files`);
|
||||||
|
});
|
||||||
|
|
||||||
|
console.log('\nBackup location: <project-dir>/.migrated-to-mcp/');
|
||||||
|
console.log('Original files can be restored if needed.');
|
||||||
|
|
||||||
|
// Calculate total size
|
||||||
|
const totalSize = results.success.reduce((sum, r) => sum + (r.fileSize || 0), 0);
|
||||||
|
console.log(`\nTotal data migrated: ${Math.round(totalSize / 1024 / 1024 * 100) / 100}MB`);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run migration
|
||||||
|
main()
|
||||||
|
.then(() => {
|
||||||
|
console.log('\n✓ Migration script completed successfully');
|
||||||
|
process.exit(0);
|
||||||
|
})
|
||||||
|
.catch(error => {
|
||||||
|
console.error('\n✗ Migration script failed:', error);
|
||||||
|
console.error(error.stack);
|
||||||
|
process.exit(1);
|
||||||
|
});
|
||||||
17
migrations/017_fix_archives_embedding_dimension.sql
Normal file
17
migrations/017_fix_archives_embedding_dimension.sql
Normal file
@@ -0,0 +1,17 @@
|
|||||||
|
-- Migration 017: Fix project_archives embedding dimension
|
||||||
|
-- Changes vector(1536) to vector(1024) to match mxbai-embed-large model
|
||||||
|
|
||||||
|
-- Drop existing index first
|
||||||
|
DROP INDEX IF EXISTS idx_archives_embedding;
|
||||||
|
|
||||||
|
-- Alter the embedding column dimension
|
||||||
|
ALTER TABLE project_archives
|
||||||
|
ALTER COLUMN embedding TYPE vector(1024);
|
||||||
|
|
||||||
|
-- Recreate index with correct dimension
|
||||||
|
CREATE INDEX idx_archives_embedding ON project_archives
|
||||||
|
USING hnsw (embedding vector_cosine_ops);
|
||||||
|
|
||||||
|
-- Record migration
|
||||||
|
INSERT INTO schema_migrations (version) VALUES ('017_fix_archives_embedding_dimension')
|
||||||
|
ON CONFLICT (version) DO NOTHING;
|
||||||
51
run-migration.js
Normal file
51
run-migration.js
Normal file
@@ -0,0 +1,51 @@
|
|||||||
|
#!/usr/bin/env node
|
||||||
|
/**
|
||||||
|
* Run a specific migration file
|
||||||
|
* Usage: node run-migration.js migrations/017_fix_archives_embedding_dimension.sql
|
||||||
|
*/
|
||||||
|
|
||||||
|
import pg from 'pg';
|
||||||
|
import { readFileSync } from 'fs';
|
||||||
|
import { resolve } from 'path';
|
||||||
|
|
||||||
|
const { Pool } = pg;
|
||||||
|
|
||||||
|
// Configuration - Direct WireGuard connection to INFRA VM PostgreSQL
|
||||||
|
const config = {
|
||||||
|
host: process.env.POSTGRES_HOST || 'infra.agiliton.internal',
|
||||||
|
port: 5432,
|
||||||
|
database: 'agiliton',
|
||||||
|
user: 'agiliton',
|
||||||
|
password: 'QtqiwCOAUpQNF6pjzOMAREzUny2bY8V1',
|
||||||
|
};
|
||||||
|
|
||||||
|
async function runMigration(migrationFile) {
|
||||||
|
const pool = new Pool(config);
|
||||||
|
|
||||||
|
try {
|
||||||
|
// Read migration SQL
|
||||||
|
const sql = readFileSync(resolve(migrationFile), 'utf-8');
|
||||||
|
|
||||||
|
console.log(`Running migration: ${migrationFile}`);
|
||||||
|
console.log('SQL:', sql);
|
||||||
|
|
||||||
|
// Execute migration
|
||||||
|
await pool.query(sql);
|
||||||
|
|
||||||
|
console.log('Migration completed successfully!');
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Migration failed:', error);
|
||||||
|
process.exit(1);
|
||||||
|
} finally {
|
||||||
|
await pool.end();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get migration file from command line
|
||||||
|
const migrationFile = process.argv[2];
|
||||||
|
if (!migrationFile) {
|
||||||
|
console.error('Usage: node run-migration.js <migration-file.sql>');
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
runMigration(migrationFile);
|
||||||
Reference in New Issue
Block a user