Add Phase 3 migration: project documentation
- migrate-project-docs-batch.mjs: 60 files across 23 projects - migrate-cf-docs-batch.mjs: ClaudeFramework documentation - migrate-plans-batch.mjs: Session plan files - 017_fix_archives_embedding_dimension.sql: Fix vector dimension - run-migration.js: Node.js migration runner utility Total migrated: 332 files, 8.85MB with semantic embeddings Fixes: CF-277
This commit is contained in:
Christian Gick
255
migrate-cf-docs-batch.mjs
Executable file
255
migrate-cf-docs-batch.mjs
Executable file
@@ -0,0 +1,255 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Automated batch migration of ClaudeFramework docs to task-mcp database
|
||||
*
|
||||
* Migrates:
|
||||
* - docs/investigations/*.md → project_archives (archive_type: investigation)
|
||||
* - docs/*AUDIT*.md → project_archives (archive_type: audit)
|
||||
* - docs/*.md → project_archives (archive_type: research)
|
||||
* - guides/*.md → project_archives (archive_type: research)
|
||||
* - delegation/*.md → project_archives (archive_type: research)
|
||||
*/
|
||||
|
||||
import { readFileSync, readdirSync, renameSync, mkdirSync, existsSync, statSync } from 'fs';
|
||||
import { join, basename, dirname } from 'path';
|
||||
import { homedir } from 'os';
|
||||
import dotenv from 'dotenv';
|
||||
import { archiveAdd } from './dist/tools/archives.js';
|
||||
|
||||
// Load environment
|
||||
dotenv.config();
|
||||
|
||||
// Configuration
|
||||
const CF_DIR = join(homedir(), 'Development/Infrastructure/ClaudeFramework');
|
||||
const BACKUP_DIR = join(CF_DIR, '.migrated-to-mcp');
|
||||
|
||||
// Exclusions
|
||||
const SKIP_FILES = [
|
||||
'README.md',
|
||||
'CHANGELOG.md',
|
||||
'LICENSE.md',
|
||||
'CONTRIBUTING.md',
|
||||
'CLAUDE.md', // Keep active instructions
|
||||
'planning.md', // Keep active planning
|
||||
'PRD.md' // Keep product requirements
|
||||
];
|
||||
|
||||
/**
|
||||
* Determine archive type based on file path and name
|
||||
*/
|
||||
function getArchiveType(filePath) {
|
||||
const filename = basename(filePath);
|
||||
const dirName = dirname(filePath);
|
||||
|
||||
// Investigation files
|
||||
if (dirName.includes('/investigations/')) {
|
||||
return 'investigation';
|
||||
}
|
||||
|
||||
// Audit files
|
||||
if (filename.includes('AUDIT') || filename.includes('_AUDIT')) {
|
||||
return 'audit';
|
||||
}
|
||||
|
||||
// Research/documentation files
|
||||
return 'research';
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract title from markdown content (first H1)
|
||||
*/
|
||||
function extractTitle(content, filename) {
|
||||
const lines = content.split('\n');
|
||||
for (const line of lines) {
|
||||
if (line.startsWith('# ')) {
|
||||
return line.slice(2).trim().substring(0, 500);
|
||||
}
|
||||
}
|
||||
// Fallback to filename
|
||||
return filename
|
||||
.replace('.md', '')
|
||||
.replace(/_/g, ' ')
|
||||
.replace(/-/g, ' ')
|
||||
.split(' ')
|
||||
.map(word => word.charAt(0).toUpperCase() + word.slice(1).toLowerCase())
|
||||
.join(' ')
|
||||
.substring(0, 500);
|
||||
}
|
||||
|
||||
/**
|
||||
* Migrate a single file
|
||||
*/
|
||||
async function migrateFile(filePath) {
|
||||
const filename = basename(filePath);
|
||||
|
||||
try {
|
||||
const content = readFileSync(filePath, 'utf-8');
|
||||
const title = extractTitle(content, filename);
|
||||
const fileSize = statSync(filePath).size;
|
||||
const archiveType = getArchiveType(filePath);
|
||||
|
||||
console.log(`\n[${new Date().toISOString().substring(11, 19)}] Migrating: ${filePath.replace(CF_DIR, '')}`);
|
||||
console.log(` Title: ${title}`);
|
||||
console.log(` Size: ${Math.round(fileSize / 1024)}KB`);
|
||||
console.log(` Type: ${archiveType}`);
|
||||
|
||||
// Call archive_add
|
||||
const result = await archiveAdd({
|
||||
project: 'CF',
|
||||
archive_type: archiveType,
|
||||
title,
|
||||
content,
|
||||
original_path: filePath,
|
||||
file_size: fileSize
|
||||
});
|
||||
|
||||
console.log(` ✓ Migrated successfully`);
|
||||
|
||||
// Move to backup
|
||||
const relativePath = filePath.replace(CF_DIR + '/', '');
|
||||
const backupPath = join(BACKUP_DIR, relativePath);
|
||||
const backupDir = dirname(backupPath);
|
||||
|
||||
if (!existsSync(backupDir)) {
|
||||
mkdirSync(backupDir, { recursive: true });
|
||||
}
|
||||
|
||||
renameSync(filePath, backupPath);
|
||||
console.log(` ✓ Moved to backup`);
|
||||
|
||||
return { success: true, filename, title, fileSize, archiveType };
|
||||
|
||||
} catch (error) {
|
||||
console.error(` ✗ Error: ${error.message}`);
|
||||
return { success: false, filename, error: error.message };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Find all markdown files to migrate
|
||||
*/
|
||||
function findMarkdownFiles() {
|
||||
const files = [];
|
||||
const dirs = [
|
||||
join(CF_DIR, 'docs'),
|
||||
join(CF_DIR, 'guides'),
|
||||
join(CF_DIR, 'delegation')
|
||||
];
|
||||
|
||||
function scanDir(dir) {
|
||||
if (!existsSync(dir)) return;
|
||||
|
||||
const entries = readdirSync(dir, { withFileTypes: true });
|
||||
|
||||
for (const entry of entries) {
|
||||
const fullPath = join(dir, entry.name);
|
||||
|
||||
if (entry.isDirectory()) {
|
||||
scanDir(fullPath);
|
||||
} else if (entry.isFile() && entry.name.endsWith('.md')) {
|
||||
if (!SKIP_FILES.includes(entry.name)) {
|
||||
files.push(fullPath);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
dirs.forEach(scanDir);
|
||||
return files.sort();
|
||||
}
|
||||
|
||||
/**
|
||||
* Main migration function
|
||||
*/
|
||||
async function main() {
|
||||
console.log('================================================================================');
|
||||
console.log('Batch Migration: ClaudeFramework docs → task-mcp database');
|
||||
console.log('================================================================================\n');
|
||||
|
||||
// Find all files
|
||||
const filesToMigrate = findMarkdownFiles();
|
||||
|
||||
console.log(`Found ${filesToMigrate.length} markdown files to migrate\n`);
|
||||
|
||||
if (filesToMigrate.length === 0) {
|
||||
console.log('✓ No files to migrate!');
|
||||
return;
|
||||
}
|
||||
|
||||
// Show sample paths
|
||||
console.log('Sample files:');
|
||||
filesToMigrate.slice(0, 5).forEach(f => {
|
||||
console.log(` - ${f.replace(CF_DIR, '')}`);
|
||||
});
|
||||
console.log(` ... and ${filesToMigrate.length - 5} more\n`);
|
||||
|
||||
console.log('Starting migration...\n');
|
||||
|
||||
const results = {
|
||||
success: [],
|
||||
failed: []
|
||||
};
|
||||
|
||||
// Migrate each file
|
||||
for (let i = 0; i < filesToMigrate.length; i++) {
|
||||
const filePath = filesToMigrate[i];
|
||||
const result = await migrateFile(filePath);
|
||||
|
||||
if (result.success) {
|
||||
results.success.push(result);
|
||||
} else {
|
||||
results.failed.push(result);
|
||||
}
|
||||
|
||||
// Progress indicator
|
||||
const progress = Math.round(((i + 1) / filesToMigrate.length) * 100);
|
||||
console.log(` Progress: ${i + 1}/${filesToMigrate.length} (${progress}%)`);
|
||||
|
||||
// Small delay to avoid overwhelming the database
|
||||
await new Promise(resolve => setTimeout(resolve, 50));
|
||||
}
|
||||
|
||||
// Summary
|
||||
console.log('\n================================================================================');
|
||||
console.log('Migration Complete');
|
||||
console.log('================================================================================\n');
|
||||
console.log(`✓ Successfully migrated: ${results.success.length} files`);
|
||||
console.log(`✗ Failed: ${results.failed.length} files`);
|
||||
|
||||
if (results.failed.length > 0) {
|
||||
console.log('\nFailed files:');
|
||||
results.failed.forEach(f => {
|
||||
console.log(` - ${f.filename}: ${f.error}`);
|
||||
});
|
||||
}
|
||||
|
||||
// Group by type
|
||||
const byType = {};
|
||||
results.success.forEach(r => {
|
||||
byType[r.archiveType] = (byType[r.archiveType] || 0) + 1;
|
||||
});
|
||||
|
||||
console.log('\nBy archive type:');
|
||||
Object.entries(byType).forEach(([type, count]) => {
|
||||
console.log(` - ${type}: ${count} files`);
|
||||
});
|
||||
|
||||
console.log(`\nBackup location: ${BACKUP_DIR}`);
|
||||
console.log('Original files can be restored if needed.');
|
||||
|
||||
// Calculate total size
|
||||
const totalSize = results.success.reduce((sum, r) => sum + (r.fileSize || 0), 0);
|
||||
console.log(`\nTotal data migrated: ${Math.round(totalSize / 1024 / 1024 * 100) / 100}MB`);
|
||||
}
|
||||
|
||||
// Run migration
|
||||
main()
|
||||
.then(() => {
|
||||
console.log('\n✓ Migration script completed successfully');
|
||||
process.exit(0);
|
||||
})
|
||||
.catch(error => {
|
||||
console.error('\n✗ Migration script failed:', error);
|
||||
console.error(error.stack);
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in New Issue
Block a user