25bed341e9
Three migration scripts for complete file-to-database migration: - migrate-all-docs-batch.mjs: Main migration (1,172 files) - migrate-missed-docs.mjs: Supplementary for hidden dirs (34 files) - migrate-external-archive.mjs: External archive cleanup (5 files) Total migrated: 1,211 files (~15MB) to project_archives table All with semantic embeddings for vector search Related: CF-267, CF-268 Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
280 lines
10 KiB
JavaScript
280 lines
10 KiB
JavaScript
#!/usr/bin/env node
|
|
/**
|
|
* Supplementary migration for 34 files missed by initial migration
|
|
*
|
|
* Missed due to hidden directory exclusions (.framework-backup, .project-files, .claude/commands, .playwright-mcp)
|
|
* and Archived/ directory exclusion.
|
|
*
|
|
* This script:
|
|
* - Migrates 34 specific files with hardcoded paths
|
|
* - Assigns proper archive types based on file category
|
|
* - DELETES files after successful migration (no backup needed - already have backups from main migration)
|
|
*/
|
|
|
|
import { readFileSync, unlinkSync, statSync, existsSync } from 'fs';
|
|
import { join, basename, dirname } from 'path';
|
|
import { homedir } from 'os';
|
|
import dotenv from 'dotenv';
|
|
import { archiveAdd } from './dist/tools/archives.js';
|
|
|
|
// Load environment
|
|
dotenv.config();
|
|
|
|
// Base directory
|
|
const DEV_DIR = join(homedir(), 'Development');
|
|
|
|
// Files to migrate with their categories
|
|
const FILES_TO_MIGRATE = [
|
|
// .framework-backup (10 files) - completed
|
|
{ path: 'Apps/eToroGridbot/.framework-backup/planning.md', type: 'completed', project: 'GB' },
|
|
{ path: 'Apps/eToroGridbot/.framework-backup/PRD.md', type: 'completed', project: 'GB' },
|
|
{ path: 'Apps/Circles/.framework-backup/planning.md', type: 'completed', project: 'CI' },
|
|
{ path: 'Apps/Circles/.framework-backup/PRD.md', type: 'completed', project: 'CI' },
|
|
{ path: 'Apps/VPN/.framework-backup/planning.md', type: 'completed', project: 'VPN' },
|
|
{ path: 'Apps/VPN/.framework-backup/PRD.md', type: 'completed', project: 'VPN' },
|
|
{ path: 'Apps/Cardscanner/.framework-backup/planning.md', type: 'completed', project: 'CS' },
|
|
{ path: 'Apps/Cardscanner/.framework-backup/PRD.md', type: 'completed', project: 'CS' },
|
|
{ path: 'Apps/SmartTranslate/.framework-backup/planning.md', type: 'completed', project: 'ST' },
|
|
{ path: 'Apps/SmartTranslate/.framework-backup/PRD.md', type: 'completed', project: 'ST' },
|
|
|
|
// .project-files (4 files) - research
|
|
{ path: 'Infrastructure/ClaudeFramework/.project-files/CLAUDE.md', type: 'research', project: 'CF' },
|
|
{ path: 'Infrastructure/ClaudeFramework/.project-files/LEARNINGS.md', type: 'research', project: 'CF' },
|
|
{ path: 'Infrastructure/ClaudeFramework/.project-files/TOOLS.md', type: 'research', project: 'CF' },
|
|
{ path: 'Infrastructure/ClaudeFramework/.project-files/SETUP-INSTRUCTIONS.md', type: 'research', project: 'CF' },
|
|
|
|
// .claude/commands (8 files) - research
|
|
{ path: 'Apps/eToroGridbot/.claude/commands/trading-monitor.md', type: 'research', project: 'GB' },
|
|
{ path: 'Apps/eToroGridbot/.claude/commands/trading-pi-post.md', type: 'research', project: 'GB' },
|
|
{ path: 'Apps/eToroGridbot/.claude/commands/trading-start.md', type: 'research', project: 'GB' },
|
|
{ path: 'Apps/eToroGridbot/.claude/commands/trading-newsletter.md', type: 'research', project: 'GB' },
|
|
{ path: 'Apps/eToroGridbot/conductor/.claude/commands/trading-monitor.md', type: 'research', project: 'GB' },
|
|
{ path: 'Apps/eToroGridbot/conductor/.claude/commands/trading-pi-post.md', type: 'research', project: 'GB' },
|
|
{ path: 'Apps/eToroGridbot/conductor/.claude/commands/trading-start.md', type: 'research', project: 'GB' },
|
|
{ path: 'Apps/eToroGridbot/conductor/.claude/commands/trading-newsletter.md', type: 'research', project: 'GB' },
|
|
|
|
// .playwright-mcp (1 file) - research
|
|
{ path: 'Apps/OfBullsAndBears/.playwright-mcp/base44-sync-status.md', type: 'research', project: 'OB' },
|
|
|
|
// Archived (11 files) - completed
|
|
{ path: 'Archived/CSMaps/claude.md', type: 'completed', project: 'CM' },
|
|
{ path: 'Archived/CSMaps/planning.md', type: 'completed', project: 'CM' },
|
|
{ path: 'Archived/CSMaps/PRD.md', type: 'completed', project: 'CM' },
|
|
{ path: 'Archived/CSMaps/tasks.md', type: 'completed', project: 'CM' },
|
|
{ path: 'Archived/ZorkiOS_Source/claude.md', type: 'completed', project: 'ZK' },
|
|
{ path: 'Archived/ZorkiOS_Source/planning.md', type: 'completed', project: 'ZK' },
|
|
{ path: 'Archived/ZorkiOS_Source/PRD.md', type: 'completed', project: 'ZK' },
|
|
{ path: 'Archived/ZorkiOS_Source/tasks.md', type: 'completed', project: 'ZK' },
|
|
{ path: 'Archived/MacOS-Claude/DEPLOYMENT_CONFIG.md', type: 'completed', project: 'MC' },
|
|
{ path: 'Archived/MacOS-Claude/AppStoreDeployment.md', type: 'completed', project: 'MC' },
|
|
{ path: 'Archived/MacOS-Claude/README_XPC.md', type: 'completed', project: 'MC' }
|
|
];
|
|
|
|
/**
|
|
* Extract title from markdown content (first H1)
|
|
*/
|
|
function extractTitle(content, filename, projectKey) {
|
|
const lines = content.split('\n');
|
|
|
|
// Look for first H1
|
|
for (const line of lines) {
|
|
if (line.startsWith('# ')) {
|
|
return line.slice(2).trim().substring(0, 500);
|
|
}
|
|
}
|
|
|
|
// Fallback to project key + filename
|
|
const baseName = filename.replace('.md', '').replace(/_/g, ' ');
|
|
return `${projectKey} - ${baseName}`.substring(0, 500);
|
|
}
|
|
|
|
/**
|
|
* Migrate a single file
|
|
*/
|
|
async function migrateFile(fileInfo) {
|
|
const filePath = join(DEV_DIR, fileInfo.path);
|
|
const filename = basename(filePath);
|
|
|
|
// Check if file exists
|
|
if (!existsSync(filePath)) {
|
|
console.log(`\n⚠ SKIP: ${fileInfo.path} (file not found)`);
|
|
return { success: false, filename, projectKey: fileInfo.project, error: 'File not found', filePath: fileInfo.path };
|
|
}
|
|
|
|
try {
|
|
const content = readFileSync(filePath, 'utf-8');
|
|
const title = extractTitle(content, filename, fileInfo.project);
|
|
const fileSize = statSync(filePath).size;
|
|
|
|
console.log(`\n[${new Date().toISOString().substring(11, 19)}] Migrating: ${fileInfo.path}`);
|
|
console.log(` Project: ${fileInfo.project}`);
|
|
console.log(` Title: ${title}`);
|
|
console.log(` Size: ${Math.round(fileSize / 1024)}KB`);
|
|
console.log(` Type: ${fileInfo.type}`);
|
|
|
|
// Call archive_add
|
|
const result = await archiveAdd({
|
|
project: fileInfo.project,
|
|
archive_type: fileInfo.type,
|
|
title,
|
|
content,
|
|
original_path: filePath,
|
|
file_size: fileSize
|
|
});
|
|
|
|
console.log(` ✓ Migrated to database (ID: ${result.id})`);
|
|
|
|
// DELETE the original file (no backup needed)
|
|
unlinkSync(filePath);
|
|
console.log(` ✓ Deleted original file`);
|
|
|
|
return {
|
|
success: true,
|
|
filename,
|
|
projectKey: fileInfo.project,
|
|
title,
|
|
fileSize,
|
|
archiveType: fileInfo.type,
|
|
filePath: fileInfo.path,
|
|
archiveId: result.id
|
|
};
|
|
|
|
} catch (error) {
|
|
console.error(` ✗ Error: ${error.message}`);
|
|
return {
|
|
success: false,
|
|
filename,
|
|
projectKey: fileInfo.project,
|
|
archiveType: fileInfo.type,
|
|
error: error.message,
|
|
filePath: fileInfo.path
|
|
};
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Main migration function
|
|
*/
|
|
async function main() {
|
|
console.log('================================================================================');
|
|
console.log('Supplementary Migration - 34 Missed Files → task-mcp database');
|
|
console.log('================================================================================\n');
|
|
|
|
const results = {
|
|
frameworkBackup: [],
|
|
projectFiles: [],
|
|
claudeCommands: [],
|
|
playwrightMcp: [],
|
|
archived: []
|
|
};
|
|
|
|
let totalSuccess = 0;
|
|
let totalFailed = 0;
|
|
let totalSize = 0;
|
|
|
|
// Migrate .framework-backup files
|
|
console.log('\n📁 Category: .framework-backup (10 files)');
|
|
console.log('─'.repeat(80));
|
|
for (const file of FILES_TO_MIGRATE.slice(0, 10)) {
|
|
const result = await migrateFile(file);
|
|
results.frameworkBackup.push(result);
|
|
if (result.success) {
|
|
totalSuccess++;
|
|
totalSize += result.fileSize || 0;
|
|
} else {
|
|
totalFailed++;
|
|
}
|
|
}
|
|
|
|
// Migrate .project-files
|
|
console.log('\n📁 Category: .project-files (4 files)');
|
|
console.log('─'.repeat(80));
|
|
for (const file of FILES_TO_MIGRATE.slice(10, 14)) {
|
|
const result = await migrateFile(file);
|
|
results.projectFiles.push(result);
|
|
if (result.success) {
|
|
totalSuccess++;
|
|
totalSize += result.fileSize || 0;
|
|
} else {
|
|
totalFailed++;
|
|
}
|
|
}
|
|
|
|
// Migrate .claude/commands
|
|
console.log('\n📁 Category: .claude/commands (8 files)');
|
|
console.log('─'.repeat(80));
|
|
for (const file of FILES_TO_MIGRATE.slice(14, 22)) {
|
|
const result = await migrateFile(file);
|
|
results.claudeCommands.push(result);
|
|
if (result.success) {
|
|
totalSuccess++;
|
|
totalSize += result.fileSize || 0;
|
|
} else {
|
|
totalFailed++;
|
|
}
|
|
}
|
|
|
|
// Migrate .playwright-mcp
|
|
console.log('\n📁 Category: .playwright-mcp (1 file)');
|
|
console.log('─'.repeat(80));
|
|
for (const file of FILES_TO_MIGRATE.slice(22, 23)) {
|
|
const result = await migrateFile(file);
|
|
results.playwrightMcp.push(result);
|
|
if (result.success) {
|
|
totalSuccess++;
|
|
totalSize += result.fileSize || 0;
|
|
} else {
|
|
totalFailed++;
|
|
}
|
|
}
|
|
|
|
// Migrate Archived
|
|
console.log('\n📁 Category: Archived projects (11 files)');
|
|
console.log('─'.repeat(80));
|
|
for (const file of FILES_TO_MIGRATE.slice(23)) {
|
|
const result = await migrateFile(file);
|
|
results.archived.push(result);
|
|
if (result.success) {
|
|
totalSuccess++;
|
|
totalSize += result.fileSize || 0;
|
|
} else {
|
|
totalFailed++;
|
|
}
|
|
}
|
|
|
|
// Summary
|
|
console.log('\n\n================================================================================');
|
|
console.log('Migration Summary');
|
|
console.log('================================================================================\n');
|
|
|
|
console.log(`Total files processed: ${FILES_TO_MIGRATE.length}`);
|
|
console.log(`✓ Successfully migrated: ${totalSuccess}`);
|
|
console.log(`✗ Failed: ${totalFailed}`);
|
|
console.log(`Total size migrated: ${Math.round(totalSize / 1024)}KB\n`);
|
|
|
|
console.log('By category:');
|
|
console.log(` .framework-backup: ${results.frameworkBackup.filter(r => r.success).length}/10`);
|
|
console.log(` .project-files: ${results.projectFiles.filter(r => r.success).length}/4`);
|
|
console.log(` .claude/commands: ${results.claudeCommands.filter(r => r.success).length}/8`);
|
|
console.log(` .playwright-mcp: ${results.playwrightMcp.filter(r => r.success).length}/1`);
|
|
console.log(` Archived: ${results.archived.filter(r => r.success).length}/11\n`);
|
|
|
|
if (totalFailed > 0) {
|
|
console.log('\n❌ Failed files:');
|
|
const allResults = [...results.frameworkBackup, ...results.projectFiles, ...results.claudeCommands, ...results.playwrightMcp, ...results.archived];
|
|
allResults.filter(r => !r.success).forEach(r => {
|
|
console.log(` - ${r.filePath}: ${r.error}`);
|
|
});
|
|
}
|
|
|
|
console.log('\n✓ Supplementary migration complete!');
|
|
console.log('✓ All successfully migrated files have been DELETED from filesystem');
|
|
console.log('✓ Next step: Cleanup .migrated-to-mcp directories from main migration\n');
|
|
}
|
|
|
|
// Run migration
|
|
main().catch(error => {
|
|
console.error('Fatal error:', error);
|
|
process.exit(1);
|
|
});
|