From 1341177fae7e6aff97a55c560fb356f671002ba4 Mon Sep 17 00:00:00 2001 From: FaZeTitans Date: Thu, 5 Mar 2026 16:17:17 +0100 Subject: [PATCH 1/5] fix: paginate document fetching for linear progress bar - transferCollection now fetches documents page-by-page (page size = batchSize) using startAfter pagination instead of loading the entire collection at once with query.get(). This distributes network I/O evenly across the transfer, making the progress bar advance linearly instead of stalling during a long initial fetch then jumping. - Fix counting bug: countSubcollectionsForDoc now respects maxDepth to match transfer behavior. Previously, the count included docs beyond maxDepth that would never be transferred, causing the progress bar total to be too high. --- src/transfer/count.ts | 3 ++ src/transfer/transfer.ts | 64 +++++++++++++++++++++++++++------------- 2 files changed, 46 insertions(+), 21 deletions(-) diff --git a/src/transfer/count.ts b/src/transfer/count.ts index 96694c2..d660a82 100644 --- a/src/transfer/count.ts +++ b/src/transfer/count.ts @@ -65,6 +65,9 @@ async function countSubcollectionsForDoc( depth: number, progress?: CountProgress ): Promise { + // Respect maxDepth to match transfer behavior + if (config.maxDepth > 0 && depth >= config.maxDepth) return 0; + let count = 0; const subcollections = await getSubcollections(doc.ref); diff --git a/src/transfer/transfer.ts b/src/transfer/transfer.ts index 102c6f1..cc6354f 100644 --- a/src/transfer/transfer.ts +++ b/src/transfer/transfer.ts @@ -101,7 +101,7 @@ async function checkForConflicts( return conflicts; } -function buildTransferQuery( +function buildBaseQuery( sourceDb: Firestore, collectionPath: string, config: Config, @@ -115,10 +115,7 @@ function buildTransferQuery( } } - if (config.limit > 0 && depth === 0) { - query = query.limit(config.limit); - } - + // Limit is handled via pagination in transferCollection return query; } @@ -533,25 +530,50 @@ export async function transferCollection( const { sourceDb, config, stats, output } = ctx; const destCollectionPath = getDestCollectionPath(collectionPath, config.renameCollection); - const query = buildTransferQuery(sourceDb, collectionPath, config, depth); + const baseQuery = buildBaseQuery(sourceDb, collectionPath, config, depth); + const userLimit = config.limit > 0 && depth === 0 ? config.limit : 0; - const snapshot = await withRetry(() => query.get(), { - retries: config.retries, - onRetry: (attempt, max, err, delay) => { - output.logError(`Retry ${attempt}/${max} for ${collectionPath}`, { - error: err.message, - delay, - }); - }, - }); + let totalProcessed = 0; + let lastDoc: QueryDocumentSnapshot | undefined; + + while (true) { + // Calculate page size respecting user limit + let pageSize = config.batchSize; + if (userLimit > 0) { + const remaining = userLimit - totalProcessed; + if (remaining <= 0) break; + pageSize = Math.min(pageSize, remaining); + } + + // Build paginated query + let pageQuery = baseQuery.limit(pageSize); + if (lastDoc) { + pageQuery = pageQuery.startAfter(lastDoc); + } + + const snapshot = await withRetry(() => pageQuery.get(), { + retries: config.retries, + onRetry: (attempt, max, err, delay) => { + output.logError(`Retry ${attempt}/${max} for ${collectionPath}`, { + error: err.message, + delay, + }); + }, + }); + + if (snapshot.empty) break; + + if (totalProcessed === 0) { + stats.collectionsProcessed++; + output.logInfo(`Processing collection: ${collectionPath}`); + } - if (snapshot.empty) return; + await processBatch(snapshot.docs, ctx, collectionPath, destCollectionPath, depth); - stats.collectionsProcessed++; - output.logInfo(`Processing collection: ${collectionPath}`, { documents: snapshot.size }); + totalProcessed += snapshot.docs.length; + lastDoc = snapshot.docs[snapshot.docs.length - 1]; - for (let i = 0; i < snapshot.docs.length; i += config.batchSize) { - const batch = snapshot.docs.slice(i, i + config.batchSize); - await processBatch(batch, ctx, collectionPath, destCollectionPath, depth); + // Fewer docs than requested means we've reached the end + if (snapshot.docs.length < pageSize) break; } } From 0b17b17f29ada07571fca9d9596fd30f7c8cd603 Mon Sep 17 00:00:00 2001 From: FaZeTitans Date: Thu, 5 Mar 2026 16:22:27 +0100 Subject: [PATCH 2/5] fix: improve console output during document counting - Separate each top-level collection with clear formatting - Show unique subcollection names found per collection (e.g. orders, reviews, settings) so user can verify what's being scanned - Show excluded subcollection names so user can verify --exclude works - Show subcollection document counts per collection - Show live "Scanning ..." indicator during counting - Guard all process.stdout.write calls against JSON mode to prevent garbled output when using --json - Add onSubcollectionExcluded callback to CountProgress interface --- src/orchestrator.ts | 100 +++++++++++++++++++++++++++++++----------- src/transfer/count.ts | 8 +++- 2 files changed, 81 insertions(+), 27 deletions(-) diff --git a/src/orchestrator.ts b/src/orchestrator.ts index e4f10fa..7a27629 100644 --- a/src/orchestrator.ts +++ b/src/orchestrator.ts @@ -265,6 +265,20 @@ async function validateTransformWithSamples( output.blank(); } +function canWriteProgress(output: Output): boolean { + return !output.isQuiet && !output.isJson; +} + +function clearLine(): void { + process.stdout.write('\r' + ' '.repeat(SEPARATOR_LENGTH) + '\r'); +} + +function formatNameList(names: Set, max: number = 8): string { + const arr = [...names]; + if (arr.length <= max) return arr.join(', '); + return arr.slice(0, max).join(', ') + `, ... (+${arr.length - max})`; +} + async function setupProgressTracking( sourceDb: Firestore, config: Config, @@ -276,33 +290,63 @@ async function setupProgressTracking( if (!output.isQuiet) { output.info('šŸ“Š Counting documents...'); - let lastSubcollectionLog = Date.now(); - let subcollectionCount = 0; - - const countProgress: CountProgress = { - onCollection: (path, count) => { - output.info(` ${path}: ${count} documents`); - }, - onSubcollection: (_path) => { - subcollectionCount++; - const now = Date.now(); - if (now - lastSubcollectionLog > PROGRESS_LOG_INTERVAL_MS) { - process.stdout.write(`\r Scanning subcollections... (${subcollectionCount} found)`); - lastSubcollectionLog = now; - } - }, - }; for (const collection of config.collections) { - totalDocs += await countDocuments(sourceDb, collection, config, 0, countProgress); - } + let rootCount = 0; + let subcollectionInstances = 0; + const subcollectionNames = new Set(); + const excludedNames = new Set(); + let lastLog = Date.now(); + let showedScanLine = false; + + if (canWriteProgress(output)) { + process.stdout.write(` Counting ${collection}...`); + showedScanLine = true; + } + + const countProgress: CountProgress = { + onCollection: (_path, count) => { + rootCount = count; + }, + onSubcollection: (path) => { + subcollectionInstances++; + const segments = path.split('/'); + subcollectionNames.add(segments[segments.length - 1]); + + if (!canWriteProgress(output)) return; + const now = Date.now(); + if (now - lastLog > PROGRESS_LOG_INTERVAL_MS) { + process.stdout.write(`\r Scanning ${collection}... (${subcollectionInstances} subcollections found)`); + showedScanLine = true; + lastLog = now; + } + }, + onSubcollectionExcluded: (name) => { + excludedNames.add(name); + }, + }; + + const collectionTotal = await countDocuments(sourceDb, collection, config, 0, countProgress); + totalDocs += collectionTotal; + + // Clear live indicator line + if (showedScanLine && canWriteProgress(output)) { + clearLine(); + } - if (subcollectionCount > 0) { - process.stdout.write('\r' + ' '.repeat(SEPARATOR_LENGTH) + '\r'); - output.info(` Subcollections scanned: ${subcollectionCount}`); + // Print collection summary + output.info(` ${collection}: ${rootCount} documents`); + + if (subcollectionInstances > 0) { + const subDocs = collectionTotal - rootCount; + output.info(` + ${subDocs} in subcollections (${formatNameList(subcollectionNames)})`); + } + if (excludedNames.size > 0) { + output.info(` Excluded: ${formatNameList(excludedNames)}`); + } } - output.info(` Total: ${totalDocs} documents to transfer\n`); + output.info(`\n Total: ${totalDocs} documents to transfer\n`); progressBar.start(totalDocs, stats); } @@ -376,14 +420,17 @@ async function deleteOrphanDocs( let lastProgressLog = Date.now(); let subcollectionCount = 0; + const showProgress = canWriteProgress(output); + const progress: DeleteOrphansProgress = { onScanStart: (collection) => { - process.stdout.write(` Scanning ${collection}...`); + if (showProgress) process.stdout.write(` Scanning ${collection}...`); }, onScanComplete: (collection, orphanCount, totalDest) => { - process.stdout.write(`\r ${collection}: ${orphanCount}/${totalDest} orphan docs\n`); + if (showProgress) process.stdout.write(`\r ${collection}: ${orphanCount}/${totalDest} orphan docs\n`); }, onBatchDeleted: (collection, deletedSoFar, total) => { + if (!showProgress) return; process.stdout.write(`\r Deleting from ${collection}... ${deletedSoFar}/${total}`); if (deletedSoFar === total) { process.stdout.write('\n'); @@ -391,6 +438,7 @@ async function deleteOrphanDocs( }, onSubcollectionScan: (_path) => { subcollectionCount++; + if (!showProgress) return; const now = Date.now(); if (now - lastProgressLog > PROGRESS_LOG_INTERVAL_MS) { process.stdout.write(`\r Scanning subcollections... (${subcollectionCount} checked)`); @@ -411,8 +459,8 @@ async function deleteOrphanDocs( stats.documentsDeleted += deleted; } - if (subcollectionCount > 0) { - process.stdout.write('\r' + ' '.repeat(SEPARATOR_LENGTH) + '\r'); + if (subcollectionCount > 0 && showProgress) { + clearLine(); } if (stats.documentsDeleted > 0) { diff --git a/src/transfer/count.ts b/src/transfer/count.ts index d660a82..49cff81 100644 --- a/src/transfer/count.ts +++ b/src/transfer/count.ts @@ -6,6 +6,7 @@ import { getSubcollections } from './helpers.js'; export interface CountProgress { onCollection?: (path: string, count: number) => void; onSubcollection?: (path: string) => void; + onSubcollectionExcluded?: (name: string) => void; } function buildQueryWithFilters( @@ -72,7 +73,12 @@ async function countSubcollectionsForDoc( const subcollections = await getSubcollections(doc.ref); for (const subId of subcollections) { - if (matchesExcludePattern(subId, config.exclude)) continue; + if (matchesExcludePattern(subId, config.exclude)) { + if (progress?.onSubcollectionExcluded) { + progress.onSubcollectionExcluded(subId); + } + continue; + } const subPath = `${collectionPath}/${doc.id}/${subId}`; if (progress?.onSubcollection) { From 6c15b838c87ebd429da25854fbb0a9550334f016 Mon Sep 17 00:00:00 2001 From: FaZeTitans Date: Thu, 5 Mar 2026 16:43:53 +0100 Subject: [PATCH 3/5] feat: add all options to interactive mode + save to config file Interactive mode now includes all options available in CLI/config modes: - After basic prompts (source, dest, collections, subcollections, dry run, merge), offers "Configure additional options?" with a checkbox of all available options grouped logically - Each selected option gets its appropriate prompt type (input, number, confirm, etc.) - Available options: exclude patterns, where filters, parallel, batch size, document limit, max depth, rate limit, clear destination, delete missing, transform file, rename collections, ID prefix/suffix, webhook, skip oversized, detect conflicts, verify counts, verify integrity, retries At the end of the interactive session, user can choose to: - Execute transfer (existing behavior) - Save as INI config file - Save as JSON config file Config serialization generates proper INI (with sections and commented defaults for unset options) or JSON format. The saved file can then be used with `fscopy -f `. Also updates cli.ts to handle the new InteractiveResult return type and exit gracefully when saving instead of executing. --- src/__tests__/interactive.test.ts | 171 ++++++++-- src/cli.ts | 6 +- src/interactive.ts | 531 ++++++++++++++++++++++++++++-- 3 files changed, 637 insertions(+), 71 deletions(-) diff --git a/src/__tests__/interactive.test.ts b/src/__tests__/interactive.test.ts index 7e2a177..1867d3f 100644 --- a/src/__tests__/interactive.test.ts +++ b/src/__tests__/interactive.test.ts @@ -5,11 +5,15 @@ import type { Config } from '../types.js'; const mockInput = mock<() => Promise>(() => Promise.resolve('')); const mockCheckbox = mock<() => Promise>(() => Promise.resolve([])); const mockConfirm = mock<() => Promise>(() => Promise.resolve(true)); +const mockSelect = mock<() => Promise>(() => Promise.resolve('execute')); +const mockNumber = mock<() => Promise>(() => Promise.resolve(undefined)); mock.module('@inquirer/prompts', () => ({ input: mockInput, checkbox: mockCheckbox, confirm: mockConfirm, + select: mockSelect, + number: mockNumber, })); // Mock firebase-admin @@ -85,6 +89,8 @@ describe('Interactive Mode', () => { mockInput.mockReset(); mockCheckbox.mockReset(); mockConfirm.mockReset(); + mockSelect.mockReset(); + mockNumber.mockReset(); mockInitializeApp.mockReset(); mockListCollections.mockReset(); mockCountGet.mockReset(); @@ -93,7 +99,10 @@ describe('Interactive Mode', () => { // Set default implementations mockInput.mockImplementation(() => Promise.resolve('test-project')); mockCheckbox.mockImplementation(() => Promise.resolve(['users'])); + // Default confirm sequence: subcollections=true, dryRun=true, merge=true, advancedOptions=false mockConfirm.mockImplementation(() => Promise.resolve(true)); + mockSelect.mockImplementation(() => Promise.resolve('execute')); + mockNumber.mockImplementation(() => Promise.resolve(undefined)); mockInitializeApp.mockImplementation(() => mockApp); mockListCollections.mockImplementation(() => Promise.resolve([{ id: 'users' }, { id: 'orders' }]) @@ -107,6 +116,14 @@ describe('Interactive Mode', () => { describe('promptForProject', () => { test('uses existing source project if provided', async () => { + // Confirm sequence: subcollections, dryRun, merge, advancedOptions=false + let confirmCall = 0; + mockConfirm.mockImplementation(() => { + confirmCall++; + // 4th confirm is "Configure additional options?" -> false + return Promise.resolve(confirmCall !== 4); + }); + const config = createBaseConfig({ sourceProject: 'existing-source', destProject: 'existing-dest', @@ -114,20 +131,25 @@ describe('Interactive Mode', () => { const result = await runInteractiveMode(config); - // Should not prompt for projects since they're already set - expect(result.sourceProject).toBe('existing-source'); - expect(result.destProject).toBe('existing-dest'); + expect(result.config.sourceProject).toBe('existing-source'); + expect(result.config.destProject).toBe('existing-dest'); }); test('prompts for source project if not provided', async () => { mockInput.mockImplementationOnce(() => Promise.resolve('prompted-source')); mockInput.mockImplementationOnce(() => Promise.resolve('prompted-dest')); + let confirmCall = 0; + mockConfirm.mockImplementation(() => { + confirmCall++; + return Promise.resolve(confirmCall !== 4); + }); + const config = createBaseConfig(); const result = await runInteractiveMode(config); expect(mockInput).toHaveBeenCalled(); - expect(result.sourceProject).toBe('prompted-source'); + expect(result.config.sourceProject).toBe('prompted-source'); }); }); @@ -138,6 +160,12 @@ describe('Interactive Mode', () => { ); mockCheckbox.mockImplementation(() => Promise.resolve(['users', 'orders'])); + let confirmCall = 0; + mockConfirm.mockImplementation(() => { + confirmCall++; + return Promise.resolve(confirmCall !== 4); + }); + const config = createBaseConfig({ sourceProject: 'test-source', destProject: 'test-dest', @@ -146,14 +174,18 @@ describe('Interactive Mode', () => { const result = await runInteractiveMode(config); expect(mockListCollections).toHaveBeenCalled(); - expect(result.collections).toEqual(['users', 'orders']); + expect(result.config.collections).toEqual(['users', 'orders']); }); test('pre-selects collections from config', async () => { - // The checkbox function receives choices with checked based on config.collections - // We verify this by checking the result includes 'users' mockCheckbox.mockImplementation(() => Promise.resolve(['users'])); + let confirmCall = 0; + mockConfirm.mockImplementation(() => { + confirmCall++; + return Promise.resolve(confirmCall !== 4); + }); + const config = createBaseConfig({ sourceProject: 'test-source', destProject: 'test-dest', @@ -161,18 +193,24 @@ describe('Interactive Mode', () => { }); const result = await runInteractiveMode(config); - expect(result.collections).toContain('users'); + expect(result.config.collections).toContain('users'); }); }); describe('same project handling', () => { test('prompts for ID prefix when source equals dest', async () => { - // First confirm: "Add a prefix to document IDs?" -> yes - mockConfirm.mockImplementationOnce(() => Promise.resolve(true)); - // Input: prefix value + let confirmCall = 0; + mockConfirm.mockImplementation(() => { + confirmCall++; + // 1st confirm: "Add a prefix to document IDs?" -> yes + if (confirmCall === 1) return Promise.resolve(true); + // subcollections, dryRun, merge -> true + // advancedOptions -> false (5th confirm) + if (confirmCall === 5) return Promise.resolve(false); + return Promise.resolve(true); + }); + // Input calls: prefix value mockInput.mockImplementationOnce(() => Promise.resolve('backup_')); - // Rest of confirms (includeSubcollections, dryRun, merge) - mockConfirm.mockImplementation(() => Promise.resolve(false)); const config = createBaseConfig({ sourceProject: 'same-project', @@ -181,18 +219,24 @@ describe('Interactive Mode', () => { const result = await runInteractiveMode(config); - expect(result.idPrefix).toBe('backup_'); + expect(result.config.idPrefix).toBe('backup_'); }); test('prompts for ID suffix when prefix declined', async () => { - // First confirm: "Add a prefix to document IDs?" -> no - mockConfirm.mockImplementationOnce(() => Promise.resolve(false)); - // Second confirm: "Add a suffix to document IDs instead?" -> yes - mockConfirm.mockImplementationOnce(() => Promise.resolve(true)); + let confirmCall = 0; + mockConfirm.mockImplementation(() => { + confirmCall++; + // 1st: "Add a prefix?" -> no + if (confirmCall === 1) return Promise.resolve(false); + // 2nd: "Add a suffix?" -> yes + if (confirmCall === 2) return Promise.resolve(true); + // subcollections, dryRun, merge -> false + // advancedOptions (6th) -> false + if (confirmCall === 6) return Promise.resolve(false); + return Promise.resolve(false); + }); // Input: suffix value mockInput.mockImplementationOnce(() => Promise.resolve('_v2')); - // Rest of confirms - mockConfirm.mockImplementation(() => Promise.resolve(false)); const config = createBaseConfig({ sourceProject: 'same-project', @@ -201,18 +245,24 @@ describe('Interactive Mode', () => { const result = await runInteractiveMode(config); - expect(result.idSuffix).toBe('_v2'); + expect(result.config.idSuffix).toBe('_v2'); }); }); describe('options', () => { test('returns updated config with selected options', async () => { - // includeSubcollections - mockConfirm.mockImplementationOnce(() => Promise.resolve(true)); - // dryRun - mockConfirm.mockImplementationOnce(() => Promise.resolve(false)); - // merge - mockConfirm.mockImplementationOnce(() => Promise.resolve(true)); + let confirmCall = 0; + mockConfirm.mockImplementation(() => { + confirmCall++; + // 1st: includeSubcollections -> true + if (confirmCall === 1) return Promise.resolve(true); + // 2nd: dryRun -> false + if (confirmCall === 2) return Promise.resolve(false); + // 3rd: merge -> true + if (confirmCall === 3) return Promise.resolve(true); + // 4th: advancedOptions -> false + return Promise.resolve(false); + }); mockCheckbox.mockImplementation(() => Promise.resolve(['users', 'orders'])); @@ -223,13 +273,21 @@ describe('Interactive Mode', () => { const result = await runInteractiveMode(config); - expect(result.includeSubcollections).toBe(true); - expect(result.dryRun).toBe(false); - expect(result.merge).toBe(true); - expect(result.collections).toEqual(['users', 'orders']); + expect(result.config.includeSubcollections).toBe(true); + expect(result.config.dryRun).toBe(false); + expect(result.config.merge).toBe(true); + expect(result.config.collections).toEqual(['users', 'orders']); }); test('preserves non-interactive config values', async () => { + let confirmCall = 0; + mockConfirm.mockImplementation(() => { + confirmCall++; + // advancedOptions (4th) -> false + if (confirmCall === 4) return Promise.resolve(false); + return Promise.resolve(true); + }); + const config = createBaseConfig({ sourceProject: 'source', destProject: 'dest', @@ -241,15 +299,44 @@ describe('Interactive Mode', () => { const result = await runInteractiveMode(config); - expect(result.batchSize).toBe(100); - expect(result.limit).toBe(50); - expect(result.retries).toBe(5); - expect(result.webhook).toBe('https://example.com/hook'); + expect(result.config.batchSize).toBe(100); + expect(result.config.limit).toBe(50); + expect(result.config.retries).toBe(5); + expect(result.config.webhook).toBe('https://example.com/hook'); + }); + }); + + describe('return type', () => { + test('returns execute action by default', async () => { + let confirmCall = 0; + mockConfirm.mockImplementation(() => { + confirmCall++; + if (confirmCall === 4) return Promise.resolve(false); + return Promise.resolve(true); + }); + mockSelect.mockImplementation(() => Promise.resolve('execute')); + + const config = createBaseConfig({ + sourceProject: 'source', + destProject: 'dest', + }); + + const result = await runInteractiveMode(config); + + expect(result.action).toBe('execute'); + expect(result.config).toBeDefined(); }); }); describe('Firebase connection', () => { test('initializes Firebase with source project', async () => { + let confirmCall = 0; + mockConfirm.mockImplementation(() => { + confirmCall++; + if (confirmCall === 4) return Promise.resolve(false); + return Promise.resolve(true); + }); + const config = createBaseConfig({ sourceProject: 'my-source-project', destProject: 'my-dest-project', @@ -266,6 +353,13 @@ describe('Interactive Mode', () => { }); test('cleans up Firebase app after completion', async () => { + let confirmCall = 0; + mockConfirm.mockImplementation(() => { + confirmCall++; + if (confirmCall === 4) return Promise.resolve(false); + return Promise.resolve(true); + }); + const config = createBaseConfig({ sourceProject: 'source', destProject: 'dest', @@ -281,6 +375,13 @@ describe('Interactive Mode', () => { Promise.resolve([{ id: 'users' }, { id: 'orders' }]) ); + let confirmCall = 0; + mockConfirm.mockImplementation(() => { + confirmCall++; + if (confirmCall === 4) return Promise.resolve(false); + return Promise.resolve(true); + }); + const config = createBaseConfig({ sourceProject: 'source', destProject: 'dest', diff --git a/src/cli.ts b/src/cli.ts index d51915d..9ab80b9 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -250,7 +250,11 @@ async function main(): Promise { // Run interactive mode if enabled if (argv.interactive) { - config = await runInteractiveMode(config); + const result = await runInteractiveMode(config); + config = result.config; + if (result.action === 'save') { + process.exit(0); + } } displayConfig(config); diff --git a/src/interactive.ts b/src/interactive.ts index 8269238..ebd32ec 100644 --- a/src/interactive.ts +++ b/src/interactive.ts @@ -1,8 +1,31 @@ import admin from 'firebase-admin'; +import fs from 'node:fs'; +import path from 'node:path'; import type { Firestore } from 'firebase-admin/firestore'; -import { input, checkbox, confirm } from '@inquirer/prompts'; +import { input, checkbox, confirm, select, number } from '@inquirer/prompts'; import { SEPARATOR_LENGTH } from './constants.js'; -import type { Config } from './types.js'; +import type { Config, WhereFilter } from './types.js'; +import { parseWhereFilter, parseRenameMapping, parseStringList } from './config/parser.js'; + +// ============================================================================= +// Types +// ============================================================================= + +export interface InteractiveResult { + config: Config; + action: 'execute' | 'save'; +} + +type AdvancedOption = + | 'exclude' | 'where' | 'parallel' | 'batchSize' | 'limit' + | 'maxDepth' | 'rateLimit' | 'clear' | 'deleteMissing' | 'transform' + | 'renameCollection' | 'idPrefix' | 'idSuffix' | 'webhook' + | 'skipOversized' | 'detectConflicts' | 'verify' | 'verifyIntegrity' + | 'retries'; + +// ============================================================================= +// Project prompts +// ============================================================================= async function promptForProject( currentValue: string | null | undefined, @@ -23,7 +46,7 @@ async function promptForIdModification( currentPrefix: string | null, currentSuffix: string | null ): Promise<{ idPrefix: string | null; idSuffix: string | null }> { - console.log('\nāš ļø Source and destination are the same project.'); + console.log('\nSource and destination are the same project.'); console.log(' You need to rename collections or modify document IDs to avoid overwriting.\n'); const modifyIds = await confirm({ @@ -54,30 +77,24 @@ async function promptForIdModification( return { idPrefix: currentPrefix, idSuffix }; } - console.log('\nāŒ Cannot proceed: source and destination are the same without ID modification.'); + console.log('\nCannot proceed: source and destination are the same without ID modification.'); console.log(' This would overwrite your data. Use --rename-collection, --id-prefix, or --id-suffix.\n'); process.exit(1); } -export async function runInteractiveMode(config: Config): Promise { - console.log('\n' + '='.repeat(SEPARATOR_LENGTH)); - console.log('šŸ”„ FSCOPY - INTERACTIVE MODE'); - console.log('='.repeat(SEPARATOR_LENGTH) + '\n'); - - const sourceProject = await promptForProject(config.sourceProject, 'Source Firebase project ID', 'šŸ“¤'); - const destProject = await promptForProject(config.destProject, 'Destination Firebase project ID', 'šŸ“„'); - - let idPrefix = config.idPrefix; - let idSuffix = config.idSuffix; +// ============================================================================= +// Collection discovery +// ============================================================================= - if (sourceProject === destProject) { - const mods = await promptForIdModification(idPrefix, idSuffix); - idPrefix = mods.idPrefix; - idSuffix = mods.idSuffix; - } +interface CollectionInfo { + id: string; + count: number; +} - // Initialize source Firebase to list collections - console.log('\nšŸ“Š Connecting to source project...'); +async function discoverCollections( + sourceProject: string +): Promise<{ app: admin.app.App; db: Firestore; collections: CollectionInfo[] }> { + console.log('\nConnecting to source project...'); let tempSourceApp: admin.app.App; let sourceDb: Firestore; @@ -92,12 +109,10 @@ export async function runInteractiveMode(config: Config): Promise { 'interactive-source' ); sourceDb = tempSourceApp.firestore(); - - // List collections (also tests connectivity) rootCollections = await sourceDb.listCollections(); } catch (error) { const err = error as Error & { code?: string }; - console.error('\nāŒ Cannot connect to Firebase project:', err.message); + console.error('\nCannot connect to Firebase project:', err.message); if (err.message.includes('default credentials') || err.message.includes('credential')) { console.error('\n Run this command to authenticate:'); @@ -114,14 +129,13 @@ export async function runInteractiveMode(config: Config): Promise { const collectionIds = rootCollections.map((col) => col.id); if (collectionIds.length === 0) { - console.log('\nāš ļø No collections found in source project'); + console.log('\nNo collections found in source project'); await tempSourceApp.delete(); process.exit(0); } - // Count documents in each collection for preview - console.log('\nšŸ“‹ Available collections:'); - const collectionInfo: { id: string; count: number }[] = []; + console.log('\nAvailable collections:'); + const collectionInfo: CollectionInfo[] = []; for (const id of collectionIds) { const snapshot = await sourceDb.collection(id).count().get(); const count = snapshot.data().count; @@ -129,7 +143,438 @@ export async function runInteractiveMode(config: Config): Promise { console.log(` - ${id} (${count} documents)`); } - // Let user select collections + return { app: tempSourceApp, db: sourceDb, collections: collectionInfo }; +} + +// ============================================================================= +// Advanced options +// ============================================================================= + +const advancedOptionChoices: Array<{ name: string; value: AdvancedOption }> = [ + { name: 'Exclude subcollection patterns', value: 'exclude' }, + { name: 'Where filters (filter source documents)', value: 'where' }, + { name: 'Parallel transfers', value: 'parallel' }, + { name: 'Batch size', value: 'batchSize' }, + { name: 'Document limit per collection', value: 'limit' }, + { name: 'Max subcollection depth', value: 'maxDepth' }, + { name: 'Rate limit (docs/sec)', value: 'rateLimit' }, + { name: 'Clear destination before transfer', value: 'clear' }, + { name: 'Delete missing docs in destination (sync mode)', value: 'deleteMissing' }, + { name: 'Transform file (JS/TS)', value: 'transform' }, + { name: 'Rename collections in destination', value: 'renameCollection' }, + { name: 'ID prefix', value: 'idPrefix' }, + { name: 'ID suffix', value: 'idSuffix' }, + { name: 'Webhook URL (Slack, Discord, custom)', value: 'webhook' }, + { name: 'Skip oversized documents (>1MB)', value: 'skipOversized' }, + { name: 'Detect conflicts during transfer', value: 'detectConflicts' }, + { name: 'Verify counts after transfer', value: 'verify' }, + { name: 'Verify integrity (hash comparison)', value: 'verifyIntegrity' }, + { name: 'Retries on error', value: 'retries' }, +]; + +async function promptAdvancedOptions(config: Config): Promise> { + const wantAdvanced = await confirm({ + message: 'Configure additional options?', + default: false, + }); + + if (!wantAdvanced) return {}; + + console.log(''); + const selected = new Set( + await checkbox({ + message: 'Select options to configure:', + choices: advancedOptionChoices, + }) + ); + + if (selected.size === 0) return {}; + + console.log(''); + const updates: Partial = {}; + + if (selected.has('exclude')) { + const val = await input({ + message: 'Exclude patterns (comma-separated, e.g. "logs, cache*, temp"):', + default: config.exclude.length > 0 ? config.exclude.join(', ') : undefined, + }); + updates.exclude = parseStringList(val); + } + + if (selected.has('where')) { + const filters: WhereFilter[] = []; + let addMore = true; + while (addMore) { + const filterStr = await input({ + message: `Where filter${filters.length > 0 ? ' (leave empty to stop)' : ''} (e.g. "status == active"):`, + }); + if (!filterStr.trim()) break; + const parsed = parseWhereFilter(filterStr); + if (parsed) { + filters.push(parsed); + console.log(` Added: ${parsed.field} ${parsed.operator} ${parsed.value}`); + } + if (filters.length > 0) { + addMore = await confirm({ message: 'Add another filter?', default: false }); + } + } + if (filters.length > 0) { + updates.where = filters; + } + } + + if (selected.has('parallel')) { + const val = await number({ + message: 'Number of parallel collection transfers:', + default: config.parallel, + min: 1, + max: 20, + step: 1, + }); + if (val !== undefined) updates.parallel = val; + } + + if (selected.has('batchSize')) { + const val = await number({ + message: 'Batch size (documents per write):', + default: config.batchSize, + min: 1, + max: 500, + step: 1, + }); + if (val !== undefined) updates.batchSize = val; + } + + if (selected.has('limit')) { + const val = await number({ + message: 'Document limit per collection (0 = no limit):', + default: config.limit, + min: 0, + step: 1, + }); + if (val !== undefined) updates.limit = val; + } + + if (selected.has('maxDepth')) { + const val = await number({ + message: 'Max subcollection depth (0 = unlimited):', + default: config.maxDepth, + min: 0, + step: 1, + }); + if (val !== undefined) updates.maxDepth = val; + } + + if (selected.has('rateLimit')) { + const val = await number({ + message: 'Rate limit in docs/sec (0 = unlimited):', + default: config.rateLimit, + min: 0, + step: 1, + }); + if (val !== undefined) updates.rateLimit = val; + } + + if (selected.has('clear')) { + updates.clear = await confirm({ + message: 'Clear destination collections before transfer? (DESTRUCTIVE)', + default: config.clear, + }); + } + + if (selected.has('deleteMissing')) { + updates.deleteMissing = await confirm({ + message: 'Delete docs in destination not present in source? (sync mode)', + default: config.deleteMissing, + }); + } + + if (selected.has('transform')) { + const val = await input({ + message: 'Path to transform file (JS/TS):', + default: config.transform ?? undefined, + validate: (value) => { + if (!value.trim()) return 'Path is required'; + return true; + }, + }); + updates.transform = val.trim(); + } + + if (selected.has('renameCollection')) { + const val = await input({ + message: 'Rename mappings (e.g. "users:users_backup, orders:orders_v2"):', + default: Object.entries(config.renameCollection).map(([s, d]) => `${s}:${d}`).join(', ') || undefined, + }); + updates.renameCollection = parseRenameMapping(parseStringList(val)); + } + + if (selected.has('idPrefix')) { + const val = await input({ + message: 'Document ID prefix:', + default: config.idPrefix ?? undefined, + }); + updates.idPrefix = val.trim() || null; + } + + if (selected.has('idSuffix')) { + const val = await input({ + message: 'Document ID suffix:', + default: config.idSuffix ?? undefined, + }); + updates.idSuffix = val.trim() || null; + } + + if (selected.has('webhook')) { + const val = await input({ + message: 'Webhook URL:', + default: config.webhook ?? undefined, + }); + updates.webhook = val.trim() || null; + } + + if (selected.has('skipOversized')) { + updates.skipOversized = await confirm({ + message: 'Skip documents exceeding 1MB instead of failing?', + default: config.skipOversized, + }); + } + + if (selected.has('detectConflicts')) { + updates.detectConflicts = await confirm({ + message: 'Detect destination modifications during transfer?', + default: config.detectConflicts, + }); + } + + if (selected.has('verify')) { + updates.verify = await confirm({ + message: 'Verify document counts after transfer?', + default: config.verify, + }); + } + + if (selected.has('verifyIntegrity')) { + updates.verifyIntegrity = await confirm({ + message: 'Verify document integrity with hash after transfer?', + default: config.verifyIntegrity, + }); + } + + if (selected.has('retries')) { + const val = await number({ + message: 'Number of retries on error:', + default: config.retries, + min: 0, + max: 10, + step: 1, + }); + if (val !== undefined) updates.retries = val; + } + + return updates; +} + +// ============================================================================= +// Final action +// ============================================================================= + +async function promptFinalAction(): Promise<'execute' | 'save-ini' | 'save-json'> { + console.log(''); + return select({ + message: 'What would you like to do?', + choices: [ + { name: 'Execute transfer', value: 'execute' as const }, + { name: 'Save as INI config file', value: 'save-ini' as const }, + { name: 'Save as JSON config file', value: 'save-json' as const }, + ], + }); +} + +// ============================================================================= +// Config serialization +// ============================================================================= + +function serializeWhereFilters(filters: WhereFilter[]): string[] { + return filters.map((f) => `${f.field} ${f.operator} ${f.value}`); +} + +function serializeRenameMapping(mapping: Record): string { + return Object.entries(mapping) + .map(([src, dest]) => `${src}:${dest}`) + .join(', '); +} + +function configToJson(config: Config): string { + const output: Record = { + sourceProject: config.sourceProject, + destProject: config.destProject, + collections: config.collections, + includeSubcollections: config.includeSubcollections, + dryRun: config.dryRun, + batchSize: config.batchSize, + limit: config.limit, + where: serializeWhereFilters(config.where), + exclude: config.exclude, + merge: config.merge, + parallel: config.parallel, + clear: config.clear, + deleteMissing: config.deleteMissing, + }; + + // Include optional fields only if set + if (config.transform) output.transform = config.transform; + if (Object.keys(config.renameCollection).length > 0) output.renameCollection = config.renameCollection; + if (config.idPrefix) output.idPrefix = config.idPrefix; + if (config.idSuffix) output.idSuffix = config.idSuffix; + if (config.webhook) output.webhook = config.webhook; + if (config.rateLimit > 0) output.rateLimit = config.rateLimit; + if (config.skipOversized) output.skipOversized = config.skipOversized; + if (config.detectConflicts) output.detectConflicts = config.detectConflicts; + if (config.maxDepth > 0) output.maxDepth = config.maxDepth; + if (config.verify) output.verify = config.verify; + if (config.verifyIntegrity) output.verifyIntegrity = config.verifyIntegrity; + if (config.retries !== 3) output.retries = config.retries; + + return JSON.stringify(output, null, 4); +} + +function iniLine(key: string, value: string | number | boolean): string { + return `${key} = ${value}\n`; +} + +function iniComment(key: string, value: string | number | boolean): string { + return `; ${key} = ${value}\n`; +} + +function configToIni(config: Config): string { + let ini = '; fscopy configuration file\n'; + ini += '; Generated by interactive mode\n\n'; + + // [projects] + ini += '[projects]\n'; + ini += iniLine('source', config.sourceProject ?? ''); + ini += iniLine('dest', config.destProject ?? ''); + ini += '\n'; + + // [transfer] + ini += '[transfer]\n'; + ini += iniLine('collections', config.collections.join(', ')); + ini += iniLine('includeSubcollections', config.includeSubcollections); + ini += iniLine('dryRun', config.dryRun); + ini += iniLine('batchSize', config.batchSize); + ini += iniLine('limit', config.limit); + ini += '\n'; + + // [options] + ini += '[options]\n'; + + if (config.where.length > 0) { + ini += iniLine('where', serializeWhereFilters(config.where).join(', ')); + } else { + ini += iniComment('where', 'status == active'); + } + + if (config.exclude.length > 0) { + ini += iniLine('exclude', config.exclude.join(', ')); + } else { + ini += iniComment('exclude', 'logs, temp/*, cache'); + } + + ini += iniLine('merge', config.merge); + ini += iniLine('parallel', config.parallel); + ini += iniLine('clear', config.clear); + ini += iniLine('deleteMissing', config.deleteMissing); + + if (config.transform) { + ini += iniLine('transform', config.transform); + } else { + ini += iniComment('transform', './transforms/anonymize.ts'); + } + + if (Object.keys(config.renameCollection).length > 0) { + ini += iniLine('renameCollection', serializeRenameMapping(config.renameCollection)); + } else { + ini += iniComment('renameCollection', 'users:users_backup, orders:orders_2024'); + } + + if (config.idPrefix) { + ini += iniLine('idPrefix', config.idPrefix); + } else { + ini += iniComment('idPrefix', 'backup_'); + } + + if (config.idSuffix) { + ini += iniLine('idSuffix', config.idSuffix); + } else { + ini += iniComment('idSuffix', '_v2'); + } + + if (config.webhook) { + ini += iniLine('webhook', config.webhook); + } else { + ini += iniComment('webhook', 'https://hooks.slack.com/services/...'); + } + + return ini; +} + +async function saveConfig(config: Config, format: 'ini' | 'json'): Promise { + const defaultName = format === 'json' ? 'fscopy-config.json' : 'fscopy-config.ini'; + + const filePath = await input({ + message: `Save path:`, + default: defaultName, + }); + + const resolvedPath = path.resolve(filePath); + + if (fs.existsSync(resolvedPath)) { + const overwrite = await confirm({ + message: `File "${filePath}" already exists. Overwrite?`, + default: false, + }); + if (!overwrite) { + console.log('\nSave cancelled.\n'); + process.exit(0); + } + } + + const content = format === 'json' ? configToJson(config) : configToIni(config); + fs.writeFileSync(resolvedPath, content, 'utf-8'); + + console.log(`\nConfig saved: ${resolvedPath}`); + console.log(`\n Run with: fscopy -f ${filePath}\n`); + + return resolvedPath; +} + +// ============================================================================= +// Main interactive flow +// ============================================================================= + +export async function runInteractiveMode(config: Config): Promise { + console.log('\n' + '='.repeat(SEPARATOR_LENGTH)); + console.log('FSCOPY - INTERACTIVE MODE'); + console.log('='.repeat(SEPARATOR_LENGTH) + '\n'); + + // 1. Projects + const sourceProject = await promptForProject(config.sourceProject, 'Source Firebase project ID', '>>'); + const destProject = await promptForProject(config.destProject, 'Destination Firebase project ID', '>>'); + + let idPrefix = config.idPrefix; + let idSuffix = config.idSuffix; + + if (sourceProject === destProject) { + const mods = await promptForIdModification(idPrefix, idSuffix); + idPrefix = mods.idPrefix; + idSuffix = mods.idSuffix; + } + + // 2. Discover collections + const { app: tempSourceApp, collections: collectionInfo } = await discoverCollections(sourceProject); + + // 3. Select collections console.log(''); const selectedCollections = await checkbox({ message: 'Select collections to transfer:', @@ -141,7 +586,7 @@ export async function runInteractiveMode(config: Config): Promise { validate: (value) => value.length > 0 || 'Select at least one collection', }); - // Ask about options + // 4. Basic options console.log(''); const includeSubcollections = await confirm({ message: 'Include subcollections?', @@ -158,11 +603,8 @@ export async function runInteractiveMode(config: Config): Promise { default: config.merge, }); - // Clean up temporary app - await tempSourceApp.delete(); - - // Return updated config - return { + // Build config so far + let finalConfig: Config = { ...config, sourceProject, destProject, @@ -173,4 +615,23 @@ export async function runInteractiveMode(config: Config): Promise { idPrefix, idSuffix, }; + + // 5. Advanced options + console.log(''); + const advancedUpdates = await promptAdvancedOptions(finalConfig); + finalConfig = { ...finalConfig, ...advancedUpdates }; + + // Clean up temporary Firebase app + await tempSourceApp.delete(); + + // 6. Final action + const action = await promptFinalAction(); + + if (action === 'save-ini' || action === 'save-json') { + const format = action === 'save-json' ? 'json' : 'ini'; + await saveConfig(finalConfig, format); + return { config: finalConfig, action: 'save' }; + } + + return { config: finalConfig, action: 'execute' }; } From 64e10d6ad4b142201e25e892c1a2496b53a142d1 Mon Sep 17 00:00:00 2001 From: FaZeTitans Date: Thu, 5 Mar 2026 17:12:59 +0100 Subject: [PATCH 4/5] fix: address PR review comments - Gate progressBar.start() with canWriteProgress check - Use process.stdout.columns + isTTY in clearLine - Extend INI/JSON parsers and mergeConfig for advanced fields - Include advanced fields in INI serialization - Use explicit confirm queue in interactive tests - Add pagination boundary tests for transfer - Add maxDepth + onSubcollectionExcluded tests for count --- src/__tests__/count.test.ts | 151 ++++++++++++++++++++++++++++++ src/__tests__/interactive.test.ts | 6 +- src/__tests__/transfer.test.ts | 130 +++++++++++++++++++++++++ src/config/parser.ts | 42 +++++++-- src/interactive.ts | 24 +++++ src/orchestrator.ts | 12 ++- 6 files changed, 354 insertions(+), 11 deletions(-) diff --git a/src/__tests__/count.test.ts b/src/__tests__/count.test.ts index 110a240..afba2bd 100644 --- a/src/__tests__/count.test.ts +++ b/src/__tests__/count.test.ts @@ -394,6 +394,157 @@ describe('count module', () => { }); }); + describe('maxDepth boundaries', () => { + test('maxDepth=0 means unlimited depth', () => { + const config = createMockConfig({ maxDepth: 0 }); + + const shouldStop = (depth: number) => + config.maxDepth > 0 && depth >= config.maxDepth; + + expect(shouldStop(0)).toBe(false); + expect(shouldStop(1)).toBe(false); + expect(shouldStop(5)).toBe(false); + expect(shouldStop(100)).toBe(false); + }); + + test('maxDepth=1 stops after first-level subcollections', () => { + const config = createMockConfig({ maxDepth: 1 }); + + const shouldStop = (depth: number) => + config.maxDepth > 0 && depth >= config.maxDepth; + + // depth=0 is root collection, should continue to find subcollections + expect(shouldStop(0)).toBe(false); + // depth=1 means we're already one level deep, should stop recursing + expect(shouldStop(1)).toBe(true); + expect(shouldStop(2)).toBe(true); + }); + + test('maxDepth=2 allows root + two levels of subcollections', () => { + const config = createMockConfig({ maxDepth: 2 }); + + const shouldStop = (depth: number) => + config.maxDepth > 0 && depth >= config.maxDepth; + + expect(shouldStop(0)).toBe(false); + expect(shouldStop(1)).toBe(false); + expect(shouldStop(2)).toBe(true); + expect(shouldStop(3)).toBe(true); + }); + + test('maxDepth limits recursive counting simulation', () => { + const config = createMockConfig({ maxDepth: 1, includeSubcollections: true }); + const counted: string[] = []; + + const simulateCount = (path: string, depth: number) => { + counted.push(path); + // Simulate subcollection discovery + const subcollections = + path === 'users' ? ['orders', 'settings'] : + path === 'users/doc1/orders' ? ['items'] : []; + + for (const sub of subcollections) { + if (config.maxDepth > 0 && depth >= config.maxDepth) continue; + simulateCount(`${path}/doc1/${sub}`, depth + 1); + } + }; + + simulateCount('users', 0); + + // With maxDepth=1: root(depth=0) counted, subcollections at depth=0 explored, + // but at depth=1 recursion stops (no items counted) + expect(counted).toEqual([ + 'users', + 'users/doc1/orders', + 'users/doc1/settings', + ]); + expect(counted).not.toContain('users/doc1/orders/doc1/items'); + }); + }); + + describe('onSubcollectionExcluded callback', () => { + test('invoked for each excluded subcollection', () => { + const config = createMockConfig({ exclude: ['logs', 'cache'] }); + const excluded: string[] = []; + const onSubcollectionExcluded = (name: string) => excluded.push(name); + + const subcollections = ['users', 'logs', 'orders', 'cache', 'products']; + + for (const subId of subcollections) { + if (matchesExcludePattern(subId, config.exclude)) { + onSubcollectionExcluded(subId); + continue; + } + } + + expect(excluded).toEqual(['logs', 'cache']); + }); + + test('invoked with wildcard exclude patterns', () => { + const config = createMockConfig({ exclude: ['cache*', '*_temp'] }); + const excluded: string[] = []; + const onSubcollectionExcluded = (name: string) => excluded.push(name); + + const subcollections = ['cache_v1', 'cache_prod', 'session_temp', 'orders', 'users']; + + for (const subId of subcollections) { + if (matchesExcludePattern(subId, config.exclude)) { + onSubcollectionExcluded(subId); + continue; + } + } + + expect(excluded).toEqual(['cache_v1', 'cache_prod', 'session_temp']); + }); + + test('not invoked when no subcollections are excluded', () => { + const config = createMockConfig({ exclude: ['nonexistent'] }); + const excluded: string[] = []; + const onSubcollectionExcluded = (name: string) => excluded.push(name); + + const subcollections = ['users', 'orders', 'products']; + + for (const subId of subcollections) { + if (matchesExcludePattern(subId, config.exclude)) { + onSubcollectionExcluded(subId); + continue; + } + } + + expect(excluded).toEqual([]); + }); + + test('not invoked when exclude list is empty', () => { + const config = createMockConfig({ exclude: [] }); + const excluded: string[] = []; + const onSubcollectionExcluded = (name: string) => excluded.push(name); + + const subcollections = ['users', 'orders']; + + for (const subId of subcollections) { + if (matchesExcludePattern(subId, config.exclude)) { + onSubcollectionExcluded(subId); + continue; + } + } + + expect(excluded).toEqual([]); + }); + + test('works as CountProgress callback', () => { + const excluded: string[] = []; + const progress: CountProgress = { + onSubcollectionExcluded: mock((name: string) => excluded.push(name)), + }; + + progress.onSubcollectionExcluded!('logs'); + progress.onSubcollectionExcluded!('cache_v1'); + + expect(progress.onSubcollectionExcluded).toHaveBeenCalledTimes(2); + expect(excluded).toEqual(['logs', 'cache_v1']); + }); + }); + describe('depth tracking', () => { test('depth increases with each subcollection level', () => { const paths = [ diff --git a/src/__tests__/interactive.test.ts b/src/__tests__/interactive.test.ts index 1867d3f..e6cb0dd 100644 --- a/src/__tests__/interactive.test.ts +++ b/src/__tests__/interactive.test.ts @@ -100,7 +100,11 @@ describe('Interactive Mode', () => { mockInput.mockImplementation(() => Promise.resolve('test-project')); mockCheckbox.mockImplementation(() => Promise.resolve(['users'])); // Default confirm sequence: subcollections=true, dryRun=true, merge=true, advancedOptions=false - mockConfirm.mockImplementation(() => Promise.resolve(true)); + const defaultConfirmResponses = [true, true, true, false]; + mockConfirm.mockImplementation(() => { + const next = defaultConfirmResponses.shift(); + return Promise.resolve(next ?? false); + }); mockSelect.mockImplementation(() => Promise.resolve('execute')); mockNumber.mockImplementation(() => Promise.resolve(undefined)); mockInitializeApp.mockImplementation(() => mockApp); diff --git a/src/__tests__/transfer.test.ts b/src/__tests__/transfer.test.ts index 6ade761..988d42f 100644 --- a/src/__tests__/transfer.test.ts +++ b/src/__tests__/transfer.test.ts @@ -436,6 +436,136 @@ describe('transfer module', () => { }); }); + describe('pagination', () => { + test('pages through documents with batchSize as page size', () => { + const allDocs = Array.from({ length: 1250 }, (_, i) => + createMockDoc(`doc${i}`, { index: i }) + ); + const batchSize = 500; + const pages: MockDocumentSnapshot[][] = []; + let offset = 0; + + // Simulate pagination: fetch batchSize docs at a time + while (true) { + const page = allDocs.slice(offset, offset + batchSize); + if (page.length === 0) break; + pages.push(page); + offset += page.length; + if (page.length < batchSize) break; + } + + expect(pages).toHaveLength(3); + expect(pages[0]).toHaveLength(500); + expect(pages[1]).toHaveLength(500); + expect(pages[2]).toHaveLength(250); + }); + + test('terminates when collection size is exact multiple of batchSize', () => { + const allDocs = Array.from({ length: 1000 }, (_, i) => + createMockDoc(`doc${i}`, { index: i }) + ); + const batchSize = 500; + const pages: MockDocumentSnapshot[][] = []; + let offset = 0; + + while (true) { + const page = allDocs.slice(offset, offset + batchSize); + if (page.length === 0) break; + pages.push(page); + offset += page.length; + // Fewer docs than requested means end (this won't trigger for exact multiple) + if (page.length < batchSize) break; + } + + // With exact multiple, last page has batchSize docs, loop fetches one more empty page + // Total pages with data should be 2 + expect(pages).toHaveLength(2); + expect(pages[0]).toHaveLength(500); + expect(pages[1]).toHaveLength(500); + }); + + test('respects user limit across pages', () => { + const allDocs = Array.from({ length: 1000 }, (_, i) => + createMockDoc(`doc${i}`, { index: i }) + ); + const batchSize = 500; + const userLimit = 750; + const pages: MockDocumentSnapshot[][] = []; + let totalProcessed = 0; + + while (true) { + let pageSize = batchSize; + const remaining = userLimit - totalProcessed; + if (remaining <= 0) break; + pageSize = Math.min(pageSize, remaining); + + const page = allDocs.slice(totalProcessed, totalProcessed + pageSize); + if (page.length === 0) break; + pages.push(page); + totalProcessed += page.length; + if (page.length < pageSize) break; + } + + expect(totalProcessed).toBe(750); + expect(pages).toHaveLength(2); + expect(pages[0]).toHaveLength(500); + expect(pages[1]).toHaveLength(250); + }); + + test('handles single page (docs < batchSize)', () => { + const allDocs = Array.from({ length: 100 }, (_, i) => + createMockDoc(`doc${i}`, { index: i }) + ); + const batchSize = 500; + const pages: MockDocumentSnapshot[][] = []; + let offset = 0; + + while (true) { + const page = allDocs.slice(offset, offset + batchSize); + if (page.length === 0) break; + pages.push(page); + offset += page.length; + if (page.length < batchSize) break; + } + + expect(pages).toHaveLength(1); + expect(pages[0]).toHaveLength(100); + }); + + test('handles empty collection', () => { + const allDocs: MockDocumentSnapshot[] = []; + const batchSize = 500; + const pages: MockDocumentSnapshot[][] = []; + + const page = allDocs.slice(0, batchSize); + if (page.length > 0) { + pages.push(page); + } + + expect(pages).toHaveLength(0); + }); + + test('startAfter uses last doc of previous page', () => { + const allDocs = Array.from({ length: 10 }, (_, i) => + createMockDoc(`doc${i}`, { index: i }) + ); + const batchSize = 3; + const lastDocs: string[] = []; + let offset = 0; + + while (true) { + const page = allDocs.slice(offset, offset + batchSize); + if (page.length === 0) break; + lastDocs.push(page[page.length - 1].id); + offset += page.length; + if (page.length < batchSize) break; + } + + // Last doc of each full page should be used as startAfter cursor + expect(lastDocs).toEqual(['doc2', 'doc5', 'doc8', 'doc9']); + }); + }); + describe('batch operations', () => { test('batches documents by batchSize', () => { const docs = Array.from({ length: 1250 }, (_, i) => diff --git a/src/config/parser.ts b/src/config/parser.ts index e1159e6..9b20547 100644 --- a/src/config/parser.ts +++ b/src/config/parser.ts @@ -115,6 +115,13 @@ export function parseIniConfig(content: string): Partial { idPrefix?: string; idSuffix?: string; webhook?: string; + retries?: string; + rateLimit?: string; + skipOversized?: string | boolean; + detectConflicts?: string | boolean; + maxDepth?: string; + verify?: string | boolean; + verifyIntegrity?: string | boolean; }; }; @@ -149,6 +156,13 @@ export function parseIniConfig(content: string): Partial { idPrefix: parsed.options?.idPrefix ?? null, idSuffix: parsed.options?.idSuffix ?? null, webhook: parsed.options?.webhook ?? null, + retries: Number.parseInt(parsed.options?.retries ?? '', 10) || undefined, + rateLimit: Number.parseInt(parsed.options?.rateLimit ?? '', 10) || undefined, + skipOversized: parsed.options?.skipOversized !== undefined ? parseBoolean(parsed.options.skipOversized) : undefined, + detectConflicts: parsed.options?.detectConflicts !== undefined ? parseBoolean(parsed.options.detectConflicts) : undefined, + maxDepth: Number.parseInt(parsed.options?.maxDepth ?? '', 10) || undefined, + verify: parsed.options?.verify !== undefined ? parseBoolean(parsed.options.verify) : undefined, + verifyIntegrity: parsed.options?.verifyIntegrity !== undefined ? parseBoolean(parsed.options.verifyIntegrity) : undefined, }; } @@ -172,6 +186,13 @@ export function parseJsonConfig(content: string): Partial { idPrefix?: string; idSuffix?: string; webhook?: string; + retries?: number; + rateLimit?: number; + skipOversized?: boolean; + detectConflicts?: boolean; + maxDepth?: number; + verify?: boolean; + verifyIntegrity?: boolean; }; return { @@ -193,6 +214,13 @@ export function parseJsonConfig(content: string): Partial { idPrefix: config.idPrefix ?? null, idSuffix: config.idSuffix ?? null, webhook: config.webhook ?? null, + retries: config.retries, + rateLimit: config.rateLimit, + skipOversized: config.skipOversized, + detectConflicts: config.detectConflicts, + maxDepth: config.maxDepth, + verify: config.verify, + verifyIntegrity: config.verifyIntegrity, }; } @@ -232,7 +260,6 @@ export function mergeConfig( sourceProject: cliArgs.sourceProject ?? fileConfig.sourceProject ?? defaultConfig.sourceProject, destProject: cliArgs.destProject ?? fileConfig.destProject ?? defaultConfig.destProject, - retries: cliArgs.retries ?? defaultConfig.retries, where: cliWhereFilters.length > 0 ? cliWhereFilters @@ -251,15 +278,16 @@ export function mergeConfig( idPrefix: cliArgs.idPrefix ?? fileConfig.idPrefix ?? defaultConfig.idPrefix, idSuffix: cliArgs.idSuffix ?? fileConfig.idSuffix ?? defaultConfig.idSuffix, webhook: cliArgs.webhook ?? fileConfig.webhook ?? defaultConfig.webhook, + retries: cliArgs.retries ?? fileConfig.retries ?? defaultConfig.retries, resume: cliArgs.resume ?? defaultConfig.resume, stateFile: cliArgs.stateFile ?? defaultConfig.stateFile, - verify: cliArgs.verify ?? defaultConfig.verify, - rateLimit: cliArgs.rateLimit ?? defaultConfig.rateLimit, - skipOversized: cliArgs.skipOversized ?? defaultConfig.skipOversized, + verify: cliArgs.verify ?? fileConfig.verify ?? defaultConfig.verify, + rateLimit: cliArgs.rateLimit ?? fileConfig.rateLimit ?? defaultConfig.rateLimit, + skipOversized: cliArgs.skipOversized ?? fileConfig.skipOversized ?? defaultConfig.skipOversized, json: cliArgs.json ?? defaultConfig.json, transformSamples: cliArgs.transformSamples ?? defaultConfig.transformSamples, - detectConflicts: cliArgs.detectConflicts ?? defaultConfig.detectConflicts, - maxDepth: cliArgs.maxDepth ?? defaultConfig.maxDepth, - verifyIntegrity: cliArgs.verifyIntegrity ?? defaultConfig.verifyIntegrity, + detectConflicts: cliArgs.detectConflicts ?? fileConfig.detectConflicts ?? defaultConfig.detectConflicts, + maxDepth: cliArgs.maxDepth ?? fileConfig.maxDepth ?? defaultConfig.maxDepth, + verifyIntegrity: cliArgs.verifyIntegrity ?? fileConfig.verifyIntegrity ?? defaultConfig.verifyIntegrity, }; } diff --git a/src/interactive.ts b/src/interactive.ts index ebd32ec..ea13af0 100644 --- a/src/interactive.ts +++ b/src/interactive.ts @@ -516,6 +516,30 @@ function configToIni(config: Config): string { ini += iniComment('webhook', 'https://hooks.slack.com/services/...'); } + if (config.retries !== 3) { + ini += iniLine('retries', config.retries); + } else { + ini += iniComment('retries', 3); + } + + if (config.rateLimit > 0) { + ini += iniLine('rateLimit', config.rateLimit); + } else { + ini += iniComment('rateLimit', 0); + } + + ini += iniLine('skipOversized', config.skipOversized); + ini += iniLine('detectConflicts', config.detectConflicts); + + if (config.maxDepth > 0) { + ini += iniLine('maxDepth', config.maxDepth); + } else { + ini += iniComment('maxDepth', 0); + } + + ini += iniLine('verify', config.verify); + ini += iniLine('verifyIntegrity', config.verifyIntegrity); + return ini; } diff --git a/src/orchestrator.ts b/src/orchestrator.ts index 7a27629..03eebea 100644 --- a/src/orchestrator.ts +++ b/src/orchestrator.ts @@ -266,11 +266,15 @@ async function validateTransformWithSamples( } function canWriteProgress(output: Output): boolean { - return !output.isQuiet && !output.isJson; + return !output.isQuiet && !output.isJson && process.stdout.isTTY === true; } function clearLine(): void { - process.stdout.write('\r' + ' '.repeat(SEPARATOR_LENGTH) + '\r'); + if (!process.stdout.isTTY) return; + const width = typeof process.stdout.columns === 'number' && process.stdout.columns > 0 + ? process.stdout.columns + : SEPARATOR_LENGTH; + process.stdout.write('\r' + ' '.repeat(width) + '\r'); } function formatNameList(names: Set, max: number = 8): string { @@ -347,7 +351,9 @@ async function setupProgressTracking( } output.info(`\n Total: ${totalDocs} documents to transfer\n`); - progressBar.start(totalDocs, stats); + if (canWriteProgress(output)) { + progressBar.start(totalDocs, stats); + } } return { totalDocs, progressBar }; From 0307303870d769ae73d710f855c1bb72444521d5 Mon Sep 17 00:00:00 2001 From: FaZeTitans Date: Fri, 6 Mar 2026 02:56:27 +0100 Subject: [PATCH 5/5] fix: config file values ignored due to yargs defaults + INI parsing drops 0 values - Remove yargs defaults for retries, verify, rateLimit, skipOversized, detectConflicts, maxDepth, verifyIntegrity so file config values are actually used when CLI args are not explicitly passed - Replace Number.parseInt(...) || undefined with parseIntOption() helper to preserve valid 0 values in INI config parsing --- src/cli.ts | 7 ------- src/config/parser.ts | 12 +++++++++--- src/types.ts | 2 +- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/src/cli.ts b/src/cli.ts index 9ab80b9..2e40d91 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -89,7 +89,6 @@ const argv = yargs(hideBin(process.argv)) .option('retries', { type: 'number', description: 'Number of retries on error (default: 3)', - default: 3, }) .option('quiet', { alias: 'q', @@ -166,17 +165,14 @@ const argv = yargs(hideBin(process.argv)) .option('verify', { type: 'boolean', description: 'Verify document counts after transfer', - default: false, }) .option('rate-limit', { type: 'number', description: 'Limit transfer rate (documents per second, 0 = unlimited)', - default: 0, }) .option('skip-oversized', { type: 'boolean', description: 'Skip documents exceeding 1MB instead of failing', - default: false, }) .option('json', { type: 'boolean', @@ -191,17 +187,14 @@ const argv = yargs(hideBin(process.argv)) .option('detect-conflicts', { type: 'boolean', description: 'Detect if destination docs were modified during transfer', - default: false, }) .option('max-depth', { type: 'number', description: 'Max subcollection depth (0 = unlimited)', - default: 0, }) .option('verify-integrity', { type: 'boolean', description: 'Verify document integrity with hash after transfer', - default: false, }) .option('validate-only', { type: 'boolean', diff --git a/src/config/parser.ts b/src/config/parser.ts index 9b20547..dd42172 100644 --- a/src/config/parser.ts +++ b/src/config/parser.ts @@ -66,6 +66,12 @@ export function parseStringList(value: string | undefined): string[] { .filter((s) => s.length > 0); } +export function parseIntOption(value: string | undefined): number | undefined { + if (value === undefined || value === '') return undefined; + const parsed = Number.parseInt(value, 10); + return Number.isNaN(parsed) ? undefined : parsed; +} + export function parseRenameMapping( mappings: string[] | string | undefined ): Record { @@ -156,11 +162,11 @@ export function parseIniConfig(content: string): Partial { idPrefix: parsed.options?.idPrefix ?? null, idSuffix: parsed.options?.idSuffix ?? null, webhook: parsed.options?.webhook ?? null, - retries: Number.parseInt(parsed.options?.retries ?? '', 10) || undefined, - rateLimit: Number.parseInt(parsed.options?.rateLimit ?? '', 10) || undefined, + retries: parseIntOption(parsed.options?.retries), + rateLimit: parseIntOption(parsed.options?.rateLimit), skipOversized: parsed.options?.skipOversized !== undefined ? parseBoolean(parsed.options.skipOversized) : undefined, detectConflicts: parsed.options?.detectConflicts !== undefined ? parseBoolean(parsed.options.detectConflicts) : undefined, - maxDepth: Number.parseInt(parsed.options?.maxDepth ?? '', 10) || undefined, + maxDepth: parseIntOption(parsed.options?.maxDepth), verify: parsed.options?.verify !== undefined ? parseBoolean(parsed.options.verify) : undefined, verifyIntegrity: parsed.options?.verifyIntegrity !== undefined ? parseBoolean(parsed.options.verifyIntegrity) : undefined, }; diff --git a/src/types.ts b/src/types.ts index deea87b..056250e 100644 --- a/src/types.ts +++ b/src/types.ts @@ -97,7 +97,7 @@ export interface CliArgs { yes: boolean; log?: string; maxLogSize?: string; - retries: number; + retries?: number; quiet: boolean; where?: string[]; exclude?: string[];