diff --git a/README.md b/README.md index 92aa0af..fcecfbc 100644 --- a/README.md +++ b/README.md @@ -36,9 +36,10 @@ AWS CLI installation is **NOT** required by this module. 3. [Sync the local file system with a remote S3 bucket](#sync-the-local-file-system-with-a-remote-s3-bucket) 4. [Sync two remote S3 buckets](#sync-two-remote-s3-buckets) 5. [Monitor transfer progress](#monitor-transfer-progress) - 6. [Use AWS SDK command input options](#use-aws-sdk-command-input-options) - 7. [Relocate objects during sync](#relocate-objects-during-sync) - 8. [Filter source files](#filter-source-files) + 6. [Monitor object status](#monitor-object-status) + 7. [Use AWS SDK command input options](#use-aws-sdk-command-input-options) + 8. [Relocate objects during sync](#relocate-objects-during-sync) + 9. [Filter source files](#filter-source-files) 1. [API Reference](#api-reference) - [Class: S3SyncClient](#class-s3-sync-client) - [new S3SyncClient(configuration)](#new-s3-sync-client) @@ -100,7 +101,6 @@ await sync('s3://my-source-bucket', 's3://my-target-bucket', { del: true }); #### Monitor transfer progress ```javascript -const EventEmitter = require('events'); const { TransferMonitor } = require('s3-sync-client'); const monitor = new TransferMonitor(); @@ -128,6 +128,35 @@ try { } ``` +#### Monitor object status + +```javascript +const { TransferMonitor } = require('s3-sync-client'); + +const monitor = new TransferMonitor(); +monitor.on('start', (data) => console.log("Start:", data)); +monitor.on('done', (data) => console.log("Done:", data)); +monitor.on('error', (data) => console.log("Error:", data)); + +await sync('s3://mybucket', '/path/to/local/dir', { monitor }); + +/* output, per object as it is processed: +... +Start: { + filePath: '...' +} +... +Done: { + filePath: '...' +} +... +Done: { + filePath: '...', + error: '...' +} +*/ +``` + #### Use AWS SDK command input options ```javascript diff --git a/lib/sync-objects/bucket-object.js b/lib/sync-objects/bucket-object.js index f2bc39a..3d74532 100644 --- a/lib/sync-objects/bucket-object.js +++ b/lib/sync-objects/bucket-object.js @@ -30,6 +30,7 @@ class BucketObject extends SyncObject { Bucket: this.bucket, Key: this.key, }, commandInput); + monitor?.emit('start', { filePath }); const { Body: readStream, LastModified } = await client.send( new GetObjectCommand(getObjectCommandInput), { abortSignal }, @@ -44,11 +45,19 @@ class BucketObject extends SyncObject { }); } return new Promise((resolve, reject) => { - writeStream.on('error', reject); + writeStream.on('error', (error) => { + monitor?.emit('error', { filePath, error }); + reject(error); + }); writeStream.on('finish', () => { - fs.utimes(filePath, LastModified, LastModified, (err) => { - if (err) reject(err); - else resolve(); + fs.utimes(filePath, LastModified, LastModified, (error) => { + if (error) { + monitor?.emit('error', { filePath, error }); + reject(error); + } else { + monitor?.emit('done', { filePath }); + resolve(); + } }); }); }); diff --git a/test/s3-sync-client.test.js b/test/s3-sync-client.test.js index a606850..febd68b 100644 --- a/test/s3-sync-client.test.js +++ b/test/s3-sync-client.test.js @@ -50,11 +50,14 @@ describe('S3SyncClient', () => { test('load bucket 2 dataset', async () => { const monitor = new TransferMonitor(); let count = 0; + let done = 0; monitor.on('progress', (progress) => { count = progress.count.current; }); + monitor.on('done', () => { done++; }); await emptyBucket(syncClient, BUCKET_2); await syncClient.sync(DATA_DIR, `s3://${BUCKET_2}`, { del: true, maxConcurrentTransfers: 200, monitor }); const objects = await syncClient.listLocalObjects(DATA_DIR); expect(count).toStrictEqual(5000); + expect(done).toStrictEqual(count); expect(objects.length).toStrictEqual(5000); }); @@ -113,11 +116,14 @@ describe('S3SyncClient', () => { test('sync a single dir with progress tracking', async () => { const monitor = new TransferMonitor(); let count = 0; + let done = 0; monitor.on('progress', (progress) => { count = progress.count.current; }); + monitor.on('done', () => { done++; }); await syncClient.bucketWithBucket(`${BUCKET_2}/def/jkl`, BUCKET, { maxConcurrentTransfers: 200, monitor }); const objects = await syncClient.listBucketObjects(BUCKET, { prefix: 'def/jkl' }); expect(hasObject(objects, 'def/jkl/xmoj')).toBe(true); expect(count).toStrictEqual(11); + expect(done).toStrictEqual(count); expect(objects.length).toStrictEqual(11); }); @@ -268,10 +274,13 @@ describe('S3SyncClient', () => { test('sync 5000 bucket objects successfully with progress tracking', async () => { const monitor = new TransferMonitor(); let count = 0; + let done = 0; monitor.on('progress', (progress) => { count = progress.count.current; }); + monitor.on('done', () => { done++; }); await syncClient.sync(`s3://${BUCKET_2}`, SYNC_DIR, { maxConcurrentTransfers: 200, monitor }); const objects = await syncClient.listLocalObjects(SYNC_DIR); expect(count).toStrictEqual(5000); + expect(done).toStrictEqual(count); expect(objects.length).toBeGreaterThanOrEqual(5000); });