feat: add aws test suites

Author: socaseinpoint

.mdeprc.js

@@ -8,6 +8,14 @@ module.exports = {
   post_exec: 'yarn coverage:report'
 };
 
+if (process.env.PROVIDER === 'aws') {
+  module.exports.tests = './test/suites/providers/aws/*.js';
+}
+
+if (process.env.PROVIDER !== 'aws') {
+  module.exports.tests = './test/suites/**/!(providers)/*.js';
+}
+
 switch (process.env.DB) {
   case 'sentinel':
     module.exports.services.push('redisSentinel');

package.json

@@ -7,9 +7,10 @@
     "compile": "rimraf ./lib && babel -d ./lib --copy-files ./src",
     "pretest": "yarn compile",
     "test": "rimraf ./coverage && yarn lint && yarn test:e2e",
-    "test:e2e": "yarn test:e2e:cluster && yarn test:e2e:sentinel",
-    "test:e2e:cluster": "DB=cluster mdep test run",
-    "test:e2e:sentinel": "DB=sentinel mdep test run --docker_compose ./test/docker-compose.sentinel.yml",
+    "test:e2e": "yarn test:e2e:cluster && yarn test:e2e:sentinel && test:e2e:aws-provider",
+    "test:e2e:cluster": "PROVIDER=gce DB=cluster mdep test run",
+    "test:e2e:sentinel": "PROVIDER=gce DB=sentinel mdep test run --docker_compose ./test/docker-compose.sentinel.yml",
+    "test:e2e:aws-provider": "PROVIDER=aws DB=sentinel mdep test run --docker_compose ./test/docker-compose.sentinel.yml",
     "start": "mfleet",
     "lint": "eslint ./src ./test",
     "prepublishOnly": "yarn compile",

src/actions/upload.js

@@ -38,6 +38,7 @@ const {
  * @return {Promise}
  */
 async function initFileUpload({ params }) {
+  console.log('init file upload');
   const {
     files,
     meta,

src/configs/core.js

@@ -93,6 +93,7 @@ exports.transport = [{
  * @returns {Provider}
  */
 exports.selectTransport = function selectTransport() {
+  // console.log('my provider 1', this.providers[0]);
   return this.providers[0];
 };
 

src/index.js

@@ -164,8 +164,8 @@ class Files extends Microfleet {
     await super.connect();
     await this.initWebhook();
     await Promise.mapSeries(this.providers, (provider) => {
-      // @todo
-      if (provider.config.name !== 'gce' || provider.config.name !== 'aws') return null;
+      if (!['aws', 'gce'].includes(provider.config.name)) return null;
+
       if (!provider.config.bucket.channel.pubsub) return null;
       return provider.subscribe(this.handleUploadNotification.bind(this));
     });

src/providers/aws.js

@@ -2,6 +2,7 @@ const Promise = require('bluebird');
 const AbstractFileTransfer = require('ms-files-transport');
 const { merge } = require('lodash');
 const S3 = require('aws-sdk/clients/s3');
+const SNS = require('aws-sdk/clients/sns');
 
 const DOWNLOAD_URL_EXPIRES_IN_SEC = 60000;
 
@@ -61,7 +62,25 @@ class AWSTransport extends AbstractFileTransfer {
    * @return {Subscription}
    */
   async subscribe() {
-    this.log.warn('the method is not implemented yet');
+    const { Topics: topics } = await new SNS({ region: this._config.aws.credentials.region }).listTopics({}).promise();
+
+    const topicArn = `arn:aws:sns:us-west-2:178085672309:${this._config.aws.credentials.topicName}`;
+
+    const topic = topics.find(
+      (_topic) => _topic.TopicArn === topicArn
+    );
+
+    if (!topic) {
+      await new SNS({ region: this._config.aws.credentials.region }).createTopic({ Name: this._config.aws.credentials.topicName }).promise();
+    }
+
+    const params = {
+      Protocol: 'https',
+      TopicArn: topicArn,
+      Endpoint: 'localhost:443',
+    };
+
+    await new SNS().subscribe(params).promise();
   }
 
   /**
@@ -93,6 +112,7 @@ class AWSTransport extends AbstractFileTransfer {
         if (!isBucketExist) {
           const bucketParams = {
             Bucket: bucketName,
+            region: this._config.aws.credentials.region,
           };
 
           await aws.createBucket(bucketParams, function handleCreateBucket(_err) {
@@ -125,11 +145,9 @@ class AWSTransport extends AbstractFileTransfer {
   }
 
   // @todo interface
-  getDownloadUrlSigned(filename, downloadName) {
-    this.log.warn(`${downloadName} is not implemented yet`);
-
+  getDownloadUrlSigned(filename) {
     const params = {
-      Bucket: this._config.bucketName,
+      Bucket: this.getBucketName(),
       Expires: DOWNLOAD_URL_EXPIRES_IN_SEC,
       Key: filename,
     };
@@ -174,11 +192,9 @@ class AWSTransport extends AbstractFileTransfer {
     return new Promise((resolve, reject) => {
       this._aws.getSignedUrl('putObject', params, (err, url) => {
         if (err) {
-          console.log('init resumable upload err', err);
           return reject(err);
         }
 
-        console.log('init resumable upload url', url);
         return resolve(url);
       });
     });
@@ -239,15 +255,11 @@ class AWSTransport extends AbstractFileTransfer {
     const params = {
       Bucket: this._config.bucket.name,
       Expires: DOWNLOAD_URL_EXPIRES_IN_SEC,
-      Key: opts.filename,
+      Key: opts.resource,
     };
 
-    params.ContentType = opts.contentType;
-
     return new Promise((resolve, reject) => {
-      console.log('signed url params', params);
       this._aws.getSignedUrl('putObject', params, (err, url) => {
-        console.log('signed url err', err);
         if (err) {
           return reject(err);
         }
@@ -305,12 +317,10 @@ class AWSTransport extends AbstractFileTransfer {
     console.log(`exists method aws for: ${filename}`);
 
     return new Promise((resolve) => {
-      this._aws.headObject({ Key: filename, Bucket: this._config.bucket.name }, (err, data) => {
+      this._aws.headObject({ Key: filename, Bucket: this._config.bucket.name }, (err) => {
         if (err) {
-          console.log('exists err', err);
           return resolve(false);
         }
-        console.log('exists data', data);
         return resolve(true);
       });
     });
@@ -322,7 +332,14 @@ class AWSTransport extends AbstractFileTransfer {
    * @return {Promise}
    */
   remove(filename) {
-    this.log.warn('the method is not implemented yet', { filename });
+    return new Promise((resolve) => {
+      this._aws.deleteObject({ Key: filename, Bucket: this._config.bucket.name }, (err) => {
+        if (err) {
+          return resolve(false);
+        }
+        return resolve(true);
+      });
+    });
   }
 }
 
@@ -348,7 +365,7 @@ AWSTransport.defaultOpts = {
         token: undefined,
       },
     },
-    mсetadata: {},
+    metadata: {},
   },
 };
 

src/utils/fetch-data.js

@@ -102,6 +102,7 @@ async function selectMaster(redis) {
  * @param  {String[]} [fieldsFilter.pick]
  */
 module.exports = function fetchData(key, fieldFilter = {}) {
+  console.log('fetch data for key', key);
   const { redis } = this;
   const timer = perf(`fetchData:${key}`);
 

src/utils/process.js

@@ -31,6 +31,7 @@ module.exports = function processFile(key, data) {
   return Promise
     .using(acquireLock(this, `postprocess:${key}`), (lock) => {
       const { uploadId } = data;
+      console.log('port srocess uploadId', uploadId);
       const { redis } = this;
 
       return Promise

test/configs/generic/core.js

@@ -19,36 +19,16 @@ exports.amqp = {
   },
 };
 
-exports.transport = [
-  //   {
-  //   name: 'gce',
-  //   options: {
-  //     gce: {
-  //       projectId: env.GCLOUD_PROJECT_ID,
-  //       credentials: {
-  //         client_email: env.GCLOUD_PROJECT_EMAIL,
-  //         private_key: env.GCLOUD_PROJECT_PK,
-  //       },
-  //     },
-  //     bucket: {
-  //       name: env.TEST_BUCKET,
-  //       metadata: {
-  //         location: env.GCLOUD_BUCKET_LOCATION || 'EUROPE-WEST1',
-  //         dra: true,
-  //       },
-  //     },
-  //     // test for direct public URLs
-  //   },
-  //   // its not a public name!
-  //   cname: 'gce',
-  // },
+const awsTransport = [
   {
     name: 'aws',
     options: {
       aws: {
         credentials: {
+          region: env.AWS_REGION,
           accessKeyId: env.AWS_ACCESS_KEY_ID,
           secretAccessKey: env.AWS_SECRET_ACCESS_KEY,
+          topicName: env.TOPIC_NAME,
         },
       },
       bucket: {
@@ -58,7 +38,35 @@ exports.transport = [
     },
     // its not a public name!
     cname: 'aws',
-  }];
+  },
+];
+
+// const gceTransport = [
+//   {
+//     name: 'gce',
+//     options: {
+//       gce: {
+//         projectId: env.GCLOUD_PROJECT_ID,
+//         credentials: {
+//           client_email: env.GCLOUD_PROJECT_EMAIL,
+//           private_key: env.GCLOUD_PROJECT_PK,
+//         },
+//       },
+//       bucket: {
+//         name: env.TEST_BUCKET,
+//         metadata: {
+//           location: env.GCLOUD_BUCKET_LOCATION || 'EUROPE-WEST1',
+//           dra: true,
+//         },
+//       },
+//       // test for direct public URLs
+//     },
+//     // its not a public name!
+//     cname: 'gce',
+//   },
+// ];
+
+exports.transport = env.PROVIDER === 'aws' ? awsTransport : awsTransport;
 
 exports.hooks = {
   // return input, assume there are models

test/docker-compose.aws-provider.yml

@@ -0,0 +1,16 @@
+version: '3'
+
+services:
+  tester:
+    volumes:
+      - ${PWD}:/src
+      - ${PWD}/test/configs/generic:/configs/core:ro
+      # - ${PWD}/test/configs/generic:/configs/aws:ro
+      - ${PWD}/test/configs/redis-sentinel:/configs/redis:ro
+    environment:
+      NODE_ENV: "test"
+      DEBUG: "${DEBUG}"
+      TEST_BUCKET: "makeomatic-test"
+      DOTENV_FILE_PATH: "/src/test/.env"
+      NCONF_FILE_PATH: '["/configs/core","/configs/redis"]'
+