feat: add aws test suites

Author: socaseinpoint

.mdeprc.js

@@ -8,6 +8,10 @@ module.exports = {
   post_exec: 'yarn coverage:report'
 };
 
+if (process.env.PROVIDER === 'aws') {
+  module.exports.tests = './test/suites/providers/aws/*.js';
+}
+
 switch (process.env.DB) {
   case 'sentinel':
     module.exports.services.push('redisSentinel');

package.json

@@ -10,6 +10,7 @@
     "test:e2e": "yarn test:e2e:cluster && yarn test:e2e:sentinel",
     "test:e2e:cluster": "DB=cluster mdep test run",
     "test:e2e:sentinel": "DB=sentinel mdep test run --docker_compose ./test/docker-compose.sentinel.yml",
+    "test:e2e:aws-provider": "PROVIDER=aws DB=sentinel mdep test run --docker_compose ./test/docker-compose.sentinel.yml",
     "start": "mfleet",
     "lint": "eslint ./src ./test",
     "prepublishOnly": "yarn compile",

src/actions/upload.js

@@ -38,6 +38,7 @@ const {
  * @return {Promise}
  */
 async function initFileUpload({ params }) {
+  console.log('init file upload');
   const {
     files,
     meta,

src/configs/core.js

@@ -93,6 +93,7 @@ exports.transport = [{
  * @returns {Provider}
  */
 exports.selectTransport = function selectTransport() {
+  // console.log('my provider 1', this.providers[0]);
   return this.providers[0];
 };
 

src/providers/aws.js

@@ -125,15 +125,15 @@ class AWSTransport extends AbstractFileTransfer {
   }
 
   // @todo interface
-  getDownloadUrlSigned(filename, downloadName) {
-    this.log.warn(`${downloadName} is not implemented yet`);
-
+  getDownloadUrlSigned(filename) {
     const params = {
-      Bucket: this._config.bucketName,
+      Bucket: this.getBucketName(),
       Expires: DOWNLOAD_URL_EXPIRES_IN_SEC,
       Key: filename,
     };
 
+    console.log('params with Key', params);
+
     return new Promise((resolve, reject) => {
       this._aws.getSignedUrl('putObject', params, (err, url) => {
         if (err) {
@@ -161,6 +161,7 @@ class AWSTransport extends AbstractFileTransfer {
    * @return {Promise}
    */
   async initResumableUpload(opts) {
+    console.log('init resumable upload');
     const params = {
       Bucket: this._config.bucket.name,
       Expires: DOWNLOAD_URL_EXPIRES_IN_SEC,
@@ -236,13 +237,18 @@ class AWSTransport extends AbstractFileTransfer {
  * @returns {Promise}
  */
   createSignedURL(opts) {
+    // console.log('createSignedURL opts 2', opts.resource.split('/')[opts.resource.length - 1]);
+    // console.log('createSignedURL opts 2', opts);
+    console.log('12345667890');
     const params = {
       Bucket: this._config.bucket.name,
       Expires: DOWNLOAD_URL_EXPIRES_IN_SEC,
-      Key: opts.filename,
+      Key: opts.resource,
     };
 
-    params.ContentType = opts.contentType;
+    console.log('createSignedURL params', params);
+
+    // params.ContentType = opts.contentType;
 
     return new Promise((resolve, reject) => {
       console.log('signed url params', params);
@@ -305,12 +311,10 @@ class AWSTransport extends AbstractFileTransfer {
     console.log(`exists method aws for: ${filename}`);
 
     return new Promise((resolve) => {
-      this._aws.headObject({ Key: filename, Bucket: this._config.bucket.name }, (err, data) => {
+      this._aws.headObject({ Key: filename, Bucket: this._config.bucket.name }, (err) => {
         if (err) {
-          console.log('exists err', err);
           return resolve(false);
         }
-        console.log('exists data', data);
         return resolve(true);
       });
     });
@@ -322,7 +326,14 @@ class AWSTransport extends AbstractFileTransfer {
    * @return {Promise}
    */
   remove(filename) {
-    this.log.warn('the method is not implemented yet', { filename });
+    return new Promise((resolve) => {
+      this._aws.deleteObject({ Key: filename, Bucket: this._config.bucket.name }, (err) => {
+        if (err) {
+          return resolve(false);
+        }
+        return resolve(true);
+      });
+    });
   }
 }
 

src/utils/fetch-data.js

@@ -102,6 +102,7 @@ async function selectMaster(redis) {
  * @param  {String[]} [fieldsFilter.pick]
  */
 module.exports = function fetchData(key, fieldFilter = {}) {
+  console.log('fetch data for key', key);
   const { redis } = this;
   const timer = perf(`fetchData:${key}`);
 

src/utils/process.js

@@ -31,6 +31,7 @@ module.exports = function processFile(key, data) {
   return Promise
     .using(acquireLock(this, `postprocess:${key}`), (lock) => {
       const { uploadId } = data;
+      console.log('port srocess uploadId', uploadId);
       const { redis } = this;
 
       return Promise

test/configs/generic/core.js

@@ -19,29 +19,7 @@ exports.amqp = {
   },
 };
 
-exports.transport = [
-  //   {
-  //   name: 'gce',
-  //   options: {
-  //     gce: {
-  //       projectId: env.GCLOUD_PROJECT_ID,
-  //       credentials: {
-  //         client_email: env.GCLOUD_PROJECT_EMAIL,
-  //         private_key: env.GCLOUD_PROJECT_PK,
-  //       },
-  //     },
-  //     bucket: {
-  //       name: env.TEST_BUCKET,
-  //       metadata: {
-  //         location: env.GCLOUD_BUCKET_LOCATION || 'EUROPE-WEST1',
-  //         dra: true,
-  //       },
-  //     },
-  //     // test for direct public URLs
-  //   },
-  //   // its not a public name!
-  //   cname: 'gce',
-  // },
+const awsTransport = [
   {
     name: 'aws',
     options: {
@@ -58,7 +36,35 @@ exports.transport = [
     },
     // its not a public name!
     cname: 'aws',
-  }];
+  },
+];
+
+// const gceTransport = [
+//   {
+//     name: 'gce',
+//     options: {
+//       gce: {
+//         projectId: env.GCLOUD_PROJECT_ID,
+//         credentials: {
+//           client_email: env.GCLOUD_PROJECT_EMAIL,
+//           private_key: env.GCLOUD_PROJECT_PK,
+//         },
+//       },
+//       bucket: {
+//         name: env.TEST_BUCKET,
+//         metadata: {
+//           location: env.GCLOUD_BUCKET_LOCATION || 'EUROPE-WEST1',
+//           dra: true,
+//         },
+//       },
+//       // test for direct public URLs
+//     },
+//     // its not a public name!
+//     cname: 'gce',
+//   },
+// ];
+
+exports.transport = env.PROVIDER === 'aws' ? awsTransport : awsTransport;
 
 exports.hooks = {
   // return input, assume there are models

test/docker-compose.aws-provider.yml

@@ -0,0 +1,16 @@
+version: '3'
+
+services:
+  tester:
+    volumes:
+      - ${PWD}:/src
+      - ${PWD}/test/configs/generic:/configs/core:ro
+      # - ${PWD}/test/configs/generic:/configs/aws:ro
+      - ${PWD}/test/configs/redis-sentinel:/configs/redis:ro
+    environment:
+      NODE_ENV: "test"
+      DEBUG: "${DEBUG}"
+      TEST_BUCKET: "makeomatic-131232"
+      DOTENV_FILE_PATH: "/src/test/.env"
+      NCONF_FILE_PATH: '["/configs/core","/configs/redis"]'
+      PROVIDER: 'aws'

test/helpers/utils.js

@@ -177,37 +177,40 @@ function modelSimpleUpload({
 // upload single file
 //
 function upload(location, file) {
+  console.log('upload location', location);
   return request.put({
     url: location,
     body: file,
-    headers: {
-      'content-length': file.length,
-    },
+    // headers: {
+    //   'content-length': file.length,
+    // },
     simple: false,
     resolveWithFullResponse: true,
   });
 }
 
-async function uploadSimple(meta, file, isPublic) {
-  const { query: { Expires } } = url.parse(meta.location);
+async function uploadSimple(meta, file) {
+  // const { query: { Expires } } = url.parse(meta.location);
 
-  const headers = {
-    'Content-MD5': meta.md5Hash,
-    'Cache-Control': `public,max-age=${Expires}`,
-    'Content-Type': meta.contentType,
-  };
+  // const headers = {
+  //   'Content-MD5': meta.md5Hash,
+  //   'Cache-Control': `public,max-age=${Expires}`,
+  //   'Content-Type': meta.contentType,
+  // };
 
-  if (isPublic) {
-    headers['x-goog-acl'] = 'public-read';
-  }
+  // if (isPublic) {
+  //   headers['x-goog-acl'] = 'public-read';
+  // }
+
+  console.log('meta.location', meta);
 
   return request.put({
     url: meta.location,
     body: file,
     // headers,
     // simple: false,
     resolveWithFullResponse: true,
-    ACL: 'public-read',
+    // ACL: 'public-read',
   });
 }
 
@@ -217,6 +220,7 @@ async function uploadSimple(meta, file, isPublic) {
 // `rsp` is response to that message
 //
 function uploadFiles(msg, rsp) {
+  console.log('upload files rsp', rsp);
   const { files } = msg;
   return Promise
     .map(rsp.files, (part, idx) => {
@@ -251,6 +255,7 @@ function initUpload(data) {
     return this.amqp
       .publishAndWait('files.upload', data.message, { timeout: 30000 })
       .tap((rsp) => {
+        console.log('files upload res', rsp);
         this.response = rsp;
       })
       .tap((rsp) => uploadFiles(data, rsp));