Skip to content

Commit 348f597

Browse files
socaseinpointsocaseinpoint
committed
feat: add aws s3 provider
1 parent 3966e90 commit 348f597

File tree

6 files changed

+192
-3
lines changed

6 files changed

+192
-3
lines changed

‎package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@
4141
"@microfleet/validation": "9",
4242
"@sentry/node": "^6.0.2",
4343
"ali-oss": "^6.13.2",
44+
"aws-sdk": "^2.914.0",
4445
"bl": "^4.0.3",
4546
"bluebird": "^3.7.2",
4647
"cluster-key-slot": "^1.1.0",

‎src/providers/aws.js‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
const AbstractFileTransfer = require('ms-files-transport');
2+
const S3 = require('aws-sdk/clients/s3');
3+
const { merge } = require('lodash');
4+
const assert = require('assert');
5+
6+
const UPLOAD_URL_EXPIRES_IN_SEC = 60000;
7+
const DOWNLOAD_URL_EXPIRES_IN_SEC = 60000;
8+
9+
class AWSTransport extends AbstractFileTransfer {
10+
constructor(opts = {}) {
11+
super();
12+
13+
const config = merge({}, AWSTransport.defaultOpts, opts);
14+
this._config = config;
15+
this._logger = this._config.logger;
16+
17+
assert(config !== undefined);
18+
assert(config.options !== undefined);
19+
assert(config.options.accessKeyId !== undefined);
20+
assert(config.options.secretAccessKey !== undefined);
21+
assert(config.options.region !== undefined);
22+
assert(config.options.bucketName !== undefined);
23+
24+
this.setupS3();
25+
}
26+
27+
setupS3() {
28+
try {
29+
this._s3 = new S3({
30+
signatureVersion: 'v4',
31+
region: this._config.region,
32+
credentials: {
33+
accessKeyId: this._config.accessKeyId,
34+
secretAccessKey: this._config.secretAccessKey,
35+
},
36+
});
37+
} catch (e) {
38+
this._logger.warn('failed to load aws-sdk/clients/s3', e);
39+
}
40+
}
41+
42+
connect() {
43+
return Promise
44+
.bind(this)
45+
.then(this.findOrCreateBucket);
46+
}
47+
48+
async findOrCreateBucket() {
49+
const { bucketName } = this._config;
50+
51+
const s3 = this._s3;
52+
53+
const buckets = s3.listBuckets();
54+
const exists = buckets.find(bucketName);
55+
56+
if (!exists) {
57+
this.log.debug('initiating createBucket: %s', bucketName);
58+
await this._s3.createBucket({
59+
Bucket: bucketName,
60+
});
61+
}
62+
}
63+
64+
async getSignedUrl(action, input) {
65+
const params = {
66+
Bucket: this._config.bucketName,
67+
Expires: action === 'putObject' ? UPLOAD_URL_EXPIRES_IN_SEC : DOWNLOAD_URL_EXPIRES_IN_SEC,
68+
Key: input.key,
69+
};
70+
71+
if (action === 'putObject') {
72+
params.ContentType = input.contentType;
73+
}
74+
75+
return new Promise((resolve, reject) => {
76+
this._s3.getSignedUrl('putObject', params, (err, url) => {
77+
if (err) {
78+
return reject(err);
79+
}
80+
return resolve(url);
81+
});
82+
});
83+
}
84+
}
85+
86+
AWSTransport.defaultOpts = {};
87+
88+
module.exports = AWSTransport;

‎src/providers/factory.js‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,14 @@ class ProviderFactory {
5757

5858
return provider;
5959
}
60+
61+
// eslint-disable-next-line class-methods-use-this
62+
getProviderAWS(transport) {
63+
const ProviderAWS = require('./aws');
64+
const provider = new ProviderAWS(transport);
65+
66+
return provider;
67+
}
6068
}
6169

6270
module.exports = ProviderFactory;

‎src/providers/index.js‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,12 @@ function initProviders(service) {
5454
factory.getProviderOSS(transport)
5555
);
5656
}
57+
58+
if (transport.name === 'aws') {
59+
service.providers.push(
60+
factory.getProviderAWS(transport)
61+
);
62+
}
5763
}
5864

5965
// create providerByBucket map for fast access

‎test/suites/providers/aws.js‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
// const { strictEqual } = require('assert');
2+
// const AWSTransport = require('../../../src/providers/aws');
3+
4+
describe('util fetch-data suite', () => {
5+
it('should be able to create instance', () => {
6+
// const provider = new AWSTransport({
7+
// options: {
8+
// accessKeyId: 'AKIASS5V3WV23VA4KYOF',
9+
// accessKeySecret: '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',
10+
// bucket: 'test',
11+
// region: 'us-west-2',
12+
// secure: true,
13+
// },
14+
// });
15+
});
16+
});

‎yarn.lock‎

Lines changed: 73 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2007,6 +2007,21 @@ atomic-sleep@^1.0.0:
20072007
resolved "https://registry.yarnpkg.com/atomic-sleep/-/atomic-sleep-1.0.0.tgz#eb85b77a601fc932cfe432c5acd364a9e2c9075b"
20082008
integrity sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ==
20092009

2010+
aws-sdk@^2.914.0:
2011+
version "2.914.0"
2012+
resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.914.0.tgz#55b303d8d685dbe13cc3f89887aaccf920e4127f"
2013+
integrity sha512-N7UZBvTMPr+lqlOmd/vZ6NMMaw1K2Y94fq3b6roZWouUQgn5unEixVpCuRhcYleOPSnikSCIrrDs2AXq5bXhLg==
2014+
dependencies:
2015+
buffer "4.9.2"
2016+
events "1.1.1"
2017+
ieee754 "1.1.13"
2018+
jmespath "0.15.0"
2019+
querystring "0.2.0"
2020+
sax "1.2.1"
2021+
url "0.10.3"
2022+
uuid "3.3.2"
2023+
xml2js "0.4.19"
2024+
20102025
aws-sign2@~0.7.0:
20112026
version "0.7.0"
20122027
resolved "https://registry.yarnpkg.com/aws-sign2/-/aws-sign2-0.7.0.tgz#b46e890934a9591f2d2f6f86d7e6a9f1b3fe76a8"
@@ -2045,7 +2060,7 @@ balanced-match@^1.0.0:
20452060
resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767"
20462061
integrity sha1-ibTRmasr7kneFk6gK4nORi1xt2c=
20472062

2048-
base64-js@^1.3.0, base64-js@^1.3.1:
2063+
base64-js@^1.0.2, base64-js@^1.3.0, base64-js@^1.3.1:
20492064
version "1.5.1"
20502065
resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a"
20512066
integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==
@@ -2200,6 +2215,15 @@ buffer-from@^1.0.0:
22002215
resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.1.tgz#32713bc028f75c02fdb710d7c7bcec1f2c6070ef"
22012216
integrity sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==
22022217

2218+
[email protected]:
2219+
version "4.9.2"
2220+
resolved "https://registry.yarnpkg.com/buffer/-/buffer-4.9.2.tgz#230ead344002988644841ab0244af8c44bbe3ef8"
2221+
integrity sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg==
2222+
dependencies:
2223+
base64-js "^1.0.2"
2224+
ieee754 "^1.1.4"
2225+
isarray "^1.0.0"
2226+
22032227
buffer@^5.5.0:
22042228
version "5.7.1"
22052229
resolved "https://registry.yarnpkg.com/buffer/-/buffer-5.7.1.tgz#ba62e7c13133053582197160851a8f648e99eed0"
@@ -3799,6 +3823,11 @@ eventemitter3@^4.0.4, eventemitter3@^4.0.7:
37993823
resolved "https://registry.yarnpkg.com/eventemitter3/-/eventemitter3-4.0.7.tgz#2de9b68f6528d5644ef5c59526a1b4a07306169f"
38003824
integrity sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==
38013825

3826+
[email protected]:
3827+
version "1.1.1"
3828+
resolved "https://registry.yarnpkg.com/events/-/events-1.1.1.tgz#9ebdb7635ad099c70dcc4c2a1f5004288e8bd924"
3829+
integrity sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=
3830+
38023831
execa@^0.7.0:
38033832
version "0.7.0"
38043833
resolved "https://registry.yarnpkg.com/execa/-/execa-0.7.0.tgz#944becd34cc41ee32a63a9faf27ad5a65fc59777"
@@ -4858,7 +4887,12 @@ iconv-lite@^0.6.2:
48584887
dependencies:
48594888
safer-buffer ">= 2.1.2 < 3.0.0"
48604889

4861-
ieee754@^1.1.13:
4890+
[email protected]:
4891+
version "1.1.13"
4892+
resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.1.13.tgz#ec168558e95aa181fd87d37f55c32bbcb6708b84"
4893+
integrity sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==
4894+
4895+
ieee754@^1.1.13, ieee754@^1.1.4:
48624896
version "1.2.1"
48634897
resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.2.1.tgz#8eb7a10a63fff25d15a57b001586d177d1b0d352"
48644898
integrity sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==
@@ -5455,7 +5489,7 @@ java-properties@^1.0.0:
54555489
resolved "https://registry.yarnpkg.com/java-properties/-/java-properties-1.0.2.tgz#ccd1fa73907438a5b5c38982269d0e771fe78211"
54565490
integrity sha512-qjdpeo2yKlYTH7nFdK0vbZWuTCesk4o63v5iVOlhMQPfuIZQfW/HI35SjfhA+4qpg36rnFSvUK5b1m+ckIblQQ==
54575491

5458-
jmespath@^0.15.0:
5492+
jmespath@0.15.0, jmespath@^0.15.0:
54595493
version "0.15.0"
54605494
resolved "https://registry.yarnpkg.com/jmespath/-/jmespath-0.15.0.tgz#a3f222a9aae9f966f5d27c796510e28091764217"
54615495
integrity sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc=
@@ -7789,6 +7823,11 @@ pumpify@^2.0.0:
77897823
inherits "^2.0.3"
77907824
pump "^3.0.0"
77917825

7826+
[email protected]:
7827+
version "1.3.2"
7828+
resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.3.2.tgz#9653a036fb7c1ee42342f2325cceefea3926c48d"
7829+
integrity sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=
7830+
77927831
punycode@^2.1.0, punycode@^2.1.1:
77937832
version "2.1.1"
77947833
resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.1.1.tgz#b58b010ac40c22c5657616c8d2c2c02c7bf479ec"
@@ -7823,6 +7862,11 @@ query-string@^6.8.2:
78237862
split-on-first "^1.0.0"
78247863
strict-uri-encode "^2.0.0"
78257864

7865+
[email protected]:
7866+
version "0.2.0"
7867+
resolved "https://registry.yarnpkg.com/querystring/-/querystring-0.2.0.tgz#b209849203bb25df820da756e747005878521620"
7868+
integrity sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=
7869+
78267870
quick-format-unescaped@^4.0.1:
78277871
version "4.0.1"
78287872
resolved "https://registry.yarnpkg.com/quick-format-unescaped/-/quick-format-unescaped-4.0.1.tgz#437a5ea1a0b61deb7605f8ab6a8fd3858dbeb701"
@@ -8322,6 +8366,11 @@ safe-regex@^2.1.1:
83228366
resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a"
83238367
integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==
83248368

8369+
[email protected]:
8370+
version "1.2.1"
8371+
resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.1.tgz#7b8e656190b228e81a66aea748480d828cd2d37a"
8372+
integrity sha1-e45lYZCyKOgaZq6nSEgNgozS03o=
8373+
83258374
sax@>=0.6.0:
83268375
version "1.2.4"
83278376
resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9"
@@ -9519,6 +9568,14 @@ url-parse-lax@^1.0.0:
95199568
dependencies:
95209569
prepend-http "^1.0.1"
95219570

9571+
[email protected]:
9572+
version "0.10.3"
9573+
resolved "https://registry.yarnpkg.com/url/-/url-0.10.3.tgz#021e4d9c7705f21bbf37d03ceb58767402774c64"
9574+
integrity sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ=
9575+
dependencies:
9576+
punycode "1.3.2"
9577+
querystring "0.2.0"
9578+
95229579
[email protected]:
95239580
version "0.4.4"
95249581
resolved "https://registry.yarnpkg.com/urlgrey/-/urlgrey-0.4.4.tgz#892fe95960805e85519f1cd4389f2cb4cbb7652f"
@@ -9794,6 +9851,14 @@ xdg-basedir@^4.0.0:
97949851
resolved "https://registry.yarnpkg.com/xdg-basedir/-/xdg-basedir-4.0.0.tgz#4bc8d9984403696225ef83a1573cbbcb4e79db13"
97959852
integrity sha512-PSNhEJDejZYV7h50BohL09Er9VaIefr2LMAf3OEmpCkjOi34eYyQYAXUTjEQtZJTKcF0E2UKTh+osDLsgNim9Q==
97969853

9854+
[email protected]:
9855+
version "0.4.19"
9856+
resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.19.tgz#686c20f213209e94abf0d1bcf1efaa291c7827a7"
9857+
integrity sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==
9858+
dependencies:
9859+
sax ">=0.6.0"
9860+
xmlbuilder "~9.0.1"
9861+
97979862
xml2js@^0.4.16:
97989863
version "0.4.23"
97999864
resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.23.tgz#a0c69516752421eb2ac758ee4d4ccf58843eac66"
@@ -9807,6 +9872,11 @@ xmlbuilder@~11.0.0:
98079872
resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-11.0.1.tgz#be9bae1c8a046e76b31127726347d0ad7002beb3"
98089873
integrity sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==
98099874

9875+
xmlbuilder@~9.0.1:
9876+
version "9.0.7"
9877+
resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-9.0.7.tgz#132ee63d2ec5565c557e20f4c22df9aca686b10d"
9878+
integrity sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0=
9879+
98109880
xorshift@^0.2.0:
98119881
version "0.2.1"
98129882
resolved "https://registry.yarnpkg.com/xorshift/-/xorshift-0.2.1.tgz#fcd82267e9351c13f0fb9c73307f25331d29c63a"

0 commit comments

Comments
 (0)