Skip to content

Commit 5ae9eb5

Browse files
nleushiparamonausiniakinaa
committed
Domains options (#624)
* domains list loader * better whitelist reload timeout handling * implement find domain pattern by url * domain role weight * domains: find and merge records * core: `options.getDomainOptions` * use `getDomainOptions` in core * read `options.getDomainOptions('mete.prerender')` to enable prerender * remove obsolete * fix prerender using logic * bugfix * bugfix using `getDomainOptions` * bugfix typo * `getDomainOptions` can read from `CONFIG.domainOptions` * do not try to run prerender on PRERENDER URLs * remove `registerDomainOptionsUsage` * `getDomainOptions` can read from query params * bugfix undefined * Update audit.log * remove unused `xregexp` * Update audit.log * [Tests] Review tests in domain plugins (#618) * clips.twitch.tv: update test URL * libsyn.com: update test URLs * Tests: v.qq.com - replace selector with fixed URLs (#619) * v.qq.com: add static test URLs, there are no hrefs in selector yet * Update audit.log * v.qq.com: revert $ in regexp --------- Co-authored-by: Nazar Leush <n.leush@gmail.com> * [Domains] fix Instagram's new reels schema (#620) * [Domains] Review tests in domain plugins (#621) * soundcloud.com: fix user profiles tests * youku.com: update page and selector * imageshack.com: update selector in tests * youku.com: add regexp for native hrefs on pages * imageshack.com: change page and selector to avoid 504 error * v.qq.com: https instead of http * better handling of 403 on SoundCloud profiles * remove redundant regex for youku * fix youku tests --------- Co-authored-by: Ivan Paramonau <i.paramonau@gmail.com> * [Tests] Review test URLs in twitch plugins (#622) * twitch.tv: add test URLs * twitch-live-fallback: update test URL * [Tests] Review Facebook post test, update test URLs in domain plugins (#623) * update test URLs in domain plugins * tumblr: update test URLs * tumblr.text: remove bad test URL * clips.twitch.tv: add parent to test URLs * twitch-live-fallback: add parent param * Update audit.log * review failing Facebook post test --------- Co-authored-by: Nazar Leush <n.leush@gmail.com> Co-authored-by: Ivan Paramonau <i.paramonau@gmail.com> --------- Co-authored-by: Ivan Paramonau <i.paramonau@gmail.com> Co-authored-by: Aliaksei <34516115+siniakinaa@users.noreply.github.com>
1 parent a2fbfd0 commit 5ae9eb5

7 files changed

Lines changed: 95 additions & 14 deletions

File tree

‎audit.log‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,5 +41,19 @@
4141
├─────────────────────┼────────────────────────────────────────────────────────┤
4242
│ More info │ https://github.com/advisories/GHSA-73rr-hh4g-fpgx │
4343
└─────────────────────┴────────────────────────────────────────────────────────┘
44-
3 vulnerabilities found
45-
Severity: 1 low | 1 moderate | 1 high
44+
┌─────────────────────┬────────────────────────────────────────────────────────┐
45+
│ low │ qs's arrayLimit bypass in comma parsing allows denial │
46+
│ │ of service │
47+
├─────────────────────┼────────────────────────────────────────────────────────┤
48+
│ Package │ qs │
49+
├─────────────────────┼────────────────────────────────────────────────────────┤
50+
│ Vulnerable versions │ >=6.7.0 <=6.14.1 │
51+
├─────────────────────┼────────────────────────────────────────────────────────┤
52+
│ Patched versions │ >=6.14.2 │
53+
├─────────────────────┼────────────────────────────────────────────────────────┤
54+
│ Paths │ .>mock-http-server>body-parser>qs │
55+
├─────────────────────┼────────────────────────────────────────────────────────┤
56+
│ More info │ https://github.com/advisories/GHSA-w7fw-mjwx-w883 │
57+
└─────────────────────┴────────────────────────────────────────────────────────┘
58+
4 vulnerabilities found
59+
Severity: 2 low | 1 moderate | 1 high

‎lib/core.js‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1612,6 +1612,51 @@
16121612
return value;
16131613
};
16141614

1615+
options.getDomainOptions = function(path, defaultValue) {
1616+
/*
1617+
p1. providerOptions: {_: {prerender: true}} -> from api get params
1618+
p2. link url: "https://domain.com/path?_prerender=true" -> urlOptions={prerender: true}
1619+
p3. domainOptions: {meta: {prerender: true}}
1620+
*/
1621+
1622+
const bits = path.split('.');
1623+
1624+
const alternatePath = path.replace(/^[^.]+\./, '_'); // "meta.prerender" -> "_.prerender"
1625+
1626+
// p1.
1627+
var value = options.getProviderOptions(alternatePath);
1628+
if (typeof value === 'undefined') {
1629+
if (!options.getProviderOptions('app.disable_url_options', false)) {
1630+
const alternatePathForUrlOptions = alternatePath.replace(/^_/, '');
1631+
// p2.
1632+
value = urlOptions && urlOptions[alternatePathForUrlOptions];
1633+
}
1634+
if (typeof value === 'undefined') {
1635+
// p3.
1636+
value = searchParamInObj(0, bits, options.domainOptions);
1637+
}
1638+
}
1639+
1640+
if (
1641+
typeof defaultValue !== "undefined"
1642+
&& typeof value !== typeof defaultValue
1643+
) {
1644+
// Type mismatch fallback.
1645+
if (typeof defaultValue === "string") {
1646+
return "" + value;
1647+
} else {
1648+
return defaultValue;
1649+
}
1650+
}
1651+
1652+
// Read from CONFIG.
1653+
if (typeof value === 'undefined' && CONFIG.domainOptions) {
1654+
value = searchParamInObj(0, bits, CONFIG.domainOptions);
1655+
}
1656+
1657+
return typeof value !== 'undefined' ? value : defaultValue;
1658+
};
1659+
16151660
options.digitize = function(params) {
16161661
return Object.keys(params).reduce((acc, k) => {
16171662
acc[k] = typeof params[k] === 'boolean' ? (params[k] ? 1 : 0) : params[k];
@@ -1969,6 +2014,10 @@
19692014
}
19702015
}
19712016

2017+
if (options.getDomainOptionsByUrl) {
2018+
options.domainOptions = options.getDomainOptionsByUrl(uri);
2019+
}
2020+
19722021
asyncMethodCb('initial');
19732022
};
19742023

‎lib/utils.js‎

Lines changed: 18 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -91,17 +91,28 @@ export function prepareRequestOptions(request_options, options) {
9191

9292
var disable_language = false;
9393

94+
function setPrerender(prerender_option) {
95+
if (options?.allowPrerender && CONFIG.PRERENDER_URL) {
96+
request_options.uri = CONFIG.PRERENDER_URL + encodeURIComponent(uri);
97+
98+
// Use string `proxy.prerender` as additional prerender param.
99+
if (typeof prerender_option === 'string') {
100+
request_options.uri += (request_options.uri.indexOf('?') > -1 ? '&' : '?' ) + `prerender=${prerender_option}`;
101+
}
102+
}
103+
}
104+
105+
var enable_domain_prerender = options?.getDomainOptions && options.getDomainOptions('meta.prerender');
106+
if (enable_domain_prerender) {
107+
setPrerender(enable_domain_prerender);
108+
}
109+
94110
if (CONFIG.PROXY || (options && options.proxy)) {
95111

96112
var proxy = (options && options.proxy) || getCustomProxyForUri(uri, options);
97113
if (proxy) {
98-
if (proxy.prerender && CONFIG.PRERENDER_URL && options?.allowPrerender) {
99-
request_options.uri = CONFIG.PRERENDER_URL + encodeURIComponent(uri);
100-
101-
// Use string `proxy.prerender` as additional prerender param.
102-
if (typeof proxy.prerender === 'string') {
103-
request_options.uri += (request_options.uri.indexOf('?') > -1 ? '&' : '?' ) + `prerender=${proxy.prerender}`;
104-
}
114+
if (!enable_domain_prerender && proxy.prerender && options?.allowPrerender) {
115+
setPrerender(proxy.prerender);
105116

106117
} else if (proxy.proxy && CONFIG.PROXY_URL) {
107118
request_options.uri = /{url}/.test(CONFIG.PROXY_URL)

‎lib/whitelist.js‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -313,6 +313,8 @@
313313

314314
}
315315

316+
var loadFileTimeout;
317+
316318
function startScanWhitelist() {
317319

318320
var watcher = chokidar.watch(WHITELIST_DIR, {
@@ -326,7 +328,8 @@
326328
// Check if newer file added.
327329
if (p.match(/iframely-.*\.json/)) {
328330
// Wait sometime to be sure write finished.
329-
setTimeout(function() {
331+
clearTimeout(loadFileTimeout);
332+
loadFileTimeout = setTimeout(function() {
330333
loadLastWhitelist();
331334
}, 5000);
332335
}
@@ -337,7 +340,8 @@
337340
// Reload last whitelist.
338341
if (p == currentWhitelistFilename) {
339342
// Wait sometime to be sure write finished.
340-
setTimeout(function() {
343+
clearTimeout(loadFileTimeout);
344+
loadFileTimeout = setTimeout(function() {
341345
readWhitelist(p);
342346
}, 5000);
343347
}

‎logging.js‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import moment from 'moment';
2+
import CONFIG from './config.loader.js';
23

34
export default function log() {
45
var args = Array.prototype.slice.apply(arguments);
@@ -20,4 +21,4 @@ export default function log() {
2021
}
2122

2223
console.log.apply(console, args);
23-
};
24+
};

‎plugins/links/prerender/prerender.js‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,8 @@ export default {
88

99
getData: function(url, __allowJSRender, iframelyRun, options, meta, cb) {
1010

11-
if (CONFIG.PRERENDER && CONFIG.PRERENDER_URL && options.user_agent === CONFIG.FB_USER_AGENT) {
11+
if (CONFIG.PRERENDER && CONFIG.PRERENDER_URL && options.user_agent === CONFIG.FB_USER_AGENT
12+
&& !url.startsWith(CONFIG.PRERENDER_URL)) {
1213

1314
var prerenderUrl = CONFIG.PRERENDER_URL + encodeURIComponent(url);
1415
var options2 = {...options, ...{

‎plugins/links/prerender/react-app-fb-fallback.js‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,8 @@ export default {
88

99
getData: function(url, __allowJSRender, iframelyRun, options, cb) {
1010

11-
if (options.user_agent === CONFIG.FB_USER_AGENT) {
11+
if (options.user_agent === CONFIG.FB_USER_AGENT
12+
|| CONFIG.PRERENDER_URL && url.startsWith(CONFIG.PRERENDER_URL)) {
1213
return cb();
1314
}
1415

0 commit comments

Comments
 (0)