diff --git a/config.js b/config.js index 4ce755f93..b547636f4 100644 --- a/config.js +++ b/config.js @@ -204,6 +204,9 @@ const CONFIG = { // STATIC_URI is the base uri where static files are hosted. STATIC_URI: process.env.TALK_STATIC_URI || process.env.TALK_ROOT_URL, + // SCRAPER_PROXY_URL is the url to be used as a proxy by the scraper + SCRAPER_PROXY_URL: process.env.TALK_SCRAPER_PROXY_URL || null, + // The keepalive timeout (in ms) that should be used to send keep alive // messages through the websocket to keep the socket alive. KEEP_ALIVE: process.env.TALK_KEEP_ALIVE || '30s', diff --git a/docs/source/02-02-advanced-configuration.md b/docs/source/02-02-advanced-configuration.md index 451865e47..3ac9b5c1b 100644 --- a/docs/source/02-02-advanced-configuration.md +++ b/docs/source/02-02-advanced-configuration.md @@ -568,3 +568,7 @@ A JSON string representing the configuration passed to the [fetch](https://www.npmjs.com/package/node-fetch) call for the scraper. It can be used to set an authorization header, or change the user agent. (Default `{}`) + +## TALK_SCRAPER_PROXY_URL + +Sets a specific HTTP/S proxy to be used by the Asset Scraper using [https-proxy-agent](https://www.npmjs.com/package/https-proxy-agent). (Default `null`) diff --git a/jobs/scraper/index.js b/jobs/scraper/index.js index 89bf8c1d3..1ae80b818 100644 --- a/jobs/scraper/index.js +++ b/jobs/scraper/index.js @@ -6,7 +6,8 @@ const logger = createLogger('jobs:scraper'); const fetch = require('node-fetch'); const { merge } = require('lodash'); const { version } = require('../../package.json'); -const { SCRAPER_HEADERS } = require('../../config'); +const { SCRAPER_HEADERS, SCRAPER_PROXY_URL } = require('../../config'); +const HttpsProxyAgent = require('https-proxy-agent'); // Load the scraper with the rules. const metascraper = require('metascraper').load([ @@ -35,12 +36,16 @@ const headers = merge( customHeaders ); +// Add proxy configuration if exists. +const agent = SCRAPER_PROXY_URL ? new HttpsProxyAgent(SCRAPER_PROXY_URL) : null; + /** * Scrapes the given asset for metadata. */ async function scrape({ url }) { const res = await fetch(url, { headers, + agent, }); const html = await res.text(); @@ -49,7 +54,6 @@ async function scrape({ url }) { html, url, }); - return metadata; } diff --git a/package.json b/package.json index a313c1ec0..7ebb04c9f 100644 --- a/package.json +++ b/package.json @@ -125,6 +125,7 @@ "history": "^3.0.0", "hjson": "^3.1.1", "hjson-loader": "^1.0.0", + "https-proxy-agent": "^2.2.0", "immutability-helper": "^2.2.0", "imports-loader": "^0.7.1", "inquirer": "^3.2.2", diff --git a/yarn.lock b/yarn.lock index 0f6606433..ff5302ff0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -239,6 +239,12 @@ agent-base@2: extend "~3.0.0" semver "~5.0.1" +agent-base@^4.1.0: + version "4.2.0" + resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-4.2.0.tgz#9838b5c3392b962bad031e6a4c5e1024abec45ce" + dependencies: + es6-promisify "^5.0.0" + ajv-keywords@^2.0.0, ajv-keywords@^2.1.0: version "2.1.1" resolved "https://registry.yarnpkg.com/ajv-keywords/-/ajv-keywords-2.1.1.tgz#617997fc5f60576894c435f940d819e135b80762" @@ -3420,10 +3426,20 @@ es6-map@^0.1.3: es6-symbol "~3.1.1" event-emitter "~0.3.5" +es6-promise@^4.0.3: + version "4.2.4" + resolved "https://registry.yarnpkg.com/es6-promise/-/es6-promise-4.2.4.tgz#dc4221c2b16518760bd8c39a52d8f356fc00ed29" + es6-promise@^4.0.5: version "4.1.1" resolved "https://registry.yarnpkg.com/es6-promise/-/es6-promise-4.1.1.tgz#8811e90915d9a0dba36274f0b242dbda78f9c92a" +es6-promisify@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/es6-promisify/-/es6-promisify-5.0.0.tgz#5109d62f3e56ea967c4b63505aef08291c8a5203" + dependencies: + es6-promise "^4.0.3" + es6-set@~0.1.5: version "0.1.5" resolved "https://registry.yarnpkg.com/es6-set/-/es6-set-0.1.5.tgz#d2b3ec5d4d800ced818db538d28974db0a73ccb1" @@ -5077,6 +5093,13 @@ https-proxy-agent@1, https-proxy-agent@^1.0.0: debug "2" extend "3" +https-proxy-agent@^2.2.0: + version "2.2.1" + resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-2.2.1.tgz#51552970fa04d723e04c56d04178c3f92592bbc0" + dependencies: + agent-base "^4.1.0" + debug "^3.1.0" + husky@^0.14.3: version "0.14.3" resolved "https://registry.yarnpkg.com/husky/-/husky-0.14.3.tgz#c69ed74e2d2779769a17ba8399b54ce0b63c12c3"