Merge branch 'master' into nvmrc

config.js

@@ -204,6 +204,9 @@ const CONFIG = {
   // STATIC_URI is the base uri where static files are hosted.
   STATIC_URI: process.env.TALK_STATIC_URI || process.env.TALK_ROOT_URL,
 
+  // SCRAPER_PROXY_URL is the url to be used as a proxy by the scraper
+  SCRAPER_PROXY_URL: process.env.TALK_SCRAPER_PROXY_URL || null,
+
   // The keepalive timeout (in ms) that should be used to send keep alive
   // messages through the websocket to keep the socket alive.
   KEEP_ALIVE: process.env.TALK_KEEP_ALIVE || '30s',

docs/source/02-02-advanced-configuration.md

@@ -568,3 +568,7 @@ A JSON string representing the configuration passed to the
 [fetch](https://www.npmjs.com/package/node-fetch) call for the scraper. It
 can be used to set an authorization header, or change the user agent. (Default
 `{}`)
+
+## TALK_SCRAPER_PROXY_URL
+
+Sets a specific HTTP/S proxy to be used by the Asset Scraper using [https-proxy-agent](https://www.npmjs.com/package/https-proxy-agent). (Default `null`)

jobs/scraper/index.js

@@ -6,7 +6,8 @@ const logger = createLogger('jobs:scraper');
 const fetch = require('node-fetch');
 const { merge } = require('lodash');
 const { version } = require('../../package.json');
-const { SCRAPER_HEADERS } = require('../../config');
+const { SCRAPER_HEADERS, SCRAPER_PROXY_URL } = require('../../config');
+const HttpsProxyAgent = require('https-proxy-agent');
 
 // Load the scraper with the rules.
 const metascraper = require('metascraper').load([
@@ -35,12 +36,16 @@ const headers = merge(
   customHeaders
 );
 
+// Add proxy configuration if exists.
+const agent = SCRAPER_PROXY_URL ? new HttpsProxyAgent(SCRAPER_PROXY_URL) : null;
+
 /**
  * Scrapes the given asset for metadata.
  */
 async function scrape({ url }) {
   const res = await fetch(url, {
     headers,
+    agent,
   });
   const html = await res.text();
 
@@ -49,7 +54,6 @@ async function scrape({ url }) {
     html,
     url,
   });
-
   return metadata;
 }
 

package.json

@@ -125,6 +125,7 @@
     "history": "^3.0.0",
     "hjson": "^3.1.1",
     "hjson-loader": "^1.0.0",
+    "https-proxy-agent": "^2.2.0",
     "immutability-helper": "^2.2.0",
     "imports-loader": "^0.7.1",
     "inquirer": "^3.2.2",

yarn.lock

@@ -239,6 +239,12 @@ agent-base@2:
     extend "~3.0.0"
     semver "~5.0.1"
 
+agent-base@^4.1.0:
+  version "4.2.0"
+  resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-4.2.0.tgz#9838b5c3392b962bad031e6a4c5e1024abec45ce"
+  dependencies:
+    es6-promisify "^5.0.0"
+
 ajv-keywords@^2.0.0, ajv-keywords@^2.1.0:
   version "2.1.1"
   resolved "https://registry.yarnpkg.com/ajv-keywords/-/ajv-keywords-2.1.1.tgz#617997fc5f60576894c435f940d819e135b80762"
@@ -3420,10 +3426,20 @@ es6-map@^0.1.3:
     es6-symbol "~3.1.1"
     event-emitter "~0.3.5"
 
+es6-promise@^4.0.3:
+  version "4.2.4"
+  resolved "https://registry.yarnpkg.com/es6-promise/-/es6-promise-4.2.4.tgz#dc4221c2b16518760bd8c39a52d8f356fc00ed29"
+
 es6-promise@^4.0.5:
   version "4.1.1"
   resolved "https://registry.yarnpkg.com/es6-promise/-/es6-promise-4.1.1.tgz#8811e90915d9a0dba36274f0b242dbda78f9c92a"
 
+es6-promisify@^5.0.0:
+  version "5.0.0"
+  resolved "https://registry.yarnpkg.com/es6-promisify/-/es6-promisify-5.0.0.tgz#5109d62f3e56ea967c4b63505aef08291c8a5203"
+  dependencies:
+    es6-promise "^4.0.3"
+
 es6-set@~0.1.5:
   version "0.1.5"
   resolved "https://registry.yarnpkg.com/es6-set/-/es6-set-0.1.5.tgz#d2b3ec5d4d800ced818db538d28974db0a73ccb1"
@@ -5077,6 +5093,13 @@ https-proxy-agent@1, https-proxy-agent@^1.0.0:
     debug "2"
     extend "3"
 
+https-proxy-agent@^2.2.0:
+  version "2.2.1"
+  resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-2.2.1.tgz#51552970fa04d723e04c56d04178c3f92592bbc0"
+  dependencies:
+    agent-base "^4.1.0"
+    debug "^3.1.0"
+
 husky@^0.14.3:
   version "0.14.3"
   resolved "https://registry.yarnpkg.com/husky/-/husky-0.14.3.tgz#c69ed74e2d2779769a17ba8399b54ce0b63c12c3"