From 382019d907e97d3ca3f78eb4f5a592f96dc32a15 Mon Sep 17 00:00:00 2001 From: Thomaz de Oliveira dos Reis Date: Tue, 4 Sep 2018 12:01:10 -0300 Subject: [PATCH] Change proxy-agent to support http/https through http/https/socks/pac proxy on scraper (#1839) --- jobs/scraper/index.js | 4 +- package.json | 2 +- yarn.lock | 94 +++++++++++++++++++++++++++++++++++++++---- 3 files changed, 89 insertions(+), 11 deletions(-) diff --git a/jobs/scraper/index.js b/jobs/scraper/index.js index 1ae80b818..77d429042 100644 --- a/jobs/scraper/index.js +++ b/jobs/scraper/index.js @@ -7,7 +7,7 @@ const fetch = require('node-fetch'); const { merge } = require('lodash'); const { version } = require('../../package.json'); const { SCRAPER_HEADERS, SCRAPER_PROXY_URL } = require('../../config'); -const HttpsProxyAgent = require('https-proxy-agent'); +const ProxyAgent = require('proxy-agent'); // Load the scraper with the rules. const metascraper = require('metascraper').load([ @@ -37,7 +37,7 @@ const headers = merge( ); // Add proxy configuration if exists. -const agent = SCRAPER_PROXY_URL ? new HttpsProxyAgent(SCRAPER_PROXY_URL) : null; +const agent = SCRAPER_PROXY_URL ? new ProxyAgent(SCRAPER_PROXY_URL) : null; /** * Scrapes the given asset for metadata. diff --git a/package.json b/package.json index f35544cf0..de6b19772 100644 --- a/package.json +++ b/package.json @@ -126,7 +126,7 @@ "history": "^3.0.0", "hjson": "^3.1.1", "hjson-loader": "^1.0.0", - "https-proxy-agent": "^2.2.0", + "proxy-agent": "^3.0.1", "immutability-helper": "^2.2.0", "imports-loader": "^0.7.1", "inquirer": "^3.2.2", diff --git a/yarn.lock b/yarn.lock index 4c17f55e7..b4e37884f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -267,6 +267,12 @@ agent-base@2: extend "~3.0.0" semver "~5.0.1" +agent-base@4, agent-base@^4.2.0, agent-base@~4.2.0: + version "4.2.1" + resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-4.2.1.tgz#d89e5999f797875674c07d87f260fc41e83e8ca9" + dependencies: + es6-promisify "^5.0.0" + agent-base@^4.1.0: version "4.2.0" resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-4.2.0.tgz#9838b5c3392b962bad031e6a4c5e1024abec45ce" @@ -2959,7 +2965,7 @@ defined@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/defined/-/defined-1.0.0.tgz#c98d9bcef75674188e110969151199e39b1fa693" -degenerator@~1.0.2: +degenerator@^1.0.4, degenerator@~1.0.2: version "1.0.4" resolved "https://registry.yarnpkg.com/degenerator/-/degenerator-1.0.4.tgz#fcf490a37ece266464d9cc431ab98c5819ced095" dependencies: @@ -4290,7 +4296,7 @@ get-stream@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-3.0.0.tgz#8e943d1358dc37555054ecbe2edb05aa174ede14" -get-uri@2: +get-uri@2, get-uri@^2.0.0: version "2.0.2" resolved "https://registry.yarnpkg.com/get-uri/-/get-uri-2.0.2.tgz#5c795e71326f6ca1286f2fc82575cd2bab2af578" dependencies: @@ -4998,6 +5004,13 @@ http-proxy-agent@1: debug "2" extend "3" +http-proxy-agent@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/http-proxy-agent/-/http-proxy-agent-2.1.0.tgz#e4821beef5b2142a2026bd73926fe537631c5405" + dependencies: + agent-base "4" + debug "3.1.0" + http-signature@~1.1.0: version "1.1.1" resolved "https://registry.yarnpkg.com/http-signature/-/http-signature-1.1.1.tgz#df72e267066cd0ac67fb76adf8e134a8fbcf91bf" @@ -5037,7 +5050,7 @@ https-proxy-agent@1: debug "2" extend "3" -https-proxy-agent@^2.2.0, https-proxy-agent@^2.2.1: +https-proxy-agent@^2.2.1: version "2.2.1" resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-2.2.1.tgz#51552970fa04d723e04c56d04178c3f92592bbc0" dependencies: @@ -6943,7 +6956,7 @@ lru-cache@^2.5.0: version "2.7.3" resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-2.7.3.tgz#6d4524e8b955f95d4f5b58851ce21dd72fb4e952" -lru-cache@^4.0.1, lru-cache@^4.1.1: +lru-cache@^4.0.1, lru-cache@^4.1.1, lru-cache@^4.1.2: version "4.1.3" resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-4.1.3.tgz#a1175cf3496dfc8436c156c334b4955992bce69c" dependencies: @@ -7568,7 +7581,7 @@ neo-async@^2.5.0: version "2.5.1" resolved "https://registry.yarnpkg.com/neo-async/-/neo-async-2.5.1.tgz#acb909e327b1e87ec9ef15f41b8a269512ad41ee" -netmask@~1.0.4: +netmask@^1.0.6, netmask@~1.0.4: version "1.0.6" resolved "https://registry.yarnpkg.com/netmask/-/netmask-1.0.6.tgz#20297e89d86f6f6400f250d9f4f6b4c1945fcd35" @@ -8210,6 +8223,29 @@ pac-proxy-agent@1: raw-body "2" socks-proxy-agent "2" +pac-proxy-agent@^2.0.1: + version "2.0.2" + resolved "https://registry.yarnpkg.com/pac-proxy-agent/-/pac-proxy-agent-2.0.2.tgz#90d9f6730ab0f4d2607dcdcd4d3d641aa26c3896" + dependencies: + agent-base "^4.2.0" + debug "^3.1.0" + get-uri "^2.0.0" + http-proxy-agent "^2.1.0" + https-proxy-agent "^2.2.1" + pac-resolver "^3.0.0" + raw-body "^2.2.0" + socks-proxy-agent "^3.0.0" + +pac-resolver@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/pac-resolver/-/pac-resolver-3.0.0.tgz#6aea30787db0a891704deb7800a722a7615a6f26" + dependencies: + co "^4.6.0" + degenerator "^1.0.4" + ip "^1.1.5" + netmask "^1.0.6" + thunkify "^2.1.2" + pac-resolver@~2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/pac-resolver/-/pac-resolver-2.0.0.tgz#99b88d2f193fbdeefc1c9a529c1f3260ab5277cd" @@ -9083,6 +9119,23 @@ proxy-agent@2.0.0: pac-proxy-agent "1" socks-proxy-agent "2" +proxy-agent@^3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/proxy-agent/-/proxy-agent-3.0.1.tgz#4fb7b61b1476d0fe8e3a3384d90e2460bbded3f9" + dependencies: + agent-base "^4.2.0" + debug "^3.1.0" + http-proxy-agent "^2.1.0" + https-proxy-agent "^2.2.1" + lru-cache "^4.1.2" + pac-proxy-agent "^2.0.1" + proxy-from-env "^1.0.0" + socks-proxy-agent "^4.0.1" + +proxy-from-env@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/proxy-from-env/-/proxy-from-env-1.0.0.tgz#33c50398f70ea7eb96d21f7b817630a55791c7ee" + prr@~1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/prr/-/prr-1.0.1.tgz#d3fc114ba06995a45ec6893f484ceb1d78f5f476" @@ -9337,7 +9390,7 @@ range-parser@~1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/range-parser/-/range-parser-1.2.0.tgz#f49be6b487894ddc40dcc94a322f611092e00d5e" -raw-body@2, raw-body@2.3.3: +raw-body@2, raw-body@2.3.3, raw-body@^2.2.0: version "2.3.3" resolved "https://registry.yarnpkg.com/raw-body/-/raw-body-2.3.3.tgz#1b324ece6b5706e153855bc1148c65bb7f6ea0c3" dependencies: @@ -10420,6 +10473,10 @@ smart-buffer@^1.0.13, smart-buffer@^1.0.4: version "1.1.15" resolved "https://registry.yarnpkg.com/smart-buffer/-/smart-buffer-1.1.15.tgz#7f114b5b65fab3e2a35aa775bb12f0d1c649bf16" +smart-buffer@^4.0.1: + version "4.0.1" + resolved "https://registry.yarnpkg.com/smart-buffer/-/smart-buffer-4.0.1.tgz#07ea1ca8d4db24eb4cac86537d7d18995221ace3" + smartquotes@~2.3.1: version "2.3.1" resolved "https://registry.yarnpkg.com/smartquotes/-/smartquotes-2.3.1.tgz#01ebb595d6c7a9e24d90e8cb95c17d0e1af49407" @@ -10482,6 +10539,20 @@ socks-proxy-agent@2: extend "3" socks "~1.1.5" +socks-proxy-agent@^3.0.0: + version "3.0.1" + resolved "https://registry.yarnpkg.com/socks-proxy-agent/-/socks-proxy-agent-3.0.1.tgz#2eae7cf8e2a82d34565761539a7f9718c5617659" + dependencies: + agent-base "^4.1.0" + socks "^1.1.10" + +socks-proxy-agent@^4.0.1: + version "4.0.1" + resolved "https://registry.yarnpkg.com/socks-proxy-agent/-/socks-proxy-agent-4.0.1.tgz#5936bf8b707a993079c6f37db2091821bffa6473" + dependencies: + agent-base "~4.2.0" + socks "~2.2.0" + socks@1.1.9: version "1.1.9" resolved "https://registry.yarnpkg.com/socks/-/socks-1.1.9.tgz#628d7e4d04912435445ac0b6e459376cb3e6d691" @@ -10489,13 +10560,20 @@ socks@1.1.9: ip "^1.1.2" smart-buffer "^1.0.4" -socks@~1.1.5: +socks@^1.1.10, socks@~1.1.5: version "1.1.10" resolved "https://registry.yarnpkg.com/socks/-/socks-1.1.10.tgz#5b8b7fc7c8f341c53ed056e929b7bf4de8ba7b5a" dependencies: ip "^1.1.4" smart-buffer "^1.0.13" +socks@~2.2.0: + version "2.2.1" + resolved "https://registry.yarnpkg.com/socks/-/socks-2.2.1.tgz#68ad678b3642fbc5d99c64c165bc561eab0215f9" + dependencies: + ip "^1.1.5" + smart-buffer "^4.0.1" + sort-keys@^1.0.0: version "1.1.2" resolved "https://registry.yarnpkg.com/sort-keys/-/sort-keys-1.1.2.tgz#441b6d4d346798f1b4e49e8920adfba0e543f9ad" @@ -11053,7 +11131,7 @@ through@2, "through@>=2.2.7 <3", through@^2.3.6, through@~2.3, through@~2.3.1: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" -thunkify@~2.1.1: +thunkify@^2.1.2, thunkify@~2.1.1: version "2.1.2" resolved "https://registry.yarnpkg.com/thunkify/-/thunkify-2.1.2.tgz#faa0e9d230c51acc95ca13a361ac05ca7e04553d"