wassname/talk
1
0
Fork 0
mirror of https://github.com/wassname/talk.git synced 2026-07-08 15:45:09 +08:00
Code Issues Packages Projects Releases Wiki Activity

Change proxy-agent to support http/https through http/https/socks/pac proxy on scraper (#1839)

Browse Source
This commit is contained in:
Thomaz de Oliveira dos Reis
2018-09-04 12:01:10 -03:00
committed by Wyatt Johnson
parent e6107173fb
commit 382019d907
3 changed files with 89 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
jobs/scraper/index.js
+2 -2
View File
@@ -7,7 +7,7 @@ const fetch = require('node-fetch');
const { merge } = require('lodash');
const { version } = require('../../package.json');
const { SCRAPER_HEADERS, SCRAPER_PROXY_URL } = require('../../config');
const HttpsProxyAgent = require('https-proxy-agent');
const ProxyAgent = require('proxy-agent');
// Load the scraper with the rules.
const metascraper = require('metascraper').load([
@@ -37,7 +37,7 @@ const headers = merge(
);
// Add proxy configuration if exists.
const agent = SCRAPER_PROXY_URL ? new HttpsProxyAgent(SCRAPER_PROXY_URL) : null;
const agent = SCRAPER_PROXY_URL ? new ProxyAgent(SCRAPER_PROXY_URL) : null;
/**
* Scrapes the given asset for metadata.
package.json
+1 -1
View File
@@ -126,7 +126,7 @@
"history": "^3.0.0",
"hjson": "^3.1.1",
"hjson-loader": "^1.0.0",
"https-proxy-agent": "^2.2.0",
"proxy-agent": "^3.0.1",
"immutability-helper": "^2.2.0",
"imports-loader": "^0.7.1",
"inquirer": "^3.2.2",
yarn.lock
+86 -8
View File
@@ -267,6 +267,12 @@ agent-base@2:
extend "~3.0.0"
semver "~5.0.1"
agent-base@4, agent-base@^4.2.0, agent-base@~4.2.0:
version "4.2.1"
resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-4.2.1.tgz#d89e5999f797875674c07d87f260fc41e83e8ca9"
dependencies:
es6-promisify "^5.0.0"
agent-base@^4.1.0:
version "4.2.0"
resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-4.2.0.tgz#9838b5c3392b962bad031e6a4c5e1024abec45ce"
@@ -2959,7 +2965,7 @@ defined@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/defined/-/defined-1.0.0.tgz#c98d9bcef75674188e110969151199e39b1fa693"
degenerator@~1.0.2:
degenerator@^1.0.4, degenerator@~1.0.2:
version "1.0.4"
resolved "https://registry.yarnpkg.com/degenerator/-/degenerator-1.0.4.tgz#fcf490a37ece266464d9cc431ab98c5819ced095"
dependencies:
@@ -4290,7 +4296,7 @@ get-stream@^3.0.0:
version "3.0.0"
resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-3.0.0.tgz#8e943d1358dc37555054ecbe2edb05aa174ede14"
get-uri@2:
get-uri@2, get-uri@^2.0.0:
version "2.0.2"
resolved "https://registry.yarnpkg.com/get-uri/-/get-uri-2.0.2.tgz#5c795e71326f6ca1286f2fc82575cd2bab2af578"
dependencies:
@@ -4998,6 +5004,13 @@ http-proxy-agent@1:
debug "2"
extend "3"
http-proxy-agent@^2.1.0:
version "2.1.0"
resolved "https://registry.yarnpkg.com/http-proxy-agent/-/http-proxy-agent-2.1.0.tgz#e4821beef5b2142a2026bd73926fe537631c5405"
dependencies:
agent-base "4"
debug "3.1.0"
http-signature@~1.1.0:
version "1.1.1"
resolved "https://registry.yarnpkg.com/http-signature/-/http-signature-1.1.1.tgz#df72e267066cd0ac67fb76adf8e134a8fbcf91bf"
@@ -5037,7 +5050,7 @@ https-proxy-agent@1:
debug "2"
extend "3"
https-proxy-agent@^2.2.0, https-proxy-agent@^2.2.1:
https-proxy-agent@^2.2.1:
version "2.2.1"
resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-2.2.1.tgz#51552970fa04d723e04c56d04178c3f92592bbc0"
dependencies:
@@ -6943,7 +6956,7 @@ lru-cache@^2.5.0:
version "2.7.3"
resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-2.7.3.tgz#6d4524e8b955f95d4f5b58851ce21dd72fb4e952"
lru-cache@^4.0.1, lru-cache@^4.1.1:
lru-cache@^4.0.1, lru-cache@^4.1.1, lru-cache@^4.1.2:
version "4.1.3"
resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-4.1.3.tgz#a1175cf3496dfc8436c156c334b4955992bce69c"
dependencies:
@@ -7568,7 +7581,7 @@ neo-async@^2.5.0:
version "2.5.1"
resolved "https://registry.yarnpkg.com/neo-async/-/neo-async-2.5.1.tgz#acb909e327b1e87ec9ef15f41b8a269512ad41ee"
netmask@~1.0.4:
netmask@^1.0.6, netmask@~1.0.4:
version "1.0.6"
resolved "https://registry.yarnpkg.com/netmask/-/netmask-1.0.6.tgz#20297e89d86f6f6400f250d9f4f6b4c1945fcd35"
@@ -8210,6 +8223,29 @@ pac-proxy-agent@1:
raw-body "2"
socks-proxy-agent "2"
pac-proxy-agent@^2.0.1:
version "2.0.2"
resolved "https://registry.yarnpkg.com/pac-proxy-agent/-/pac-proxy-agent-2.0.2.tgz#90d9f6730ab0f4d2607dcdcd4d3d641aa26c3896"
dependencies:
agent-base "^4.2.0"
debug "^3.1.0"
get-uri "^2.0.0"
http-proxy-agent "^2.1.0"
https-proxy-agent "^2.2.1"
pac-resolver "^3.0.0"
raw-body "^2.2.0"
socks-proxy-agent "^3.0.0"
pac-resolver@^3.0.0:
version "3.0.0"
resolved "https://registry.yarnpkg.com/pac-resolver/-/pac-resolver-3.0.0.tgz#6aea30787db0a891704deb7800a722a7615a6f26"
dependencies:
co "^4.6.0"
degenerator "^1.0.4"
ip "^1.1.5"
netmask "^1.0.6"
thunkify "^2.1.2"
pac-resolver@~2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/pac-resolver/-/pac-resolver-2.0.0.tgz#99b88d2f193fbdeefc1c9a529c1f3260ab5277cd"
@@ -9083,6 +9119,23 @@ proxy-agent@2.0.0:
pac-proxy-agent "1"
socks-proxy-agent "2"
proxy-agent@^3.0.1:
version "3.0.1"
resolved "https://registry.yarnpkg.com/proxy-agent/-/proxy-agent-3.0.1.tgz#4fb7b61b1476d0fe8e3a3384d90e2460bbded3f9"
dependencies:
agent-base "^4.2.0"
debug "^3.1.0"
http-proxy-agent "^2.1.0"
https-proxy-agent "^2.2.1"
lru-cache "^4.1.2"
pac-proxy-agent "^2.0.1"
proxy-from-env "^1.0.0"
socks-proxy-agent "^4.0.1"
proxy-from-env@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/proxy-from-env/-/proxy-from-env-1.0.0.tgz#33c50398f70ea7eb96d21f7b817630a55791c7ee"
prr@~1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/prr/-/prr-1.0.1.tgz#d3fc114ba06995a45ec6893f484ceb1d78f5f476"
@@ -9337,7 +9390,7 @@ range-parser@~1.2.0:
version "1.2.0"
resolved "https://registry.yarnpkg.com/range-parser/-/range-parser-1.2.0.tgz#f49be6b487894ddc40dcc94a322f611092e00d5e"
raw-body@2, raw-body@2.3.3:
raw-body@2, raw-body@2.3.3, raw-body@^2.2.0:
version "2.3.3"
resolved "https://registry.yarnpkg.com/raw-body/-/raw-body-2.3.3.tgz#1b324ece6b5706e153855bc1148c65bb7f6ea0c3"
dependencies:
@@ -10420,6 +10473,10 @@ smart-buffer@^1.0.13, smart-buffer@^1.0.4:
version "1.1.15"
resolved "https://registry.yarnpkg.com/smart-buffer/-/smart-buffer-1.1.15.tgz#7f114b5b65fab3e2a35aa775bb12f0d1c649bf16"
smart-buffer@^4.0.1:
version "4.0.1"
resolved "https://registry.yarnpkg.com/smart-buffer/-/smart-buffer-4.0.1.tgz#07ea1ca8d4db24eb4cac86537d7d18995221ace3"
smartquotes@~2.3.1:
version "2.3.1"
resolved "https://registry.yarnpkg.com/smartquotes/-/smartquotes-2.3.1.tgz#01ebb595d6c7a9e24d90e8cb95c17d0e1af49407"
@@ -10482,6 +10539,20 @@ socks-proxy-agent@2:
extend "3"
socks "~1.1.5"
socks-proxy-agent@^3.0.0:
version "3.0.1"
resolved "https://registry.yarnpkg.com/socks-proxy-agent/-/socks-proxy-agent-3.0.1.tgz#2eae7cf8e2a82d34565761539a7f9718c5617659"
dependencies:
agent-base "^4.1.0"
socks "^1.1.10"
socks-proxy-agent@^4.0.1:
version "4.0.1"
resolved "https://registry.yarnpkg.com/socks-proxy-agent/-/socks-proxy-agent-4.0.1.tgz#5936bf8b707a993079c6f37db2091821bffa6473"
dependencies:
agent-base "~4.2.0"
socks "~2.2.0"
socks@1.1.9:
version "1.1.9"
resolved "https://registry.yarnpkg.com/socks/-/socks-1.1.9.tgz#628d7e4d04912435445ac0b6e459376cb3e6d691"
@@ -10489,13 +10560,20 @@ socks@1.1.9:
ip "^1.1.2"
smart-buffer "^1.0.4"
socks@~1.1.5:
socks@^1.1.10, socks@~1.1.5:
version "1.1.10"
resolved "https://registry.yarnpkg.com/socks/-/socks-1.1.10.tgz#5b8b7fc7c8f341c53ed056e929b7bf4de8ba7b5a"
dependencies:
ip "^1.1.4"
smart-buffer "^1.0.13"
socks@~2.2.0:
version "2.2.1"
resolved "https://registry.yarnpkg.com/socks/-/socks-2.2.1.tgz#68ad678b3642fbc5d99c64c165bc561eab0215f9"
dependencies:
ip "^1.1.5"
smart-buffer "^4.0.1"
sort-keys@^1.0.0:
version "1.1.2"
resolved "https://registry.yarnpkg.com/sort-keys/-/sort-keys-1.1.2.tgz#441b6d4d346798f1b4e49e8920adfba0e543f9ad"
@@ -11053,7 +11131,7 @@ through@2, "through@>=2.2.7 <3", through@^2.3.6, through@~2.3, through@~2.3.1:
version "2.3.8"
resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5"
thunkify@~2.1.1:
thunkify@^2.1.2, thunkify@~2.1.1:
version "2.1.2"
resolved "https://registry.yarnpkg.com/thunkify/-/thunkify-2.1.2.tgz#faa0e9d230c51acc95ca13a361ac05ca7e04553d"