diff --git a/package-lock.json b/package-lock.json index e7e583ba1..a54ec9a0e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3365,6 +3365,16 @@ "@types/node": "*" } }, + "@types/agent-base": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/@types/agent-base/-/agent-base-4.2.0.tgz", + "integrity": "sha512-8mrhPstU+ZX0Ugya8tl5DsDZ1I5ZwQzbL/8PA0z8Gj0k9nql7nkaMzmPVLj+l/nixWaliXi+EBiLA8bptw3z7Q==", + "dev": true, + "requires": { + "@types/events": "*", + "@types/node": "*" + } + }, "@types/anymatch": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/@types/anymatch/-/anymatch-1.3.1.tgz", @@ -4519,6 +4529,14 @@ "integrity": "sha512-z55ocwKBRLryBs394Sm3ushTtBeg6VAeuku7utSoSnsJKvKcnXFIyC6vh27n3rXyxSgkJBBCAvyOn7gSUcTYjg==", "dev": true }, + "agent-base": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-4.2.1.tgz", + "integrity": "sha512-JVwXMr9nHYTUXsBFKUqhJwvlcYU/blreOEUkhNR2eXZIvwd+c+o5V4MgDPKWnMS/56awN3TRzIP+KoPn+roQtg==", + "requires": { + "es6-promisify": "^5.0.0" + } + }, "airbnb-prop-types": { "version": "2.13.2", "resolved": "https://registry.npmjs.org/airbnb-prop-types/-/airbnb-prop-types-2.13.2.tgz", @@ -5198,8 +5216,7 @@ "ast-types": { "version": "0.11.7", "resolved": "https://registry.npmjs.org/ast-types/-/ast-types-0.11.7.tgz", - "integrity": "sha512-2mP3TwtkY/aTv5X3ZsMpNAbOnyoC/aMJwJSoaELPkHId0nSQgFcnU4dRW3isxiz7+zBexk0ym3WNVjMiQBnJSw==", - "dev": true + "integrity": "sha512-2mP3TwtkY/aTv5X3ZsMpNAbOnyoC/aMJwJSoaELPkHId0nSQgFcnU4dRW3isxiz7+zBexk0ym3WNVjMiQBnJSw==" }, "astral-regex": { "version": "1.0.0", @@ -9798,6 +9815,21 @@ "assert-plus": "^1.0.0" } }, + "data-uri-to-buffer": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-2.0.1.tgz", + "integrity": "sha512-OkVVLrerfAKZlW2ZZ3Ve2y65jgiWqBKsTfUIAFbn8nVbPcCZg6l6gikKlEYv0kXcmzqGm6mFq/Jf2vriuEkv8A==", + "requires": { + "@types/node": "^8.0.7" + }, + "dependencies": { + "@types/node": { + "version": "8.10.48", + "resolved": "https://registry.npmjs.org/@types/node/-/node-8.10.48.tgz", + "integrity": "sha512-c35YEBTkL4rzXY2ucpSKy+UYHjUBIIkuJbWYbsGIrKLEWU5dgJMmLkkIb3qeC3O3Tpb1ZQCwecscvJTDjDjkRw==" + } + } + }, "data-urls": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-1.1.0.tgz", @@ -10249,6 +10281,23 @@ } } }, + "degenerator": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/degenerator/-/degenerator-1.0.4.tgz", + "integrity": "sha1-/PSQo37OJmRk2cxDGrmMWBnO0JU=", + "requires": { + "ast-types": "0.x.x", + "escodegen": "1.x.x", + "esprima": "3.x.x" + }, + "dependencies": { + "esprima": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-3.1.3.tgz", + "integrity": "sha1-/cpRzuYTOJXjyI1TXOSdv/YqRjM=" + } + } + }, "del": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/del/-/del-4.1.1.tgz", @@ -13006,6 +13055,21 @@ "integrity": "sha1-lu258v2wGZWCKyY92KratnSBgbw=", "dev": true }, + "es6-promisify": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/es6-promisify/-/es6-promisify-5.0.0.tgz", + "integrity": "sha1-UQnWLz5W6pZ8S2NQWu8IKRyKUgM=", + "requires": { + "es6-promise": "^4.0.3" + }, + "dependencies": { + "es6-promise": { + "version": "4.2.6", + "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-4.2.6.tgz", + "integrity": "sha512-aRVgGdnmW2OiySVPUC9e6m+plolMAJKjZnQlCwNSuK5yQ0JN61DZSO1X1Ufd1foqWRAlig0rhduTCHe7sVtK5Q==" + } + } + }, "es6-symbol": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/es6-symbol/-/es6-symbol-3.1.1.tgz", @@ -13047,7 +13111,6 @@ "version": "1.10.0", "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-1.10.0.tgz", "integrity": "sha512-fjUOf8johsv23WuIKdNQU4P9t9jhQ4Qzx6pC2uW890OloK3Zs1ZAoCNpg/2larNF501jLl3UNy0kIRcF6VI22g==", - "dev": true, "requires": { "esprima": "^3.1.3", "estraverse": "^4.2.0", @@ -13059,14 +13122,12 @@ "esprima": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/esprima/-/esprima-3.1.3.tgz", - "integrity": "sha1-/cpRzuYTOJXjyI1TXOSdv/YqRjM=", - "dev": true + "integrity": "sha1-/cpRzuYTOJXjyI1TXOSdv/YqRjM=" }, "source-map": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", - "dev": true, "optional": true } } @@ -13641,6 +13702,11 @@ "schema-utils": "^1.0.0" } }, + "file-uri-to-path": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", + "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==" + }, "filename-regex": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/filename-regex/-/filename-regex-2.0.1.tgz", @@ -13989,6 +14055,11 @@ "requires": { "ms": "^2.1.1" } + }, + "semver": { + "version": "5.7.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.0.tgz", + "integrity": "sha512-Ya52jSX2u7QKghxeoFGpLwCtGlt7j0oY9DYb5apt9nPlJ42ID+ulTXESnt/qAQcoSERyZ5sl3LDIOw0nAn/5DA==" } } }, @@ -15058,6 +15129,43 @@ } } }, + "ftp": { + "version": "0.3.10", + "resolved": "https://registry.npmjs.org/ftp/-/ftp-0.3.10.tgz", + "integrity": "sha1-kZfYYa2BQvPmPVqDv+TFn3MwiF0=", + "requires": { + "readable-stream": "1.1.x", + "xregexp": "2.0.0" + }, + "dependencies": { + "isarray": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", + "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=" + }, + "readable-stream": { + "version": "1.1.14", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz", + "integrity": "sha1-fPTFTvZI44EwhMY23SB54WbAgdk=", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.1", + "isarray": "0.0.1", + "string_decoder": "~0.10.x" + } + }, + "string_decoder": { + "version": "0.10.31", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", + "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=" + }, + "xregexp": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/xregexp/-/xregexp-2.0.0.tgz", + "integrity": "sha1-UqY+VsoLhKfzpfPWGHLxJq16WUM=" + } + } + }, "function-bind": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", @@ -15112,6 +15220,11 @@ "number-is-nan": "^1.0.0" } }, + "mkdirp": { + "version": "0.5.1", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", + "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=" + }, "string-width": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", @@ -15171,6 +15284,44 @@ "integrity": "sha1-jpQ9E1jcN1VQVOy+LtsFqhdO3hQ=", "dev": true }, + "get-uri": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/get-uri/-/get-uri-2.0.3.tgz", + "integrity": "sha512-x5j6Ks7FOgLD/GlvjKwgu7wdmMR55iuRHhn8hj/+gA+eSbxQvZ+AEomq+3MgVEZj1vpi738QahGbCCSIDtXtkw==", + "requires": { + "data-uri-to-buffer": "2", + "debug": "4", + "extend": "~3.0.2", + "file-uri-to-path": "1", + "ftp": "~0.3.10", + "readable-stream": "3" + }, + "dependencies": { + "debug": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", + "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "requires": { + "ms": "^2.1.1" + } + }, + "extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" + }, + "readable-stream": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.3.0.tgz", + "integrity": "sha512-EsI+s3k3XsW+fU8fQACLN59ky34AZ14LoeVZpYwmZvldCFo0r0gnelwF2TcMjLor/BTL5aDJVBMkss0dthToPw==", + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + } + } + }, "get-value": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/get-value/-/get-value-2.0.6.tgz", @@ -15317,9 +15468,9 @@ }, "dependencies": { "glob": { - "version": "7.1.3", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.3.tgz", - "integrity": "sha512-vcfuiIxogLV4DlGBHIUOwI0IbrJ8HWPc4MU7HzviGeNho/UJDfi6B5p3sHeWIQ0KGIU0Jpxi5ZHxemQfLkkAwQ==", + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.2.tgz", + "integrity": "sha512-MJTUg1kjuLeQCJ+ccE4Vpa6kKVXkPYJ2mOCQyUuKLcLQsdrMCpBPUi8qVE6+YuaJkozeA9NusTAw3hLr8Xe5EQ==", "dev": true, "requires": { "fs.realpath": "^1.0.0", @@ -17353,6 +17504,30 @@ "requires-port": "^1.0.0" } }, + "http-proxy-agent": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-2.1.0.tgz", + "integrity": "sha512-qwHbBLV7WviBl0rQsOzH6o5lwyOIvwp/BdFnvVxXORldu5TmjFfjzBcWUWS5kWAZhmv+JtiDhSuQCp4sBfbIgg==", + "requires": { + "agent-base": "4", + "debug": "3.1.0" + }, + "dependencies": { + "debug": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", + "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==", + "requires": { + "ms": "2.0.0" + } + }, + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + } + } + }, "http-proxy-middleware": { "version": "0.19.1", "resolved": "https://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-0.19.1.tgz", @@ -17381,6 +17556,25 @@ "integrity": "sha1-7AbBDgo0wPL68Zn3/X/Hj//QPHM=", "dev": true }, + "https-proxy-agent": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-2.2.1.tgz", + "integrity": "sha512-HPCTS1LW51bcyMYbxUIOO4HEOlQ1/1qRaFWcyxvwaqUS9TY88aoEuHUY33kuAh1YhVVaDQhLZsnPd+XNARWZlQ==", + "requires": { + "agent-base": "^4.1.0", + "debug": "^3.1.0" + }, + "dependencies": { + "debug": { + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz", + "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==", + "requires": { + "ms": "^2.1.1" + } + } + } + }, "humanize-duration": { "version": "3.18.0", "resolved": "https://registry.npmjs.org/humanize-duration/-/humanize-duration-3.18.0.tgz", @@ -17879,8 +18073,7 @@ "ip": { "version": "1.1.5", "resolved": "https://registry.npmjs.org/ip/-/ip-1.1.5.tgz", - "integrity": "sha1-vd7XARQpCCjAoDnnLvJfWq7ENUo=", - "dev": true + "integrity": "sha1-vd7XARQpCCjAoDnnLvJfWq7ENUo=" }, "ip-regex": { "version": "1.0.3", @@ -22395,6 +22588,11 @@ "integrity": "sha512-AO81vsIO1k1sM4Zrd6Hu7regmJN1NSiAja10gc4bX3F0wd+9rQmcuHQaHVQCYIEC8iFXnE+mavh23GOt7wBgug==", "dev": true }, + "netmask": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/netmask/-/netmask-1.0.6.tgz", + "integrity": "sha1-ICl+idhvb2QA8lDZ9Pa0wZRfzTU=" + }, "next-tick": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/next-tick/-/next-tick-1.0.0.tgz", @@ -23800,6 +23998,43 @@ "integrity": "sha1-y8ec26+P1CKOE/Yh8rGiN8GyB7M=", "dev": true }, + "pac-proxy-agent": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pac-proxy-agent/-/pac-proxy-agent-3.0.0.tgz", + "integrity": "sha512-AOUX9jES/EkQX2zRz0AW7lSx9jD//hQS8wFXBvcnd/J2Py9KaMJMqV/LPqJssj1tgGufotb2mmopGPR15ODv1Q==", + "requires": { + "agent-base": "^4.2.0", + "debug": "^3.1.0", + "get-uri": "^2.0.0", + "http-proxy-agent": "^2.1.0", + "https-proxy-agent": "^2.2.1", + "pac-resolver": "^3.0.0", + "raw-body": "^2.2.0", + "socks-proxy-agent": "^4.0.1" + }, + "dependencies": { + "debug": { + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz", + "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==", + "requires": { + "ms": "^2.1.1" + } + } + } + }, + "pac-resolver": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pac-resolver/-/pac-resolver-3.0.0.tgz", + "integrity": "sha512-tcc38bsjuE3XZ5+4vP96OfhOugrX+JcnpUbhfuc4LuXBLQhoTthOstZeoQJBDnQUDYzYmdImKsbz0xSl1/9qeA==", + "requires": { + "co": "^4.6.0", + "degenerator": "^1.0.4", + "ip": "^1.1.5", + "netmask": "^1.0.6", + "thunkify": "^2.1.2" + } + }, "pako": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.6.tgz", @@ -27475,6 +27710,36 @@ "ipaddr.js": "1.6.0" } }, + "proxy-agent": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/proxy-agent/-/proxy-agent-3.1.0.tgz", + "integrity": "sha512-IkbZL4ClW3wwBL/ABFD2zJ8iP84CY0uKMvBPk/OceQe/cEjrxzN1pMHsLwhbzUoRhG9QbSxYC+Z7LBkTiBNvrA==", + "requires": { + "agent-base": "^4.2.0", + "debug": "^3.1.0", + "http-proxy-agent": "^2.1.0", + "https-proxy-agent": "^2.2.1", + "lru-cache": "^4.1.2", + "pac-proxy-agent": "^3.0.0", + "proxy-from-env": "^1.0.0", + "socks-proxy-agent": "^4.0.1" + }, + "dependencies": { + "debug": { + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz", + "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==", + "requires": { + "ms": "^2.1.1" + } + } + } + }, + "proxy-from-env": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.0.0.tgz", + "integrity": "sha1-M8UDmPcOp+uW0h97gXYwpVeRx+4=" + }, "prr": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/prr/-/prr-1.0.1.tgz", @@ -30752,6 +31017,11 @@ "resolved": "https://registry.npmjs.org/slick/-/slick-1.12.2.tgz", "integrity": "sha1-vQSN23TefRymkV+qSldXCzVQwtc=" }, + "smart-buffer": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.0.2.tgz", + "integrity": "sha512-JDhEpTKzXusOqXZ0BUIdH+CjFdO/CR3tLlf5CN34IypI+xMmXW1uB16OOY8z3cICbJlDAVJzNbwBhNO0wt9OAw==" + }, "smartquotes": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/smartquotes/-/smartquotes-2.3.1.tgz", @@ -30909,6 +31179,24 @@ } } }, + "socks": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/socks/-/socks-2.3.2.tgz", + "integrity": "sha512-pCpjxQgOByDHLlNqlnh/mNSAxIUkyBBuwwhTcV+enZGbDaClPvHdvm6uvOwZfFJkam7cGhBNbb4JxiP8UZkRvQ==", + "requires": { + "ip": "^1.1.5", + "smart-buffer": "4.0.2" + } + }, + "socks-proxy-agent": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-4.0.2.tgz", + "integrity": "sha512-NT6syHhI9LmuEMSK6Kd2V7gNv5KFZoLE7V5udWmn0de+3Mkj3UMA/AJPLyeNUVmElCurSHtUdM3ETpR3z770Wg==", + "requires": { + "agent-base": "~4.2.1", + "socks": "~2.3.2" + } + }, "sort-keys": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/sort-keys/-/sort-keys-2.0.0.tgz", @@ -32282,6 +32570,11 @@ "xtend": "~4.0.0" } }, + "thunkify": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/thunkify/-/thunkify-2.1.2.tgz", + "integrity": "sha1-+qDp0jDFGsyVyhOjYawFyn4EVT0=" + }, "thunky": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/thunky/-/thunky-1.0.3.tgz", diff --git a/package.json b/package.json index 0fd59db14..3958cc65c 100644 --- a/package.json +++ b/package.json @@ -116,6 +116,7 @@ "performance-now": "^2.1.0", "permit": "^0.2.4", "prom-client": "^11.3.0", + "proxy-agent": "^3.1.0", "querystringify": "^2.1.0", "react-relay-network-modern": "^2.4.0", "source-map-support": "^0.5.12", @@ -140,6 +141,7 @@ "@coralproject/rte": "^0.10.15", "@intervolga/optimize-cssnano-plugin": "^1.0.6", "@types/basic-auth": "^1.1.2", + "@types/agent-base": "^4.2.0", "@types/bcryptjs": "^2.4.1", "@types/bull": "^3.5.12", "@types/bunyan": "^1.8.4", diff --git a/src/core/client/admin/routes/configure/sections/advanced/components/PermittedDomainsConfig.tsx b/src/core/client/admin/routes/configure/sections/advanced/components/PermittedDomainsConfig.tsx index 603e1fb12..17eef6f1f 100644 --- a/src/core/client/admin/routes/configure/sections/advanced/components/PermittedDomainsConfig.tsx +++ b/src/core/client/admin/routes/configure/sections/advanced/components/PermittedDomainsConfig.tsx @@ -21,7 +21,9 @@ const PermittedDomainsConfig: FunctionComponent = ({ disabled }) => ( -
}> +
} + > Permitted Domains
@@ -34,7 +36,11 @@ const PermittedDomainsConfig: FunctionComponent = ({ disabled }) => ( use is localhost, staging.yourdomain.com, yourdomain.com, etc. - + {({ input, meta }) => ( <> { const enhanced = withFragmentContainer({ settings: graphql` fragment PermittedDomainsConfigContainer_settings on Settings { - domains + allowedDomains } `, })(PermittedDomainsConfigContainer); diff --git a/src/core/client/admin/test/configure/__snapshots__/advanced.spec.tsx.snap b/src/core/client/admin/test/configure/__snapshots__/advanced.spec.tsx.snap index 956d30b1c..96613e39c 100644 --- a/src/core/client/admin/test/configure/__snapshots__/advanced.spec.tsx.snap +++ b/src/core/client/admin/test/configure/__snapshots__/advanced.spec.tsx.snap @@ -152,7 +152,7 @@ exports[`renders configure advanced 1`] = ` > @@ -172,8 +172,8 @@ yourdomain.com, etc. autoCorrect="off" className="TextField-input TextField-colorRegular" disabled={false} - id="configure-advanced-domains" - name="domains" + id="configure-advanced-allowedDomains" + name="allowedDomains" onChange={[Function]} placeholder="" spellCheck={false} diff --git a/src/core/client/admin/test/configure/advanced.spec.tsx b/src/core/client/admin/test/configure/advanced.spec.tsx index 92777ae38..fe18779b6 100644 --- a/src/core/client/admin/test/configure/advanced.spec.tsx +++ b/src/core/client/admin/test/configure/advanced.spec.tsx @@ -146,7 +146,7 @@ it("change permitted domains to be empty", async () => { const resolvers = createResolversStub({ Mutation: { updateSettings: ({ variables }) => { - expectAndFail(variables.settings.domains).toEqual([]); + expectAndFail(variables.settings.allowedDomains).toEqual([]); return { settings: pureMerge(settings, variables.settings), }; @@ -190,7 +190,7 @@ it("change permitted domains to include more domains", async () => { const resolvers = createResolversStub({ Mutation: { updateSettings: ({ variables }) => { - expectAndFail(variables.settings.domains).toEqual([ + expectAndFail(variables.settings.allowedDomains).toEqual([ "localhost:8080", "localhost:3000", ]); diff --git a/src/core/client/admin/test/fixtures.ts b/src/core/client/admin/test/fixtures.ts index b23ae92c5..068a841d7 100644 --- a/src/core/client/admin/test/fixtures.ts +++ b/src/core/client/admin/test/fixtures.ts @@ -40,7 +40,7 @@ export const settings = createFixture({ message: "Comments are closed on this story.", }, customCSSURL: "", - domains: ["localhost:8080"], + allowedDomains: ["localhost:8080"], editCommentWindowLength: 30000, communityGuidelines: { enabled: false, diff --git a/src/core/client/framework/rest/install.ts b/src/core/client/framework/rest/install.ts index 81949036f..28b09df04 100644 --- a/src/core/client/framework/rest/install.ts +++ b/src/core/client/framework/rest/install.ts @@ -7,7 +7,7 @@ export interface InstallInput { contactEmail: string; url: string; }; - domains: string[]; + allowedDomains: string[]; }; user: { username: string; diff --git a/src/core/client/install/containers/WizardContainer.tsx b/src/core/client/install/containers/WizardContainer.tsx index be6d2bb58..84ed0d309 100644 --- a/src/core/client/install/containers/WizardContainer.tsx +++ b/src/core/client/install/containers/WizardContainer.tsx @@ -18,7 +18,7 @@ interface FormData { username: string; password: string; confirmPassword: string; - domains: string[]; + allowedDomains: string[]; } interface WizardContainerState { @@ -31,7 +31,7 @@ function shapeFinalData(data: FormData): InstallInput { organizationName, organizationContactEmail, organizationURL, - domains, + allowedDomains, username, password, email, @@ -44,7 +44,7 @@ function shapeFinalData(data: FormData): InstallInput { contactEmail: organizationContactEmail, url: organizationURL, }, - domains, + allowedDomains, }, user: { username, @@ -69,7 +69,7 @@ class WizardContainer extends Component { username: "", password: "", confirmPassword: "", - domains: [], + allowedDomains: [], }, }; diff --git a/src/core/client/install/steps/components/PermittedDomains.tsx b/src/core/client/install/steps/components/PermittedDomains.tsx index 5120eb527..e9b182abd 100644 --- a/src/core/client/install/steps/components/PermittedDomains.tsx +++ b/src/core/client/install/steps/components/PermittedDomains.tsx @@ -19,14 +19,14 @@ import { import BackButton from "./BackButton"; interface FormProps { - domains: string; + allowedDomains: string; } export interface PermittedDomainsForm { onSubmit: OnSubmit; onGoToPreviousStep: () => void; data: { - domains: string[]; + allowedDomains: string[]; }; } @@ -35,7 +35,7 @@ const PermittedDomains: FunctionComponent = props => {
{({ handleSubmit, submitting, submitError }) => ( @@ -60,7 +60,7 @@ const PermittedDomains: FunctionComponent = props => { )} - + {({ input, meta }) => ( diff --git a/src/core/client/install/steps/containers/PermittedDomainsContainer.tsx b/src/core/client/install/steps/containers/PermittedDomainsContainer.tsx index 6ede2b867..ccda7a75e 100644 --- a/src/core/client/install/steps/containers/PermittedDomainsContainer.tsx +++ b/src/core/client/install/steps/containers/PermittedDomainsContainer.tsx @@ -21,8 +21,8 @@ class PermittedDomainsContainer extends Component< > { private onSubmit: PermittedDomainsForm["onSubmit"] = async (input, form) => { try { - const domains = input.domains.split(","); - await this.props.onInstall({ domains }); + const allowedDomains = input.allowedDomains.split(","); + await this.props.onInstall({ allowedDomains }); return this.props.onGoToNextStep(); } catch (error) { return { [FORM_ERROR]: error.message }; diff --git a/src/core/server/app/handlers/api/install.ts b/src/core/server/app/handlers/api/install.ts index 05df04fd0..627ac8134 100644 --- a/src/core/server/app/handlers/api/install.ts +++ b/src/core/server/app/handlers/api/install.ts @@ -31,7 +31,7 @@ const TenantInstallBodySchema = Joi.object().keys({ .lowercase() .email(), }), - domains: Joi.array().items( + allowedDomains: Joi.array().items( Joi.string() .trim() .uri() diff --git a/src/core/server/app/middleware/csp/tenant.spec.ts b/src/core/server/app/middleware/csp/tenant.spec.ts index 8fab5eb94..4b4c0cc0c 100644 --- a/src/core/server/app/middleware/csp/tenant.spec.ts +++ b/src/core/server/app/middleware/csp/tenant.spec.ts @@ -2,14 +2,14 @@ import { generateFrameOptions } from "coral-server/app/middleware/csp/tenant"; import { Request } from "coral-server/types/express"; it("denies when the tenant has no specified domains", () => { - const tenant = { domains: [] }; + const tenant = { allowedDomains: [] }; const req = {} as Request; expect(generateFrameOptions(req, tenant)).toEqual("deny"); }); it("allow-from single domain when there is one domain", () => { - const tenant = { domains: ["https://coralproject.net"] }; + const tenant = { allowedDomains: ["https://coralproject.net"] }; const req = {} as Request; expect(generateFrameOptions(req, tenant)).toEqual( @@ -19,7 +19,10 @@ it("allow-from single domain when there is one domain", () => { it("deny from the domain when it does not provide and there are multiple tenants domains", () => { const tenant = { - domains: ["https://coralproject.net", "https://news.coralproject.net"], + allowedDomains: [ + "https://coralproject.net", + "https://news.coralproject.net", + ], }; const req = { headers: {}, query: { parentUrl: "" } } as Request; @@ -28,7 +31,10 @@ it("deny from the domain when it does not provide and there are multiple tenants it("allows from the domain when it does not provide a match and there are multiple tenants domains", () => { const tenant = { - domains: ["https://coralproject.net", "https://news.coralproject.net"], + allowedDomains: [ + "https://coralproject.net", + "https://news.coralproject.net", + ], }; const req = { headers: {}, @@ -40,7 +46,10 @@ it("allows from the domain when it does not provide a match and there are multip it("allows from the domain when it does provide a match and there are multiple tenants domains", () => { const tenant = { - domains: ["https://coralproject.net", "https://news.coralproject.net"], + allowedDomains: [ + "https://coralproject.net", + "https://news.coralproject.net", + ], }; const req = { headers: {}, @@ -54,7 +63,7 @@ it("allows from the domain when it does provide a match and there are multiple t it("it prefixes domains of the tenant when generating the frame option", () => { const tenant = { - domains: ["coralproject.net", "news.coralproject.net"], + allowedDomains: ["coralproject.net", "news.coralproject.net"], }; const req = { headers: {}, @@ -68,7 +77,7 @@ it("it prefixes domains of the tenant when generating the frame option", () => { it("it prefixes domains of the tenant when generating the frame option", () => { const tenant = { - domains: ["coralproject.net", "news.coralproject.net"], + allowedDomains: ["coralproject.net", "news.coralproject.net"], }; const req = { headers: {}, @@ -82,7 +91,7 @@ it("it prefixes domains of the tenant when generating the frame option", () => { it("it prefixes domains of the tenant when generating the frame option and denies based on it", () => { const tenant = { - domains: ["coralproject.net", "news.coralproject.net"], + allowedDomains: ["coralproject.net", "news.coralproject.net"], }; const req = { headers: {}, diff --git a/src/core/server/app/middleware/csp/tenant.ts b/src/core/server/app/middleware/csp/tenant.ts index 6fbd9ec81..8373efe9a 100644 --- a/src/core/server/app/middleware/csp/tenant.ts +++ b/src/core/server/app/middleware/csp/tenant.ts @@ -35,13 +35,13 @@ export const cspTenantMiddleware: RequestHandler = (req, res, next) => { function generateContentSecurityPolicy( req: Request, - tenant: Pick + tenant: Pick ) { const directives: Record = {}; // Only the domains that are allowed by the tenant may embed Coral. directives.frameAncestors = - tenant.domains.length > 0 ? tenant.domains : ["'none'"]; + tenant.allowedDomains.length > 0 ? tenant.allowedDomains : ["'none'"]; // Build the directive. const directive = builder({ directives }); @@ -51,20 +51,20 @@ function generateContentSecurityPolicy( export function generateFrameOptions( req: Request, - tenant: Pick + tenant: Pick ) { // If there aren't any domains, then we reject it. - if (tenant.domains.length === 0) { + if (tenant.allowedDomains.length === 0) { return "deny"; } // If there is only one domain on the tenant, and we don't require // prefixing, then return it! if ( - tenant.domains.length === 1 && - !doesRequireSchemePrefixing(tenant.domains[0]) + tenant.allowedDomains.length === 1 && + !doesRequireSchemePrefixing(tenant.allowedDomains[0]) ) { - return `allow-from ${getOrigin(tenant.domains[0])}`; + return `allow-from ${getOrigin(tenant.allowedDomains[0])}`; } const parentsURL = extractParentsURL(req); @@ -87,11 +87,11 @@ export function generateFrameOptions( // If there is only one domain on the tenant, and we require prefixing, then // return it with prefixing! if ( - tenant.domains.length === 1 && - !doesRequireSchemePrefixing(tenant.domains[0]) + tenant.allowedDomains.length === 1 && + !doesRequireSchemePrefixing(tenant.allowedDomains[0]) ) { return `allow-from ${getOrigin( - prefixSchemeIfRequired(parentSecure, tenant.domains[0]) + prefixSchemeIfRequired(parentSecure, tenant.allowedDomains[0]) )}`; } @@ -104,7 +104,7 @@ export function generateFrameOptions( // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Frame-Options // We need to find the domain that is asking so we can respond with the right // result, sort of like CORS! - const allowFrom = tenant.domains + const allowFrom = tenant.allowedDomains .map(domain => getOrigin(prefixSchemeIfRequired(parentSecure, domain))) .find(origin => origin === parentsOrigin); if (!allowFrom) { diff --git a/src/core/server/errors/index.ts b/src/core/server/errors/index.ts index 287020403..13944ad4f 100644 --- a/src/core/server/errors/index.ts +++ b/src/core/server/errors/index.ts @@ -237,7 +237,7 @@ export class URLInvalidError extends CoralError { ...properties }: { url: string; - tenantDomains: string[]; + allowedDomains: string[]; tenantDomain?: string; }) { super({ @@ -250,7 +250,7 @@ export class URLInvalidError extends CoralError { export class StoryURLInvalidError extends CoralError { constructor(properties: { storyURL: string; - tenantDomains: string[]; + allowedDomains: string[]; tenantDomain?: string; }) { super({ diff --git a/src/core/server/graph/tenant/loaders/Stories.ts b/src/core/server/graph/tenant/loaders/Stories.ts index 52c905cae..a227b546e 100644 --- a/src/core/server/graph/tenant/loaders/Stories.ts +++ b/src/core/server/graph/tenant/loaders/Stories.ts @@ -3,20 +3,25 @@ import DataLoader from "dataloader"; import TenantContext from "coral-server/graph/tenant/context"; import { GQLSTORY_STATUS, - GQLStoryMetadata, QueryToStoriesArgs, } from "coral-server/graph/tenant/schema/__generated__/types"; import { Connection } from "coral-server/models/helpers/connection"; import { - FindOrCreateStoryInput, retrieveManyStories, retrieveStoryConnection, Story, StoryConnectionInput, } from "coral-server/models/story"; -import { findOrCreate } from "coral-server/services/stories"; +import { + find, + findOrCreate, + FindOrCreateStory, + FindStory, +} from "coral-server/services/stories"; import { scraper } from "coral-server/services/stories/scraper"; +import { createManyBatchLoadFn } from "./util"; + const statusFilter = ( status?: GQLSTORY_STATUS ): StoryConnectionInput["filter"] => { @@ -61,15 +66,21 @@ const primeStoriesFromConnection = (ctx: TenantContext) => ( export default (ctx: TenantContext) => ({ findOrCreate: new DataLoader( - (inputs: FindOrCreateStoryInput[]) => - Promise.all( - inputs.map(input => - findOrCreate(ctx.mongo, ctx.tenant, input, ctx.scraperQueue, ctx.now) - ) - ), + createManyBatchLoadFn((input: FindOrCreateStory) => + findOrCreate(ctx.mongo, ctx.tenant, input, ctx.scraperQueue, ctx.now) + ), { // TODO: (wyattjoh) see if there's something we can do to improve the cache key - cacheKeyFn: (input: FindOrCreateStoryInput) => `${input.id}:${input.url}`, + cacheKeyFn: (input: FindOrCreateStory) => `${input.id}:${input.url}`, + } + ), + find: new DataLoader( + createManyBatchLoadFn((input: FindStory) => + find(ctx.mongo, ctx.tenant, input) + ), + { + // TODO: (wyattjoh) see if there's something we can do to improve the cache key + cacheKeyFn: (input: FindStory) => `${input.id}:${input.url}`, } ), story: new DataLoader(ids => @@ -87,7 +98,7 @@ export default (ctx: TenantContext) => ({ ...queryFilter(query), }, }).then(primeStoriesFromConnection(ctx)), - debugScrapeMetadata: new DataLoader(urls => - Promise.all(urls.map(url => scraper.scrape(url))) + debugScrapeMetadata: new DataLoader( + createManyBatchLoadFn((url: string) => scraper.scrape(url)) ), }); diff --git a/src/core/server/graph/tenant/loaders/util.ts b/src/core/server/graph/tenant/loaders/util.ts index aaf93286a..8a51683ac 100644 --- a/src/core/server/graph/tenant/loaders/util.ts +++ b/src/core/server/graph/tenant/loaders/util.ts @@ -24,3 +24,9 @@ export class SingletonResolver { return promise; } } + +export function createManyBatchLoadFn( + batchLoadFn: (input: U) => Promise +) { + return (inputs: U[]) => Promise.all(inputs.map(input => batchLoadFn(input))); +} diff --git a/src/core/server/graph/tenant/resolvers/Query.ts b/src/core/server/graph/tenant/resolvers/Query.ts index 57d837f91..60e77ee8b 100644 --- a/src/core/server/graph/tenant/resolvers/Query.ts +++ b/src/core/server/graph/tenant/resolvers/Query.ts @@ -3,7 +3,10 @@ import { GQLQueryTypeResolver } from "coral-server/graph/tenant/schema/__generat import { moderationQueuesResolver } from "./ModerationQueues"; export const Query: Required> = { - story: (source, args, ctx) => ctx.loaders.Stories.findOrCreate.load(args), + story: (source, args, ctx) => + ctx.tenant.stories.disableLazy + ? ctx.loaders.Stories.find.load(args) + : ctx.loaders.Stories.findOrCreate.load(args), stories: (source, args, ctx) => ctx.loaders.Stories.connection(args), user: (source, args, ctx) => ctx.loaders.Users.user.load(args.id), users: (source, args, ctx) => ctx.loaders.Users.connection(args), diff --git a/src/core/server/graph/tenant/schema/schema.graphql b/src/core/server/graph/tenant/schema/schema.graphql index 9ba3d0263..9e7e43fd1 100644 --- a/src/core/server/graph/tenant/schema/schema.graphql +++ b/src/core/server/graph/tenant/schema/schema.graphql @@ -1023,6 +1023,39 @@ type Organization { url: String! } +""" +StoryScrapingConfiguration stores the configuration around story scraping. +""" +type StoryScrapingConfiguration { + """ + enabled, when true, enables stories to be scraped. When disabled, stories will + only be looked up instead, and must be created via the API directly. + """ + enabled: Boolean! + + """ + proxyURL when specified, allows scraping requests to use the provided proxy. + All requests will then be passed through the appropriote proxy as parsed by + the [proxy-agent](https://www.npmjs.com/package/proxy-agent) package. + """ + proxyURL: String +} + +""" +StoryConfiguration stores the configuration for working with stories. +""" +type StoryConfiguration { + """ + scraping stores configuration around story scraping. + """ + scraping: StoryScrapingConfiguration! + + """ + disableLazy when true, will only allow lookups of stories created via the API. + """ + disableLazy: Boolean! +} + """ Settings stores the global settings for a given Tenant. """ @@ -1038,9 +1071,9 @@ type Settings { domain: String! @auth(roles: [ADMIN]) """ - domains will return a given list of permitted domains. + allowedDomains is the list of domains that stories can come from. """ - domains: [String!]! @auth(roles: [ADMIN]) + allowedDomains: [String!]! @auth(roles: [ADMIN]) """ locale is the specified locale for this Tenant. @@ -1126,6 +1159,11 @@ type Settings { """ reaction: ReactionConfiguration! + """ + stories stores the configuration around stories. + """ + stories: StoryConfiguration! @auth(roles: [ADMIN]) + """ createdAt is the time that the Settings was created at. """ @@ -2744,14 +2782,47 @@ input SettingsCloseCommentingInput { message: String } +""" +StoryScrapingConfigurationInput stores the configuration around story scraping. +""" +input StoryScrapingConfigurationInput { + """ + enabled, when true, enables stories to be scraped. When disabled, stories will + only be looked up instead, and must be created via the API directly. + """ + enabled: Boolean + + """ + proxyURL when specified, allows scraping requests to use the provided proxy. + All requests will then be passed through the appropriote proxy as parsed by + the [proxy-agent](https://www.npmjs.com/package/proxy-agent) package. + """ + proxyURL: String +} + +""" +StoryConfiguration stores the configuration for working with stories. +""" +input StoryConfigurationInput { + """ + scraping stores configuration around story scraping. + """ + scraping: StoryScrapingConfigurationInput + + """ + disableLazy when true, will only allow lookups of stories created via the API. + """ + disableLazy: Boolean +} + """ SettingsInput is the partial type of the Settings type for performing mutations. """ input SettingsInput { """ - domains will return a given list of domains that hosts Stories. + allowedDomains is the list of domains that stories can come from. """ - domains: [String!] + allowedDomains: [String!] """ moderation is the moderation mode for all Stories on the site. @@ -2826,6 +2897,11 @@ input SettingsInput { charCount stores the character count moderation settings. """ charCount: SettingsCharCountInput + + """ + stories stores the configuration around stories. + """ + stories: StoryConfigurationInput } """ diff --git a/src/core/server/models/settings.ts b/src/core/server/models/settings.ts index 9211cc390..39db154cd 100644 --- a/src/core/server/models/settings.ts +++ b/src/core/server/models/settings.ts @@ -81,6 +81,7 @@ export type Settings = GlobalModerationSettings & | "editCommentWindowLength" | "customCSSURL" | "communityGuidelines" + | "stories" | "createdAt" > & { /** diff --git a/src/core/server/models/story/index.ts b/src/core/server/models/story/index.ts index 78bd6a451..a3f4c1303 100644 --- a/src/core/server/models/story/index.ts +++ b/src/core/server/models/story/index.ts @@ -161,6 +161,29 @@ export async function upsertStory( return result.value || null; } +export interface FindStoryInput { + id?: string; + url?: string; +} + +export async function findStory( + mongo: Db, + tenantID: string, + { id, url }: FindStoryInput +) { + if (id) { + return retrieveStory(mongo, tenantID, id); + } + + if (url) { + return retrieveStoryByURL(mongo, tenantID, url); + } + + // Story can't be found with that ID/URL combination and scraping is + // disabled, so we fail here. + return null; +} + export interface FindOrCreateStoryInput { id?: string; url?: string; diff --git a/src/core/server/models/tenant.ts b/src/core/server/models/tenant.ts index 9516740b9..5f39c88b1 100644 --- a/src/core/server/models/tenant.ts +++ b/src/core/server/models/tenant.ts @@ -29,7 +29,7 @@ export interface TenantResource { } export interface TenantSettings - extends Pick { + extends Pick { readonly id: string; /** @@ -60,7 +60,7 @@ export async function createTenantIndexes(mongo: Db) { */ export type CreateTenantInput = Pick< Tenant, - "domain" | "domains" | "locale" | "organization" + "domain" | "allowedDomains" | "locale" | "organization" >; /** @@ -180,6 +180,12 @@ export async function createTenant( sortLabel: "Most Respected", icon: "thumb_up", }, + stories: { + scraping: { + enabled: true, + }, + disableLazy: false, + }, createdAt: now, }; diff --git a/src/core/server/services/stories/index.ts b/src/core/server/services/stories/index.ts index c1900f893..78623ae19 100644 --- a/src/core/server/services/stories/index.ts +++ b/src/core/server/services/stories/index.ts @@ -21,6 +21,8 @@ import { CreateStoryInput, findOrCreateStory, FindOrCreateStoryInput, + findStory, + FindStoryInput, mergeCommentStatusCount, openStory, removeStories, @@ -42,6 +44,21 @@ import { scrape } from "coral-server/services/stories/scraper"; import { AugmentedRedis } from "../redis"; import { isURLPermitted } from "../tenant/url"; +export type FindStory = FindStoryInput; + +export async function find(mongo: Db, tenant: Tenant, input: FindStory) { + // If the URL is provided, and the url is not on a allowed domain, then refuse + // to create the Asset. + if (input.url && !isURLPermitted(tenant, input.url)) { + throw new StoryURLInvalidError({ + storyURL: input.url, + allowedDomains: tenant.allowedDomains, + }); + } + + return findStory(mongo, tenant.id, input); +} + export type FindOrCreateStory = FindOrCreateStoryInput; export async function findOrCreate( @@ -56,20 +73,16 @@ export async function findOrCreate( if (input.url && !isURLPermitted(tenant, input.url)) { throw new StoryURLInvalidError({ storyURL: input.url, - tenantDomains: tenant.domains, + allowedDomains: tenant.allowedDomains, }); } - // TODO: check to see if the tenant has enabled lazy story creation, if they haven't, switch to find only. - const story = await findOrCreateStory(mongo, tenant.id, input, now); if (!story) { return null; } - // TODO: check to see if the tenant has scraping enabled. - - if (!story.metadata && !story.scrapedAt) { + if (tenant.stories.scraping.enabled && !story.metadata && !story.scrapedAt) { // If the scraper has not scraped this story, and we have no metadata, we // need to scrape it now! await scraper.add({ @@ -155,7 +168,10 @@ export async function create( ) { // Ensure that the given URL is allowed. if (!isURLPermitted(tenant, storyURL)) { - throw new StoryURLInvalidError({ storyURL, tenantDomains: tenant.domains }); + throw new StoryURLInvalidError({ + storyURL, + allowedDomains: tenant.allowedDomains, + }); } // Construct the input payload. @@ -173,7 +189,7 @@ export async function create( input, now ); - if (!metadata) { + if (!metadata && tenant.stories.scraping.enabled) { // If the scraper has not scraped this story and story metadata was not // provided, we need to scrape it now! newStory = await scrape(mongo, tenant.id, newStory.id, storyURL); @@ -195,7 +211,7 @@ export async function update( if (input.url && !isURLPermitted(tenant, input.url)) { throw new StoryURLInvalidError({ storyURL: input.url, - tenantDomains: tenant.domains, + allowedDomains: tenant.allowedDomains, }); } diff --git a/src/core/server/services/stories/scraper/index.ts b/src/core/server/services/stories/scraper/index.ts index 6d054b07e..558b9a0b8 100644 --- a/src/core/server/services/stories/scraper/index.ts +++ b/src/core/server/services/stories/scraper/index.ts @@ -6,10 +6,14 @@ import descriptionScraper from "metascraper-description"; import imageScraper from "metascraper-image"; import titleScraper from "metascraper-title"; import { Db } from "mongodb"; +import fetch, { RequestInit } from "node-fetch"; +import ProxyAgent from "proxy-agent"; +import { version } from "coral-common/version"; import { GQLStoryMetadata } from "coral-server/graph/tenant/schema/__generated__/types"; import logger from "coral-server/logger"; import { retrieveStory, updateStory } from "coral-server/models/story"; +import { retrieveTenant } from "coral-server/models/tenant"; import { modifiedScraper } from "./rules/modified"; import { sectionScraper } from "./rules/section"; @@ -30,16 +34,31 @@ class Scraper { this.log = logger.child({ taskName: "scraper" }); } - public async scrape(url: string): Promise { + public async scrape( + url: string, + proxyURL?: string + ): Promise { // Grab the page HTML. const log = this.log.child({ storyURL: url }); + const options: RequestInit = { + headers: { + "User-Agent": `Talk Scraper/${version}`, + }, + }; + if (proxyURL) { + // Force the type here because there's a slight mismatch. + options.agent = (new ProxyAgent( + proxyURL + ) as unknown) as RequestInit["agent"]; + log.debug("using proxy for scrape"); + } + const start = Date.now(); log.debug("starting scrape of Story"); - // TODO: investigate adding scraping proxy support based on the Tenant. - const res = await fetch(url, {}); + const res = await fetch(url, options); if (res.status !== 200) { log.warn( { statusCode: res.status }, @@ -117,6 +136,12 @@ export async function scrape( storyID: string, storyURL?: string ) { + // Grab the Tenant. + const tenant = await retrieveTenant(mongo, tenantID); + if (!tenant) { + throw new Error("tenant not found"); + } + // If the URL wasn't provided, grab it from the database. if (!storyURL) { const retrievedStory = await retrieveStory(mongo, tenantID, storyID); @@ -129,7 +154,10 @@ export async function scrape( } // Get the metadata from the scraped html. - const metadata = await scraper.scrape(storyURL); + const metadata = await scraper.scrape( + storyURL, + tenant.stories.scraping.proxyURL + ); if (!metadata) { throw new Error("story at specified url not found"); } diff --git a/src/core/server/services/tenant/url.spec.ts b/src/core/server/services/tenant/url.spec.ts index dc1c4faf7..cc1637fa9 100644 --- a/src/core/server/services/tenant/url.spec.ts +++ b/src/core/server/services/tenant/url.spec.ts @@ -1,43 +1,49 @@ import { Tenant } from "coral-server/models/tenant"; import { isURLPermitted } from "coral-server/services/tenant/url"; -type PartialTenant = Pick; +type PartialTenant = Pick; function createTenant(input: Partial = {}): PartialTenant { if (!input.domain) { input.domain = ""; } - if (!input.domains) { - input.domains = []; + if (!input.allowedDomains) { + input.allowedDomains = []; } return input as PartialTenant; } it("denies when the tenant has no specified domains", () => { - const tenant = { domains: [] }; + const tenant = { allowedDomains: [] }; expect(isURLPermitted(tenant, "")).toEqual(false); }); it("denies when tenant has a domain but not a valid url", () => { - const tenant = createTenant({ domains: ["https://coralproject.net"] }); + const tenant = createTenant({ allowedDomains: ["https://coralproject.net"] }); expect(isURLPermitted(tenant, "")).toEqual(false); }); -it("denies when there are multiple tenants domains and not a valid url", () => { +it("denies when there are multiple tenants allowedDomains and not a valid url", () => { const tenant = createTenant({ - domains: ["https://coralproject.net", "https://news.coralproject.net"], + allowedDomains: [ + "https://coralproject.net", + "https://news.coralproject.net", + ], }); expect(isURLPermitted(tenant, "")).toEqual(false); }); -it("denies when there are multiple tenants domains and a invalid url", () => { +it("denies when there are multiple tenants allowedDomains and a invalid url", () => { const tenant = createTenant({ - domains: ["https://coralproject.net", "https://news.coralproject.net"], + allowedDomains: [ + "https://coralproject.net", + "https://news.coralproject.net", + ], }); expect( @@ -45,9 +51,12 @@ it("denies when there are multiple tenants domains and a invalid url", () => { ).toEqual(false); }); -it("allows when there are multiple tenants domains and a valid url", () => { +it("allows when there are multiple tenants allowedDomains and a valid url", () => { const tenant = createTenant({ - domains: ["https://coralproject.net", "https://news.coralproject.net"], + allowedDomains: [ + "https://coralproject.net", + "https://news.coralproject.net", + ], }); expect( @@ -55,9 +64,9 @@ it("allows when there are multiple tenants domains and a valid url", () => { ).toEqual(true); }); -it("allows when there are multiple prefix domains and a valid url", () => { +it("allows when there are multiple prefix allowedDomains and a valid url", () => { const tenant = createTenant({ - domains: ["coralproject.net", "news.coralproject.net"], + allowedDomains: ["coralproject.net", "news.coralproject.net"], }); expect(isURLPermitted(tenant, "http://news.coralproject.net/a/page")).toEqual( @@ -65,9 +74,9 @@ it("allows when there are multiple prefix domains and a valid url", () => { ); }); -it("allows when there are some prefix domains and a valid url", () => { +it("allows when there are some prefix allowedDomains and a valid url", () => { const tenant = createTenant({ - domains: ["http://coralproject.net", "news.coralproject.net"], + allowedDomains: ["http://coralproject.net", "news.coralproject.net"], }); expect( diff --git a/src/core/server/services/tenant/url.ts b/src/core/server/services/tenant/url.ts index 8bba4d8a1..40f442b0b 100644 --- a/src/core/server/services/tenant/url.ts +++ b/src/core/server/services/tenant/url.ts @@ -7,13 +7,13 @@ import { import { Tenant } from "coral-server/models/tenant"; export function isURLPermitted( - tenant: Pick, + tenant: Pick, targetURL: string, includeTenantDomain?: false ): boolean; export function isURLPermitted( - tenant: Pick, + tenant: Pick, targetURL: string, includeTenantDomain: true ): boolean; @@ -23,13 +23,13 @@ export function isURLPermitted( * the Tenant's domain configuration. */ export function isURLPermitted( - tenant: Pick, + tenant: Pick, targetURL: string, includeTenantDomain: boolean = false ) { // If there aren't any domains, then we reject it, because no url we have can // satisfy those requirements. - if (tenant.domains.length === 0 && !includeTenantDomain) { + if (tenant.allowedDomains.length === 0 && !includeTenantDomain) { return false; } @@ -48,8 +48,8 @@ export function isURLPermitted( // Create the list of domains to check against. const domains = includeTenantDomain - ? [tenant.domain, ...tenant.domains] - : tenant.domains; + ? [tenant.domain, ...tenant.allowedDomains] + : tenant.allowedDomains; // Loop over all the Tenant domains provided. Prefix the domain of each if it // is required with the target url scheme. Return if at least one match is