wassname/talk
1
0
Fork 0
mirror of https://github.com/wassname/talk.git synced 2026-08-16 11:29:31 +08:00
Code Issues Packages Projects Releases Wiki Activity

[CORL-147] Lazy Story Creation + Proxy (#2302)

Browse Source 
* feat: added toggle for lazy stories and proxy

* feat: improve story loading logic
This commit is contained in:
Wyatt Johnson
2019-05-31 22:28:40 +00:00
committed by GitHub
parent 4ce7512c4b
commit 1302ea1918
26 changed files with 595 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files
package-lock.json
Generated
+304 -11
View File
@@ -3365,6 +3365,16 @@
"@types/node": "*"
}
},
"@types/agent-base": {
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/@types/agent-base/-/agent-base-4.2.0.tgz",
"integrity": "sha512-8mrhPstU+ZX0Ugya8tl5DsDZ1I5ZwQzbL/8PA0z8Gj0k9nql7nkaMzmPVLj+l/nixWaliXi+EBiLA8bptw3z7Q==",
"dev": true,
"requires": {
"@types/events": "*",
"@types/node": "*"
}
},
"@types/anymatch": {
"version": "1.3.1",
"resolved": "https://registry.npmjs.org/@types/anymatch/-/anymatch-1.3.1.tgz",
@@ -4519,6 +4529,14 @@
"integrity": "sha512-z55ocwKBRLryBs394Sm3ushTtBeg6VAeuku7utSoSnsJKvKcnXFIyC6vh27n3rXyxSgkJBBCAvyOn7gSUcTYjg==",
"dev": true
},
"agent-base": {
"version": "4.2.1",
"resolved": "https://registry.npmjs.org/agent-base/-/agent-base-4.2.1.tgz",
"integrity": "sha512-JVwXMr9nHYTUXsBFKUqhJwvlcYU/blreOEUkhNR2eXZIvwd+c+o5V4MgDPKWnMS/56awN3TRzIP+KoPn+roQtg==",
"requires": {
"es6-promisify": "^5.0.0"
}
},
"airbnb-prop-types": {
"version": "2.13.2",
"resolved": "https://registry.npmjs.org/airbnb-prop-types/-/airbnb-prop-types-2.13.2.tgz",
@@ -5198,8 +5216,7 @@
"ast-types": {
"version": "0.11.7",
"resolved": "https://registry.npmjs.org/ast-types/-/ast-types-0.11.7.tgz",
"integrity": "sha512-2mP3TwtkY/aTv5X3ZsMpNAbOnyoC/aMJwJSoaELPkHId0nSQgFcnU4dRW3isxiz7+zBexk0ym3WNVjMiQBnJSw==",
"dev": true
"integrity": "sha512-2mP3TwtkY/aTv5X3ZsMpNAbOnyoC/aMJwJSoaELPkHId0nSQgFcnU4dRW3isxiz7+zBexk0ym3WNVjMiQBnJSw=="
},
"astral-regex": {
"version": "1.0.0",
@@ -9798,6 +9815,21 @@
"assert-plus": "^1.0.0"
}
},
"data-uri-to-buffer": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-2.0.1.tgz",
"integrity": "sha512-OkVVLrerfAKZlW2ZZ3Ve2y65jgiWqBKsTfUIAFbn8nVbPcCZg6l6gikKlEYv0kXcmzqGm6mFq/Jf2vriuEkv8A==",
"requires": {
"@types/node": "^8.0.7"
},
"dependencies": {
"@types/node": {
"version": "8.10.48",
"resolved": "https://registry.npmjs.org/@types/node/-/node-8.10.48.tgz",
"integrity": "sha512-c35YEBTkL4rzXY2ucpSKy+UYHjUBIIkuJbWYbsGIrKLEWU5dgJMmLkkIb3qeC3O3Tpb1ZQCwecscvJTDjDjkRw=="
}
}
},
"data-urls": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/data-urls/-/data-urls-1.1.0.tgz",
@@ -10249,6 +10281,23 @@
}
}
},
"degenerator": {
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/degenerator/-/degenerator-1.0.4.tgz",
"integrity": "sha1-/PSQo37OJmRk2cxDGrmMWBnO0JU=",
"requires": {
"ast-types": "0.x.x",
"escodegen": "1.x.x",
"esprima": "3.x.x"
},
"dependencies": {
"esprima": {
"version": "3.1.3",
"resolved": "https://registry.npmjs.org/esprima/-/esprima-3.1.3.tgz",
"integrity": "sha1-/cpRzuYTOJXjyI1TXOSdv/YqRjM="
}
}
},
"del": {
"version": "4.1.1",
"resolved": "https://registry.npmjs.org/del/-/del-4.1.1.tgz",
@@ -13006,6 +13055,21 @@
"integrity": "sha1-lu258v2wGZWCKyY92KratnSBgbw=",
"dev": true
},
"es6-promisify": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/es6-promisify/-/es6-promisify-5.0.0.tgz",
"integrity": "sha1-UQnWLz5W6pZ8S2NQWu8IKRyKUgM=",
"requires": {
"es6-promise": "^4.0.3"
},
"dependencies": {
"es6-promise": {
"version": "4.2.6",
"resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-4.2.6.tgz",
"integrity": "sha512-aRVgGdnmW2OiySVPUC9e6m+plolMAJKjZnQlCwNSuK5yQ0JN61DZSO1X1Ufd1foqWRAlig0rhduTCHe7sVtK5Q=="
}
}
},
"es6-symbol": {
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/es6-symbol/-/es6-symbol-3.1.1.tgz",
@@ -13047,7 +13111,6 @@
"version": "1.10.0",
"resolved": "https://registry.npmjs.org/escodegen/-/escodegen-1.10.0.tgz",
"integrity": "sha512-fjUOf8johsv23WuIKdNQU4P9t9jhQ4Qzx6pC2uW890OloK3Zs1ZAoCNpg/2larNF501jLl3UNy0kIRcF6VI22g==",
"dev": true,
"requires": {
"esprima": "^3.1.3",
"estraverse": "^4.2.0",
@@ -13059,14 +13122,12 @@
"esprima": {
"version": "3.1.3",
"resolved": "https://registry.npmjs.org/esprima/-/esprima-3.1.3.tgz",
"integrity": "sha1-/cpRzuYTOJXjyI1TXOSdv/YqRjM=",
"dev": true
"integrity": "sha1-/cpRzuYTOJXjyI1TXOSdv/YqRjM="
},
"source-map": {
"version": "0.6.1",
"resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
"integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
"dev": true,
"optional": true
}
}
@@ -13641,6 +13702,11 @@
"schema-utils": "^1.0.0"
}
},
"file-uri-to-path": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz",
"integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw=="
},
"filename-regex": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/filename-regex/-/filename-regex-2.0.1.tgz",
@@ -13989,6 +14055,11 @@
"requires": {
"ms": "^2.1.1"
}
},
"semver": {
"version": "5.7.0",
"resolved": "https://registry.npmjs.org/semver/-/semver-5.7.0.tgz",
"integrity": "sha512-Ya52jSX2u7QKghxeoFGpLwCtGlt7j0oY9DYb5apt9nPlJ42ID+ulTXESnt/qAQcoSERyZ5sl3LDIOw0nAn/5DA=="
}
}
},
@@ -15058,6 +15129,43 @@
}
}
},
"ftp": {
"version": "0.3.10",
"resolved": "https://registry.npmjs.org/ftp/-/ftp-0.3.10.tgz",
"integrity": "sha1-kZfYYa2BQvPmPVqDv+TFn3MwiF0=",
"requires": {
"readable-stream": "1.1.x",
"xregexp": "2.0.0"
},
"dependencies": {
"isarray": {
"version": "0.0.1",
"resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz",
"integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8="
},
"readable-stream": {
"version": "1.1.14",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz",
"integrity": "sha1-fPTFTvZI44EwhMY23SB54WbAgdk=",
"requires": {
"core-util-is": "~1.0.0",
"inherits": "~2.0.1",
"isarray": "0.0.1",
"string_decoder": "~0.10.x"
}
},
"string_decoder": {
"version": "0.10.31",
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz",
"integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ="
},
"xregexp": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/xregexp/-/xregexp-2.0.0.tgz",
"integrity": "sha1-UqY+VsoLhKfzpfPWGHLxJq16WUM="
}
}
},
"function-bind": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz",
@@ -15112,6 +15220,11 @@
"number-is-nan": "^1.0.0"
}
},
"mkdirp": {
"version": "0.5.1",
"resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz",
"integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM="
},
"string-width": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz",
@@ -15171,6 +15284,44 @@
"integrity": "sha1-jpQ9E1jcN1VQVOy+LtsFqhdO3hQ=",
"dev": true
},
"get-uri": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/get-uri/-/get-uri-2.0.3.tgz",
"integrity": "sha512-x5j6Ks7FOgLD/GlvjKwgu7wdmMR55iuRHhn8hj/+gA+eSbxQvZ+AEomq+3MgVEZj1vpi738QahGbCCSIDtXtkw==",
"requires": {
"data-uri-to-buffer": "2",
"debug": "4",
"extend": "~3.0.2",
"file-uri-to-path": "1",
"ftp": "~0.3.10",
"readable-stream": "3"
},
"dependencies": {
"debug": {
"version": "4.1.1",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz",
"integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==",
"requires": {
"ms": "^2.1.1"
}
},
"extend": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz",
"integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g=="
},
"readable-stream": {
"version": "3.3.0",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.3.0.tgz",
"integrity": "sha512-EsI+s3k3XsW+fU8fQACLN59ky34AZ14LoeVZpYwmZvldCFo0r0gnelwF2TcMjLor/BTL5aDJVBMkss0dthToPw==",
"requires": {
"inherits": "^2.0.3",
"string_decoder": "^1.1.1",
"util-deprecate": "^1.0.1"
}
}
}
},
"get-value": {
"version": "2.0.6",
"resolved": "https://registry.npmjs.org/get-value/-/get-value-2.0.6.tgz",
@@ -15317,9 +15468,9 @@
},
"dependencies": {
"glob": {
"version": "7.1.3",
"resolved": "https://registry.npmjs.org/glob/-/glob-7.1.3.tgz",
"integrity": "sha512-vcfuiIxogLV4DlGBHIUOwI0IbrJ8HWPc4MU7HzviGeNho/UJDfi6B5p3sHeWIQ0KGIU0Jpxi5ZHxemQfLkkAwQ==",
"version": "7.1.2",
"resolved": "https://registry.npmjs.org/glob/-/glob-7.1.2.tgz",
"integrity": "sha512-MJTUg1kjuLeQCJ+ccE4Vpa6kKVXkPYJ2mOCQyUuKLcLQsdrMCpBPUi8qVE6+YuaJkozeA9NusTAw3hLr8Xe5EQ==",
"dev": true,
"requires": {
"fs.realpath": "^1.0.0",
@@ -17353,6 +17504,30 @@
"requires-port": "^1.0.0"
}
},
"http-proxy-agent": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-2.1.0.tgz",
"integrity": "sha512-qwHbBLV7WviBl0rQsOzH6o5lwyOIvwp/BdFnvVxXORldu5TmjFfjzBcWUWS5kWAZhmv+JtiDhSuQCp4sBfbIgg==",
"requires": {
"agent-base": "4",
"debug": "3.1.0"
},
"dependencies": {
"debug": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz",
"integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==",
"requires": {
"ms": "2.0.0"
}
},
"ms": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
"integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
}
}
},
"http-proxy-middleware": {
"version": "0.19.1",
"resolved": "https://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-0.19.1.tgz",
@@ -17381,6 +17556,25 @@
"integrity": "sha1-7AbBDgo0wPL68Zn3/X/Hj//QPHM=",
"dev": true
},
"https-proxy-agent": {
"version": "2.2.1",
"resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-2.2.1.tgz",
"integrity": "sha512-HPCTS1LW51bcyMYbxUIOO4HEOlQ1/1qRaFWcyxvwaqUS9TY88aoEuHUY33kuAh1YhVVaDQhLZsnPd+XNARWZlQ==",
"requires": {
"agent-base": "^4.1.0",
"debug": "^3.1.0"
},
"dependencies": {
"debug": {
"version": "3.2.6",
"resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz",
"integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==",
"requires": {
"ms": "^2.1.1"
}
}
}
},
"humanize-duration": {
"version": "3.18.0",
"resolved": "https://registry.npmjs.org/humanize-duration/-/humanize-duration-3.18.0.tgz",
@@ -17879,8 +18073,7 @@
"ip": {
"version": "1.1.5",
"resolved": "https://registry.npmjs.org/ip/-/ip-1.1.5.tgz",
"integrity": "sha1-vd7XARQpCCjAoDnnLvJfWq7ENUo=",
"dev": true
"integrity": "sha1-vd7XARQpCCjAoDnnLvJfWq7ENUo="
},
"ip-regex": {
"version": "1.0.3",
@@ -22395,6 +22588,11 @@
"integrity": "sha512-AO81vsIO1k1sM4Zrd6Hu7regmJN1NSiAja10gc4bX3F0wd+9rQmcuHQaHVQCYIEC8iFXnE+mavh23GOt7wBgug==",
"dev": true
},
"netmask": {
"version": "1.0.6",
"resolved": "https://registry.npmjs.org/netmask/-/netmask-1.0.6.tgz",
"integrity": "sha1-ICl+idhvb2QA8lDZ9Pa0wZRfzTU="
},
"next-tick": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/next-tick/-/next-tick-1.0.0.tgz",
@@ -23800,6 +23998,43 @@
"integrity": "sha1-y8ec26+P1CKOE/Yh8rGiN8GyB7M=",
"dev": true
},
"pac-proxy-agent": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/pac-proxy-agent/-/pac-proxy-agent-3.0.0.tgz",
"integrity": "sha512-AOUX9jES/EkQX2zRz0AW7lSx9jD//hQS8wFXBvcnd/J2Py9KaMJMqV/LPqJssj1tgGufotb2mmopGPR15ODv1Q==",
"requires": {
"agent-base": "^4.2.0",
"debug": "^3.1.0",
"get-uri": "^2.0.0",
"http-proxy-agent": "^2.1.0",
"https-proxy-agent": "^2.2.1",
"pac-resolver": "^3.0.0",
"raw-body": "^2.2.0",
"socks-proxy-agent": "^4.0.1"
},
"dependencies": {
"debug": {
"version": "3.2.6",
"resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz",
"integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==",
"requires": {
"ms": "^2.1.1"
}
}
}
},
"pac-resolver": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/pac-resolver/-/pac-resolver-3.0.0.tgz",
"integrity": "sha512-tcc38bsjuE3XZ5+4vP96OfhOugrX+JcnpUbhfuc4LuXBLQhoTthOstZeoQJBDnQUDYzYmdImKsbz0xSl1/9qeA==",
"requires": {
"co": "^4.6.0",
"degenerator": "^1.0.4",
"ip": "^1.1.5",
"netmask": "^1.0.6",
"thunkify": "^2.1.2"
}
},
"pako": {
"version": "1.0.6",
"resolved": "https://registry.npmjs.org/pako/-/pako-1.0.6.tgz",
@@ -27475,6 +27710,36 @@
"ipaddr.js": "1.6.0"
}
},
"proxy-agent": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/proxy-agent/-/proxy-agent-3.1.0.tgz",
"integrity": "sha512-IkbZL4ClW3wwBL/ABFD2zJ8iP84CY0uKMvBPk/OceQe/cEjrxzN1pMHsLwhbzUoRhG9QbSxYC+Z7LBkTiBNvrA==",
"requires": {
"agent-base": "^4.2.0",
"debug": "^3.1.0",
"http-proxy-agent": "^2.1.0",
"https-proxy-agent": "^2.2.1",
"lru-cache": "^4.1.2",
"pac-proxy-agent": "^3.0.0",
"proxy-from-env": "^1.0.0",
"socks-proxy-agent": "^4.0.1"
},
"dependencies": {
"debug": {
"version": "3.2.6",
"resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz",
"integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==",
"requires": {
"ms": "^2.1.1"
}
}
}
},
"proxy-from-env": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.0.0.tgz",
"integrity": "sha1-M8UDmPcOp+uW0h97gXYwpVeRx+4="
},
"prr": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/prr/-/prr-1.0.1.tgz",
@@ -30752,6 +31017,11 @@
"resolved": "https://registry.npmjs.org/slick/-/slick-1.12.2.tgz",
"integrity": "sha1-vQSN23TefRymkV+qSldXCzVQwtc="
},
"smart-buffer": {
"version": "4.0.2",
"resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.0.2.tgz",
"integrity": "sha512-JDhEpTKzXusOqXZ0BUIdH+CjFdO/CR3tLlf5CN34IypI+xMmXW1uB16OOY8z3cICbJlDAVJzNbwBhNO0wt9OAw=="
},
"smartquotes": {
"version": "2.3.1",
"resolved": "https://registry.npmjs.org/smartquotes/-/smartquotes-2.3.1.tgz",
@@ -30909,6 +31179,24 @@
}
}
},
"socks": {
"version": "2.3.2",
"resolved": "https://registry.npmjs.org/socks/-/socks-2.3.2.tgz",
"integrity": "sha512-pCpjxQgOByDHLlNqlnh/mNSAxIUkyBBuwwhTcV+enZGbDaClPvHdvm6uvOwZfFJkam7cGhBNbb4JxiP8UZkRvQ==",
"requires": {
"ip": "^1.1.5",
"smart-buffer": "4.0.2"
}
},
"socks-proxy-agent": {
"version": "4.0.2",
"resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-4.0.2.tgz",
"integrity": "sha512-NT6syHhI9LmuEMSK6Kd2V7gNv5KFZoLE7V5udWmn0de+3Mkj3UMA/AJPLyeNUVmElCurSHtUdM3ETpR3z770Wg==",
"requires": {
"agent-base": "~4.2.1",
"socks": "~2.3.2"
}
},
"sort-keys": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/sort-keys/-/sort-keys-2.0.0.tgz",
@@ -32282,6 +32570,11 @@
"xtend": "~4.0.0"
}
},
"thunkify": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/thunkify/-/thunkify-2.1.2.tgz",
"integrity": "sha1-+qDp0jDFGsyVyhOjYawFyn4EVT0="
},
"thunky": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/thunky/-/thunky-1.0.3.tgz",
package.json
+2
View File
@@ -116,6 +116,7 @@
"performance-now": "^2.1.0",
"permit": "^0.2.4",
"prom-client": "^11.3.0",
"proxy-agent": "^3.1.0",
"querystringify": "^2.1.0",
"react-relay-network-modern": "^2.4.0",
"source-map-support": "^0.5.12",
@@ -140,6 +141,7 @@
"@coralproject/rte": "^0.10.15",
"@intervolga/optimize-cssnano-plugin": "^1.0.6",
"@types/basic-auth": "^1.1.2",
"@types/agent-base": "^4.2.0",
"@types/bcryptjs": "^2.4.1",
"@types/bull": "^3.5.12",
"@types/bunyan": "^1.8.4",
src/core/client/admin/routes/configure/sections/advanced/components/PermittedDomainsConfig.tsx
+8 -2
View File
@@ -21,7 +21,9 @@ const PermittedDomainsConfig: FunctionComponent<Props> = ({ disabled }) => (
<FormField>
<HorizontalGutter size="full">
<Localized id="configure-advanced-permittedDomains">
<Header container={<label htmlFor="configure-advanced-domains" />}>
<Header
container={<label htmlFor="configure-advanced-allowedDomains" />}
>
Permitted Domains
</Header>
</Localized>
@@ -34,7 +36,11 @@ const PermittedDomainsConfig: FunctionComponent<Props> = ({ disabled }) => (
use is localhost, staging.yourdomain.com, yourdomain.com, etc.
</Typography>
</Localized>
<Field name="domains" parse={parseStringList} format={formatStringList}>
<Field
name="allowedDomains"
parse={parseStringList}
format={formatStringList}
>
{({ input, meta }) => (
<>
<TextField
src/core/client/admin/routes/configure/sections/advanced/containers/PermittedDomainsConfigContainer.tsx
+1 -1
View File
@@ -27,7 +27,7 @@ class PermittedDomainsConfigContainer extends React.Component<Props> {
const enhanced = withFragmentContainer<Props>({
settings: graphql`
fragment PermittedDomainsConfigContainer_settings on Settings {
domains
allowedDomains
}
`,
})(PermittedDomainsConfigContainer);
src/core/client/admin/test/configure/__snapshots__/advanced.spec.tsx.snap
+3 -3
View File
@@ -152,7 +152,7 @@ exports[`renders configure advanced 1`] = `
>
<label
className="Typography-root Typography-heading1 Typography-colorTextPrimary Header-root"
htmlFor="configure-advanced-domains"
htmlFor="configure-advanced-allowedDomains"
>
Permitted Domains
</label>
@@ -172,8 +172,8 @@ yourdomain.com, etc.
autoCorrect="off"
className="TextField-input TextField-colorRegular"
disabled={false}
id="configure-advanced-domains"
name="domains"
id="configure-advanced-allowedDomains"
name="allowedDomains"
onChange={[Function]}
placeholder=""
spellCheck={false}
src/core/client/admin/test/configure/advanced.spec.tsx
+2 -2
View File
@@ -146,7 +146,7 @@ it("change permitted domains to be empty", async () => {
const resolvers = createResolversStub<GQLResolver>({
Mutation: {
updateSettings: ({ variables }) => {
expectAndFail(variables.settings.domains).toEqual([]);
expectAndFail(variables.settings.allowedDomains).toEqual([]);
return {
settings: pureMerge(settings, variables.settings),
};
@@ -190,7 +190,7 @@ it("change permitted domains to include more domains", async () => {
const resolvers = createResolversStub<GQLResolver>({
Mutation: {
updateSettings: ({ variables }) => {
expectAndFail(variables.settings.domains).toEqual([
expectAndFail(variables.settings.allowedDomains).toEqual([
"localhost:8080",
"localhost:3000",
]);
src/core/client/admin/test/fixtures.ts
+1 -1
View File
@@ -40,7 +40,7 @@ export const settings = createFixture<GQLSettings>({
message: "Comments are closed on this story.",
},
customCSSURL: "",
domains: ["localhost:8080"],
allowedDomains: ["localhost:8080"],
editCommentWindowLength: 30000,
communityGuidelines: {
enabled: false,
src/core/client/framework/rest/install.ts
+1 -1
View File
@@ -7,7 +7,7 @@ export interface InstallInput {
contactEmail: string;
url: string;
};
domains: string[];
allowedDomains: string[];
};
user: {
username: string;
src/core/client/install/containers/WizardContainer.tsx
+4 -4
View File
@@ -18,7 +18,7 @@ interface FormData {
username: string;
password: string;
confirmPassword: string;
domains: string[];
allowedDomains: string[];
}
interface WizardContainerState {
@@ -31,7 +31,7 @@ function shapeFinalData(data: FormData): InstallInput {
organizationName,
organizationContactEmail,
organizationURL,
domains,
allowedDomains,
username,
password,
email,
@@ -44,7 +44,7 @@ function shapeFinalData(data: FormData): InstallInput {
contactEmail: organizationContactEmail,
url: organizationURL,
},
domains,
allowedDomains,
},
user: {
username,
@@ -69,7 +69,7 @@ class WizardContainer extends Component<Props, WizardContainerState> {
username: "",
password: "",
confirmPassword: "",
domains: [],
allowedDomains: [],
},
};
src/core/client/install/steps/components/PermittedDomains.tsx
+4 -4
View File
@@ -19,14 +19,14 @@ import {
import BackButton from "./BackButton";
interface FormProps {
domains: string;
allowedDomains: string;
}
export interface PermittedDomainsForm {
onSubmit: OnSubmit<FormProps>;
onGoToPreviousStep: () => void;
data: {
domains: string[];
allowedDomains: string[];
};
}
@@ -35,7 +35,7 @@ const PermittedDomains: FunctionComponent<PermittedDomainsForm> = props => {
<Form
onSubmit={props.onSubmit}
initialValues={{
domains: props.data.domains.join(","),
allowedDomains: props.data.allowedDomains.join(","),
}}
>
{({ handleSubmit, submitting, submitError }) => (
@@ -60,7 +60,7 @@ const PermittedDomains: FunctionComponent<PermittedDomainsForm> = props => {
</CallOut>
)}
<Field name="domains">
<Field name="allowedDomains">
{({ input, meta }) => (
<FormField>
<Localized id="install-permittedDomains-permittedDomains">
src/core/client/install/steps/containers/PermittedDomainsContainer.tsx
+2 -2
View File
@@ -21,8 +21,8 @@ class PermittedDomainsContainer extends Component<
> {
private onSubmit: PermittedDomainsForm["onSubmit"] = async (input, form) => {
try {
const domains = input.domains.split(",");
await this.props.onInstall({ domains });
const allowedDomains = input.allowedDomains.split(",");
await this.props.onInstall({ allowedDomains });
return this.props.onGoToNextStep();
} catch (error) {
return { [FORM_ERROR]: error.message };
src/core/server/app/handlers/api/install.ts
+1 -1
View File
@@ -31,7 +31,7 @@ const TenantInstallBodySchema = Joi.object().keys({
.lowercase()
.email(),
}),
domains: Joi.array().items(
allowedDomains: Joi.array().items(
Joi.string()
.trim()
.uri()
src/core/server/app/middleware/csp/tenant.spec.ts
+17 -8
View File
@@ -2,14 +2,14 @@ import { generateFrameOptions } from "coral-server/app/middleware/csp/tenant";
import { Request } from "coral-server/types/express";
it("denies when the tenant has no specified domains", () => {
const tenant = { domains: [] };
const tenant = { allowedDomains: [] };
const req = {} as Request;
expect(generateFrameOptions(req, tenant)).toEqual("deny");
});
it("allow-from single domain when there is one domain", () => {
const tenant = { domains: ["https://coralproject.net"] };
const tenant = { allowedDomains: ["https://coralproject.net"] };
const req = {} as Request;
expect(generateFrameOptions(req, tenant)).toEqual(
@@ -19,7 +19,10 @@ it("allow-from single domain when there is one domain", () => {
it("deny from the domain when it does not provide and there are multiple tenants domains", () => {
const tenant = {
domains: ["https://coralproject.net", "https://news.coralproject.net"],
allowedDomains: [
"https://coralproject.net",
"https://news.coralproject.net",
],
};
const req = { headers: {}, query: { parentUrl: "" } } as Request;
@@ -28,7 +31,10 @@ it("deny from the domain when it does not provide and there are multiple tenants
it("allows from the domain when it does not provide a match and there are multiple tenants domains", () => {
const tenant = {
domains: ["https://coralproject.net", "https://news.coralproject.net"],
allowedDomains: [
"https://coralproject.net",
"https://news.coralproject.net",
],
};
const req = {
headers: {},
@@ -40,7 +46,10 @@ it("allows from the domain when it does not provide a match and there are multip
it("allows from the domain when it does provide a match and there are multiple tenants domains", () => {
const tenant = {
domains: ["https://coralproject.net", "https://news.coralproject.net"],
allowedDomains: [
"https://coralproject.net",
"https://news.coralproject.net",
],
};
const req = {
headers: {},
@@ -54,7 +63,7 @@ it("allows from the domain when it does provide a match and there are multiple t
it("it prefixes domains of the tenant when generating the frame option", () => {
const tenant = {
domains: ["coralproject.net", "news.coralproject.net"],
allowedDomains: ["coralproject.net", "news.coralproject.net"],
};
const req = {
headers: {},
@@ -68,7 +77,7 @@ it("it prefixes domains of the tenant when generating the frame option", () => {
it("it prefixes domains of the tenant when generating the frame option", () => {
const tenant = {
domains: ["coralproject.net", "news.coralproject.net"],
allowedDomains: ["coralproject.net", "news.coralproject.net"],
};
const req = {
headers: {},
@@ -82,7 +91,7 @@ it("it prefixes domains of the tenant when generating the frame option", () => {
it("it prefixes domains of the tenant when generating the frame option and denies based on it", () => {
const tenant = {
domains: ["coralproject.net", "news.coralproject.net"],
allowedDomains: ["coralproject.net", "news.coralproject.net"],
};
const req = {
headers: {},
src/core/server/app/middleware/csp/tenant.ts
+11 -11
View File
@@ -35,13 +35,13 @@ export const cspTenantMiddleware: RequestHandler = (req, res, next) => {
function generateContentSecurityPolicy(
req: Request,
tenant: Pick<Tenant, "domains">
tenant: Pick<Tenant, "allowedDomains">
) {
const directives: Record<string, any> = {};
// Only the domains that are allowed by the tenant may embed Coral.
directives.frameAncestors =
tenant.domains.length > 0 ? tenant.domains : ["'none'"];
tenant.allowedDomains.length > 0 ? tenant.allowedDomains : ["'none'"];
// Build the directive.
const directive = builder({ directives });
@@ -51,20 +51,20 @@ function generateContentSecurityPolicy(
export function generateFrameOptions(
req: Request,
tenant: Pick<Tenant, "domains">
tenant: Pick<Tenant, "allowedDomains">
) {
// If there aren't any domains, then we reject it.
if (tenant.domains.length === 0) {
if (tenant.allowedDomains.length === 0) {
return "deny";
}
// If there is only one domain on the tenant, and we don't require
// prefixing, then return it!
if (
tenant.domains.length === 1 &&
!doesRequireSchemePrefixing(tenant.domains[0])
tenant.allowedDomains.length === 1 &&
!doesRequireSchemePrefixing(tenant.allowedDomains[0])
) {
return `allow-from ${getOrigin(tenant.domains[0])}`;
return `allow-from ${getOrigin(tenant.allowedDomains[0])}`;
}
const parentsURL = extractParentsURL(req);
@@ -87,11 +87,11 @@ export function generateFrameOptions(
// If there is only one domain on the tenant, and we require prefixing, then
// return it with prefixing!
if (
tenant.domains.length === 1 &&
!doesRequireSchemePrefixing(tenant.domains[0])
tenant.allowedDomains.length === 1 &&
!doesRequireSchemePrefixing(tenant.allowedDomains[0])
) {
return `allow-from ${getOrigin(
prefixSchemeIfRequired(parentSecure, tenant.domains[0])
prefixSchemeIfRequired(parentSecure, tenant.allowedDomains[0])
)}`;
}
@@ -104,7 +104,7 @@ export function generateFrameOptions(
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Frame-Options
// We need to find the domain that is asking so we can respond with the right
// result, sort of like CORS!
const allowFrom = tenant.domains
const allowFrom = tenant.allowedDomains
.map(domain => getOrigin(prefixSchemeIfRequired(parentSecure, domain)))
.find(origin => origin === parentsOrigin);
if (!allowFrom) {
src/core/server/errors/index.ts
+2 -2
View File
@@ -237,7 +237,7 @@ export class URLInvalidError extends CoralError {
...properties
}: {
url: string;
tenantDomains: string[];
allowedDomains: string[];
tenantDomain?: string;
}) {
super({
@@ -250,7 +250,7 @@ export class URLInvalidError extends CoralError {
export class StoryURLInvalidError extends CoralError {
constructor(properties: {
storyURL: string;
tenantDomains: string[];
allowedDomains: string[];
tenantDomain?: string;
}) {
super({
src/core/server/graph/tenant/loaders/Stories.ts
+23 -12
View File
@@ -3,20 +3,25 @@ import DataLoader from "dataloader";
import TenantContext from "coral-server/graph/tenant/context";
import {
GQLSTORY_STATUS,
GQLStoryMetadata,
QueryToStoriesArgs,
} from "coral-server/graph/tenant/schema/__generated__/types";
import { Connection } from "coral-server/models/helpers/connection";
import {
FindOrCreateStoryInput,
retrieveManyStories,
retrieveStoryConnection,
Story,
StoryConnectionInput,
} from "coral-server/models/story";
import { findOrCreate } from "coral-server/services/stories";
import {
find,
findOrCreate,
FindOrCreateStory,
FindStory,
} from "coral-server/services/stories";
import { scraper } from "coral-server/services/stories/scraper";
import { createManyBatchLoadFn } from "./util";
const statusFilter = (
status?: GQLSTORY_STATUS
): StoryConnectionInput["filter"] => {
@@ -61,15 +66,21 @@ const primeStoriesFromConnection = (ctx: TenantContext) => (
export default (ctx: TenantContext) => ({
findOrCreate: new DataLoader(
(inputs: FindOrCreateStoryInput[]) =>
Promise.all(
inputs.map(input =>
findOrCreate(ctx.mongo, ctx.tenant, input, ctx.scraperQueue, ctx.now)
)
),
createManyBatchLoadFn((input: FindOrCreateStory) =>
findOrCreate(ctx.mongo, ctx.tenant, input, ctx.scraperQueue, ctx.now)
),
{
// TODO: (wyattjoh) see if there's something we can do to improve the cache key
cacheKeyFn: (input: FindOrCreateStoryInput) => `${input.id}:${input.url}`,
cacheKeyFn: (input: FindOrCreateStory) => `${input.id}:${input.url}`,
}
),
find: new DataLoader(
createManyBatchLoadFn((input: FindStory) =>
find(ctx.mongo, ctx.tenant, input)
),
{
// TODO: (wyattjoh) see if there's something we can do to improve the cache key
cacheKeyFn: (input: FindStory) => `${input.id}:${input.url}`,
}
),
story: new DataLoader<string, Story | null>(ids =>
@@ -87,7 +98,7 @@ export default (ctx: TenantContext) => ({
...queryFilter(query),
},
}).then(primeStoriesFromConnection(ctx)),
debugScrapeMetadata: new DataLoader<string, GQLStoryMetadata | null>(urls =>
Promise.all(urls.map(url => scraper.scrape(url)))
debugScrapeMetadata: new DataLoader(
createManyBatchLoadFn((url: string) => scraper.scrape(url))
),
});
src/core/server/graph/tenant/loaders/util.ts
+6
View File
@@ -24,3 +24,9 @@ export class SingletonResolver<T> {
return promise;
}
}
export function createManyBatchLoadFn<U, V>(
batchLoadFn: (input: U) => Promise<V>
) {
return (inputs: U[]) => Promise.all(inputs.map(input => batchLoadFn(input)));
}
src/core/server/graph/tenant/resolvers/Query.ts
+4 -1
View File
@@ -3,7 +3,10 @@ import { GQLQueryTypeResolver } from "coral-server/graph/tenant/schema/__generat
import { moderationQueuesResolver } from "./ModerationQueues";
export const Query: Required<GQLQueryTypeResolver<void>> = {
story: (source, args, ctx) => ctx.loaders.Stories.findOrCreate.load(args),
story: (source, args, ctx) =>
ctx.tenant.stories.disableLazy
? ctx.loaders.Stories.find.load(args)
: ctx.loaders.Stories.findOrCreate.load(args),
stories: (source, args, ctx) => ctx.loaders.Stories.connection(args),
user: (source, args, ctx) => ctx.loaders.Users.user.load(args.id),
users: (source, args, ctx) => ctx.loaders.Users.connection(args),
src/core/server/graph/tenant/schema/schema.graphql
+80 -4
View File
@@ -1023,6 +1023,39 @@ type Organization {
url: String!
}
"""
StoryScrapingConfiguration stores the configuration around story scraping.
"""
type StoryScrapingConfiguration {
"""
enabled, when true, enables stories to be scraped. When disabled, stories will
only be looked up instead, and must be created via the API directly.
"""
enabled: Boolean!
"""
proxyURL when specified, allows scraping requests to use the provided proxy.
All requests will then be passed through the appropriote proxy as parsed by
the [proxy-agent](https://www.npmjs.com/package/proxy-agent) package.
"""
proxyURL: String
}
"""
StoryConfiguration stores the configuration for working with stories.
"""
type StoryConfiguration {
"""
scraping stores configuration around story scraping.
"""
scraping: StoryScrapingConfiguration!
"""
disableLazy when true, will only allow lookups of stories created via the API.
"""
disableLazy: Boolean!
}
"""
Settings stores the global settings for a given Tenant.
"""
@@ -1038,9 +1071,9 @@ type Settings {
domain: String! @auth(roles: [ADMIN])
"""
domains will return a given list of permitted domains.
allowedDomains is the list of domains that stories can come from.
"""
domains: [String!]! @auth(roles: [ADMIN])
allowedDomains: [String!]! @auth(roles: [ADMIN])
"""
locale is the specified locale for this Tenant.
@@ -1126,6 +1159,11 @@ type Settings {
"""
reaction: ReactionConfiguration!
"""
stories stores the configuration around stories.
"""
stories: StoryConfiguration! @auth(roles: [ADMIN])
"""
createdAt is the time that the Settings was created at.
"""
@@ -2744,14 +2782,47 @@ input SettingsCloseCommentingInput {
message: String
}
"""
StoryScrapingConfigurationInput stores the configuration around story scraping.
"""
input StoryScrapingConfigurationInput {
"""
enabled, when true, enables stories to be scraped. When disabled, stories will
only be looked up instead, and must be created via the API directly.
"""
enabled: Boolean
"""
proxyURL when specified, allows scraping requests to use the provided proxy.
All requests will then be passed through the appropriote proxy as parsed by
the [proxy-agent](https://www.npmjs.com/package/proxy-agent) package.
"""
proxyURL: String
}
"""
StoryConfiguration stores the configuration for working with stories.
"""
input StoryConfigurationInput {
"""
scraping stores configuration around story scraping.
"""
scraping: StoryScrapingConfigurationInput
"""
disableLazy when true, will only allow lookups of stories created via the API.
"""
disableLazy: Boolean
}
"""
SettingsInput is the partial type of the Settings type for performing mutations.
"""
input SettingsInput {
"""
domains will return a given list of domains that hosts Stories.
allowedDomains is the list of domains that stories can come from.
"""
domains: [String!]
allowedDomains: [String!]
"""
moderation is the moderation mode for all Stories on the site.
@@ -2826,6 +2897,11 @@ input SettingsInput {
charCount stores the character count moderation settings.
"""
charCount: SettingsCharCountInput
"""
stories stores the configuration around stories.
"""
stories: StoryConfigurationInput
}
"""
src/core/server/models/settings.ts
+1
View File
@@ -81,6 +81,7 @@ export type Settings = GlobalModerationSettings &
| "editCommentWindowLength"
| "customCSSURL"
| "communityGuidelines"
| "stories"
| "createdAt"
> & {
/**
src/core/server/models/story/index.ts
+23
View File
@@ -161,6 +161,29 @@ export async function upsertStory(
return result.value || null;
}
export interface FindStoryInput {
id?: string;
url?: string;
}
export async function findStory(
mongo: Db,
tenantID: string,
{ id, url }: FindStoryInput
) {
if (id) {
return retrieveStory(mongo, tenantID, id);
}
if (url) {
return retrieveStoryByURL(mongo, tenantID, url);
}
// Story can't be found with that ID/URL combination and scraping is
// disabled, so we fail here.
return null;
}
export interface FindOrCreateStoryInput {
id?: string;
url?: string;
src/core/server/models/tenant.ts
+8 -2
View File
@@ -29,7 +29,7 @@ export interface TenantResource {
}
export interface TenantSettings
extends Pick<GQLSettings, "domain" | "domains" | "organization"> {
extends Pick<GQLSettings, "domain" | "allowedDomains" | "organization"> {
readonly id: string;
/**
@@ -60,7 +60,7 @@ export async function createTenantIndexes(mongo: Db) {
*/
export type CreateTenantInput = Pick<
Tenant,
"domain" | "domains" | "locale" | "organization"
"domain" | "allowedDomains" | "locale" | "organization"
>;
/**
@@ -180,6 +180,12 @@ export async function createTenant(
sortLabel: "Most Respected",
icon: "thumb_up",
},
stories: {
scraping: {
enabled: true,
},
disableLazy: false,
},
createdAt: now,
};
src/core/server/services/stories/index.ts
+25 -9
View File
@@ -21,6 +21,8 @@ import {
CreateStoryInput,
findOrCreateStory,
FindOrCreateStoryInput,
findStory,
FindStoryInput,
mergeCommentStatusCount,
openStory,
removeStories,
@@ -42,6 +44,21 @@ import { scrape } from "coral-server/services/stories/scraper";
import { AugmentedRedis } from "../redis";
import { isURLPermitted } from "../tenant/url";
export type FindStory = FindStoryInput;
export async function find(mongo: Db, tenant: Tenant, input: FindStory) {
// If the URL is provided, and the url is not on a allowed domain, then refuse
// to create the Asset.
if (input.url && !isURLPermitted(tenant, input.url)) {
throw new StoryURLInvalidError({
storyURL: input.url,
allowedDomains: tenant.allowedDomains,
});
}
return findStory(mongo, tenant.id, input);
}
export type FindOrCreateStory = FindOrCreateStoryInput;
export async function findOrCreate(
@@ -56,20 +73,16 @@ export async function findOrCreate(
if (input.url && !isURLPermitted(tenant, input.url)) {
throw new StoryURLInvalidError({
storyURL: input.url,
tenantDomains: tenant.domains,
allowedDomains: tenant.allowedDomains,
});
}
// TODO: check to see if the tenant has enabled lazy story creation, if they haven't, switch to find only.
const story = await findOrCreateStory(mongo, tenant.id, input, now);
if (!story) {
return null;
}
// TODO: check to see if the tenant has scraping enabled.
if (!story.metadata && !story.scrapedAt) {
if (tenant.stories.scraping.enabled && !story.metadata && !story.scrapedAt) {
// If the scraper has not scraped this story, and we have no metadata, we
// need to scrape it now!
await scraper.add({
@@ -155,7 +168,10 @@ export async function create(
) {
// Ensure that the given URL is allowed.
if (!isURLPermitted(tenant, storyURL)) {
throw new StoryURLInvalidError({ storyURL, tenantDomains: tenant.domains });
throw new StoryURLInvalidError({
storyURL,
allowedDomains: tenant.allowedDomains,
});
}
// Construct the input payload.
@@ -173,7 +189,7 @@ export async function create(
input,
now
);
if (!metadata) {
if (!metadata && tenant.stories.scraping.enabled) {
// If the scraper has not scraped this story and story metadata was not
// provided, we need to scrape it now!
newStory = await scrape(mongo, tenant.id, newStory.id, storyURL);
@@ -195,7 +211,7 @@ export async function update(
if (input.url && !isURLPermitted(tenant, input.url)) {
throw new StoryURLInvalidError({
storyURL: input.url,
tenantDomains: tenant.domains,
allowedDomains: tenant.allowedDomains,
});
}
src/core/server/services/stories/scraper/index.ts
+32 -4
View File
@@ -6,10 +6,14 @@ import descriptionScraper from "metascraper-description";
import imageScraper from "metascraper-image";
import titleScraper from "metascraper-title";
import { Db } from "mongodb";
import fetch, { RequestInit } from "node-fetch";
import ProxyAgent from "proxy-agent";
import { version } from "coral-common/version";
import { GQLStoryMetadata } from "coral-server/graph/tenant/schema/__generated__/types";
import logger from "coral-server/logger";
import { retrieveStory, updateStory } from "coral-server/models/story";
import { retrieveTenant } from "coral-server/models/tenant";
import { modifiedScraper } from "./rules/modified";
import { sectionScraper } from "./rules/section";
@@ -30,16 +34,31 @@ class Scraper {
this.log = logger.child({ taskName: "scraper" });
}
public async scrape(url: string): Promise<GQLStoryMetadata | null> {
public async scrape(
url: string,
proxyURL?: string
): Promise<GQLStoryMetadata | null> {
// Grab the page HTML.
const log = this.log.child({ storyURL: url });
const options: RequestInit = {
headers: {
"User-Agent": `Talk Scraper/${version}`,
},
};
if (proxyURL) {
// Force the type here because there's a slight mismatch.
options.agent = (new ProxyAgent(
proxyURL
) as unknown) as RequestInit["agent"];
log.debug("using proxy for scrape");
}
const start = Date.now();
log.debug("starting scrape of Story");
// TODO: investigate adding scraping proxy support based on the Tenant.
const res = await fetch(url, {});
const res = await fetch(url, options);
if (res.status !== 200) {
log.warn(
{ statusCode: res.status },
@@ -117,6 +136,12 @@ export async function scrape(
storyID: string,
storyURL?: string
) {
// Grab the Tenant.
const tenant = await retrieveTenant(mongo, tenantID);
if (!tenant) {
throw new Error("tenant not found");
}
// If the URL wasn't provided, grab it from the database.
if (!storyURL) {
const retrievedStory = await retrieveStory(mongo, tenantID, storyID);
@@ -129,7 +154,10 @@ export async function scrape(
}
// Get the metadata from the scraped html.
const metadata = await scraper.scrape(storyURL);
const metadata = await scraper.scrape(
storyURL,
tenant.stories.scraping.proxyURL
);
if (!metadata) {
throw new Error("story at specified url not found");
}
src/core/server/services/tenant/url.spec.ts
+24 -15
View File
@@ -1,43 +1,49 @@
import { Tenant } from "coral-server/models/tenant";
import { isURLPermitted } from "coral-server/services/tenant/url";
type PartialTenant = Pick<Tenant, "domains" | "domain">;
type PartialTenant = Pick<Tenant, "allowedDomains" | "domain">;
function createTenant(input: Partial<PartialTenant> = {}): PartialTenant {
if (!input.domain) {
input.domain = "";
}
if (!input.domains) {
input.domains = [];
if (!input.allowedDomains) {
input.allowedDomains = [];
}
return input as PartialTenant;
}
it("denies when the tenant has no specified domains", () => {
const tenant = { domains: [] };
const tenant = { allowedDomains: [] };
expect(isURLPermitted(tenant, "")).toEqual(false);
});
it("denies when tenant has a domain but not a valid url", () => {
const tenant = createTenant({ domains: ["https://coralproject.net"] });
const tenant = createTenant({ allowedDomains: ["https://coralproject.net"] });
expect(isURLPermitted(tenant, "")).toEqual(false);
});
it("denies when there are multiple tenants domains and not a valid url", () => {
it("denies when there are multiple tenants allowedDomains and not a valid url", () => {
const tenant = createTenant({
domains: ["https://coralproject.net", "https://news.coralproject.net"],
allowedDomains: [
"https://coralproject.net",
"https://news.coralproject.net",
],
});
expect(isURLPermitted(tenant, "")).toEqual(false);
});
it("denies when there are multiple tenants domains and a invalid url", () => {
it("denies when there are multiple tenants allowedDomains and a invalid url", () => {
const tenant = createTenant({
domains: ["https://coralproject.net", "https://news.coralproject.net"],
allowedDomains: [
"https://coralproject.net",
"https://news.coralproject.net",
],
});
expect(
@@ -45,9 +51,12 @@ it("denies when there are multiple tenants domains and a invalid url", () => {
).toEqual(false);
});
it("allows when there are multiple tenants domains and a valid url", () => {
it("allows when there are multiple tenants allowedDomains and a valid url", () => {
const tenant = createTenant({
domains: ["https://coralproject.net", "https://news.coralproject.net"],
allowedDomains: [
"https://coralproject.net",
"https://news.coralproject.net",
],
});
expect(
@@ -55,9 +64,9 @@ it("allows when there are multiple tenants domains and a valid url", () => {
).toEqual(true);
});
it("allows when there are multiple prefix domains and a valid url", () => {
it("allows when there are multiple prefix allowedDomains and a valid url", () => {
const tenant = createTenant({
domains: ["coralproject.net", "news.coralproject.net"],
allowedDomains: ["coralproject.net", "news.coralproject.net"],
});
expect(isURLPermitted(tenant, "http://news.coralproject.net/a/page")).toEqual(
@@ -65,9 +74,9 @@ it("allows when there are multiple prefix domains and a valid url", () => {
);
});
it("allows when there are some prefix domains and a valid url", () => {
it("allows when there are some prefix allowedDomains and a valid url", () => {
const tenant = createTenant({
domains: ["http://coralproject.net", "news.coralproject.net"],
allowedDomains: ["http://coralproject.net", "news.coralproject.net"],
});
expect(
src/core/server/services/tenant/url.ts
+6 -6
View File
@@ -7,13 +7,13 @@ import {
import { Tenant } from "coral-server/models/tenant";
export function isURLPermitted(
tenant: Pick<Tenant, "domains">,
tenant: Pick<Tenant, "allowedDomains">,
targetURL: string,
includeTenantDomain?: false
): boolean;
export function isURLPermitted(
tenant: Pick<Tenant, "domains" | "domain">,
tenant: Pick<Tenant, "allowedDomains" | "domain">,
targetURL: string,
includeTenantDomain: true
): boolean;
@@ -23,13 +23,13 @@ export function isURLPermitted(
* the Tenant's domain configuration.
*/
export function isURLPermitted(
tenant: Pick<Tenant, "domains" | "domain">,
tenant: Pick<Tenant, "allowedDomains" | "domain">,
targetURL: string,
includeTenantDomain: boolean = false
) {
// If there aren't any domains, then we reject it, because no url we have can
// satisfy those requirements.
if (tenant.domains.length === 0 && !includeTenantDomain) {
if (tenant.allowedDomains.length === 0 && !includeTenantDomain) {
return false;
}
@@ -48,8 +48,8 @@ export function isURLPermitted(
// Create the list of domains to check against.
const domains = includeTenantDomain
? [tenant.domain, ...tenant.domains]
: tenant.domains;
? [tenant.domain, ...tenant.allowedDomains]
: tenant.allowedDomains;
// Loop over all the Tenant domains provided. Prefix the domain of each if it
// is required with the target url scheme. Return if at least one match is