diff --git a/package-lock.json b/package-lock.json index 58e2db337..a08665dc8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -69,6 +69,14 @@ "dev": true, "requires": { "ms": "2.0.0" + }, + "dependencies": { + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=", + "dev": true + } } }, "expand-brackets": { @@ -1310,6 +1318,14 @@ "dev": true, "requires": { "ms": "2.0.0" + }, + "dependencies": { + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=", + "dev": true + } } } } @@ -1911,11 +1927,26 @@ "@types/node": "*" } }, + "@types/ms": { + "version": "0.7.30", + "resolved": "https://registry.npmjs.org/@types/ms/-/ms-0.7.30.tgz", + "integrity": "sha512-OftRLCgAzJP7vmKn9by/GVjnf4hloz/pXNOwPo0vKGAfXI7GqWXJi9N2kRar4cP5s1dGwuwcagWqO6iHBTq1Mg==", + "dev": true + }, "@types/node": { "version": "10.5.2", "resolved": "https://registry.npmjs.org/@types/node/-/node-10.5.2.tgz", "integrity": "sha512-m9zXmifkZsMHZBOyxZWilMwmTlpC8x5Ty360JKTiXvlXZfBWYpsg9ZZvP/Ye+iZUh+Q+MxDLjItVTWIsfwz+8Q==" }, + "@types/node-fetch": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.1.2.tgz", + "integrity": "sha512-XroxUzLpKuL+CVkQqXlffRkEPi4Gh3Oui/mWyS7ztKiyqVxiU+h3imCW5I2NQmde5jK+3q++36/Q96cyRWsweg==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, "@types/nunjucks": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/@types/nunjucks/-/nunjucks-3.0.0.tgz", @@ -2211,6 +2242,14 @@ "dev": true, "requires": { "ms": "2.0.0" + }, + "dependencies": { + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=", + "dev": true + } } } } @@ -2243,6 +2282,14 @@ "dev": true, "requires": { "ms": "2.0.0" + }, + "dependencies": { + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=", + "dev": true + } } } } @@ -2279,6 +2326,14 @@ "dev": true, "requires": { "ms": "2.0.0" + }, + "dependencies": { + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=", + "dev": true + } } } } @@ -2309,6 +2364,14 @@ "dev": true, "requires": { "ms": "2.0.0" + }, + "dependencies": { + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=", + "dev": true + } } } } @@ -2361,6 +2424,14 @@ "dev": true, "requires": { "ms": "2.0.0" + }, + "dependencies": { + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=", + "dev": true + } } } } @@ -2398,6 +2469,14 @@ "dev": true, "requires": { "ms": "2.0.0" + }, + "dependencies": { + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=", + "dev": true + } } } } @@ -4821,6 +4900,14 @@ "dev": true, "requires": { "ms": "2.0.0" + }, + "dependencies": { + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=", + "dev": true + } } }, "expand-brackets": { @@ -6879,6 +6966,11 @@ "whatwg-fetch": "2.0.3" }, "dependencies": { + "node-fetch": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.0.0.tgz", + "integrity": "sha1-mCu6Q+zU8pIqKcwYamu7C7c/y6Y=" + }, "whatwg-fetch": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/whatwg-fetch/-/whatwg-fetch-2.0.3.tgz", @@ -7383,6 +7475,13 @@ "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", "requires": { "ms": "2.0.0" + }, + "dependencies": { + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + } } }, "debug-fabulous": { @@ -7404,6 +7503,12 @@ "requires": { "ms": "2.0.0" } + }, + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=", + "dev": true } } }, @@ -8320,6 +8425,14 @@ "dev": true, "requires": { "ms": "2.0.0" + }, + "dependencies": { + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=", + "dev": true + } } }, "decamelize": { @@ -8348,6 +8461,14 @@ "dev": true, "requires": { "ms": "2.0.0" + }, + "dependencies": { + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=", + "dev": true + } } } } @@ -9834,13 +9955,6 @@ "dev": true, "requires": { "intl-pluralrules": "github:projectfluent/IntlPluralRules#94cb0fa1c23ad943bc5aafef43cea132fa51d68b" - }, - "dependencies": { - "intl-pluralrules": { - "version": "github:projectfluent/IntlPluralRules#94cb0fa1c23ad943bc5aafef43cea132fa51d68b", - "from": "github:projectfluent/IntlPluralRules#module", - "dev": true - } } }, "fluent-langneg": { @@ -9885,6 +9999,14 @@ "dev": true, "requires": { "ms": "2.0.0" + }, + "dependencies": { + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=", + "dev": true + } } } } @@ -10045,24 +10167,24 @@ "dependencies": { "abbrev": { "version": "1.1.1", - "resolved": "", + "resolved": false, "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==", "optional": true }, "ansi-regex": { "version": "2.1.1", - "resolved": "", + "resolved": false, "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=" }, "aproba": { "version": "1.2.0", - "resolved": "", + "resolved": false, "integrity": "sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw==", "optional": true }, "are-we-there-yet": { "version": "1.1.4", - "resolved": "", + "resolved": false, "integrity": "sha1-u13KOCu5TwXhUZQ3PRb9O6HKEQ0=", "optional": true, "requires": { @@ -10072,12 +10194,12 @@ }, "balanced-match": { "version": "1.0.0", - "resolved": "", + "resolved": false, "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" }, "brace-expansion": { "version": "1.1.11", - "resolved": "", + "resolved": false, "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", "requires": { "balanced-match": "^1.0.0", @@ -10086,34 +10208,34 @@ }, "chownr": { "version": "1.0.1", - "resolved": "", + "resolved": false, "integrity": "sha1-4qdQQqlVGQi+vSW4Uj1fl2nXkYE=", "optional": true }, "code-point-at": { "version": "1.1.0", - "resolved": "", + "resolved": false, "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=" }, "concat-map": { "version": "0.0.1", - "resolved": "", + "resolved": false, "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" }, "console-control-strings": { "version": "1.1.0", - "resolved": "", + "resolved": false, "integrity": "sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4=" }, "core-util-is": { "version": "1.0.2", - "resolved": "", + "resolved": false, "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=", "optional": true }, "debug": { "version": "2.6.9", - "resolved": "", + "resolved": false, "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", "optional": true, "requires": { @@ -10122,25 +10244,25 @@ }, "deep-extend": { "version": "0.5.1", - "resolved": "", + "resolved": false, "integrity": "sha512-N8vBdOa+DF7zkRrDCsaOXoCs/E2fJfx9B9MrKnnSiHNh4ws7eSys6YQE4KvT1cecKmOASYQBhbKjeuDD9lT81w==", "optional": true }, "delegates": { "version": "1.0.0", - "resolved": "", + "resolved": false, "integrity": "sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o=", "optional": true }, "detect-libc": { "version": "1.0.3", - "resolved": "", + "resolved": false, "integrity": "sha1-+hN8S9aY7fVc1c0CrFWfkaTEups=", "optional": true }, "fs-minipass": { "version": "1.2.5", - "resolved": "", + "resolved": false, "integrity": "sha512-JhBl0skXjUPCFH7x6x61gQxrKyXsxB5gcgePLZCwfyCGGsTISMoIeObbrvVeP6Xmyaudw4TT43qV2Gz+iyd2oQ==", "optional": true, "requires": { @@ -10149,13 +10271,13 @@ }, "fs.realpath": { "version": "1.0.0", - "resolved": "", + "resolved": false, "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=", "optional": true }, "gauge": { "version": "2.7.4", - "resolved": "", + "resolved": false, "integrity": "sha1-LANAXHU4w51+s3sxcCLjJfsBi/c=", "optional": true, "requires": { @@ -10171,7 +10293,7 @@ }, "glob": { "version": "7.1.2", - "resolved": "", + "resolved": false, "integrity": "sha512-MJTUg1kjuLeQCJ+ccE4Vpa6kKVXkPYJ2mOCQyUuKLcLQsdrMCpBPUi8qVE6+YuaJkozeA9NusTAw3hLr8Xe5EQ==", "optional": true, "requires": { @@ -10185,13 +10307,13 @@ }, "has-unicode": { "version": "2.0.1", - "resolved": "", + "resolved": false, "integrity": "sha1-4Ob+aijPUROIVeCG0Wkedx3iqLk=", "optional": true }, "iconv-lite": { "version": "0.4.21", - "resolved": "", + "resolved": false, "integrity": "sha512-En5V9za5mBt2oUA03WGD3TwDv0MKAruqsuxstbMUZaj9W9k/m1CV/9py3l0L5kw9Bln8fdHQmzHSYtvpvTLpKw==", "optional": true, "requires": { @@ -10200,7 +10322,7 @@ }, "ignore-walk": { "version": "3.0.1", - "resolved": "", + "resolved": false, "integrity": "sha512-DTVlMx3IYPe0/JJcYP7Gxg7ttZZu3IInhuEhbchuqneY9wWe5Ojy2mXLBaQFUQmo0AW2r3qG7m1mg86js+gnlQ==", "optional": true, "requires": { @@ -10209,7 +10331,7 @@ }, "inflight": { "version": "1.0.6", - "resolved": "", + "resolved": false, "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", "optional": true, "requires": { @@ -10219,18 +10341,18 @@ }, "inherits": { "version": "2.0.3", - "resolved": "", + "resolved": false, "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" }, "ini": { "version": "1.3.5", - "resolved": "", + "resolved": false, "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==", "optional": true }, "is-fullwidth-code-point": { "version": "1.0.0", - "resolved": "", + "resolved": false, "integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=", "requires": { "number-is-nan": "^1.0.0" @@ -10238,13 +10360,13 @@ }, "isarray": { "version": "1.0.0", - "resolved": "", + "resolved": false, "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=", "optional": true }, "minimatch": { "version": "3.0.4", - "resolved": "", + "resolved": false, "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", "requires": { "brace-expansion": "^1.1.7" @@ -10252,12 +10374,12 @@ }, "minimist": { "version": "0.0.8", - "resolved": "", + "resolved": false, "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=" }, "minipass": { "version": "2.2.4", - "resolved": "", + "resolved": false, "integrity": "sha512-hzXIWWet/BzWhYs2b+u7dRHlruXhwdgvlTMDKC6Cb1U7ps6Ac6yQlR39xsbjWJE377YTCtKwIXIpJ5oP+j5y8g==", "requires": { "safe-buffer": "^5.1.1", @@ -10266,7 +10388,7 @@ }, "minizlib": { "version": "1.1.0", - "resolved": "", + "resolved": false, "integrity": "sha512-4T6Ur/GctZ27nHfpt9THOdRZNgyJ9FZchYO1ceg5S8Q3DNLCKYy44nCZzgCJgcvx2UM8czmqak5BCxJMrq37lA==", "optional": true, "requires": { @@ -10275,7 +10397,7 @@ }, "mkdirp": { "version": "0.5.1", - "resolved": "", + "resolved": false, "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", "requires": { "minimist": "0.0.8" @@ -10283,13 +10405,13 @@ }, "ms": { "version": "2.0.0", - "resolved": "", + "resolved": false, "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=", "optional": true }, "needle": { "version": "2.2.0", - "resolved": "", + "resolved": false, "integrity": "sha512-eFagy6c+TYayorXw/qtAdSvaUpEbBsDwDyxYFgLZ0lTojfH7K+OdBqAF7TAFwDokJaGpubpSGG0wO3iC0XPi8w==", "optional": true, "requires": { @@ -10300,7 +10422,7 @@ }, "node-pre-gyp": { "version": "0.10.0", - "resolved": "", + "resolved": false, "integrity": "sha512-G7kEonQLRbcA/mOoFoxvlMrw6Q6dPf92+t/l0DFSMuSlDoWaI9JWIyPwK0jyE1bph//CUEL65/Fz1m2vJbmjQQ==", "optional": true, "requires": { @@ -10318,7 +10440,7 @@ }, "nopt": { "version": "4.0.1", - "resolved": "", + "resolved": false, "integrity": "sha1-0NRoWv1UFRk8jHUFYC0NF81kR00=", "optional": true, "requires": { @@ -10328,13 +10450,13 @@ }, "npm-bundled": { "version": "1.0.3", - "resolved": "", + "resolved": false, "integrity": "sha512-ByQ3oJ/5ETLyglU2+8dBObvhfWXX8dtPZDMePCahptliFX2iIuhyEszyFk401PZUNQH20vvdW5MLjJxkwU80Ow==", "optional": true }, "npm-packlist": { "version": "1.1.10", - "resolved": "", + "resolved": false, "integrity": "sha512-AQC0Dyhzn4EiYEfIUjCdMl0JJ61I2ER9ukf/sLxJUcZHfo+VyEfz2rMJgLZSS1v30OxPQe1cN0LZA1xbcaVfWA==", "optional": true, "requires": { @@ -10344,7 +10466,7 @@ }, "npmlog": { "version": "4.1.2", - "resolved": "", + "resolved": false, "integrity": "sha512-2uUqazuKlTaSI/dC8AzicUck7+IrEaOnN/e0jd3Xtt1KcGpwx30v50mL7oPyr/h9bL3E4aZccVwpwP+5W9Vjkg==", "optional": true, "requires": { @@ -10356,18 +10478,18 @@ }, "number-is-nan": { "version": "1.0.1", - "resolved": "", + "resolved": false, "integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0=" }, "object-assign": { "version": "4.1.1", - "resolved": "", + "resolved": false, "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=", "optional": true }, "once": { "version": "1.4.0", - "resolved": "", + "resolved": false, "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", "requires": { "wrappy": "1" @@ -10375,19 +10497,19 @@ }, "os-homedir": { "version": "1.0.2", - "resolved": "", + "resolved": false, "integrity": "sha1-/7xJiDNuDoM94MFox+8VISGqf7M=", "optional": true }, "os-tmpdir": { "version": "1.0.2", - "resolved": "", + "resolved": false, "integrity": "sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ=", "optional": true }, "osenv": { "version": "0.1.5", - "resolved": "", + "resolved": false, "integrity": "sha512-0CWcCECdMVc2Rw3U5w9ZjqX6ga6ubk1xDVKxtBQPK7wis/0F2r9T6k4ydGYhecl7YUBxBVxhL5oisPsNxAPe2g==", "optional": true, "requires": { @@ -10397,19 +10519,19 @@ }, "path-is-absolute": { "version": "1.0.1", - "resolved": "", + "resolved": false, "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=", "optional": true }, "process-nextick-args": { "version": "2.0.0", - "resolved": "", + "resolved": false, "integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw==", "optional": true }, "rc": { "version": "1.2.7", - "resolved": "", + "resolved": false, "integrity": "sha512-LdLD8xD4zzLsAT5xyushXDNscEjB7+2ulnl8+r1pnESlYtlJtVSoCMBGr30eDRJ3+2Gq89jK9P9e4tCEH1+ywA==", "optional": true, "requires": { @@ -10421,7 +10543,7 @@ "dependencies": { "minimist": { "version": "1.2.0", - "resolved": "", + "resolved": false, "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=", "optional": true } @@ -10429,7 +10551,7 @@ }, "readable-stream": { "version": "2.3.6", - "resolved": "", + "resolved": false, "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", "optional": true, "requires": { @@ -10444,7 +10566,7 @@ }, "rimraf": { "version": "2.6.2", - "resolved": "", + "resolved": false, "integrity": "sha512-lreewLK/BlghmxtfH36YYVg1i8IAce4TI7oao75I1g245+6BctqTVQiBP3YUJ9C6DQOXJmkYR9X9fCLtCOJc5w==", "optional": true, "requires": { @@ -10453,42 +10575,42 @@ }, "safe-buffer": { "version": "5.1.1", - "resolved": "", + "resolved": false, "integrity": "sha512-kKvNJn6Mm93gAczWVJg7wH+wGYWNrDHdWvpUmHyEsgCtIwwo3bqPtV4tR5tuPaUhTOo/kvhVwd8XwwOllGYkbg==" }, "safer-buffer": { "version": "2.1.2", - "resolved": "", + "resolved": false, "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", "optional": true }, "sax": { "version": "1.2.4", - "resolved": "", + "resolved": false, "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==", "optional": true }, "semver": { "version": "5.5.0", - "resolved": "", + "resolved": false, "integrity": "sha512-4SJ3dm0WAwWy/NVeioZh5AntkdJoWKxHxcmyP622fOkgHa4z3R0TdBJICINyaSDE6uNwVc8gZr+ZinwZAH4xIA==", "optional": true }, "set-blocking": { "version": "2.0.0", - "resolved": "", + "resolved": false, "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=", "optional": true }, "signal-exit": { "version": "3.0.2", - "resolved": "", + "resolved": false, "integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0=", "optional": true }, "string-width": { "version": "1.0.2", - "resolved": "", + "resolved": false, "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=", "requires": { "code-point-at": "^1.0.0", @@ -10498,7 +10620,7 @@ }, "string_decoder": { "version": "1.1.1", - "resolved": "", + "resolved": false, "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", "optional": true, "requires": { @@ -10507,7 +10629,7 @@ }, "strip-ansi": { "version": "3.0.1", - "resolved": "", + "resolved": false, "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", "requires": { "ansi-regex": "^2.0.0" @@ -10515,13 +10637,13 @@ }, "strip-json-comments": { "version": "2.0.1", - "resolved": "", + "resolved": false, "integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo=", "optional": true }, "tar": { "version": "4.4.1", - "resolved": "", + "resolved": false, "integrity": "sha512-O+v1r9yN4tOsvl90p5HAP4AEqbYhx4036AGMm075fH9F8Qwi3oJ+v4u50FkT/KkvywNGtwkk0zRI+8eYm1X/xg==", "optional": true, "requires": { @@ -10536,13 +10658,13 @@ }, "util-deprecate": { "version": "1.0.2", - "resolved": "", + "resolved": false, "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=", "optional": true }, "wide-align": { "version": "1.1.2", - "resolved": "", + "resolved": false, "integrity": "sha512-ijDLlyQ7s6x1JgCLur53osjm/UXUYD9+0PbYKrBsYisYXzCxN+HC3mYDNy/dWdmf3AwqwU3CXwDCvsNgGK1S0w==", "optional": true, "requires": { @@ -10551,12 +10673,12 @@ }, "wrappy": { "version": "1.0.2", - "resolved": "", + "resolved": false, "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" }, "yallist": { "version": "3.0.2", - "resolved": "", + "resolved": false, "integrity": "sha1-hFK0u36Dx8GI2AQcGoN8dz1ti7k=" } } @@ -11236,15 +11358,6 @@ "is-utf8": "^0.2.0" } }, - "v8flags": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/v8flags/-/v8flags-3.1.1.tgz", - "integrity": "sha512-iw/1ViSEaff8NJ3HLyEjawk/8hjJib3E7pvG4pddVXfUg1983s3VGsiClDjhK64MQVDGqc1Q8r18S4VKQZS9EQ==", - "dev": true, - "requires": { - "homedir-polyfill": "^1.0.1" - } - }, "which-module": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/which-module/-/which-module-1.0.0.tgz", @@ -11346,113 +11459,11 @@ "integrity": "sha512-yAdfUZ+c2wetVNIFsNRn44THW+Lty6S5TwMpUfLA/UaGhiXbBv/F8E60/1hMLd0cnF/CDoWH8vzVaI5bAcHCjw==", "dev": true }, - "clone": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/clone/-/clone-2.1.2.tgz", - "integrity": "sha1-G39Ln1kfHo+DZwQBYANFoCiHQ18=", - "dev": true - }, - "clone-stats": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/clone-stats/-/clone-stats-1.0.0.tgz", - "integrity": "sha1-s3gt/4u1R04Yuba/D9/ngvh3doA=", - "dev": true - }, - "glob": { - "version": "7.1.2", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.2.tgz", - "integrity": "sha512-MJTUg1kjuLeQCJ+ccE4Vpa6kKVXkPYJ2mOCQyUuKLcLQsdrMCpBPUi8qVE6+YuaJkozeA9NusTAw3hLr8Xe5EQ==", - "dev": true, - "requires": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^3.0.4", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" - } - }, - "glob-stream": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/glob-stream/-/glob-stream-6.1.0.tgz", - "integrity": "sha1-cEXJlBOz65SIjYOrRtC0BMx73eQ=", - "dev": true, - "requires": { - "extend": "^3.0.0", - "glob": "^7.1.1", - "glob-parent": "^3.1.0", - "is-negated-glob": "^1.0.0", - "ordered-read-streams": "^1.0.0", - "pumpify": "^1.3.5", - "readable-stream": "^2.1.5", - "remove-trailing-separator": "^1.0.1", - "to-absolute-glob": "^2.0.0", - "unique-stream": "^2.0.2" - } - }, - "ordered-read-streams": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/ordered-read-streams/-/ordered-read-streams-1.0.1.tgz", - "integrity": "sha1-d8DLN8QVJdZBZtmQ/61+xqDhNj4=", - "dev": true, - "requires": { - "readable-stream": "^2.0.1" - } - }, "source-map": { "version": "0.7.3", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.7.3.tgz", "integrity": "sha512-CkCj6giN3S+n9qrYiBTX5gystlENnRW5jZeNLHpe6aue+SrHcG5VYwujhW9s4dY31mEGsxBDrHR6oI69fTXsaQ==", "dev": true - }, - "unique-stream": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/unique-stream/-/unique-stream-2.2.1.tgz", - "integrity": "sha1-WqADz76Uxf+GbE59ZouxxNuts2k=", - "dev": true, - "requires": { - "json-stable-stringify": "^1.0.0", - "through2-filter": "^2.0.0" - } - }, - "vinyl": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/vinyl/-/vinyl-2.2.0.tgz", - "integrity": "sha512-MBH+yP0kC/GQ5GwBqrTPTzEfiiLjta7hTtvQtbxBgTeSXsmKQRQecjibMbxIXzVT3Y9KJK+drOz1/k+vsu8Nkg==", - "dev": true, - "requires": { - "clone": "^2.1.1", - "clone-buffer": "^1.0.0", - "clone-stats": "^1.0.0", - "cloneable-readable": "^1.0.0", - "remove-trailing-separator": "^1.0.1", - "replace-ext": "^1.0.0" - } - }, - "vinyl-fs": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/vinyl-fs/-/vinyl-fs-3.0.3.tgz", - "integrity": "sha512-vIu34EkyNyJxmP0jscNzWBSygh7VWhqun6RmqVfXePrOwi9lhvRs//dOaGOTRUQr4tx7/zd26Tk5WeSVZitgng==", - "dev": true, - "requires": { - "fs-mkdirp-stream": "^1.0.0", - "glob-stream": "^6.1.0", - "graceful-fs": "^4.0.0", - "is-valid-glob": "^1.0.0", - "lazystream": "^1.0.0", - "lead": "^1.0.0", - "object.assign": "^4.0.4", - "pumpify": "^1.3.5", - "readable-stream": "^2.3.3", - "remove-bom-buffer": "^3.0.0", - "remove-bom-stream": "^1.2.0", - "resolve-options": "^1.1.0", - "through2": "^2.0.0", - "to-through": "^2.0.0", - "value-or-function": "^3.0.0", - "vinyl": "^2.0.0", - "vinyl-sourcemap": "^1.1.0" - } } } }, @@ -12373,6 +12384,11 @@ "integrity": "sha1-ftGxQQxqDg94z5XTuEQMY/eLhhQ=", "dev": true }, + "intl-pluralrules": { + "version": "github:projectfluent/IntlPluralRules#94cb0fa1c23ad943bc5aafef43cea132fa51d68b", + "from": "github:projectfluent/IntlPluralRules#module", + "dev": true + }, "invariant": { "version": "2.2.4", "resolved": "https://registry.npmjs.org/invariant/-/invariant-2.2.4.tgz", @@ -13093,6 +13109,14 @@ "dev": true, "requires": { "ms": "2.0.0" + }, + "dependencies": { + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=", + "dev": true + } } }, "glob": { @@ -15162,6 +15186,14 @@ "dev": true, "requires": { "ms": "2.0.0" + }, + "dependencies": { + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=", + "dev": true + } } }, "koa-compose": { @@ -15238,6 +15270,14 @@ "dev": true, "requires": { "ms": "2.0.0" + }, + "dependencies": { + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=", + "dev": true + } } } } @@ -15259,6 +15299,14 @@ "dev": true, "requires": { "ms": "2.0.0" + }, + "dependencies": { + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=", + "dev": true + } } } } @@ -16412,9 +16460,9 @@ } }, "ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz", + "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==" }, "multicast-dns": { "version": "6.2.3", @@ -16628,9 +16676,9 @@ } }, "node-fetch": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.0.0.tgz", - "integrity": "sha1-mCu6Q+zU8pIqKcwYamu7C7c/y6Y=" + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.2.0.tgz", + "integrity": "sha512-OayFWziIxiHY8bCUyLX6sTpDH8Jsbp4FfYd1j1f7vZyfgkcOnAyM4oQR16f8a0s7Gl/viMGRey8eScYk4V4EZA==" }, "node-forge": { "version": "0.7.5", @@ -17701,9 +17749,9 @@ "dev": true }, "popper.js": { - "version": "1.14.4", - "resolved": "https://registry.npmjs.org/popper.js/-/popper.js-1.14.4.tgz", - "integrity": "sha1-juwdj/AqWjoVLdQ0FKFce3n9abY=", + "version": "1.14.3", + "resolved": "https://registry.npmjs.org/popper.js/-/popper.js-1.14.3.tgz", + "integrity": "sha1-FDj5jQRqz3tNeM1QK/QYrGTU8JU=", "dev": true }, "portfinder": { @@ -22120,6 +22168,13 @@ "on-finished": "~2.3.0", "range-parser": "~1.2.0", "statuses": "~1.4.0" + }, + "dependencies": { + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + } } }, "serialize-error": { @@ -22994,6 +23049,13 @@ "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==", "requires": { "ms": "2.0.0" + }, + "dependencies": { + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + } } } } @@ -24616,6 +24678,15 @@ "integrity": "sha512-qNdTUMaCjPs4eEnM3W9H94R3sU70YCuT+/ST7nUf+id1bVOrdjrpUaeZLqPBPRph3hsgn4a4BvwpxhHZx+oSDg==", "dev": true }, + "v8flags": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/v8flags/-/v8flags-3.1.1.tgz", + "integrity": "sha512-iw/1ViSEaff8NJ3HLyEjawk/8hjJib3E7pvG4pddVXfUg1983s3VGsiClDjhK64MQVDGqc1Q8r18S4VKQZS9EQ==", + "dev": true, + "requires": { + "homedir-polyfill": "^1.0.1" + } + }, "validate-npm-package-license": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/validate-npm-package-license/-/validate-npm-package-license-3.0.3.tgz", @@ -24756,34 +24827,6 @@ "now-and-later": "^2.0.0", "remove-bom-buffer": "^3.0.0", "vinyl": "^2.0.0" - }, - "dependencies": { - "clone": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/clone/-/clone-2.1.2.tgz", - "integrity": "sha1-G39Ln1kfHo+DZwQBYANFoCiHQ18=", - "dev": true - }, - "clone-stats": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/clone-stats/-/clone-stats-1.0.0.tgz", - "integrity": "sha1-s3gt/4u1R04Yuba/D9/ngvh3doA=", - "dev": true - }, - "vinyl": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/vinyl/-/vinyl-2.2.0.tgz", - "integrity": "sha512-MBH+yP0kC/GQ5GwBqrTPTzEfiiLjta7hTtvQtbxBgTeSXsmKQRQecjibMbxIXzVT3Y9KJK+drOz1/k+vsu8Nkg==", - "dev": true, - "requires": { - "clone": "^2.1.1", - "clone-buffer": "^1.0.0", - "clone-stats": "^1.0.0", - "cloneable-readable": "^1.0.0", - "remove-trailing-separator": "^1.0.1", - "replace-ext": "^1.0.0" - } - } } }, "vinyl-sourcemaps-apply": { @@ -25145,6 +25188,14 @@ "dev": true, "requires": { "ms": "2.0.0" + }, + "dependencies": { + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=", + "dev": true + } } }, "supports-color": { @@ -25311,6 +25362,14 @@ "dev": true, "requires": { "ms": "2.0.0" + }, + "dependencies": { + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=", + "dev": true + } } }, "decamelize": { diff --git a/package.json b/package.json index 567e2aed3..5c2f34d7d 100644 --- a/package.json +++ b/package.json @@ -57,6 +57,8 @@ "lodash": "^4.17.10", "luxon": "^1.3.1", "mongodb": "^3.1.1", + "ms": "^2.1.1", + "node-fetch": "^2.2.0", "nunjucks": "^3.1.3", "passport": "^0.4.0", "passport-local": "^1.0.0", @@ -101,7 +103,9 @@ "@types/luxon": "^0.5.3", "@types/mini-css-extract-plugin": "^0.2.0", "@types/mongodb": "^3.1.1", + "@types/ms": "^0.7.30", "@types/node": "^10.5.2", + "@types/node-fetch": "^2.1.2", "@types/nunjucks": "^3.0.0", "@types/passport": "^0.4.6", "@types/passport-local": "^1.0.33", diff --git a/scripts/generateSchemaTypes.js b/scripts/generateSchemaTypes.js index f2c52a705..c52393830 100644 --- a/scripts/generateSchemaTypes.js +++ b/scripts/generateSchemaTypes.js @@ -66,7 +66,7 @@ async function main() { file.types = await generateTSTypesAsString(schema, { tabSpaces: 2, typePrefix: "GQL", - strictNulls: true, + strictNulls: false, ...file.config, }); } diff --git a/src/core/common/config.ts b/src/core/common/config.ts index 099bcc4b2..665370638 100644 --- a/src/core/common/config.ts +++ b/src/core/common/config.ts @@ -86,6 +86,14 @@ const config = convict({ env: "LOGGING_LEVEL", arg: "logging", }, + disable_tenant_caching: { + doc: + "Disables the tenant caching, all tenants will be loaded from MongoDB each time it's needed", + format: Boolean, + default: false, + env: "DISABLE_TENANT_CACHING", + arg: "disableTenantCaching", + }, }); export type Config = typeof config; diff --git a/src/core/server/app/index.ts b/src/core/server/app/index.ts index 5f7ba22e6..1559a37e5 100644 --- a/src/core/server/app/index.ts +++ b/src/core/server/app/index.ts @@ -14,6 +14,7 @@ import { handleSubscriptions } from "talk-server/graph/common/subscriptions/midd import { Schemas } from "talk-server/graph/schemas"; import TenantCache from "talk-server/services/tenant/cache"; +import { errorHandler } from "talk-server/app/middleware/error"; import { accessLogger, errorLogger } from "./middleware/logging"; import serveStatic from "./middleware/serveStatic"; import { createRouter } from "./router"; @@ -57,6 +58,7 @@ export async function createApp(options: AppOptions): Promise { // Error Handling parent.use(notFoundMiddleware); parent.use(errorLogger); + parent.use(errorHandler); return parent; } diff --git a/src/core/server/app/middleware/error.ts b/src/core/server/app/middleware/error.ts index 2f666260b..b7be782cb 100644 --- a/src/core/server/app/middleware/error.ts +++ b/src/core/server/app/middleware/error.ts @@ -4,3 +4,20 @@ export const apiErrorHandler: ErrorRequestHandler = (err, req, res, next) => { // TODO: handle better when we improve errors. res.status(500).json({ error: err.message }); }; + +export const errorHandler: ErrorRequestHandler = (err, req, res, next) => { + // TODO: handle better when we improve errors. + if (err.message === "not found") { + // TODO: handle better when we improve errors. + res + .status(404) + .send(err.message) + .end(); + } else { + // TODO: handle better when we improve errors. + res + .status(500) + .send(err.message) + .end(); + } +}; diff --git a/src/core/server/app/middleware/logging.ts b/src/core/server/app/middleware/logging.ts index 134c19df1..cc651825c 100644 --- a/src/core/server/app/middleware/logging.ts +++ b/src/core/server/app/middleware/logging.ts @@ -39,6 +39,10 @@ export const accessLogger: RequestHandler = (req, res, next) => { }; export const errorLogger: ErrorRequestHandler = (err, req, res, next) => { - logger.error(err, "http error"); + // TODO: handle better when we improve errors. + if (err.message !== "not found") { + logger.error({ err }, "http error"); + } + next(err); }; diff --git a/src/core/server/app/middleware/passport/index.ts b/src/core/server/app/middleware/passport/index.ts index a6bd0032b..ae1a3406f 100644 --- a/src/core/server/app/middleware/passport/index.ts +++ b/src/core/server/app/middleware/passport/index.ts @@ -2,6 +2,7 @@ import { NextFunction, RequestHandler, Response } from "express"; import { Db } from "mongodb"; import passport, { Authenticator } from "passport"; +import { Config } from "talk-common/config"; import { createJWTStrategy, JWTSigningConfig, @@ -22,6 +23,7 @@ export type VerifyCallback = ( ) => void; export interface PassportOptions { + config: Config; mongo: Db; signingConfig: JWTSigningConfig; tenantCache: TenantCache; diff --git a/src/core/server/app/middleware/passport/oidc.ts b/src/core/server/app/middleware/passport/oidc.ts index a835bd042..025e1bd4d 100644 --- a/src/core/server/app/middleware/passport/oidc.ts +++ b/src/core/server/app/middleware/passport/oidc.ts @@ -5,6 +5,7 @@ import { Db } from "mongodb"; import { Strategy as OAuth2Strategy, VerifyCallback } from "passport-oauth2"; import { Strategy } from "passport-strategy"; +import { Config } from "talk-common/config"; import { validate } from "talk-server/app/request/body"; import { reconstructURL } from "talk-server/app/url"; import { GQLUSER_ROLE } from "talk-server/graph/tenant/schema/__generated__/types"; @@ -12,6 +13,7 @@ import { OIDCAuthIntegration } from "talk-server/models/settings"; import { Tenant } from "talk-server/models/tenant"; import { OIDCProfile, retrieveUserWithProfile } from "talk-server/models/user"; import TenantCache from "talk-server/services/tenant/cache"; +import { TenantCacheAdapter } from "talk-server/services/tenant/cache/adapter"; import { upsert } from "talk-server/services/users"; import { Request } from "talk-server/types/express"; @@ -177,25 +179,23 @@ const OIDC_SCOPE = "openid email profile"; export interface OIDCStrategyOptions { mongo: Db; tenantCache: TenantCache; + config: Config; } export default class OIDCStrategy extends Strategy { public name = "oidc"; private mongo: Db; - private cache = new Map(); + private cache: TenantCacheAdapter; - constructor({ mongo, tenantCache }: OIDCStrategyOptions) { + constructor({ mongo, tenantCache, config }: OIDCStrategyOptions) { super(); this.mongo = mongo; + this.cache = new TenantCacheAdapter(tenantCache, config); - // Subscribe to updates with Tenants. - tenantCache.subscribe(tenant => { - // Delete the tenant cache item when the tenant changes. The refreshed - // Tenant will come in with the request. - this.cache.delete(tenant.id); - }); + // Connect the cache adapter. + this.cache.subscribe(); } private lookupJWKSClient( diff --git a/src/core/server/graph/tenant/resolvers/auth_integrations.ts b/src/core/server/graph/tenant/resolvers/auth_integrations.ts index 164e15bfe..b40d9d832 100644 --- a/src/core/server/graph/tenant/resolvers/auth_integrations.ts +++ b/src/core/server/graph/tenant/resolvers/auth_integrations.ts @@ -1,7 +1,10 @@ import { GQLAuthIntegrationsTypeResolver } from "talk-server/graph/tenant/schema/__generated__/types"; -import { AuthIntegration, AuthIntegrations } from "talk-server/models/settings"; +import { + AuthIntegrations, + EnableableIntegration, +} from "talk-server/models/settings"; -const disabled: AuthIntegration = { enabled: false }; +const disabled: EnableableIntegration = { enabled: false }; const AuthIntegrations: GQLAuthIntegrationsTypeResolver = { local: auth => auth.local || disabled, diff --git a/src/core/server/graph/tenant/schema/schema.graphql b/src/core/server/graph/tenant/schema/schema.graphql index 302b9e525..1c96051bd 100644 --- a/src/core/server/graph/tenant/schema/schema.graphql +++ b/src/core/server/graph/tenant/schema/schema.graphql @@ -38,6 +38,16 @@ enum ACTION_ITEM_TYPE { COMMENTS } +enum ACTION_GROUP { + SPAM_COMMENT + TOXIC_COMMENT + BODY_COUNT + TRUST + LINKS + BANNED_WORD + SUSPECT_WORD +} + ################################################################################ ## Settings ################################################################################ @@ -57,9 +67,9 @@ enum MODERATION_MODE { } """ -WordlistSettings describes all the available wordlists. +Wordlist describes all the available wordlists. """ -type WordlistSettings { +type Wordlist { """ banned words will by default reject the comment if it is found. """ @@ -177,6 +187,111 @@ type AuthSettings { integrations: AuthIntegrations! } +################################################################################ +## ExternalIntegrations +################################################################################ + +type AkismetExternalIntegration { + """ + enabled when True will enable the integration. + """ + enabled: Boolean! + + """ + The key for the Akismet integration. + """ + key: String + + """ + The site (blog) for the Akismet integration. + """ + site: String +} + +type PerspectiveExternalIntegration { + """ + enabled when True will enable the integration. + """ + enabled: Boolean! + + """ + The endpoint that Talk should use to communicate with the perspective API. + """ + endpoint: String + + """ + The key for the Perspective API integration. + """ + key: String + + """ + The threshold that given a specific toxic comment score, the comment will + be marked by Talk as toxic. + """ + threshold: Float + + """ + When True, comments sent will not be stored by the Google Perspective API. + """ + doNotStore: Boolean +} + +type ExternalIntegrations { + """ + akismet provides integration with the Akismet Spam detection service. + """ + akismet: AkismetExternalIntegration! + + """ + perspective provides integration with the Perspective API comment analysis + platform. + """ + perspective: PerspectiveExternalIntegration! +} + +################################################################################ +## Karma +################################################################################ + +""" +KarmaThreshold defines the bounds for which a User will become unreliable or +reliable based on their karma score. If the score is equal or less than the +unreliable value, they are unreliable. If the score is equal or more than the +reliable value, they are reliable. If they are neither reliable or unreliable +then they are neutral. +""" +type KarmaThreshold { + reliable: Int! + unreliable: Int! +} + +type KarmaThresholds { + """ + flag represents karma settings in relation to how well a User's flagging + ability aligns with the moderation decicions made by moderators. + """ + flag: KarmaThreshold! + + """ + comment represents the karma setting in relation to how well a User's comments are moderated. + """ + comment: KarmaThreshold! +} + +type Karma { + """ + When true, checks will be completed to ensure that the Karma checks are + completed. + """ + enabled: Boolean! + + """ + karmaThresholds contains the currently set thresholds for triggering Trust + beheviour. + """ + thresholds: KarmaThresholds! +} + ################################################################################ ## Settings ################################################################################ @@ -295,7 +410,7 @@ type Settings { """ wordlist will return a given list of words. """ - wordlist: WordlistSettings @auth(roles: [ADMIN, MODERATOR]) + wordlist: Wordlist @auth(roles: [ADMIN, MODERATOR]) """ domains will return a given list of whitelisted domains. @@ -306,6 +421,17 @@ type Settings { auth contains all the settings related to authentication and authorization. """ auth: AuthSettings! + + """ + integrations contains all the external integrations that can be enabled. + """ + integrations: ExternalIntegrations @auth(roles: [ADMIN]) + + """ + karma is the set of settings related to how user Trust and Karma are + handled. + """ + karma: Karma @auth(roles: [ADMIN, MODERATOR]) } ################################################################################ @@ -728,7 +854,7 @@ input SettingsInput { charCount: Int organizationName: String organizationContactEmail: String - # wordlist: WordlistSettings @auth(roles: [ADMIN, MODERATOR]) + # wordlist: Wordlist @auth(roles: [ADMIN, MODERATOR]) domains: [String!] # auth: AuthSettings! } diff --git a/src/core/server/index.ts b/src/core/server/index.ts index 47a0c2db4..b09763c26 100644 --- a/src/core/server/index.ts +++ b/src/core/server/index.ts @@ -70,7 +70,11 @@ class Server { const signingConfig = createJWTSigningConfig(this.config); // Create the TenantCache. - const tenantCache = new TenantCache(mongo, await createRedisClient(config)); + const tenantCache = new TenantCache( + mongo, + await createRedisClient(config), + config + ); // Prime the tenant cache so it'll be ready to serve now. await tenantCache.primeAll(); diff --git a/src/core/server/models/actions.ts b/src/core/server/models/actions.ts index f304da550..d162b1300 100644 --- a/src/core/server/models/actions.ts +++ b/src/core/server/models/actions.ts @@ -1,4 +1,5 @@ import { + GQLACTION_GROUP, GQLACTION_ITEM_TYPE, GQLACTION_TYPE, } from "talk-server/graph/tenant/schema/__generated__/types"; @@ -10,7 +11,7 @@ export interface Action { action_type: GQLACTION_TYPE; item_type: GQLACTION_ITEM_TYPE; item_id: string; - group_id?: string; + group_id?: GQLACTION_GROUP; user_id?: string; created_at: Date; metadata?: Record; diff --git a/src/core/server/models/comment.ts b/src/core/server/models/comment.ts index 430f85225..9a961a17a 100644 --- a/src/core/server/models/comment.ts +++ b/src/core/server/models/comment.ts @@ -34,7 +34,7 @@ export interface StatusHistoryItem { export interface Comment extends TenantResource { readonly id: string; - parent_id: string | null; + parent_id?: string; author_id: string; asset_id: string; body: string; diff --git a/src/core/server/models/settings.ts b/src/core/server/models/settings.ts index 217db50b9..9df3eaffe 100644 --- a/src/core/server/models/settings.ts +++ b/src/core/server/models/settings.ts @@ -1,13 +1,11 @@ import { + GQLExternalIntegrations, + GQLKarma, GQLMODERATION_MODE, GQLUSER_ROLE, + GQLWordlist, } from "talk-server/graph/tenant/schema/__generated__/types"; -export interface Wordlist { - banned: string[]; - suspect: string[]; -} - export interface EmailDomainRuleCondition { /** * emailDomain is the domain name component of the email addresses that should @@ -49,7 +47,7 @@ export interface AuthRules { restrictTo?: EmailDomainRuleCondition[]; } -export interface AuthIntegration { +export interface EnableableIntegration { enabled: boolean; } @@ -63,7 +61,7 @@ export interface DisplayNameAuthIntegration { * embed to allow single sign on. */ export interface SSOAuthIntegration - extends AuthIntegration, + extends EnableableIntegration, DisplayNameAuthIntegration { key: string; } @@ -73,7 +71,7 @@ export interface SSOAuthIntegration * will be used in the admin to provide staff logins for users. */ export interface OIDCAuthIntegration - extends AuthIntegration, + extends EnableableIntegration, DisplayNameAuthIntegration { clientID: string; clientSecret: string; @@ -83,17 +81,17 @@ export interface OIDCAuthIntegration tokenURL: string; } -export interface FacebookAuthIntegration extends AuthIntegration { +export interface FacebookAuthIntegration extends EnableableIntegration { clientID: string; clientSecret: string; } -export interface GoogleAuthIntegration extends AuthIntegration { +export interface GoogleAuthIntegration extends EnableableIntegration { clientID: string; clientSecret: string; } -export type LocalAuthIntegration = AuthIntegration; +export type LocalAuthIntegration = EnableableIntegration; /** * AuthIntegrations describes all of the possible auth integration @@ -130,33 +128,6 @@ export interface Auth { integrations: AuthIntegrations; } -/** - * Akismet provides integration with the Akismet Spam detection service. - */ -export interface AkismetIntegration { - /** - * When true, it will enable comments to be checked by Akismet. - */ - enabled: boolean; - - /** - * The key for the Akismet integration. - */ - key?: string; - - /** - * The site (blog) for the Akismet integration. - */ - site?: string; -} - -export interface ExternalIntegrations { - /** - * akismet provides integration with the Akismet Spam detection service. - */ - akismet: AkismetIntegration; -} - export interface ModerationSettings { moderation: GQLMODERATION_MODE; requireEmailConfirmation: boolean; @@ -175,45 +146,6 @@ export interface ModerationSettings { charCount?: number; } -/** - * KarmaThreshold defines the bounds for which a User will become unreliable or - * reliable based on their karma score. If the score is equal or less than the - * unreliable value, they are unreliable. If the score is equal or more than the - * reliable value, they are reliable. If they are neither reliable or unreliable - * then they are neutral. - */ -export interface KarmaThreshold { - reliable: number; - unreliable: number; -} - -export interface KarmaThresholds { - /** - * flag represents karma settings in relation to how well a User's flagging - * ability aligns with the moderation decicions made by moderators. - */ - flag: KarmaThreshold; - - /** - * comment represents the karma setting in relation to how well a User's comments are moderated. - */ - comment: KarmaThreshold; -} - -export interface Karma { - /** - * When true, checks will be completed to ensure that the Karma checks are - * completed. - */ - enabled: boolean; - - /** - * karmaThresholds contains the currently set thresholds for triggering Trust - * beheviour. - */ - thresholds: KarmaThresholds; -} - export interface Settings extends ModerationSettings { customCssUrl?: string; @@ -227,12 +159,12 @@ export interface Settings extends ModerationSettings { * karma is the set of settings related to how user Trust and Karma are * handled. */ - karma: Karma; + karma: GQLKarma; /** * wordlist stores all the banned/suspect words. */ - wordlist: Wordlist; + wordlist: GQLWordlist; /** * Set of configured authentication integrations. @@ -242,5 +174,5 @@ export interface Settings extends ModerationSettings { /** * Various integrations with external services. */ - integrations: ExternalIntegrations; + integrations: GQLExternalIntegrations; } diff --git a/src/core/server/models/tenant.ts b/src/core/server/models/tenant.ts index bd67c99b4..f98b62727 100644 --- a/src/core/server/models/tenant.ts +++ b/src/core/server/models/tenant.ts @@ -91,6 +91,9 @@ export async function createTenant(db: Db, input: CreateTenantInput) { akismet: { enabled: false, }, + perspective: { + enabled: false, + }, }, }; diff --git a/src/core/server/services/comments/index.ts b/src/core/server/services/comments/index.ts index 6bacbd415..1c4531f3e 100644 --- a/src/core/server/services/comments/index.ts +++ b/src/core/server/services/comments/index.ts @@ -14,7 +14,7 @@ import { Request } from "talk-server/types/express"; export type CreateComment = Omit< CreateCommentInput, - "status" | "action_counts" + "status" | "action_counts" | "metadata" >; export async function create( @@ -44,7 +44,7 @@ export async function create( } // Run the comment through the moderation phases. - const { status } = await processForModeration({ + const { status, metadata } = await processForModeration({ asset, tenant, comment: input, @@ -55,9 +55,10 @@ export async function create( // TODO: (wyattjoh) use the actions somehow. const comment = await createComment(mongo, tenant.id, { + ...input, status, action_counts: {}, - ...input, + metadata, }); if (input.parent_id) { diff --git a/src/core/server/services/comments/moderation/index.spec.ts b/src/core/server/services/comments/moderation/index.spec.ts new file mode 100644 index 000000000..4b0ce2ebf --- /dev/null +++ b/src/core/server/services/comments/moderation/index.spec.ts @@ -0,0 +1,105 @@ +import { + GQLACTION_GROUP, + GQLACTION_TYPE, + GQLCOMMENT_STATUS, +} from "talk-server/graph/tenant/schema/__generated__/types"; +import { + compose, + ModerationPhaseContext, +} from "talk-server/services/comments/moderation"; + +describe("compose", () => { + it("handles when a phase throws an error", async () => { + const err = new Error("this is an error"); + const enhanced = compose([ + () => { + throw err; + }, + ]); + + await expect(enhanced({} as ModerationPhaseContext)).rejects.toEqual(err); + }); + + it("handles when it returns a status", async () => { + const status = GQLCOMMENT_STATUS.ACCEPTED; + const enhanced = compose([() => ({ status })]); + + await expect(enhanced({} as ModerationPhaseContext)).resolves.toEqual({ + status, + metadata: {}, + actions: [], + }); + }); + + it("merges the metadata", async () => { + const status = GQLCOMMENT_STATUS.ACCEPTED; + const enhanced = compose([ + () => ({ metadata: { first: true } }), + () => ({ status, metadata: { second: true } }), + () => ({ metadata: { third: true } }), + ]); + + await expect(enhanced({} as ModerationPhaseContext)).resolves.toEqual({ + status, + metadata: { first: true, second: true }, + actions: [], + }); + }); + + it("merges actions", async () => { + const status = GQLCOMMENT_STATUS.ACCEPTED; + + const flags = [ + { + action_type: GQLACTION_TYPE.FLAG, + group_id: GQLACTION_GROUP.TOXIC_COMMENT, + }, + { + action_type: GQLACTION_TYPE.FLAG, + group_id: GQLACTION_GROUP.SPAM_COMMENT, + }, + ]; + + const enhanced = compose([ + () => ({ + actions: [flags[0]], + }), + () => ({ + status, + actions: [flags[1]], + }), + () => ({ + actions: [ + { + action_type: GQLACTION_TYPE.FLAG, + group_id: GQLACTION_GROUP.LINKS, + }, + ], + }), + ]); + + const final = await enhanced({} as ModerationPhaseContext); + + for (const flag of flags) { + expect(final.actions).toContainEqual(flag); + } + + expect(final.actions).not.toContainEqual({ + action_type: GQLACTION_TYPE.FLAG, + group_id: GQLACTION_GROUP.LINKS, + }); + }); + + it("handles when it does not return a status", async () => { + const enhanced = compose([ + () => ({ metadata: { first: true } }), + () => ({ metadata: { second: true } }), + ]); + + await expect(enhanced({} as ModerationPhaseContext)).resolves.toEqual({ + status: GQLCOMMENT_STATUS.NONE, + metadata: { first: true, second: true }, + actions: [], + }); + }); +}); diff --git a/src/core/server/services/comments/moderation/index.ts b/src/core/server/services/comments/moderation/index.ts index cfd7f2c80..33f286320 100644 --- a/src/core/server/services/comments/moderation/index.ts +++ b/src/core/server/services/comments/moderation/index.ts @@ -3,13 +3,13 @@ import { GQLCOMMENT_STATUS } from "talk-server/graph/tenant/schema/__generated__ import { Action } from "talk-server/models/actions"; import { Asset } from "talk-server/models/asset"; import { Tenant } from "talk-server/models/tenant"; -import { CreateComment } from "talk-server/services/comments"; - import { User } from "talk-server/models/user"; +import { CreateComment } from "talk-server/services/comments"; import { Request } from "talk-server/types/express"; + import { moderationPhases } from "./phases"; -// TODO: (wyattjoh) move into actions module. +// TODO: (wyattjoh) move into actions module once we have action methods. export type CreateAction = Omit< Action, "id" | "item_type" | "item_id" | "created_at" @@ -18,6 +18,7 @@ export type CreateAction = Omit< export interface PhaseResult { actions: CreateAction[]; status: GQLCOMMENT_STATUS; + metadata: Record; } export interface ModerationPhaseContext { @@ -42,31 +43,47 @@ export type IntermediateModerationPhase = ( * compose will create a moderation pipeline for which is executable with the * passed actions. */ -const compose = ( +export const compose = ( phases: IntermediateModerationPhase[] ): ModerationPhase => async context => { - const actions: CreateAction[] = []; + const final: PhaseResult = { + status: GQLCOMMENT_STATUS.NONE, + actions: [], + metadata: {}, + }; // Loop over all the moderation phases and see if we've resolved the status. for (const phase of phases) { const result = await phase(context); if (result) { - if (result.actions) { - actions.push(...result.actions); + // If this result contained actions, then we should push it into the + // other actions. + const { actions } = result; + if (actions) { + final.actions.push(...actions); + } + + // If this result contained metadata, then we should merge it into the + // other metadata. + const { metadata } = result; + if (metadata) { + final.metadata = { + ...final.metadata, + ...metadata, + }; } // If this result contained a status, then we've finished resolving // phases! const { status } = result; if (status) { - return { status, actions }; + final.status = status; + break; } } } - // If we didn't determine a different comment from a previous itteration, set - // it to 'NONE'. - return { status: GQLCOMMENT_STATUS.NONE, actions }; + return final; }; /** diff --git a/src/core/server/services/comments/moderation/phases/assetClosed.ts b/src/core/server/services/comments/moderation/phases/assetClosed.ts index 0755949ac..2aec0db0b 100644 --- a/src/core/server/services/comments/moderation/phases/assetClosed.ts +++ b/src/core/server/services/comments/moderation/phases/assetClosed.ts @@ -1,12 +1,15 @@ -import { IntermediateModerationPhase } from "talk-server/services/comments/moderation"; +import { + IntermediateModerationPhase, + IntermediatePhaseResult, +} from "talk-server/services/comments/moderation"; // This phase checks to see if the asset being processed is closed or not. -export const assetClosed: IntermediateModerationPhase = ({ asset }) => { +export const assetClosed: IntermediateModerationPhase = ({ + asset, +}): IntermediatePhaseResult | void => { // Check to see if the asset has closed commenting... if (asset.closedAt && asset.closedAt.valueOf() <= Date.now()) { // TODO: (wyattjoh) return better error. throw new Error("asset is currently closed for commenting"); } - - return; }; diff --git a/src/core/server/services/comments/moderation/phases/commentLength.ts b/src/core/server/services/comments/moderation/phases/commentLength.ts index d714950b2..82a4ada04 100644 --- a/src/core/server/services/comments/moderation/phases/commentLength.ts +++ b/src/core/server/services/comments/moderation/phases/commentLength.ts @@ -1,18 +1,22 @@ import { + GQLACTION_GROUP, GQLACTION_TYPE, GQLCOMMENT_STATUS, } from "talk-server/graph/tenant/schema/__generated__/types"; import { ModerationSettings } from "talk-server/models/settings"; -import { IntermediateModerationPhase } from "talk-server/services/comments/moderation"; +import { + IntermediateModerationPhase, + IntermediatePhaseResult, +} from "talk-server/services/comments/moderation"; const testCharCount = (settings: Partial, length: number) => settings.charCountEnable && settings.charCount && length > settings.charCount; -export const commentLength: IntermediateModerationPhase = async ({ +export const commentLength: IntermediateModerationPhase = ({ asset, tenant, comment, -}) => { +}): IntermediatePhaseResult | void => { const length = comment.body.length; // Check to see if the body is too short, if it is, then complain about it! @@ -32,7 +36,7 @@ export const commentLength: IntermediateModerationPhase = async ({ actions: [ { action_type: GQLACTION_TYPE.FLAG, - group_id: "BODY_COUNT", + group_id: GQLACTION_GROUP.BODY_COUNT, metadata: { count: length, }, @@ -40,6 +44,4 @@ export const commentLength: IntermediateModerationPhase = async ({ ], }; } - - return; }; diff --git a/src/core/server/services/comments/moderation/phases/commentingDisabled.ts b/src/core/server/services/comments/moderation/phases/commentingDisabled.ts index 742d97974..07ad21ecb 100644 --- a/src/core/server/services/comments/moderation/phases/commentingDisabled.ts +++ b/src/core/server/services/comments/moderation/phases/commentingDisabled.ts @@ -1,5 +1,8 @@ import { ModerationSettings } from "talk-server/models/settings"; -import { IntermediateModerationPhase } from "talk-server/services/comments/moderation"; +import { + IntermediateModerationPhase, + IntermediatePhaseResult, +} from "talk-server/services/comments/moderation"; const testDisabledCommenting = (settings: Partial) => settings.disableCommenting; @@ -7,7 +10,7 @@ const testDisabledCommenting = (settings: Partial) => export const commentingDisabled: IntermediateModerationPhase = ({ asset, tenant, -}) => { +}): IntermediatePhaseResult | void => { // Check to see if the asset has closed commenting. if ( testDisabledCommenting(tenant) || diff --git a/src/core/server/services/comments/moderation/phases/index.ts b/src/core/server/services/comments/moderation/phases/index.ts index 6cb656903..a0a724cc6 100644 --- a/src/core/server/services/comments/moderation/phases/index.ts +++ b/src/core/server/services/comments/moderation/phases/index.ts @@ -1,6 +1,7 @@ import { IntermediateModerationPhase } from "talk-server/services/comments/moderation"; import { premod } from "talk-server/services/comments/moderation/phases/premod"; +import { toxic } from "talk-server/services/comments/moderation/phases/toxic"; import { assetClosed } from "./assetClosed"; import { commentingDisabled } from "./commentingDisabled"; import { commentLength } from "./commentLength"; @@ -22,5 +23,6 @@ export const moderationPhases: IntermediateModerationPhase[] = [ links, karma, spam, + toxic, premod, ]; diff --git a/src/core/server/services/comments/moderation/phases/karma.ts b/src/core/server/services/comments/moderation/phases/karma.ts index 3a4d8fa84..f5f4d9e37 100755 --- a/src/core/server/services/comments/moderation/phases/karma.ts +++ b/src/core/server/services/comments/moderation/phases/karma.ts @@ -1,8 +1,12 @@ import { + GQLACTION_GROUP, GQLACTION_TYPE, GQLCOMMENT_STATUS, } from "talk-server/graph/tenant/schema/__generated__/types"; -import { IntermediateModerationPhase } from "talk-server/services/comments/moderation"; +import { + IntermediateModerationPhase, + IntermediatePhaseResult, +} from "talk-server/services/comments/moderation"; import { getCommentTrustScore, isReliableCommenter, @@ -10,7 +14,10 @@ import { // This phase checks to see if the user making the comment is allowed to do so // considering their reliability (Trust) status. -export const karma: IntermediateModerationPhase = ({ tenant, author }) => { +export const karma: IntermediateModerationPhase = ({ + tenant, + author, +}): IntermediatePhaseResult | void => { // If the user is not a reliable commenter (passed the unreliability // threshold by having too many rejected comments) then we can change the // status of the comment to `SYSTEM_WITHHELD`, therefore pushing the user's @@ -27,7 +34,7 @@ export const karma: IntermediateModerationPhase = ({ tenant, author }) => { actions: [ { action_type: GQLACTION_TYPE.FLAG, - group_id: "TRUST", + group_id: GQLACTION_GROUP.TRUST, metadata: { trust: getCommentTrustScore(author), }, @@ -35,6 +42,4 @@ export const karma: IntermediateModerationPhase = ({ tenant, author }) => { ], }; } - - return; }; diff --git a/src/core/server/services/comments/moderation/phases/links.ts b/src/core/server/services/comments/moderation/phases/links.ts index 3db715d5f..18628920a 100755 --- a/src/core/server/services/comments/moderation/phases/links.ts +++ b/src/core/server/services/comments/moderation/phases/links.ts @@ -2,11 +2,15 @@ import linkify from "linkify-it"; import tlds from "tlds"; import { + GQLACTION_GROUP, GQLACTION_TYPE, GQLCOMMENT_STATUS, } from "talk-server/graph/tenant/schema/__generated__/types"; import { ModerationSettings } from "talk-server/models/settings"; -import { IntermediateModerationPhase } from "talk-server/services/comments/moderation"; +import { + IntermediateModerationPhase, + IntermediatePhaseResult, +} from "talk-server/services/comments/moderation"; /** * The preloaded linkify instance with common tlds. @@ -24,8 +28,7 @@ export const links: IntermediateModerationPhase = ({ asset, tenant, comment, - author, -}) => { +}): IntermediatePhaseResult | void => { if ( testPremodLinksEnable(tenant, comment.body) || (asset.settings && testPremodLinksEnable(asset.settings, comment.body)) @@ -36,7 +39,7 @@ export const links: IntermediateModerationPhase = ({ actions: [ { action_type: GQLACTION_TYPE.FLAG, - group_id: "LINKS", + group_id: GQLACTION_GROUP.LINKS, metadata: { links: comment.body, }, @@ -44,6 +47,4 @@ export const links: IntermediateModerationPhase = ({ ], }; } - - return; }; diff --git a/src/core/server/services/comments/moderation/phases/premod.ts b/src/core/server/services/comments/moderation/phases/premod.ts index c12fb582d..7ccdcfb73 100755 --- a/src/core/server/services/comments/moderation/phases/premod.ts +++ b/src/core/server/services/comments/moderation/phases/premod.ts @@ -3,14 +3,20 @@ import { GQLMODERATION_MODE, } from "talk-server/graph/tenant/schema/__generated__/types"; import { ModerationSettings } from "talk-server/models/settings"; -import { IntermediateModerationPhase } from "talk-server/services/comments/moderation"; +import { + IntermediateModerationPhase, + IntermediatePhaseResult, +} from "talk-server/services/comments/moderation"; const testModerationMode = (settings: Partial) => settings.moderation === GQLMODERATION_MODE.PRE; // This phase checks to see if the settings have premod enabled, if they do, // the comment is premod, otherwise, it's just none. -export const premod: IntermediateModerationPhase = ({ asset, tenant }) => { +export const premod: IntermediateModerationPhase = ({ + asset, + tenant, +}): IntermediatePhaseResult | void => { // If the settings say that we're in premod mode, then the comment is in // premod status. @@ -23,6 +29,4 @@ export const premod: IntermediateModerationPhase = ({ asset, tenant }) => { status: GQLCOMMENT_STATUS.PREMOD, }; } - - return; }; diff --git a/src/core/server/services/comments/moderation/phases/spam.ts b/src/core/server/services/comments/moderation/phases/spam.ts index f44a4e85b..d5fc85fc5 100644 --- a/src/core/server/services/comments/moderation/phases/spam.ts +++ b/src/core/server/services/comments/moderation/phases/spam.ts @@ -1,10 +1,15 @@ import { Client } from "akismet-api"; import { + GQLACTION_GROUP, GQLACTION_TYPE, GQLCOMMENT_STATUS, } from "talk-server/graph/tenant/schema/__generated__/types"; -import { IntermediateModerationPhase } from "talk-server/services/comments/moderation"; +import logger from "talk-server/logger"; +import { + IntermediateModerationPhase, + IntermediatePhaseResult, +} from "talk-server/services/comments/moderation"; export const spam: IntermediateModerationPhase = async ({ asset, @@ -12,16 +17,26 @@ export const spam: IntermediateModerationPhase = async ({ comment, author, req, -}) => { +}): Promise => { const integration = tenant.integrations.akismet; // We can only check for spam if this comment originated from a graphql // request via an HTTP call. - if (!req || !integration.enabled) { + if (!req) { + logger.debug({ tenant_id: tenant.id }, "request was not available"); + return; + } + + if (!integration.enabled) { + logger.debug({ tenant_id: tenant.id }, "akismet integration was disabled"); return; } if (!integration.key || !integration.site) { + logger.error( + { tenant_id: tenant.id }, + "akismet integration was enabled but configuration was missing" + ); return; } @@ -34,41 +49,73 @@ export const spam: IntermediateModerationPhase = async ({ // Grab the properties we need. const userIP = req.ip; if (!userIP) { + logger.debug( + { tenant_id: tenant.id }, + "request did not contain ip address, aborting spam check" + ); return; } const userAgent = req.get("User-Agent"); if (!userAgent || userAgent.length === 0) { + logger.debug( + { tenant_id: tenant.id }, + "request did not contain User-Agent header, aborting spam check" + ); return; } const referrer = req.get("Referrer"); if (!referrer || referrer.length === 0) { + logger.debug( + { tenant_id: tenant.id }, + "request did not contain Referrer header, aborting spam check" + ); return; } - // Check the comment for spam. - const isSpam = await client.checkSpam({ - user_ip: userIP, // REQUIRED - referrer, // REQUIRED - user_agent: userAgent, // REQUIRED - comment_content: comment.body, - permalink: asset.url, - comment_author: author.displayName || author.username || "", - comment_type: "comment", - is_test: false, - }); - if (isSpam) { - return { - status: GQLCOMMENT_STATUS.SYSTEM_WITHHELD, - actions: [ - { - action_type: GQLACTION_TYPE.FLAG, - group_id: "SPAM_COMMENT", - }, - ], - }; - } + try { + logger.trace({ tenant_id: tenant.id }, "checking comment for spam"); - return; + // Check the comment for spam. + const isSpam = await client.checkSpam({ + user_ip: userIP, // REQUIRED + referrer, // REQUIRED + user_agent: userAgent, // REQUIRED + comment_content: comment.body, + permalink: asset.url, + comment_author: author.displayName || author.username || "", + comment_type: "comment", + is_test: false, + }); + if (isSpam) { + logger.trace( + { tenant_id: tenant.id, is_spam: isSpam }, + "comment contained spam" + ); + return { + status: GQLCOMMENT_STATUS.SYSTEM_WITHHELD, + actions: [ + { + action_type: GQLACTION_TYPE.FLAG, + group_id: GQLACTION_GROUP.SPAM_COMMENT, + }, + ], + metadata: { + // Store the spam result from Akismet in the Comment metadata. + akismet: spam, + }, + }; + } + + logger.trace( + { tenant_id: tenant.id, is_spam: isSpam }, + "comment did not contain spam" + ); + } catch (err) { + logger.error( + { tenant_id: tenant.id, err }, + "could not determine if comment contained spam" + ); + } }; diff --git a/src/core/server/services/comments/moderation/phases/staff.ts b/src/core/server/services/comments/moderation/phases/staff.ts index 607e49c82..aad2a931d 100755 --- a/src/core/server/services/comments/moderation/phases/staff.ts +++ b/src/core/server/services/comments/moderation/phases/staff.ts @@ -2,7 +2,10 @@ import { GQLCOMMENT_STATUS, GQLUSER_ROLE, } from "talk-server/graph/tenant/schema/__generated__/types"; -import { IntermediateModerationPhase } from "talk-server/services/comments/moderation"; +import { + IntermediateModerationPhase, + IntermediatePhaseResult, +} from "talk-server/services/comments/moderation"; // If a given user is a staff member, always approve their comment. export const staff: IntermediateModerationPhase = ({ @@ -10,12 +13,10 @@ export const staff: IntermediateModerationPhase = ({ tenant, comment, author, -}) => { +}): IntermediatePhaseResult | void => { if (author.role !== GQLUSER_ROLE.COMMENTER) { return { status: GQLCOMMENT_STATUS.ACCEPTED, }; } - - return; }; diff --git a/src/core/server/services/comments/moderation/phases/toxic.ts b/src/core/server/services/comments/moderation/phases/toxic.ts new file mode 100644 index 000000000..1690458f6 --- /dev/null +++ b/src/core/server/services/comments/moderation/phases/toxic.ts @@ -0,0 +1,183 @@ +import { isNil } from "lodash"; +import ms from "ms"; +import fetch from "node-fetch"; + +import { Omit } from "talk-common/types"; +import { + GQLACTION_GROUP, + GQLACTION_TYPE, + GQLCOMMENT_STATUS, + GQLPerspectiveExternalIntegration, +} from "talk-server/graph/tenant/schema/__generated__/types"; +import logger from "talk-server/logger"; +import { + IntermediateModerationPhase, + IntermediatePhaseResult, +} from "talk-server/services/comments/moderation"; + +export const toxic: IntermediateModerationPhase = async ({ + tenant, + comment, +}): Promise => { + const integration = tenant.integrations.perspective; + + if (!integration.enabled) { + // The Toxic comment plugin is not enabled. + logger.debug( + { tenant_id: tenant.id }, + "perspective integration was disabled" + ); + return; + } + + if (!integration.key) { + // The Toxic comment requires a key in order to communicate with the API. + logger.error( + { tenant_id: tenant.id }, + "perspective integration was enabled but configuration was missing" + ); + return; + } + + let endpoint = integration.endpoint; + if (isNil(endpoint)) { + // TODO: (wyattjoh) replace hardcoded default with config. + endpoint = "https://commentanalyzer.googleapis.com/v1alpha1"; + + logger.trace( + { tenant_id: tenant.id, endpoint }, + "endpoint missing in integration settings, using defaults" + ); + } + + let threshold = integration.threshold; + if (isNil(threshold)) { + // TODO: (wyattjoh) replace hardcoded default with config. + threshold = 0.8; + + logger.trace( + { tenant_id: tenant.id, threshold }, + "threshold missing in integration settings, using defaults" + ); + } + + let doNotStore = integration.doNotStore; + if (isNil(doNotStore)) { + doNotStore = true; + + logger.trace( + { tenant_id: tenant.id, do_not_store: doNotStore }, + "doNotStore missing in integration settings, using defaults" + ); + } + + // TODO: (wyattjoh) replace hardcoded default with config. + const timeout = ms("300ms"); + + try { + logger.trace({ tenant_id: tenant.id }, "checking comment toxicity"); + + // Call into the Toxic comment API. + const scores = await getScores( + comment.body, + { + endpoint, + key: integration.key, + doNotStore, + }, + timeout + ); + + const score = scores.SEVERE_TOXICITY.summaryScore; + const isToxic = score > threshold; + if (isToxic) { + logger.trace( + { tenant_id: tenant.id, score, is_toxic: isToxic, threshold }, + "comment was toxic" + ); + return { + status: GQLCOMMENT_STATUS.SYSTEM_WITHHELD, + actions: [ + { + action_type: GQLACTION_TYPE.FLAG, + group_id: GQLACTION_GROUP.TOXIC_COMMENT, + }, + ], + metadata: { + // Store the scores from perspective in the Comment metadata. + perspective: scores, + }, + }; + } + + logger.trace( + { tenant_id: tenant.id, score, is_toxic: isToxic, threshold }, + "comment was not toxic" + ); + } catch (err) { + logger.error( + { tenant_id: tenant.id, err }, + "could not determine comment toxicity" + ); + } +}; + +/** + * getScores will return the toxicity scores for the comment text. + * + * @param text comment text to check for toxicity + * @param settings integration settings used to communicate with the perspective api + * @param timeout timeout for communicating with the perspective api + */ +async function getScores( + text: string, + { + key, + endpoint, + doNotStore, + }: Required>, + timeout: number +) { + try { + const response = await fetch(`${endpoint}/comments:analyze?key=${key}`, { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + timeout, + body: JSON.stringify({ + comment: { + text, + }, + // TODO: (wyattjoh) support other languages. + languages: ["en"], + doNotStore, + requestedAttributes: { + TOXICITY: {}, + SEVERE_TOXICITY: {}, + }, + }), + }); + + // Grab the data out of the Perspective API. + const data = await response.json(); + + // Reformat the scores. + return { + TOXICITY: { + summaryScore: data.attributeScores.TOXICITY.summaryScore.value, + }, + SEVERE_TOXICITY: { + summaryScore: data.attributeScores.SEVERE_TOXICITY.summaryScore.value, + }, + }; + } catch (err) { + // Ensure that the API key doesn't get leaked to the logs by accident. + if (err.message) { + err.message = err.message.replace(key, "***"); + } + + // Rethrow the error. + throw err; + } +} diff --git a/src/core/server/services/comments/moderation/phases/wordlist.ts b/src/core/server/services/comments/moderation/phases/wordlist.ts index 0c62f3261..a817d3fea 100755 --- a/src/core/server/services/comments/moderation/phases/wordlist.ts +++ b/src/core/server/services/comments/moderation/phases/wordlist.ts @@ -1,17 +1,19 @@ import { + GQLACTION_GROUP, GQLACTION_TYPE, GQLCOMMENT_STATUS, } from "talk-server/graph/tenant/schema/__generated__/types"; -import { IntermediateModerationPhase } from "talk-server/services/comments/moderation"; +import { + IntermediateModerationPhase, + IntermediatePhaseResult, +} from "talk-server/services/comments/moderation"; import { containsMatchingPhrase } from "talk-server/services/comments/moderation/wordlist"; // This phase checks the comment against the wordlist. export const wordlist: IntermediateModerationPhase = ({ - asset, tenant, comment, - author, -}) => { +}): IntermediatePhaseResult | void => { // Decide the status based on whether or not the current asset/settings // has pre-mod enabled or not. If the comment was rejected based on the // wordlist, then reject it, otherwise if the moderation setting is @@ -23,7 +25,7 @@ export const wordlist: IntermediateModerationPhase = ({ actions: [ { action_type: GQLACTION_TYPE.FLAG, - group_id: "BANNED_WORD", + group_id: GQLACTION_GROUP.BANNED_WORD, }, ], }; @@ -40,11 +42,9 @@ export const wordlist: IntermediateModerationPhase = ({ actions: [ { action_type: GQLACTION_TYPE.FLAG, - group_id: "SUSPECT_WORD", + group_id: GQLACTION_GROUP.SUSPECT_WORD, }, ], }; } - - return; }; diff --git a/src/core/server/services/tenant/cache/adapter.ts b/src/core/server/services/tenant/cache/adapter.ts new file mode 100644 index 000000000..feb6269a9 --- /dev/null +++ b/src/core/server/services/tenant/cache/adapter.ts @@ -0,0 +1,92 @@ +import { Config } from "talk-common/config"; +import TenantCache from "talk-server/services/tenant/cache"; + +export type DeconstructionFn = (tenantID: string, value: T) => Promise; + +/** + * TenantCacheAdapter is designed to allow additional services to cache entries + * that are related to tenants that could possibly be cached. When caching of + * tenants are enabled, this acts as a map to store entries, and will + * automatically invalidate tenants that have been updated. + */ +export class TenantCacheAdapter { + private cache = new Map(); + private tenantCache: TenantCache; + private isCachingEnabled: boolean; + + private unsubscribeFn?: () => void; + private deconstructionFn?: DeconstructionFn; + + constructor( + tenantCache: TenantCache, + config: Config, + deconstructionFn?: DeconstructionFn + ) { + this.tenantCache = tenantCache; + this.isCachingEnabled = !config.get("disable_tenant_caching"); + this.deconstructionFn = deconstructionFn; + } + + public subscribe() { + if (this.isCachingEnabled) { + // Unsubscribe from updates if we + this.unsubscribe(); + + this.unsubscribeFn = this.tenantCache.subscribe(async tenant => { + // Get the current set value for the item in the cache. + const value = this.get(tenant.id); + + // Delete the tenant cache item when the tenant changes. + this.cache.delete(tenant.id); + + if (this.deconstructionFn) { + // The deconstruction function is set. We will check that the value + // exists, and if it does, we will call the function with the given + // identifier, this allows the caller to attach deconstruction + // components to the tenant being removed. The only side affect to + // note is that by the time that the deconstruction function is + // called, the tenant has already been purged from the cache. + if (typeof value !== "undefined") { + await this.deconstructionFn(tenant.id, value); + } + } + }); + } + } + + /** + * This will disconnect the map/cache from getting updates. + */ + public unsubscribe() { + if (this.isCachingEnabled && this.unsubscribeFn) { + this.unsubscribeFn(); + } + } + + /** + * get will return the cached entry keyed on the tenantID. + * + * @param tenantID the tenantID for the cached item + */ + public get(tenantID: string): T | undefined { + if (this.isCachingEnabled) { + return this.cache.get(tenantID); + } + + return; + } + + /** + * set will set the cached entry into the map (if caching is enabled). + * + * @param tenantID the tenantID for the cached item + * @param value the value to set in the map (if caching is enabled) + */ + public set(tenantID: string, value: T): TenantCacheAdapter { + if (this.isCachingEnabled) { + this.cache.set(tenantID, value); + } + + return this; + } +} diff --git a/src/core/server/services/tenant/cache.ts b/src/core/server/services/tenant/cache/index.ts similarity index 72% rename from src/core/server/services/tenant/cache.ts rename to src/core/server/services/tenant/cache/index.ts index 58b1a4a1a..9b14ee35a 100644 --- a/src/core/server/services/tenant/cache.ts +++ b/src/core/server/services/tenant/cache/index.ts @@ -4,6 +4,7 @@ import { Db } from "mongodb"; import uuid from "uuid"; import { EventEmitter } from "events"; +import { Config } from "talk-common/config"; import logger from "talk-server/logger"; import { retrieveAllTenants, @@ -29,21 +30,13 @@ export default class TenantCache { /** * tenantsByID reference the tenants that have been cached/retrieved by ID. */ - private tenantsByID = new DataLoader | null>(ids => { - logger.debug({ ids: ids.length }, "now loading tenants"); - return retrieveManyTenants(this.mongo, ids); - }); + private tenantsByID: DataLoader | null>; /** * tenantsByDomain reference the tenants that have been cached/retrieved by * Domain. */ - private tenantsByDomain = new DataLoader | null>( - domains => { - logger.debug({ domains: domains.length }, "now loading tenants"); - return retrieveManyTenantsByDomain(this.mongo, domains); - } - ); + private tenantsByDomain: DataLoader | null>; /** * Create a new client application ID. This prevents duplicated messages @@ -54,23 +47,70 @@ export default class TenantCache { private mongo: Db; private emitter = new EventEmitter(); + private cachingEnabled: boolean; + + constructor(mongo: Db, subscriber: Redis, config: Config) { + this.cachingEnabled = !config.get("disable_tenant_caching"); + if (!this.cachingEnabled) { + logger.warn("tenant caching is disabled"); + } else { + logger.debug("tenant caching is enabled"); + } - constructor(mongo: Db, subscriber: Redis) { // Save the Db reference. this.mongo = mongo; - // Attach to messages on this connection so we can receive updates when - // the tenant are changed. - subscriber.on("message", this.onMessage); + // Configure the data loaders. + this.tenantsByID = new DataLoader( + async ids => { + logger.debug({ ids: ids.length }, "now loading tenants"); + const tenants = await retrieveManyTenants(this.mongo, ids); + logger.debug( + { tenants: tenants.filter(t => t !== null).length }, + "loaded tenants" + ); + return tenants; + }, + { + cache: this.cachingEnabled, + } + ); - // Subscribe to tenant notifications. - subscriber.subscribe(TENANT_UPDATE_CHANNEL); + this.tenantsByDomain = new DataLoader( + async domains => { + logger.debug({ domains: domains.length }, "now loading tenants"); + const tenants = await retrieveManyTenantsByDomain(this.mongo, domains); + logger.debug( + { tenants: tenants.filter(t => t !== null).length }, + "loaded tenants" + ); + return tenants; + }, + { + cache: this.cachingEnabled, + } + ); + + // We don't need updates if we aren't synced to tenant updates. + if (this.cachingEnabled) { + // Attach to messages on this connection so we can receive updates when + // the tenant are changed. + subscriber.on("message", this.onMessage); + + // Subscribe to tenant notifications. + subscriber.subscribe(TENANT_UPDATE_CHANNEL); + } } /** * primeAll will load all the tenants into the cache on startup. */ public async primeAll() { + if (!this.cachingEnabled) { + logger.debug("tenants not primed, caching disabled"); + return; + } + // Grab all the tenants for this node. const tenants = await retrieveAllTenants(this.mongo); diff --git a/src/core/server/services/users/karma.ts b/src/core/server/services/users/karma.ts index 4a17538b0..387c44125 100644 --- a/src/core/server/services/users/karma.ts +++ b/src/core/server/services/users/karma.ts @@ -1,13 +1,13 @@ import { get } from "lodash"; -import { KarmaThresholds } from "talk-server/models/settings"; +import { GQLKarmaThresholds } from "talk-server/graph/tenant/schema/__generated__/types"; import { User } from "talk-server/models/user"; export const getCommentTrustScore = (user: User): number => get(user, "metadata.trust.comment.karma", 0); export const isReliableCommenter = ( - thresholds: KarmaThresholds, + thresholds: GQLKarmaThresholds, user: User ): boolean | null => { const score = getCommentTrustScore(user);