From 6f538d3235cc532b7e92a087a09eb81311650c0c Mon Sep 17 00:00:00 2001 From: Wyatt Johnson Date: Wed, 12 Dec 2018 23:14:29 +0000 Subject: [PATCH] [next] DOM Purification (#2115) * feat: added purify + linkify + better link detection * fix: linting + tests * chore: refactor RequireProperty Type --- package-lock.json | 212 ++++++++++-------- package.json | 9 +- .../client/stream/components/HTMLContent.tsx | 10 +- .../__snapshots__/HTMLContent.spec.tsx.snap | 2 +- src/core/common/types.ts | 3 + .../utils/__snapshots__/purify.spec.ts.snap | 37 +++ src/core/common/utils/purify.spec.ts | 46 ++++ src/core/common/utils/purify.ts | 49 ++++ src/core/server/services/comments/index.ts | 7 +- .../services/comments/pipeline/index.spec.ts | 18 +- .../services/comments/pipeline/index.ts | 23 +- .../comments/pipeline/phases/commentLength.ts | 2 +- .../phases/{links.ts => detectLinks.ts} | 25 +-- .../comments/pipeline/phases/index.ts | 8 +- .../comments/pipeline/phases/linkify.ts | 18 ++ .../comments/pipeline/phases/purify.ts | 24 ++ .../pipeline/phases/storyClosed.spec.ts | 48 ++-- src/types/dompurify.d.ts | 18 ++ 18 files changed, 406 insertions(+), 153 deletions(-) create mode 100644 src/core/common/utils/__snapshots__/purify.spec.ts.snap create mode 100644 src/core/common/utils/purify.spec.ts create mode 100644 src/core/common/utils/purify.ts rename src/core/server/services/comments/pipeline/phases/{links.ts => detectLinks.ts} (68%) create mode 100644 src/core/server/services/comments/pipeline/phases/linkify.ts create mode 100644 src/core/server/services/comments/pipeline/phases/purify.ts create mode 100644 src/types/dompurify.d.ts diff --git a/package-lock.json b/package-lock.json index 6ac28d8eb..0a1368d6c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1882,12 +1882,6 @@ "@types/node": "*" } }, - "@types/dompurify": { - "version": "0.0.31", - "resolved": "https://registry.npmjs.org/@types/dompurify/-/dompurify-0.0.31.tgz", - "integrity": "sha512-Vi2VBl262MjJuxpROjQU3AuBv1NgA7oaL9jqLJRgF60tps4GR/9qzXN0P4pRYPXsBA0b10AF1M6wO74uwGqtmw==", - "dev": true - }, "@types/dotenv": { "version": "4.0.3", "resolved": "https://registry.npmjs.org/@types/dotenv/-/dotenv-4.0.3.tgz", @@ -2055,9 +2049,9 @@ "dev": true }, "@types/jsdom": { - "version": "11.0.6", - "resolved": "https://registry.npmjs.org/@types/jsdom/-/jsdom-11.0.6.tgz", - "integrity": "sha512-6sw9iNdXak3Dk4iN1osdZeZcV8cTe2SNIVwbnt8MjFFAI8+nBWDNcDfjUgZZqycF3Twlr7xXlKntnOrruxNRvg==", + "version": "11.12.0", + "resolved": "https://registry.npmjs.org/@types/jsdom/-/jsdom-11.12.0.tgz", + "integrity": "sha512-XHMNZFQ0Ih3A4/NTWAO15+OsQafPKnQCanN0FYGbsTM/EoI5EoEAvvkF51/DQC2BT5low4tomp7k2RLMlriA5Q==", "dev": true, "requires": { "@types/events": "*", @@ -2081,6 +2075,12 @@ "integrity": "sha512-WRj1rJPkj3fyDME63xUkWvcWzq0XjpiqjJGNCH4Y6Fbiv2fVMDCqeIA6ch/UUG3VYrfGq1VNFLsz6Sat6FjsPw==", "dev": true }, + "@types/linkifyjs": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@types/linkifyjs/-/linkifyjs-2.1.0.tgz", + "integrity": "sha512-iPV2Ha51/vNZrleyRWaw4MHxL4gw5li0vkE4K7eT9e7kBEDv1YN7GIFJZj3PrFiTrUGobk7P92lgtGFhrS5NHw==", + "dev": true + }, "@types/lodash": { "version": "4.14.118", "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.118.tgz", @@ -2352,9 +2352,9 @@ "dev": true }, "@types/tough-cookie": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-2.3.3.tgz", - "integrity": "sha512-MDQLxNFRLasqS4UlkWMSACMKeSm1x4Q3TxzUC7KQUsh6RK1ZrQ0VEyE3yzXcBu+K8ejVj4wuX32eUG02yNp+YQ==", + "version": "2.3.4", + "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-2.3.4.tgz", + "integrity": "sha512-Set5ZdrAaKI/qHdFlVMgm/GsAv/wkXhSTuZFkJ+JI7HK+wIkIlOaUXSXieIvJ0+OvGIqtREFoE+NHJtEq0gtEw==", "dev": true }, "@types/uglify-js": { @@ -2870,9 +2870,9 @@ "integrity": "sha1-OOgxnXk3niRiiEW1O5ZyKyng5Hw=" }, "abab": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/abab/-/abab-1.0.4.tgz", - "integrity": "sha1-X6rZwsB/YN12dw9xzwJbYqY8/U4=", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/abab/-/abab-2.0.0.tgz", + "integrity": "sha512-sY5AXXVZv4Y1VACTtR11UJCPHHudgY5i26Qj5TypE6DKlIApbwb5uqhXcJ5UUGbvZNRh7EeIoW+LrJumBsKp7w==", "dev": true }, "abbrev": { @@ -2906,14 +2906,29 @@ } }, "acorn-globals": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/acorn-globals/-/acorn-globals-4.1.0.tgz", - "integrity": "sha512-KjZwU26uG3u6eZcfGbTULzFcsoz6pegNKtHPksZPOUsiKo5bUmiBPa38FuHZ/Eun+XYh/JCCkS9AS3Lu4McQOQ==", + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/acorn-globals/-/acorn-globals-4.3.0.tgz", + "integrity": "sha512-hMtHj3s5RnuhvHPowpBYvJVj3rAar82JiDQHvGs1zO0l10ocX/xEdBShNHTJaboucJUsScghp74pH3s7EnHHQw==", "dev": true, "requires": { - "acorn": "^5.0.0" + "acorn": "^6.0.1", + "acorn-walk": "^6.0.1" + }, + "dependencies": { + "acorn": { + "version": "6.0.4", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-6.0.4.tgz", + "integrity": "sha512-VY4i5EKSKkofY2I+6QLTbTTN/UvEQPCo6eiwzzSaSWfpaDhOmStMCMod6wmuPciNq+XS0faCglFu2lHZpdHUtg==", + "dev": true + } } }, + "acorn-walk": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-6.1.1.tgz", + "integrity": "sha512-OtUw6JUTgxA2QoqqmrmQ7F2NYqiBPi/L2jqHyFtllhOUvXYQXf0Z1CYUinIfyT4bTCGmrA7gX9FvHA81uzCoVw==", + "dev": true + }, "address": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/address/-/address-1.0.3.tgz", @@ -5775,9 +5790,9 @@ } }, "browser-process-hrtime": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/browser-process-hrtime/-/browser-process-hrtime-0.1.2.tgz", - "integrity": "sha1-Ql1opY00R/AqBKqJQYf86K+Le44=", + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/browser-process-hrtime/-/browser-process-hrtime-0.1.3.tgz", + "integrity": "sha512-bRFnI4NnjO6cnyLmOV/7PVoDEMJChlcfN0z4s1YMBY989/SvlfMI1lgCnkFUs53e9gQF+w7qu7XdllSTiSl8Aw==", "dev": true }, "browser-resolve": { @@ -7887,9 +7902,9 @@ "dev": true }, "cssstyle": { - "version": "0.3.1", - "resolved": "https://registry.npmjs.org/cssstyle/-/cssstyle-0.3.1.tgz", - "integrity": "sha512-tNvaxM5blOnxanyxI6panOsnfiyLRj3HV4qjqqS45WPNS1usdYWRUQjqTEEELK73lpeP/1KoIGYUwrBn/VcECA==", + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/cssstyle/-/cssstyle-1.1.1.tgz", + "integrity": "sha512-364AI1l/M5TYcFH83JnOH/pSqgaNnKmYgKrm0didZMGKWjQB60dymwWy1rKUgL3J1ffdq9xVi2yGLHdSjjSNog==", "dev": true, "requires": { "cssom": "0.3.x" @@ -7934,14 +7949,27 @@ } }, "data-urls": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-1.0.0.tgz", - "integrity": "sha512-ai40PPQR0Fn1lD2PPie79CibnlMN2AYiDhwFX/rZHVsxbs5kNJSjegqXIprhouGXlRdEnfybva7kqRGnB6mypA==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-1.1.0.tgz", + "integrity": "sha512-YTWYI9se1P55u58gL5GkQHW4P6VJBJ5iBT+B5a7i2Tjadhv52paJG0qHX4A0OR6/t52odI64KP2YvFpkDOi3eQ==", "dev": true, "requires": { - "abab": "^1.0.4", - "whatwg-mimetype": "^2.0.0", - "whatwg-url": "^6.4.0" + "abab": "^2.0.0", + "whatwg-mimetype": "^2.2.0", + "whatwg-url": "^7.0.0" + }, + "dependencies": { + "whatwg-url": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-7.0.0.tgz", + "integrity": "sha512-37GeVSIJ3kn1JgKyjiYNmSLP1yzbpb29jdmwBSgkD9h40/hyrR/OifpVUndji3tmwGgD8qpw7iQu3RSbCrBpsQ==", + "dev": true, + "requires": { + "lodash.sortby": "^4.7.0", + "tr46": "^1.0.1", + "webidl-conversions": "^4.0.2" + } + } } }, "dataloader": { @@ -9454,9 +9482,9 @@ } }, "dompurify": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-1.0.7.tgz", - "integrity": "sha512-1xK0JEda/jvIm3SgqHXKvRCh3AbEKCyBbUAGpNCMVIljBD145cPvBR66JSj3O4SdscFUx5NXsDkJpz6vDT8KLg==", + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-1.0.8.tgz", + "integrity": "sha512-vetRFbN1SXSPfP3ClIiYnxTrXquSqakBEOoB5JESn0SVcSYzpu6ougjakpKnskGctYdlNpwf+riUHSkG7d4XUw==", "dev": true }, "domutils": { @@ -12334,7 +12362,7 @@ }, "handle-thing": { "version": "1.2.5", - "resolved": "https://registry.npmjs.org/handle-thing/-/handle-thing-1.2.5.tgz", + "resolved": "http://registry.npmjs.org/handle-thing/-/handle-thing-1.2.5.tgz", "integrity": "sha1-/Xqtcmvxpf0W38KbL3pmAdJxOcQ=", "dev": true }, @@ -12803,7 +12831,7 @@ }, "http-proxy-middleware": { "version": "0.18.0", - "resolved": "https://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-0.18.0.tgz", + "resolved": "http://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-0.18.0.tgz", "integrity": "sha512-Fs25KVMPAIIcgjMZkVHJoKg9VcXcC1C8yb9JUgeDvVXY0S/zgVIhMb+qVswDIgtJe2DfckMSY2d6TuTEutlk6Q==", "dev": true, "requires": { @@ -15288,6 +15316,12 @@ "topo": "3.x.x" } }, + "jquery": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/jquery/-/jquery-3.3.1.tgz", + "integrity": "sha512-Ubldcmxp5np52/ENotGxlLe6aGMvmF4R8S6tZjsP6Knsaxd/xp3Zrh50cG93lR6nPXyUFwzN3ZSOQI0wRJNdGg==", + "optional": true + }, "js-base64": { "version": "2.4.5", "resolved": "https://registry.npmjs.org/js-base64/-/js-base64-2.4.5.tgz", @@ -15315,49 +15349,37 @@ "optional": true }, "jsdom": { - "version": "11.11.0", - "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-11.11.0.tgz", - "integrity": "sha512-ou1VyfjwsSuWkudGxb03FotDajxAto6USAlmMZjE2lc0jCznt7sBWkhfRBRaWwbnmDqdMSTKTLT5d9sBFkkM7A==", + "version": "11.12.0", + "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-11.12.0.tgz", + "integrity": "sha512-y8Px43oyiBM13Zc1z780FrfNLJCXTL40EWlty/LXUtcjykRBNgLlCjWXpfSPBl2iv+N7koQN+dvqszHZgT/Fjw==", "dev": true, "requires": { - "abab": "^1.0.4", - "acorn": "^5.3.0", + "abab": "^2.0.0", + "acorn": "^5.5.3", "acorn-globals": "^4.1.0", "array-equal": "^1.0.0", "cssom": ">= 0.3.2 < 0.4.0", - "cssstyle": ">= 0.3.1 < 0.4.0", + "cssstyle": "^1.0.0", "data-urls": "^1.0.0", - "domexception": "^1.0.0", - "escodegen": "^1.9.0", + "domexception": "^1.0.1", + "escodegen": "^1.9.1", "html-encoding-sniffer": "^1.0.2", - "left-pad": "^1.2.0", - "nwsapi": "^2.0.0", + "left-pad": "^1.3.0", + "nwsapi": "^2.0.7", "parse5": "4.0.0", "pn": "^1.1.0", - "request": "^2.83.0", + "request": "^2.87.0", "request-promise-native": "^1.0.5", "sax": "^1.2.4", "symbol-tree": "^3.2.2", - "tough-cookie": "^2.3.3", + "tough-cookie": "^2.3.4", "w3c-hr-time": "^1.0.1", "webidl-conversions": "^4.0.2", "whatwg-encoding": "^1.0.3", "whatwg-mimetype": "^2.1.0", "whatwg-url": "^6.4.1", - "ws": "^4.0.0", + "ws": "^5.2.0", "xml-name-validator": "^3.0.0" - }, - "dependencies": { - "ws": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/ws/-/ws-4.1.0.tgz", - "integrity": "sha512-ZGh/8kF9rrRNffkLFV4AzhvooEclrOH0xaugmqGsIfFgOE/pIz4fMc4Ef+5HSQqTEug2S9JZIWDR47duDSLfaA==", - "dev": true, - "requires": { - "async-limiter": "~1.0.0", - "safe-buffer": "~5.1.0" - } - } } }, "jsesc": { @@ -15842,6 +15864,16 @@ "uc.micro": "^1.0.1" } }, + "linkifyjs": { + "version": "2.1.7", + "resolved": "https://registry.npmjs.org/linkifyjs/-/linkifyjs-2.1.7.tgz", + "integrity": "sha512-Cbn77BnYEslpAObZZoP6GVQHF1j5T6RsDydNq5RVxIy4eiZAiADRx7qHfWzfEMQecc1PtZFog1AsCGGX2WjQLA==", + "requires": { + "jquery": "^3.3.1", + "react": "^16.4.2", + "react-dom": "^16.4.2" + } + }, "lint-staged": { "version": "7.3.0", "resolved": "https://registry.npmjs.org/lint-staged/-/lint-staged-7.3.0.tgz", @@ -17798,9 +17830,9 @@ } }, "nwsapi": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.0.4.tgz", - "integrity": "sha512-Zt6HRR6RcJkuj5/N9zeE7FN6YitRW//hK2wTOwX274IBphbY3Zf5+yn5mZ9v/SzAOTMjQNxZf9KkmPLWn0cV4g==", + "version": "2.0.9", + "resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.0.9.tgz", + "integrity": "sha512-nlWFSCTYQcHk/6A9FFnfhKc14c3aFhfdNBXgo8Qgi9QTBu/qg3Ww+Uiz9wMzXd1T8GFxPc2QIHB6Qtf2XFryFQ==", "dev": true }, "oauth": { @@ -21027,9 +21059,9 @@ "integrity": "sha1-8FKijacOYYkX7wqKw0wa5aaChrM=" }, "psl": { - "version": "1.1.28", - "resolved": "https://registry.npmjs.org/psl/-/psl-1.1.28.tgz", - "integrity": "sha512-+AqO1Ae+N/4r7Rvchrdm432afjT9hqJRyBN3DQv9At0tPz4hIFSGKbq64fN9dVoCow4oggIIax5/iONx0r9hZw==", + "version": "1.1.29", + "resolved": "https://registry.npmjs.org/psl/-/psl-1.1.29.tgz", + "integrity": "sha512-AeUmQ0oLN02flVHXWh9sSJF7mcdFq0ppid/JkErufc3hGIV/AMa8Fo9VgDo/cT2jFdOWoFvHp90qqBH54W+gjQ==", "dev": true }, "pstree.remy": { @@ -21269,7 +21301,6 @@ "version": "16.5.2", "resolved": "https://registry.npmjs.org/react/-/react-16.5.2.tgz", "integrity": "sha512-FDCSVd3DjVTmbEAjUNX6FgfAmQ+ypJfHUsqUJOYNCBUp1h8lqmtC+0mXJ+JjsWx4KAVTkk1vKd1hLQPvEviSuw==", - "dev": true, "requires": { "loose-envify": "^1.1.0", "object-assign": "^4.1.1", @@ -21582,7 +21613,6 @@ "version": "16.5.2", "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-16.5.2.tgz", "integrity": "sha512-RC8LDw8feuZOHVgzEf7f+cxBr/DnKdqp56VU0lAs1f4UfKc4cU8wU4fTq/mgnvynLQo8OtlPC19NUFh/zjZPuA==", - "dev": true, "requires": { "loose-envify": "^1.1.0", "object-assign": "^4.1.1", @@ -23033,7 +23063,6 @@ "version": "0.5.0", "resolved": "https://registry.npmjs.org/schedule/-/schedule-0.5.0.tgz", "integrity": "sha512-HUcJicG5Ou8xfR//c2rPT0lPIRR09vVvN81T9fqfVgBmhERUbDEQoYKjpBxbueJnCPpSu2ujXzOnRQt6x9o/jw==", - "dev": true, "requires": { "object-assign": "^4.1.1" } @@ -24549,21 +24578,13 @@ } }, "tough-cookie": { - "version": "2.4.3", - "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.4.3.tgz", - "integrity": "sha512-Q5srk/4vDM54WJsJio3XNn6K2sCG+CQ8G5Wz6bZhRZoAe/+TxjWB/GlFAnYEbkYVlON9FMk/fE3h2RLpPXo4lQ==", + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.5.0.tgz", + "integrity": "sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g==", "dev": true, "requires": { - "psl": "^1.1.24", - "punycode": "^1.4.1" - }, - "dependencies": { - "punycode": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", - "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4=", - "dev": true - } + "psl": "^1.1.28", + "punycode": "^2.1.1" } }, "tr46": { @@ -26729,12 +26750,23 @@ "dev": true }, "whatwg-encoding": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-1.0.3.tgz", - "integrity": "sha512-jLBwwKUhi8WtBfsMQlL4bUUcT8sMkAtQinscJAe/M4KHCkHuUJAF6vuB0tueNIw4c8ziO6AkRmgY+jL3a0iiPw==", + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-1.0.5.tgz", + "integrity": "sha512-b5lim54JOPN9HtzvK9HFXvBma/rnfFeqsic0hSpjtDbVxR3dJKLc+KB4V6GgiGOvl7CY/KNh8rxSo9DKQrnUEw==", "dev": true, "requires": { - "iconv-lite": "0.4.19" + "iconv-lite": "0.4.24" + }, + "dependencies": { + "iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "dev": true, + "requires": { + "safer-buffer": ">= 2.1.2 < 3" + } + } } }, "whatwg-fetch": { @@ -26744,9 +26776,9 @@ "dev": true }, "whatwg-mimetype": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-2.1.0.tgz", - "integrity": "sha512-FKxhYLytBQiUKjkYteN71fAUA3g6KpNXoho1isLiLSB3N1G4F35Q5vUxWfKFhBwi5IWF27VE6WxhrnnC+m0Mew==", + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-2.3.0.tgz", + "integrity": "sha512-M4yMwr6mAnQz76TbJm914+gPpB/nCwvZbJU28cUD6dR004SAxDLOOSUaB1JDRqLtaOV/vi0IC5lEAGFgrjGv/g==", "dev": true }, "whatwg-url": { diff --git a/package.json b/package.json index 659bf7469..e2c019c92 100644 --- a/package.json +++ b/package.json @@ -56,6 +56,7 @@ "content-security-policy-builder": "^2.0.0", "convict": "^4.3.1", "cors": "^2.8.4", + "dompurify": "^1.0.8", "dataloader": "^1.4.0", "dotenv": "^6.0.0", "dotenv-expand": "^4.2.0", @@ -78,6 +79,7 @@ "jsonwebtoken": "^8.3.0", "jwks-rsa": "^1.3.0", "linkify-it": "^2.0.3", + "linkifyjs": "^2.1.7", "lodash": "^4.17.10", "luxon": "^1.3.1", "metascraper-author": "^3.11.8", @@ -126,7 +128,6 @@ "@types/convict": "^4.2.0", "@types/cors": "^2.8.4", "@types/cross-spawn": "^6.0.0", - "@types/dompurify": "0.0.31", "@types/dotenv": "^4.0.3", "@types/enzyme": "^3.1.11", "@types/enzyme-adapter-react-16": "^1.0.2", @@ -139,9 +140,10 @@ "@types/ioredis": "^3.2.12", "@types/jest": "^23.1.5", "@types/joi": "^13.0.8", - "@types/jsdom": "^11.0.6", + "@types/jsdom": "^11.12.0", "@types/jsonwebtoken": "^7.2.7", "@types/linkify-it": "^2.0.3", + "@types/linkifyjs": "^2.1.0", "@types/lodash": "^4.14.118", "@types/luxon": "^0.5.3", "@types/mini-css-extract-plugin": "^0.2.0", @@ -194,7 +196,6 @@ "css-loader": "^0.28.11", "del": "^3.0.0", "docz": "^0.5.8", - "dompurify": "^1.0.7", "enzyme": "^3.7.0", "enzyme-adapter-react-16": "^1.6.0", "enzyme-to-json": "^3.3.4", @@ -217,7 +218,7 @@ "jest-junit": "^5.1.0", "jest-localstorage-mock": "^2.2.0", "jest-mock-console": "^0.4.0", - "jsdom": "^11.11.0", + "jsdom": "^11.12.0", "lint-staged": "^7.3.0", "loader-utils": "^1.1.0", "material-design-icons": "^3.0.1", diff --git a/src/core/client/stream/components/HTMLContent.tsx b/src/core/client/stream/components/HTMLContent.tsx index a8d20c0ca..3af8fde6b 100644 --- a/src/core/client/stream/components/HTMLContent.tsx +++ b/src/core/client/stream/components/HTMLContent.tsx @@ -1,9 +1,15 @@ import cn from "classnames"; -import dompurify from "dompurify"; import React, { StatelessComponent } from "react"; +import { createPurify } from "talk-common/utils/purify"; + import styles from "./HTMLContent.css"; +/** + * Create a purify instance that will be used to handle HTML content. + */ +const purify = createPurify(window, false); + interface HTMLContentProps { children: string; className?: string; @@ -15,7 +21,7 @@ const HTMLContent: StatelessComponent = ({ }) => (
); diff --git a/src/core/client/stream/components/__snapshots__/HTMLContent.spec.tsx.snap b/src/core/client/stream/components/__snapshots__/HTMLContent.spec.tsx.snap index fac21ba7d..28545b740 100644 --- a/src/core/client/stream/components/__snapshots__/HTMLContent.spec.tsx.snap +++ b/src/core/client/stream/components/__snapshots__/HTMLContent.spec.tsx.snap @@ -16,7 +16,7 @@ exports[`sanitizes evil html 1`] = ` className="HTMLContent-root" dangerouslySetInnerHTML={ Object { - "__html": "Hello world + "__html": "Hello world ", } } diff --git a/src/core/common/types.ts b/src/core/common/types.ts index 96334afe7..16cc0b08a 100644 --- a/src/core/common/types.ts +++ b/src/core/common/types.ts @@ -7,6 +7,9 @@ export type Overwrite = Pick> & U; export type Sub = Pick>; +export type RequireProperty = Omit, P> & + Required>; + /** * Make all properties in T writeable */ diff --git a/src/core/common/utils/__snapshots__/purify.spec.ts.snap b/src/core/common/utils/__snapshots__/purify.spec.ts.snap new file mode 100644 index 000000000..29c58bd28 --- /dev/null +++ b/src/core/common/utils/__snapshots__/purify.spec.ts.snap @@ -0,0 +1,37 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`allows anchor links 1`] = ` +Object { + "body": "This is a link", + "linkCount": 1, +} +`; + +exports[`allows anchor tags and counts them correctly 1`] = ` +" + " +`; + +exports[`sanitizes out attributes not allowed 1`] = ` +Object { + "body": "
Test
", + "linkCount": 0, +} +`; + +exports[`sanitizes out tags not allowed 1`] = ` +Object { + "body": "", + "linkCount": 0, +} +`; + +exports[`sanitizes out tags not allowed 2`] = ` +Object { + "body": "", + "linkCount": 0, +} +`; diff --git a/src/core/common/utils/purify.spec.ts b/src/core/common/utils/purify.spec.ts new file mode 100644 index 000000000..46ed1e43b --- /dev/null +++ b/src/core/common/utils/purify.spec.ts @@ -0,0 +1,46 @@ +import { JSDOM } from "jsdom"; + +import { createPurify, sanitizeCommentBody } from "./purify"; + +const window = new JSDOM("", {}).window; +const DOMPurify = createPurify(window); + +it("sanitizes out tags not allowed", () => { + expect( + sanitizeCommentBody( + DOMPurify, + "" + ) + ).toMatchSnapshot(); + + expect( + sanitizeCommentBody(DOMPurify, "") + ).toMatchSnapshot(); +}); + +it("sanitizes out attributes not allowed", () => { + expect( + sanitizeCommentBody(DOMPurify, '
Test
') + ).toMatchSnapshot(); +}); + +it("allows anchor links", () => { + expect( + sanitizeCommentBody(DOMPurify, 'This is a link') + ).toMatchSnapshot(); +}); + +it("allows anchor tags and counts them correctly", () => { + const { body, linkCount } = sanitizeCommentBody( + DOMPurify, + ` + + ` + ); + + expect(body).toMatchSnapshot(); + expect(linkCount).toEqual(2); +}); diff --git a/src/core/common/utils/purify.ts b/src/core/common/utils/purify.ts new file mode 100644 index 000000000..89ad2a5ab --- /dev/null +++ b/src/core/common/utils/purify.ts @@ -0,0 +1,49 @@ +import newDOMPurify, { DOMPurify } from "dompurify"; + +export function createPurify( + window: Window, + returnDOM: T = true as T +) { + // Initializing JSDOM and DOMPurify + const purify = newDOMPurify(window); + + // Setting our DOMPurify config. + purify.setConfig({ + // Only forward anchor tags, bold, italics, blockquote, breaks, divs, and + // spans. + ALLOWED_TAGS: ["a", "strong", "i", "blockquote", "br", "div", "span"], + // Only allow href tags for anchor tags. + ALLOWED_ATTR: ["href"], + // Always return the DOM to the caller of sanitize. + RETURN_DOM: returnDOM, + }); + + // Ensure that each anchor tag has a "target" and "rel" attributes set, and + // strip the "href" attribute from all non-anchor tags. + purify.addHook("afterSanitizeAttributes", node => { + if (node.nodeName === "A") { + // Ensure we wrap all the links with the target + rel set. + node.setAttribute("target", "_blank"); + node.setAttribute("rel", "noopener noreferrer"); + } else { + // The only tag that's allowed attributes is the "A" tag. + node.removeAttribute("href"); + } + }); + + return purify; +} + +export function sanitizeCommentBody(purify: DOMPurify, source: string) { + // Sanitize and return the HTMLBodyElement for the parsed source. + const sanitized = purify.sanitize(source); + + // Count the total number of anchor links in the sanitized output, this is the + // number of links. + const linkCount = sanitized.getElementsByTagName("a").length; + + return { + body: sanitized.innerHTML, + linkCount, + }; +} diff --git a/src/core/server/services/comments/index.ts b/src/core/server/services/comments/index.ts index 7a06a84a1..c287448cc 100644 --- a/src/core/server/services/comments/index.ts +++ b/src/core/server/services/comments/index.ts @@ -86,7 +86,7 @@ export async function create( } // Run the comment through the moderation phases. - const { actions, status, metadata } = await processForModeration({ + const { actions, body, status, metadata } = await processForModeration({ story, tenant, comment: input, @@ -118,6 +118,7 @@ export async function create( // Create the comment! const comment = await createComment(mongo, tenant.id, { ...input, + body, status, grandparentIDs, metadata, @@ -234,7 +235,7 @@ export async function edit( } // Run the comment through the moderation phases. - const { status, metadata, actions } = await processForModeration({ + const { body, status, metadata, actions } = await processForModeration({ story, tenant, comment: input, @@ -258,7 +259,7 @@ export async function edit( const result = await editComment(mongo, tenant.id, { id: input.id, authorID: author.id, - body: input.body, + body, status, metadata, actionCounts, diff --git a/src/core/server/services/comments/pipeline/index.spec.ts b/src/core/server/services/comments/pipeline/index.spec.ts index 15545e3e1..c85d45f43 100644 --- a/src/core/server/services/comments/pipeline/index.spec.ts +++ b/src/core/server/services/comments/pipeline/index.spec.ts @@ -8,6 +8,10 @@ import { ModerationPhaseContext, } from "talk-server/services/comments/pipeline"; +const context = { + comment: { body: "This is a test" }, +} as ModerationPhaseContext; + describe("compose", () => { it("handles when a phase throws an error", async () => { const err = new Error("this is an error"); @@ -17,14 +21,15 @@ describe("compose", () => { }, ]); - await expect(enhanced({} as ModerationPhaseContext)).rejects.toEqual(err); + await expect(enhanced(context)).rejects.toEqual(err); }); it("handles when it returns a status", async () => { const status = GQLCOMMENT_STATUS.ACCEPTED; const enhanced = compose([() => ({ status })]); - await expect(enhanced({} as ModerationPhaseContext)).resolves.toEqual({ + await expect(enhanced(context)).resolves.toEqual({ + body: context.comment.body, status, metadata: {}, actions: [], @@ -39,7 +44,8 @@ describe("compose", () => { () => ({ metadata: { third: true } }), ]); - await expect(enhanced({} as ModerationPhaseContext)).resolves.toEqual({ + await expect(enhanced(context)).resolves.toEqual({ + body: context.comment.body, status, metadata: { first: true, second: true }, actions: [], @@ -81,13 +87,14 @@ describe("compose", () => { }), ]); - const final = await enhanced({} as ModerationPhaseContext); + const final = await enhanced(context); for (const flag of flags) { expect(final.actions).toContainEqual(flag); } expect(final.actions).not.toContainEqual({ + body: context.comment.body, actionType: ACTION_TYPE.FLAG, reason: GQLCOMMENT_FLAG_REASON.COMMENT_DETECTED_LINKS, }); @@ -99,7 +106,8 @@ describe("compose", () => { () => ({ metadata: { second: true } }), ]); - await expect(enhanced({} as ModerationPhaseContext)).resolves.toEqual({ + await expect(enhanced(context)).resolves.toEqual({ + body: context.comment.body, status: GQLCOMMENT_STATUS.NONE, metadata: { first: true, second: true }, actions: [], diff --git a/src/core/server/services/comments/pipeline/index.ts b/src/core/server/services/comments/pipeline/index.ts index 131865b28..4857cb1bb 100644 --- a/src/core/server/services/comments/pipeline/index.ts +++ b/src/core/server/services/comments/pipeline/index.ts @@ -1,4 +1,4 @@ -import { Omit, Promiseable } from "talk-common/types"; +import { Omit, Promiseable, RequireProperty } from "talk-common/types"; import { GQLCOMMENT_STATUS } from "talk-server/graph/tenant/schema/__generated__/types"; import { CreateActionInput } from "talk-server/models/action/comment"; import { EditCommentInput } from "talk-server/models/comment"; @@ -18,12 +18,13 @@ export interface PhaseResult { actions: ModerationAction[]; status: GQLCOMMENT_STATUS; metadata: Record; + body: string; } export interface ModerationPhaseContext { story: Story; tenant: Tenant; - comment: Partial; + comment: RequireProperty, "body">; author: User; req?: Request; } @@ -47,13 +48,21 @@ export const compose = ( ): ModerationPhase => async context => { const final: PhaseResult = { status: GQLCOMMENT_STATUS.NONE, + body: context.comment.body, actions: [], - metadata: {}, + metadata: context.comment.metadata || {}, }; // Loop over all the moderation phases and see if we've resolved the status. for (const phase of phases) { - const result = await phase(context); + const result = await phase({ + ...context, + comment: { + ...context.comment, + body: final.body, + metadata: final.metadata, + }, + }); if (result) { // If this result contained actions, then we should push it into the // other actions. @@ -72,6 +81,12 @@ export const compose = ( }; } + // If the result modified the comment body, we should replace it. + const { body } = result; + if (body) { + final.body = body; + } + // If this result contained a status, then we've finished resolving // phases! const { status } = result; diff --git a/src/core/server/services/comments/pipeline/phases/commentLength.ts b/src/core/server/services/comments/pipeline/phases/commentLength.ts index b20373e19..2a8427073 100644 --- a/src/core/server/services/comments/pipeline/phases/commentLength.ts +++ b/src/core/server/services/comments/pipeline/phases/commentLength.ts @@ -38,7 +38,7 @@ export const commentLength: IntermediateModerationPhase = ({ tenant, comment, }): IntermediatePhaseResult | void => { - const length = comment.body ? striptags(comment.body).length : 0; + const length = striptags(comment.body).length; // Reject if the comment is too long or too short. testCharCount(tenant, length); diff --git a/src/core/server/services/comments/pipeline/phases/links.ts b/src/core/server/services/comments/pipeline/phases/detectLinks.ts similarity index 68% rename from src/core/server/services/comments/pipeline/phases/links.ts rename to src/core/server/services/comments/pipeline/phases/detectLinks.ts index e6ec86952..92c1100d2 100755 --- a/src/core/server/services/comments/pipeline/phases/links.ts +++ b/src/core/server/services/comments/pipeline/phases/detectLinks.ts @@ -1,38 +1,32 @@ -import linkify from "linkify-it"; -import tlds from "tlds"; - import { GQLCOMMENT_FLAG_REASON, GQLCOMMENT_STATUS, } from "talk-server/graph/tenant/schema/__generated__/types"; import { ACTION_TYPE } from "talk-server/models/action/comment"; +import { Comment } from "talk-server/models/comment"; import { ModerationSettings } from "talk-server/models/settings"; import { IntermediateModerationPhase, IntermediatePhaseResult, } from "talk-server/services/comments/pipeline"; -/** - * The preloaded linkify instance with common tlds. - */ -const testForLinks = linkify().tlds(tlds); - const testPremodLinksEnable = ( settings: Partial, - body: string -) => settings.premodLinksEnable && testForLinks.test(body); + comment: Pick +) => + settings.premodLinksEnable && comment.metadata && comment.metadata.linkCount; // This phase checks the comment if it has any links in it if the check is // enabled. -export const links: IntermediateModerationPhase = ({ +export const detectLinks: IntermediateModerationPhase = ({ story, tenant, comment, }): IntermediatePhaseResult | void => { if ( - comment.body && - (testPremodLinksEnable(tenant, comment.body) || - (story.settings && testPremodLinksEnable(story.settings, comment.body))) + comment && + (testPremodLinksEnable(tenant, comment) || + (story.settings && testPremodLinksEnable(story.settings, comment))) ) { // Add the flag related to Trust to the comment. return { @@ -42,9 +36,6 @@ export const links: IntermediateModerationPhase = ({ userID: null, actionType: ACTION_TYPE.FLAG, reason: GQLCOMMENT_FLAG_REASON.COMMENT_DETECTED_LINKS, - metadata: { - links: comment.body, - }, }, ], }; diff --git a/src/core/server/services/comments/pipeline/phases/index.ts b/src/core/server/services/comments/pipeline/phases/index.ts index 8948d8210..d3d9aac7c 100644 --- a/src/core/server/services/comments/pipeline/phases/index.ts +++ b/src/core/server/services/comments/pipeline/phases/index.ts @@ -2,9 +2,11 @@ import { IntermediateModerationPhase } from "talk-server/services/comments/pipel import { commentingDisabled } from "./commentingDisabled"; import { commentLength } from "./commentLength"; +import { detectLinks } from "./detectLinks"; import { karma } from "./karma"; -import { links } from "./links"; +import { linkify } from "./linkify"; import { preModerate } from "./preModerate"; +import { purify } from "./purify"; import { spam } from "./spam"; import { staff } from "./staff"; import { storyClosed } from "./storyClosed"; @@ -18,9 +20,11 @@ export const moderationPhases: IntermediateModerationPhase[] = [ commentLength, storyClosed, commentingDisabled, + linkify, + purify, + detectLinks, wordList, staff, - links, karma, spam, toxic, diff --git a/src/core/server/services/comments/pipeline/phases/linkify.ts b/src/core/server/services/comments/pipeline/phases/linkify.ts new file mode 100644 index 000000000..51842db85 --- /dev/null +++ b/src/core/server/services/comments/pipeline/phases/linkify.ts @@ -0,0 +1,18 @@ +import linkifyjs from "linkifyjs/html"; + +import { + IntermediateModerationPhase, + IntermediatePhaseResult, +} from "talk-server/services/comments/pipeline"; + +// linkify configuration. +const config = { + className: "", + tagName: "a", +}; + +export const linkify: IntermediateModerationPhase = async ({ + comment, +}): Promise => ({ + body: linkifyjs(comment.body, config), +}); diff --git a/src/core/server/services/comments/pipeline/phases/purify.ts b/src/core/server/services/comments/pipeline/phases/purify.ts new file mode 100644 index 000000000..978fdc05f --- /dev/null +++ b/src/core/server/services/comments/pipeline/phases/purify.ts @@ -0,0 +1,24 @@ +import { JSDOM } from "jsdom"; + +import { createPurify, sanitizeCommentBody } from "talk-common/utils/purify"; +import { + IntermediateModerationPhase, + IntermediatePhaseResult, +} from "talk-server/services/comments/pipeline"; + +// Initializing JSDOM and DOMPurify +const window = new JSDOM("", {}).window; +const DOMPurify = createPurify(window); + +export const purify: IntermediateModerationPhase = async ({ + comment, +}): Promise => { + const { body, linkCount } = sanitizeCommentBody(DOMPurify, comment.body); + + return { + body, + metadata: { + linkCount, + }, + }; +}; diff --git a/src/core/server/services/comments/pipeline/phases/storyClosed.spec.ts b/src/core/server/services/comments/pipeline/phases/storyClosed.spec.ts index c300b088a..30aa4719a 100644 --- a/src/core/server/services/comments/pipeline/phases/storyClosed.spec.ts +++ b/src/core/server/services/comments/pipeline/phases/storyClosed.spec.ts @@ -1,35 +1,35 @@ import { DateTime } from "luxon"; -import { Comment } from "talk-server/models/comment"; -import { Story } from "talk-server/models/story"; -import { Tenant } from "talk-server/models/tenant"; -import { User } from "talk-server/models/user"; import { storyClosed } from "talk-server/services/comments/pipeline/phases/storyClosed"; +import { ModerationPhaseContext } from ".."; describe("storyClosed", () => { it("throws an error when the story is closed", () => { expect(() => storyClosed({ - story: { closedAt: new Date() } as Story, - tenant: {} as Tenant, - comment: {} as Comment, - author: {} as User, + story: { closedAt: new Date() } as ModerationPhaseContext["story"], + tenant: {} as ModerationPhaseContext["tenant"], + comment: {} as ModerationPhaseContext["comment"], + author: {} as ModerationPhaseContext["author"], }) ).toThrow(); storyClosed({ - story: {} as Story, - tenant: { autoCloseStream: true } as Tenant, - comment: {} as Comment, - author: {} as User, + story: {} as ModerationPhaseContext["story"], + tenant: { autoCloseStream: true } as ModerationPhaseContext["tenant"], + comment: {} as ModerationPhaseContext["comment"], + author: {} as ModerationPhaseContext["author"], }); expect(() => storyClosed({ - story: { createdAt: new Date() } as Story, - tenant: { autoCloseStream: true, closedTimeout: -6000 } as Tenant, - comment: {} as Comment, - author: {} as User, + story: { createdAt: new Date() } as ModerationPhaseContext["story"], + tenant: { + autoCloseStream: true, + closedTimeout: -6000, + } as ModerationPhaseContext["tenant"], + comment: {} as ModerationPhaseContext["comment"], + author: {} as ModerationPhaseContext["author"], }) ).toThrow(); }); @@ -43,19 +43,19 @@ describe("storyClosed", () => { closedAt: DateTime.fromJSDate(now) .plus(60000) .toJSDate(), - } as Story, - tenant: {} as Tenant, - comment: {} as Comment, - author: {} as User, + } as ModerationPhaseContext["story"], + tenant: {} as ModerationPhaseContext["tenant"], + comment: {} as ModerationPhaseContext["comment"], + author: {} as ModerationPhaseContext["author"], }) ).toBeUndefined(); expect( storyClosed({ - story: {} as Story, - tenant: {} as Tenant, - comment: {} as Comment, - author: {} as User, + story: {} as ModerationPhaseContext["story"], + tenant: {} as ModerationPhaseContext["tenant"], + comment: {} as ModerationPhaseContext["comment"], + author: {} as ModerationPhaseContext["author"], }) ).toBeUndefined(); }); diff --git a/src/types/dompurify.d.ts b/src/types/dompurify.d.ts new file mode 100644 index 000000000..b0c05f149 --- /dev/null +++ b/src/types/dompurify.d.ts @@ -0,0 +1,18 @@ +declare module "dompurify" { + interface Config { + ALLOWED_ATTR?: string[]; + ALLOWED_TAGS?: string[]; + RETURN_DOM?: boolean; + RETURN_DOM_FRAGMENT?: T; + } + + class DOMPurify { + public setConfig(config: Config): void; + public sanitize(source: string): T extends true ? HTMLBodyElement : string; + public addHook(name: string, callback: (node: Element) => void): void; + } + + export default function createDOMPurify( + window: any + ): DOMPurify; +}