feat: expanded regexp generation, locale support, caching (#2869)

Co-authored-by: Kim Gardner <kgardnr@gmail.com>

src/core/client/admin/components/ModerateCard/CommentContent.spec.tsx

@@ -7,8 +7,13 @@ import CommentContent from "./CommentContent";
 
 it("renders correctly", () => {
   const props: PropTypesOf<typeof CommentContent> = {
-    suspectWords: ["worse"],
-    bannedWords: ["bad"],
+    phrases: {
+      locale: "en-US",
+      wordList: {
+        suspect: ["worse"],
+        banned: ["bad"],
+      },
+    },
     className: "custom",
     children: "Hello <b>Bob</b>, you bad guy",
   };
@@ -19,8 +24,13 @@ it("renders correctly", () => {
 
 it("renders empty words correctly", () => {
   const props: PropTypesOf<typeof CommentContent> = {
-    suspectWords: [],
-    bannedWords: [],
+    phrases: {
+      locale: "en-US",
+      wordList: {
+        suspect: [],
+        banned: [],
+      },
+    },
     className: "custom",
     children: "Hello <b>Bob</b>, you bad guy",
   };

src/core/client/admin/components/ModerateCard/CommentContent.tsx

@@ -1,7 +1,7 @@
 import cn from "classnames";
-import { memoize } from "lodash";
-import React, { FunctionComponent } from "react";
+import React, { FunctionComponent, useMemo } from "react";
 
+import { getPhrasesRegExp, GetPhrasesRegExpOptions } from "coral-admin/helpers";
 import { createPurify } from "coral-common/utils/purify";
 
 import styles from "./CommentContent.css";
@@ -14,8 +14,7 @@ const purify = createPurify(window, false);
 interface Props {
   className?: string;
   children: string | React.ReactElement;
-  suspectWords: ReadonlyArray<string>;
-  bannedWords: ReadonlyArray<string>;
+  phrases: GetPhrasesRegExpOptions;
 }
 
 function escapeHTML(unsafe: string) {
@@ -27,50 +26,11 @@ function escapeHTML(unsafe: string) {
     .replace(/'/g, "&#039;");
 }
 
-function escapeRegExp(str: string) {
-  return str.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); // $& means the whole matched string
-}
-
-// generate a regulare expression that catches the `phrases`.
-function generateRegExp(phrases: ReadonlyArray<string>) {
-  const inner = phrases
-    .map(phrase =>
-      phrase
-        .split(/\s+/)
-        .map(word => escapeRegExp(word))
-        .join('[\\s"?!.]+')
-    )
-    .join("|");
-
-  const pattern = `(^|[^\\w])(${inner})(?=[^\\w]|$)`;
-  try {
-    return new RegExp(pattern, "iu");
-  } catch (_err) {
-    // IE does not support unicode support, so we'll create one without.
-    return new RegExp(pattern, "i");
-  }
-}
-
-// Generate a regular expression detecting `suspectWords` and `bannedWords` phrases.
-function getPhrasesRegexp(
-  suspectWords: ReadonlyArray<string>,
-  bannedWords: ReadonlyArray<string>
-) {
-  return generateRegExp([...suspectWords, ...bannedWords]);
-}
-
-// Memoized version as arguments rarely change.
-const getPhrasesRegexpMemoized = memoize(getPhrasesRegexp);
-
-// markPhrasesHTML looks for `supsectWords` and `bannedWords` inside `text` and highlights them by returning
-// a HTML string.
-function markPhrasesHTML(
-  text: string,
-  suspectWords: ReadonlyArray<string>,
-  bannedWords: ReadonlyArray<string>
-) {
-  const regexp = getPhrasesRegexpMemoized(suspectWords, bannedWords);
-  const tokens = text.split(regexp);
+// markPhrasesHTML looks for `supsect` and `banned` words inside `text` given
+// the settings applied for the locale and highlights them by returning an HTML
+// string.
+function markPhrasesHTML(text: string, expression: RegExp) {
+  const tokens = text.split(expression);
   if (tokens.length === 1) {
     return text;
   }
@@ -87,45 +47,42 @@ function markPhrasesHTML(
 
 // markHTMLNode manipulates the node by looking for #text nodes and adding markers
 // for `supsectWords` and `bannedWords`.
-function markHTMLNode(
-  parentNode: Node,
-  suspectWords: ReadonlyArray<string>,
-  bannedWords: ReadonlyArray<string>
-) {
+function markHTMLNode(parentNode: Node, expression: RegExp) {
   parentNode.childNodes.forEach(node => {
     if (node.nodeName === "#text") {
-      const newContent = markPhrasesHTML(
-        node.textContent!,
-        suspectWords,
-        bannedWords
-      );
+      const newContent = markPhrasesHTML(node.textContent!, expression);
       if (newContent !== node.textContent) {
         const newNode = document.createElement("span");
         newNode.innerHTML = newContent;
         parentNode.replaceChild(newNode, node);
       }
     } else {
-      markHTMLNode(node, suspectWords, bannedWords);
+      markHTMLNode(node, expression);
     }
   });
 }
 
 const CommentContent: FunctionComponent<Props> = ({
-  suspectWords,
-  bannedWords,
+  phrases,
   className,
   children,
 }) => {
+  // Cache the expression used via memo. This will reduce duplicate renders of
+  // this comment content when the children change but the phrase configuration
+  // does not change. The regExp is already cached on a deeper level
+  // automatically, this is just lessening that impact further.
+  const expression = useMemo(() => getPhrasesRegExp(phrases), [phrases]);
+
   if (typeof children === "string") {
     // We create a Shadow DOM Tree with the HTML body content and
     // use it as a parser.
     const node = document.createElement("div");
     node.innerHTML = purify.sanitize(children);
 
-    if (suspectWords.length || bannedWords.length) {
+    if (expression) {
       // Then we traverse it recursively and manipulate it to highlight suspect words
       // and banned words.
-      markHTMLNode(node, suspectWords, bannedWords);
+      markHTMLNode(node, expression);
     }
 
     // Finally we render the content of the Shadow DOM Tree

src/core/client/admin/components/ModerateCard/CommentRevisionContainer.tsx

@@ -30,10 +30,7 @@ const CommentRevisionContainer: FunctionComponent<Props> = ({
         .map(c => (
           <div key={c.id}>
             <Timestamp>{c.createdAt}</Timestamp>
-            <CommentContent
-              suspectWords={settings.wordList.suspect}
-              bannedWords={settings.wordList.banned}
-            >
+            <CommentContent phrases={settings}>
               {c.body ? c.body : ""}
             </CommentContent>
           </div>
@@ -57,6 +54,7 @@ const enhanced = withFragmentContainer<Props>({
   `,
   settings: graphql`
     fragment CommentRevisionContainer_settings on Settings {
+      locale
       wordList {
         banned
         suspect

src/core/client/admin/components/ModerateCard/ModerateCard.spec.tsx

@@ -21,8 +21,13 @@ const baseProps: PropTypesOf<typeof ModerateCardN> = {
   status: "undecided",
   featured: false,
   viewContextHref: "http://localhost/comment",
-  suspectWords: ["suspect"],
-  bannedWords: ["banned"],
+  phrases: {
+    locale: "en-US",
+    wordList: {
+      suspect: ["suspect"],
+      banned: ["banned"],
+    },
+  },
   siteName: null,
   onApprove: noop,
   onReject: noop,

src/core/client/admin/components/ModerateCard/ModerateCard.tsx

@@ -10,6 +10,7 @@ import React, {
 } from "react";
 
 import { HOTKEYS } from "coral-admin/constants";
+import { GetPhrasesRegExpOptions } from "coral-admin/helpers";
 import { PropTypesOf } from "coral-framework/types";
 import {
   BaseButton,
@@ -48,8 +49,7 @@ interface Props {
   featured: boolean;
   moderatedBy: React.ReactNode | null;
   viewContextHref: string;
-  suspectWords: ReadonlyArray<string>;
-  bannedWords: ReadonlyArray<string>;
+  phrases: GetPhrasesRegExpOptions;
   showStory: boolean;
   storyTitle?: React.ReactNode;
   storyHref?: string;
@@ -87,8 +87,7 @@ const ModerateCard: FunctionComponent<Props> = ({
   viewContextHref,
   status,
   featured,
-  suspectWords,
-  bannedWords,
+  phrases,
   onApprove,
   onReject,
   onFeature,
@@ -219,11 +218,7 @@ const ModerateCard: FunctionComponent<Props> = ({
             )}
           </div>
           <div className={styles.contentArea}>
-            <CommentContent
-              suspectWords={suspectWords}
-              bannedWords={bannedWords}
-              className={styles.content}
-            >
+            <CommentContent phrases={phrases} className={styles.content}>
               {commentBody}
             </CommentContent>
             <div className={styles.viewContext}>

src/core/client/admin/components/ModerateCard/ModerateCardContainer.tsx

@@ -222,8 +222,7 @@ const ModerateCardContainer: FunctionComponent<Props> = ({
           status={getStatus(comment)}
           featured={isFeatured(comment)}
           viewContextHref={comment.permalink}
-          suspectWords={settings.wordList.suspect}
-          bannedWords={settings.wordList.banned}
+          phrases={settings}
           onApprove={handleApprove}
           onReject={handleReject}
           onFeature={onFeature}
@@ -319,6 +318,7 @@ const enhanced = withFragmentContainer<Props>({
   `,
   settings: graphql`
     fragment ModerateCardContainer_settings on Settings {
+      locale
       wordList {
         banned
         suspect

src/core/client/admin/components/ModerateCard/__snapshots__/ModerateCard.spec.tsx.snap

@@ -47,16 +47,19 @@ exports[`renders approved correctly 1`] = `
         className="ModerateCard-contentArea"
       >
         <CommentContent
-          bannedWords={
-            Array [
-              "banned",
-            ]
-          }
           className="ModerateCard-content"
-          suspectWords={
-            Array [
-              "suspect",
-            ]
+          phrases={
+            Object {
+              "locale": "en-US",
+              "wordList": Object {
+                "banned": Array [
+                  "banned",
+                ],
+                "suspect": Array [
+                  "suspect",
+                ],
+              },
+            }
           }
         >
           content
@@ -177,16 +180,19 @@ exports[`renders correctly 1`] = `
         className="ModerateCard-contentArea"
       >
         <CommentContent
-          bannedWords={
-            Array [
-              "banned",
-            ]
-          }
           className="ModerateCard-content"
-          suspectWords={
-            Array [
-              "suspect",
-            ]
+          phrases={
+            Object {
+              "locale": "en-US",
+              "wordList": Object {
+                "banned": Array [
+                  "banned",
+                ],
+                "suspect": Array [
+                  "suspect",
+                ],
+              },
+            }
           }
         >
           content
@@ -307,16 +313,19 @@ exports[`renders dangling correctly 1`] = `
         className="ModerateCard-contentArea"
       >
         <CommentContent
-          bannedWords={
-            Array [
-              "banned",
-            ]
-          }
           className="ModerateCard-content"
-          suspectWords={
-            Array [
-              "suspect",
-            ]
+          phrases={
+            Object {
+              "locale": "en-US",
+              "wordList": Object {
+                "banned": Array [
+                  "banned",
+                ],
+                "suspect": Array [
+                  "suspect",
+                ],
+              },
+            }
           }
         >
           content
@@ -437,16 +446,19 @@ exports[`renders rejected correctly 1`] = `
         className="ModerateCard-contentArea"
       >
         <CommentContent
-          bannedWords={
-            Array [
-              "banned",
-            ]
-          }
           className="ModerateCard-content"
-          suspectWords={
-            Array [
-              "suspect",
-            ]
+          phrases={
+            Object {
+              "locale": "en-US",
+              "wordList": Object {
+                "banned": Array [
+                  "banned",
+                ],
+                "suspect": Array [
+                  "suspect",
+                ],
+              },
+            }
           }
         >
           content
@@ -576,16 +588,19 @@ exports[`renders reply correctly 1`] = `
         className="ModerateCard-contentArea"
       >
         <CommentContent
-          bannedWords={
-            Array [
-              "banned",
-            ]
-          }
           className="ModerateCard-content"
-          suspectWords={
-            Array [
-              "suspect",
-            ]
+          phrases={
+            Object {
+              "locale": "en-US",
+              "wordList": Object {
+                "banned": Array [
+                  "banned",
+                ],
+                "suspect": Array [
+                  "suspect",
+                ],
+              },
+            }
           }
         >
           content
@@ -706,16 +721,19 @@ exports[`renders story info 1`] = `
         className="ModerateCard-contentArea"
       >
         <CommentContent
-          bannedWords={
-            Array [
-              "banned",
-            ]
-          }
           className="ModerateCard-content"
-          suspectWords={
-            Array [
-              "suspect",
-            ]
+          phrases={
+            Object {
+              "locale": "en-US",
+              "wordList": Object {
+                "banned": Array [
+                  "banned",
+                ],
+                "suspect": Array [
+                  "suspect",
+                ],
+              },
+            }
           }
         >
           content
@@ -874,16 +892,19 @@ exports[`renders tombstoned when comment is deleted 1`] = `
         className="ModerateCard-contentArea"
       >
         <CommentContent
-          bannedWords={
-            Array [
-              "banned",
-            ]
-          }
           className="ModerateCard-content"
-          suspectWords={
-            Array [
-              "suspect",
-            ]
+          phrases={
+            Object {
+              "locale": "en-US",
+              "wordList": Object {
+                "banned": Array [
+                  "banned",
+                ],
+                "suspect": Array [
+                  "suspect",
+                ],
+              },
+            }
           }
         >
           <Localized

src/core/client/admin/helpers/getPhrasesRegExp.ts

@@ -0,0 +1,64 @@
+import { LanguageCode } from "coral-common/helpers";
+import { createWordListRegExp } from "coral-common/utils";
+
+export interface GetPhrasesRegExpOptions {
+  locale: string;
+  wordList: {
+    banned: ReadonlyArray<string>;
+    suspect: ReadonlyArray<string>;
+  };
+}
+
+export function getPhrasesRegExp({
+  locale,
+  wordList: { banned, suspect },
+}: GetPhrasesRegExpOptions) {
+  if (banned.length === 0 && suspect.length === 0) {
+    return null;
+  }
+
+  return createWordListRegExp(locale as LanguageCode, [...banned, ...suspect]);
+}
+
+// cache is used as a global validator to the cached RegExp used by the
+// application. We expect that generally, there is only ever one word list used
+// by the client at a time, so this ensures that we only re-create the word list
+// if we must.
+const cache = {
+  keys: {
+    locale: "",
+    suspect: [] as ReadonlyArray<string>,
+    banned: [] as ReadonlyArray<string>,
+  },
+  value: null as RegExp | null,
+};
+
+export default function(options: GetPhrasesRegExpOptions) {
+  // We assume that the cache is valid unless one of the below checks fails.
+  let expired = false;
+
+  // Check the locale.
+  if (cache.keys.locale !== options.locale) {
+    cache.keys.locale = options.locale;
+    expired = true;
+  }
+
+  // Check the banned words list.
+  if (cache.keys.banned !== options.wordList.banned) {
+    cache.keys.banned = options.wordList.banned;
+    expired = true;
+  }
+
+  // Check the suspect words list.
+  if (cache.keys.suspect !== options.wordList.suspect) {
+    cache.keys.suspect = options.wordList.suspect;
+    expired = true;
+  }
+
+  // If the cache is expired, or the value doesn't exist, regenerate it.
+  if (expired) {
+    cache.value = getPhrasesRegExp(options);
+  }
+
+  return cache.value;
+}

src/core/client/admin/helpers/index.ts

@@ -1 +1,5 @@
 export { default as getQueueConnection } from "./getQueueConnection";
+export {
+  default as getPhrasesRegExp,
+  GetPhrasesRegExpOptions,
+} from "./getPhrasesRegExp";