From eadf7bde438525a350530d58147ee1005250114a Mon Sep 17 00:00:00 2001 From: Wyatt Johnson Date: Thu, 25 Oct 2018 10:55:22 -0600 Subject: [PATCH] feat: added memoization to the regexp generation --- .../comments/moderation/phases/wordList.ts | 8 +-- .../comments/moderation/phases/wordlist.ts | 55 ------------------- .../services/comments/moderation/wordList.ts | 14 +++++ .../services/comments/moderation/wordlist.ts | 25 --------- 4 files changed, 18 insertions(+), 84 deletions(-) delete mode 100755 src/core/server/services/comments/moderation/phases/wordlist.ts delete mode 100644 src/core/server/services/comments/moderation/wordlist.ts diff --git a/src/core/server/services/comments/moderation/phases/wordList.ts b/src/core/server/services/comments/moderation/phases/wordList.ts index 71f8eb860..a014bff94 100755 --- a/src/core/server/services/comments/moderation/phases/wordList.ts +++ b/src/core/server/services/comments/moderation/phases/wordList.ts @@ -7,7 +7,7 @@ import { IntermediateModerationPhase, IntermediatePhaseResult, } from "talk-server/services/comments/moderation"; -import { containsMatchingPhrase } from "talk-server/services/comments/moderation/wordlist"; +import { containsMatchingPhraseMemoized } from "talk-server/services/comments/moderation/wordlist"; // This phase checks the comment against the wordList. export const wordList: IntermediateModerationPhase = ({ @@ -23,7 +23,7 @@ export const wordList: IntermediateModerationPhase = ({ // has pre-mod enabled or not. If the comment was rejected based on the // wordList, then reject it, otherwise if the moderation setting is // premod, set it to `premod`. - if (containsMatchingPhrase(tenant.wordList.banned, comment.body)) { + if (containsMatchingPhraseMemoized(tenant.wordList.banned, comment.body)) { // Add the flag related to Trust to the comment. return { status: GQLCOMMENT_STATUS.REJECTED, @@ -40,9 +40,9 @@ export const wordList: IntermediateModerationPhase = ({ // flag to it to indicate that it needs to be looked at. // Otherwise just return the new comment. - // If the wordlist has matched the suspect word filter and we haven't disabled + // If the wordList has matched the suspect word filter and we haven't disabled // auto-flagging suspect words, then we should flag the comment! - if (containsMatchingPhrase(tenant.wordList.suspect, comment.body)) { + if (containsMatchingPhraseMemoized(tenant.wordList.suspect, comment.body)) { return { actions: [ { diff --git a/src/core/server/services/comments/moderation/phases/wordlist.ts b/src/core/server/services/comments/moderation/phases/wordlist.ts deleted file mode 100755 index 71f8eb860..000000000 --- a/src/core/server/services/comments/moderation/phases/wordlist.ts +++ /dev/null @@ -1,55 +0,0 @@ -import { - GQLCOMMENT_FLAG_REASON, - GQLCOMMENT_STATUS, -} from "talk-server/graph/tenant/schema/__generated__/types"; -import { ACTION_TYPE } from "talk-server/models/action"; -import { - IntermediateModerationPhase, - IntermediatePhaseResult, -} from "talk-server/services/comments/moderation"; -import { containsMatchingPhrase } from "talk-server/services/comments/moderation/wordlist"; - -// This phase checks the comment against the wordList. -export const wordList: IntermediateModerationPhase = ({ - tenant, - comment, -}): IntermediatePhaseResult | void => { - // If there isn't a body, there can't be a bad word! - if (!comment.body) { - return; - } - - // Decide the status based on whether or not the current asset/settings - // has pre-mod enabled or not. If the comment was rejected based on the - // wordList, then reject it, otherwise if the moderation setting is - // premod, set it to `premod`. - if (containsMatchingPhrase(tenant.wordList.banned, comment.body)) { - // Add the flag related to Trust to the comment. - return { - status: GQLCOMMENT_STATUS.REJECTED, - actions: [ - { - action_type: ACTION_TYPE.FLAG, - reason: GQLCOMMENT_FLAG_REASON.COMMENT_DETECTED_BANNED_WORD, - }, - ], - }; - } - - // If the comment has a suspect word or a link, we need to add a - // flag to it to indicate that it needs to be looked at. - // Otherwise just return the new comment. - - // If the wordlist has matched the suspect word filter and we haven't disabled - // auto-flagging suspect words, then we should flag the comment! - if (containsMatchingPhrase(tenant.wordList.suspect, comment.body)) { - return { - actions: [ - { - action_type: ACTION_TYPE.FLAG, - reason: GQLCOMMENT_FLAG_REASON.COMMENT_DETECTED_SUSPECT_WORD, - }, - ], - }; - } -}; diff --git a/src/core/server/services/comments/moderation/wordList.ts b/src/core/server/services/comments/moderation/wordList.ts index 30cc124ac..47f50d09e 100644 --- a/src/core/server/services/comments/moderation/wordList.ts +++ b/src/core/server/services/comments/moderation/wordList.ts @@ -1,3 +1,9 @@ +import { memoize } from "lodash"; + +// TODO: reintroduce this when we have https://github.com/DefinitelyTyped/DefinitelyTyped/pull/30035 merged +// // Replace `memoize.Cache`. +// memoize.Cache = WeakMap; + /** * Escape string for special regular expression characters. */ @@ -21,5 +27,13 @@ export function generateRegExp(phrases: string[]) { return new RegExp(`(^|[^\\w])(${inner})(?=[^\\w]|$)`, "iu"); } +export const generateRegExpMemoized = memoize(generateRegExp); + export const containsMatchingPhrase = (phrases: string[], testString: string) => phrases.length > 0 ? generateRegExp(phrases).test(testString) : false; + +export const containsMatchingPhraseMemoized = ( + phrases: string[], + testString: string +) => + phrases.length > 0 ? generateRegExpMemoized(phrases).test(testString) : false; diff --git a/src/core/server/services/comments/moderation/wordlist.ts b/src/core/server/services/comments/moderation/wordlist.ts deleted file mode 100644 index 30cc124ac..000000000 --- a/src/core/server/services/comments/moderation/wordlist.ts +++ /dev/null @@ -1,25 +0,0 @@ -/** - * Escape string for special regular expression characters. - */ -export function escapeRegExp(str: string) { - return str.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); // $& means the whole matched string -} - -/** - * Generate a regular expression that catches the `phrases`. - */ -export function generateRegExp(phrases: string[]) { - const inner = phrases - .map(phrase => - phrase - .split(/\s+/) - .map(word => escapeRegExp(word)) - .join('[\\s"?!.]+') - ) - .join("|"); - - return new RegExp(`(^|[^\\w])(${inner})(?=[^\\w]|$)`, "iu"); -} - -export const containsMatchingPhrase = (phrases: string[], testString: string) => - phrases.length > 0 ? generateRegExp(phrases).test(testString) : false;