diff --git a/src/core/server/services/comments/moderation/phases/wordList.ts b/src/core/server/services/comments/moderation/phases/wordList.ts new file mode 100755 index 000000000..71f8eb860 --- /dev/null +++ b/src/core/server/services/comments/moderation/phases/wordList.ts @@ -0,0 +1,55 @@ +import { + GQLCOMMENT_FLAG_REASON, + GQLCOMMENT_STATUS, +} from "talk-server/graph/tenant/schema/__generated__/types"; +import { ACTION_TYPE } from "talk-server/models/action"; +import { + IntermediateModerationPhase, + IntermediatePhaseResult, +} from "talk-server/services/comments/moderation"; +import { containsMatchingPhrase } from "talk-server/services/comments/moderation/wordlist"; + +// This phase checks the comment against the wordList. +export const wordList: IntermediateModerationPhase = ({ + tenant, + comment, +}): IntermediatePhaseResult | void => { + // If there isn't a body, there can't be a bad word! + if (!comment.body) { + return; + } + + // Decide the status based on whether or not the current asset/settings + // has pre-mod enabled or not. If the comment was rejected based on the + // wordList, then reject it, otherwise if the moderation setting is + // premod, set it to `premod`. + if (containsMatchingPhrase(tenant.wordList.banned, comment.body)) { + // Add the flag related to Trust to the comment. + return { + status: GQLCOMMENT_STATUS.REJECTED, + actions: [ + { + action_type: ACTION_TYPE.FLAG, + reason: GQLCOMMENT_FLAG_REASON.COMMENT_DETECTED_BANNED_WORD, + }, + ], + }; + } + + // If the comment has a suspect word or a link, we need to add a + // flag to it to indicate that it needs to be looked at. + // Otherwise just return the new comment. + + // If the wordlist has matched the suspect word filter and we haven't disabled + // auto-flagging suspect words, then we should flag the comment! + if (containsMatchingPhrase(tenant.wordList.suspect, comment.body)) { + return { + actions: [ + { + action_type: ACTION_TYPE.FLAG, + reason: GQLCOMMENT_FLAG_REASON.COMMENT_DETECTED_SUSPECT_WORD, + }, + ], + }; + } +}; diff --git a/src/core/server/services/comments/moderation/wordList.spec.ts b/src/core/server/services/comments/moderation/wordList.spec.ts new file mode 100644 index 000000000..d23705ecf --- /dev/null +++ b/src/core/server/services/comments/moderation/wordList.spec.ts @@ -0,0 +1,46 @@ +import { containsMatchingPhrase } from "talk-server/services/comments/moderation/wordlist"; + +const phrases = [ + "cookies", + "how to do bad things", + "how to do really bad things", + "s h i t", + "$hit", + "p**ch", + "p*ch", +]; + +describe("containsMatchingPhrase", () => { + it("does match on a word in the list", () => { + [ + "how to do really bad things", + "what is cookies", + "cookies", + "COOKIES.", + "how to do bad things", + "How To do bad things!", + "This stuff is $hit!", + "That's a p**ch!", + ].forEach(word => { + expect(containsMatchingPhrase(phrases, word)).toEqual(true); + }); + }); + + it("does not match on a word not in the list", () => { + [ + "how to", + "cookie", + "how to be a great person?", + "how to not do really bad things?", + "i have $100 dollars.", + "I have bad $ hit lling", + "That's a p***ch!", + ].forEach(word => { + expect(containsMatchingPhrase(phrases, word)).toEqual(false); + }); + }); + + it("allows an empty list", () => { + expect(containsMatchingPhrase([], "test")).toEqual(false); + }); +}); diff --git a/src/core/server/services/comments/moderation/wordList.ts b/src/core/server/services/comments/moderation/wordList.ts new file mode 100644 index 000000000..30cc124ac --- /dev/null +++ b/src/core/server/services/comments/moderation/wordList.ts @@ -0,0 +1,25 @@ +/** + * Escape string for special regular expression characters. + */ +export function escapeRegExp(str: string) { + return str.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); // $& means the whole matched string +} + +/** + * Generate a regular expression that catches the `phrases`. + */ +export function generateRegExp(phrases: string[]) { + const inner = phrases + .map(phrase => + phrase + .split(/\s+/) + .map(word => escapeRegExp(word)) + .join('[\\s"?!.]+') + ) + .join("|"); + + return new RegExp(`(^|[^\\w])(${inner})(?=[^\\w]|$)`, "iu"); +} + +export const containsMatchingPhrase = (phrases: string[], testString: string) => + phrases.length > 0 ? generateRegExp(phrases).test(testString) : false;