[next] Error and Logging Improvements (#2152)

* feat: added locale support for Tenant

* feat: added secret scrubbing to logs

* chore: cleanup logger

* chore: logger improvements

* feat: re-introduce scoped pretty logger

* feat: added initial error support

* refactor: replace trace-error.TraceError with talk.InternalError

* fix: fixed error logging

* refactor: replaced Error with VError

* fix: repaired issue with error management on api

* fix: patched bug with not found handler

* feat: added translations

* feat: added location path to invalid entries

* refactor: refactored error handling on graph

* fix: moved indexing operations to master node

* refactor: added throw for when the message isn't found in testing

* fix: removed duplicate log

* fix: fixed naming on environment variable

src/core/server/services/i18n/index.ts

@@ -0,0 +1,126 @@
+import { FluentBundle } from "fluent/compat";
+import fs from "fs-extra";
+import path from "path";
+
+import { LanguageCode, LOCALES } from "talk-common/helpers/i18n/locales";
+import config from "talk-server/config";
+
+/**
+ * isLanguageCode will return true if the string is a `LanguageCode`.
+ *
+ * @param locale the string that is being tested if it's a `LanguageCode`
+ */
+function isLanguageCode(locale: string): locale is LanguageCode {
+  return LOCALES.some(code => code === locale);
+}
+
+// pathToLocales is the path where the server stores the locales.
+const pathToLocales = path.join(__dirname, "..", "..", "locales");
+
+export class I18n {
+  private bundles: Partial<Record<LanguageCode, FluentBundle>> = {};
+  private defaultLang: LanguageCode;
+
+  constructor(defaultLocale: LanguageCode) {
+    this.defaultLang = defaultLocale;
+  }
+
+  /**
+   * load will read all the translations located in the server locales folder.
+   */
+  public async load() {
+    // Load all the locales from the server locales folder.
+
+    // Load all the locales from the locales folders.
+    const folders = await fs.readdir(pathToLocales);
+
+    // Load all the translation files for each of the folders.
+    for (const folder of folders) {
+      // Parse out the language code.
+      const locale = path.basename(folder);
+      if (!isLanguageCode(locale)) {
+        throw new Error(`invalid language code: ${locale}`);
+      }
+
+      // Now we have a language code.
+      const bundle = new FluentBundle(locale);
+
+      // Load all the translations in the folder.
+      const files = await fs.readdir(path.join(pathToLocales, folder));
+
+      for (const file of files) {
+        const messages = await fs.readFile(
+          path.join(pathToLocales, folder, file),
+          "utf8"
+        );
+
+        bundle.addMessages(messages);
+      }
+
+      this.bundles[locale] = bundle;
+    }
+  }
+
+  /**
+   * getBundle will return a bundle keyed on the language.
+   *
+   * @param lang the locale to get the bundle for
+   */
+  public getBundle(lang: LanguageCode): FluentBundle {
+    const bundle = this.bundles[lang];
+    if (!bundle) {
+      throw new Error(`bundle for language "${lang}" not found`);
+    }
+
+    return bundle;
+  }
+
+  /**
+   * getDefaultLang will return the default language.
+   */
+  public getDefaultLang(): Readonly<LanguageCode> {
+    return this.defaultLang;
+  }
+
+  /**
+   * getDefaultBundle will return the default bundle to use.
+   */
+  public getDefaultBundle(): FluentBundle {
+    return this.getBundle(this.getDefaultLang());
+  }
+}
+
+/**
+ * translate will attempt a translation but fallback to the defaultValue if it
+ * can't be translated.
+ *
+ * @param bundle the bundle to use for translations
+ * @param defaultValue the default value if the message or translation isn't
+ * available
+ * @param id the ID for the translation
+ * @param args the args to be used in the translation
+ * @param errors the errors to for the translation bundle
+ */
+export function translate(
+  bundle: FluentBundle,
+  defaultValue: string,
+  id: string,
+  args?: object,
+  errors?: string[]
+): string {
+  const message = bundle.getMessage(id);
+  if (!message) {
+    if (config.get("env") === "test") {
+      throw new Error(`the message for ${id} is missing`);
+    }
+
+    return defaultValue;
+  }
+
+  const value = bundle.format(message, args, errors);
+  if (!value) {
+    return defaultValue;
+  }
+
+  return value;
+}

src/core/server/services/i18n/translation.spec.ts

@@ -0,0 +1,7 @@
+import { I18n } from ".";
+
+it("loads the translations without error", async () => {
+  const translation = new I18n("en-US");
+  await translation.load();
+  expect(translation.getBundle("en-US")).toBeDefined();
+});

src/core/server/services/jwt/index.ts

@@ -37,7 +37,7 @@ export function createAsymmetricSigningConfig(
 ): JWTSigningConfig {
   return {
     // Secrets have their newlines encoded with newline literals.
-    secret: Buffer.from(secret.replace(/\\n/g, "\n")),
+    secret: Buffer.from(secret.replace(/\\n/g, "\n"), "utf8"),
     algorithm,
   };
 }
@@ -47,7 +47,7 @@ export function createSymmetricSigningConfig(
   secret: string
 ): JWTSigningConfig {
   return {
-    secret: new Buffer(secret),
+    secret: Buffer.from(secret, "utf8"),
     algorithm,
   };
 }
@@ -121,27 +121,23 @@ export function extractJWTFromRequest(req: Request) {
   return permit.check(req) || null;
 }
 
-function generateJTIBlacklistKey(jti: string) {
-  // jtib: JTI Blacklist namespace.
-  return `jtib:${jti}`;
+function generateJTIRevokedKey(jti: string) {
+  // jtir: JTI Revoked namespace.
+  return `jtir:${jti}`;
 }
 
-export async function blacklistJWT(
-  redis: Redis,
-  jti: string,
-  validFor: number
-) {
+export async function revokeJWT(redis: Redis, jti: string, validFor: number) {
   await redis.setex(
-    generateJTIBlacklistKey(jti),
+    generateJTIRevokedKey(jti),
     Math.ceil(validFor),
     Date.now()
   );
 }
 
-export async function checkBlacklistJWT(redis: Redis, jti: string) {
-  const expiredAtString = await redis.get(generateJTIBlacklistKey(jti));
+export async function checkJWTRevoked(redis: Redis, jti: string) {
+  const expiredAtString = await redis.get(generateJTIRevokedKey(jti));
   if (expiredAtString) {
     // TODO: (wyattjoh) return a better error.
-    throw new Error("JWT exists in blacklist");
+    throw new Error("JWT was revoked");
   }
 }

src/core/server/services/mongodb/index.ts

@@ -1,11 +1,17 @@
 import { Db, MongoClient } from "mongodb";
+
 import { Config } from "talk-server/config";
+import { InternalError } from "talk-server/errors";
 
 export async function createMongoClient(config: Config): Promise<MongoClient> {
-  return MongoClient.connect(
-    config.get("mongodb"),
-    { useNewUrlParser: true }
-  );
+  try {
+    return await MongoClient.connect(
+      config.get("mongodb"),
+      { useNewUrlParser: true }
+    );
+  } catch (err) {
+    throw new InternalError(err, "could not connect to mongodb");
+  }
 }
 
 /**

src/core/server/services/redis/index.ts

@@ -2,6 +2,8 @@ import RedisClient, { Pipeline, Redis } from "ioredis";
 
 import { Omit } from "talk-common/types";
 import { Config } from "talk-server/config";
+import { InternalError } from "talk-server/errors";
+import logger from "talk-server/logger";
 
 export interface AugmentedRedisCommands {
   mhincrby(key: string, ...args: any[]): Promise<void>;
@@ -15,6 +17,11 @@ export type AugmentedRedis = Omit<Redis, "pipeline"> &
   };
 
 function configureRedisClient(redis: Redis) {
+  // Attach to the error event.
+  redis.on("error", (err: Error) => {
+    logger.error({ err }, "an error occurred with redis");
+  });
+
   // mhincrby will increment many hash values.
   redis.defineCommand("mhincrby", {
     numberOfKeys: 1,
@@ -31,11 +38,22 @@ function configureRedisClient(redis: Redis) {
  *
  * @param config application configuration.
  */
-export function createRedisClient(config: Config): AugmentedRedis {
-  const redis = new RedisClient(config.get("redis"), {});
+export async function createRedisClient(
+  config: Config
+): Promise<AugmentedRedis> {
+  try {
+    const redis = new RedisClient(config.get("redis"), {
+      lazyConnect: true,
+    });
 
-  // Configure the redis client for use with the custom commands.
-  configureRedisClient(redis);
+    // Configure the redis client for use with the custom commands.
+    configureRedisClient(redis);
 
-  return redis as AugmentedRedis;
+    // Connect the redis client.
+    await redis.connect();
+
+    return redis as AugmentedRedis;
+  } catch (err) {
+    throw new InternalError(err, "could not connect to redis");
+  }
 }

src/core/server/services/stories/index.ts

@@ -7,6 +7,7 @@ import {
   isURLSecure,
   prefixSchemeIfRequired,
 } from "talk-server/app/url";
+import { StoryURLInvalidError } from "talk-server/errors";
 import logger from "talk-server/logger";
 import {
   countTotalActionCounts,
@@ -53,11 +54,10 @@ export async function findOrCreate(
   // If the URL is provided, and the url is not on a allowed domain, then refuse
   // to create the Asset.
   if (input.url && !isURLPermitted(tenant, input.url)) {
-    logger.warn(
-      { story_url: input.url, tenant_domains: tenant.domains },
-      "provided story url was not in the list of permitted tenant domains, story not found"
-    );
-    return null;
+    throw new StoryURLInvalidError({
+      storyURL: input.url,
+      tenantDomains: tenant.domains,
+    });
   }
 
   // TODO: check to see if the tenant has enabled lazy story creation, if they haven't, switch to find only.
@@ -189,11 +189,7 @@ export async function create(
 ) {
   // Ensure that the given URL is allowed.
   if (!isURLPermitted(tenant, storyURL)) {
-    logger.warn(
-      { storyURL, tenantDomains: tenant.domains },
-      "provided story url was not in the list of permitted tenant domains, story not created"
-    );
-    return null;
+    throw new StoryURLInvalidError({ storyURL, tenantDomains: tenant.domains });
   }
 
   // Create the story in the database.
@@ -217,11 +213,10 @@ export async function update(
 ) {
   // Ensure that the given URL is allowed.
   if (input.url && !isURLPermitted(tenant, input.url)) {
-    logger.warn(
-      { storyURL: input.url, tenantDomains: tenant.domains },
-      "provided story url was not in the list of permitted tenant domains, story not updated"
-    );
-    return null;
+    throw new StoryURLInvalidError({
+      storyURL: input.url,
+      tenantDomains: tenant.domains,
+    });
   }
 
   return updateStory(mongo, tenant.id, storyID, input);

src/core/server/services/tenant/index.ts

@@ -12,6 +12,7 @@ import {
 } from "talk-server/models/tenant";
 
 import { discover } from "talk-server/app/middleware/passport/strategies/oidc/discover";
+import { TenantInstalledAlreadyError } from "talk-server/errors";
 import logger from "talk-server/logger";
 import TenantCache from "./cache";
 
@@ -44,26 +45,22 @@ export async function install(
   input: InstallTenant
 ) {
   if (await isInstalled(cache)) {
-    // TODO: (wyattjoh) return better error
-    throw new Error(
-      "tenant already setup, setup multi-tenant mode if you want to install more than one tenant"
-    );
+    throw new TenantInstalledAlreadyError();
   }
 
   // TODO: (wyattjoh) perform any pending migrations.
 
   // TODO: (wyattjoh) setup database indexes.
 
+  logger.info({ tenant: input }, "installing tenant");
+
   // Create the Tenant.
   const tenant = await createTenant(mongo, input);
 
   // Update the tenant cache.
   await cache.update(redis, tenant);
 
-  logger.info(
-    { tenantID: tenant.id, tenantDomain: tenant.domain },
-    "a tenant has been installed"
-  );
+  logger.info({ tenant }, "a tenant has been installed");
 
   return tenant;
 }

src/core/server/services/users/index.ts

@@ -7,6 +7,21 @@ import {
   USERNAME_MIN_LENGTH,
   USERNAME_REGEX,
 } from "talk-common/helpers/validate";
+import {
+  DisplayNameExceedsMaxLengthError,
+  EmailAlreadySetError,
+  EmailExceedsMaxLengthError,
+  EmailInvalidFormatError,
+  EmailNotSetError,
+  LocalProfileAlreadySetError,
+  LocalProfileNotSetError,
+  PasswordTooShortError,
+  TokenNotFoundError,
+  UsernameAlreadySetError,
+  UsernameContainsInvalidCharactersError,
+  UsernameExceedsMaxLengthError,
+  UsernameTooShortError,
+} from "talk-server/errors";
 import { GQLUSER_ROLE } from "talk-server/graph/tenant/schema/__generated__/types";
 import { Tenant } from "talk-server/models/tenant";
 import {
@@ -26,6 +41,7 @@ import {
   UpsertUserInput,
   User,
 } from "talk-server/models/user";
+
 import { JWTSigningConfig, signPATString } from "../jwt";
 
 /**
@@ -40,15 +56,18 @@ function validateUsername(tenant: Tenant, username: string) {
   // TODO: replace these static regex/length with database options in the Tenant eventually
 
   if (!USERNAME_REGEX.test(username)) {
-    throw new Error("username contained illegal characters");
+    throw new UsernameContainsInvalidCharactersError();
   }
 
   if (username.length > USERNAME_MAX_LENGTH) {
-    throw new Error("username exceeded maximum length");
+    throw new UsernameExceedsMaxLengthError(
+      username.length,
+      USERNAME_MAX_LENGTH
+    );
   }
 
   if (username.length < USERNAME_MIN_LENGTH) {
-    throw new Error("username is too short");
+    throw new UsernameTooShortError(username.length, USERNAME_MIN_LENGTH);
   }
 }
 
@@ -64,7 +83,10 @@ function validateDisplayName(tenant: Tenant, displayName: string) {
   // TODO: replace these static regex/length with database options in the Tenant eventually
 
   if (displayName.length > DISPLAY_NAME_MAX_LENGTH) {
-    throw new Error("displayName exceeded maximum length");
+    throw new DisplayNameExceedsMaxLengthError(
+      displayName.length,
+      DISPLAY_NAME_MAX_LENGTH
+    );
   }
 }
 
@@ -79,10 +101,12 @@ function validateDisplayName(tenant: Tenant, displayName: string) {
 function validatePassword(tenant: Tenant, password: string) {
   // TODO: replace these static length with database options in the Tenant eventually
   if (password.length < PASSWORD_MIN_LENGTH) {
-    throw new Error("password is too short");
+    throw new PasswordTooShortError(password.length, PASSWORD_MIN_LENGTH);
   }
 }
 
+const EMAIL_MAX_LENGTH = 100;
+
 /**
  * validateEmail will validate that the email is valid. Current implementation
  * uses a length statically, future versions will expose this as configuration.
@@ -91,9 +115,13 @@ function validatePassword(tenant: Tenant, password: string) {
  * @param email the email to be tested
  */
 function validateEmail(tenant: Tenant, email: string) {
-  // TODO: replace these static length with database options in the Tenant eventually
   if (!EMAIL_REGEX.test(email)) {
-    throw new Error("email is in an invalid format");
+    throw new EmailInvalidFormatError();
+  }
+
+  // TODO: replace these static length with database options in the Tenant eventually
+  if (email.length > EMAIL_MAX_LENGTH) {
+    throw new EmailExceedsMaxLengthError(email.length, EMAIL_MAX_LENGTH);
   }
 }
 
@@ -123,7 +151,6 @@ export async function upsert(mongo: Db, tenant: Tenant, input: UpsertUser) {
     validatePassword(tenant, localProfile.password);
 
     if (input.email !== localProfile.id) {
-      // TODO: (wyattjoh) return better error.
       throw new Error("email addresses don't match profile");
     }
   }
@@ -150,7 +177,7 @@ export async function setUsername(
 ) {
   // We require that the username is not defined in order to use this method.
   if (user.username) {
-    throw new Error("username already associated with user");
+    throw new UsernameAlreadySetError();
   }
 
   validateUsername(tenant, username);
@@ -176,7 +203,7 @@ export async function setEmail(
   // We requires that the email address is not defined in order to use this
   // method.
   if (user.email) {
-    throw new Error("email address already associated with user");
+    throw new EmailAlreadySetError();
   }
 
   validateEmail(tenant, email);
@@ -204,13 +231,13 @@ export async function setPassword(
 ) {
   // We require that the email address for the user be defined for this method.
   if (!user.email) {
-    throw new Error("no email address associated with user");
+    throw new EmailNotSetError();
   }
 
   // We also don't allow this method to be used by users that already have a
   // local profile.
   if (user.profiles.some(({ type }) => type === "local")) {
-    throw new Error("user already has local profile");
+    throw new LocalProfileAlreadySetError();
   }
 
   validatePassword(tenant, password);
@@ -237,7 +264,7 @@ export async function updatePassword(
 ) {
   // We require that the email address for the user be defined for this method.
   if (!user.email) {
-    throw new Error("no email address associated with user");
+    throw new EmailNotSetError();
   }
 
   // We also don't allow this method to be used by users that don't have a local
@@ -245,7 +272,7 @@ export async function updatePassword(
   if (
     !user.profiles.some(({ id, type }) => type === "local" && id === user.email)
   ) {
-    throw new Error("user does not have a local profile");
+    throw new LocalProfileNotSetError();
   }
 
   validatePassword(tenant, password);
@@ -302,8 +329,7 @@ export async function deactivateToken(
   id: string
 ) {
   if (!user.tokens.find(t => t.id === id)) {
-    // TODO: (wyattjoh) return better error
-    throw new Error("token not found on user");
+    throw new TokenNotFoundError();
   }
 
   return deactivateUserToken(mongo, tenant.id, user.id, id);