[CORL-750] SSO Migration Script Bug (#2715)

* fix: fixed bug with sso migration script

* feat: refactored sso schema

* fix: resolved issue with data race and migration/install ordering

src/core/server/app/handlers/api/install.ts

@@ -205,6 +205,11 @@ export const installHandler = ({
         locale = config.get("default_locale") as LanguageCode;
       }
 
+      // Execute the pending migrations now, as the schema and types are already
+      // current for the new tenant being installed now. No point in creating
+      // a tenant when migrations have not been ran yet.
+      await migrationManager.executePendingMigrations(mongo, redis, true);
+
       // Install will throw if it can not create a Tenant, or it has already been
       // installed.
       const tenant = await install(
@@ -248,9 +253,6 @@ export const installHandler = ({
         req.coral.now
       );
 
-      // Execute pending migrations to get everything installed.
-      await migrationManager.executePendingMigrations(mongo, true);
-
       // Send back the Tenant.
       return res.sendStatus(204);
     } catch (err) {

src/core/server/app/middleware/passport/strategies/verifiers/sso.ts

@@ -5,10 +5,7 @@ import { Db } from "mongodb";
 
 import { validate } from "coral-server/app/request/body";
 import { IntegrationDisabled, TokenInvalidError } from "coral-server/errors";
-import {
-  RequiredSSOKey,
-  SSOAuthIntegration,
-} from "coral-server/models/settings";
+import { SSOAuthIntegration, SSOKey } from "coral-server/models/settings";
 import { Tenant } from "coral-server/models/tenant";
 import {
   retrieveUserWithProfile,
@@ -177,23 +174,15 @@ export function getRelevantSSOKeys(
   tokenString: string,
   now: Date,
   kid?: string
-): RequiredSSOKey[] {
+): SSOKey[] {
   // Collect all the current valid keys.
   const keys = integration.keys.filter(k => {
-    if (!k.secret) {
-      return false;
-    }
-
-    if (k.deletedAt) {
-      return false;
-    }
-
-    if (k.deprecateAt && now >= k.deprecateAt) {
+    if (k.inactiveAt && now >= k.inactiveAt) {
       return false;
     }
 
     return k;
-  }) as RequiredSSOKey[];
+  });
 
   // If there is only one key, that's all we can use!
   if (keys.length === 1) {

src/core/server/graph/tenant/resolvers/SSOAuthIntegration.ts

@@ -3,9 +3,8 @@ import * as settings from "coral-server/models/settings";
 import { GQLSSOAuthIntegrationTypeResolver } from "coral-server/graph/tenant/schema/__generated__/types";
 
 function getActiveSSOKey(keys: settings.SSOKey[]) {
-  return keys.find(
-    key => Boolean(key.secret) && !key.deletedAt && !key.deprecateAt
-  );
+  // Any key that has been rotated cannot be the active key.
+  return keys.find(key => !key.rotatedAt);
 }
 
 export const SSOAuthIntegration: GQLSSOAuthIntegrationTypeResolver<

src/core/server/index.ts

@@ -203,7 +203,10 @@ class Server {
 
     // Run migrations if there is already a Tenant installed.
     if (await isInstalled(this.tenantCache)) {
-      await this.migrationManager.executePendingMigrations(this.mongo);
+      await this.migrationManager.executePendingMigrations(
+        this.mongo,
+        this.redis
+      );
       await this.tenantCache.primeAll();
     } else {
       logger.info("no tenants are installed, skipping running migrations");

src/core/server/models/invite.ts

@@ -47,7 +47,7 @@ export async function createInvite(
   };
 
   // Insert it into the database. This may throw an error.
-  await collection(mongo).insert(invite);
+  await collection(mongo).insertOne(invite);
 
   return invite;
 }

src/core/server/models/settings.ts

@@ -1,4 +1,4 @@
-import { Omit, RequireProperty } from "coral-common/types";
+import { Omit } from "coral-common/types";
 
 import {
   GQLAuth,
@@ -46,30 +46,27 @@ export interface SSOKey {
   kid: string;
 
   /**
-   * secret is the actual underlying secret used to verify the tokens with. When
-   * this is not available, it indicates that the token secret was deleted.
+   * secret is the actual underlying secret used to verify the tokens with.
    */
-  secret?: string;
+  secret: string;
 
   /**
-   * createdAt is the time that this key was created at.
+   * createdAt is the date that the key was created at.
    */
   createdAt: Date;
 
   /**
-   * deprecateAt when provided is the time that the token should no longer be
-   * valid at.
+   * rotatedAt is the time that the token was rotated out.
    */
-  deprecateAt?: Date;
+  rotatedAt?: Date;
 
   /**
-   * deletedAt is the timestamp that the token was revoked.
+   * inactiveAt is the date that the token can no longer be used to validate
+   * tokens.
    */
-  deletedAt?: Date;
+  inactiveAt?: Date;
 }
 
-export type RequiredSSOKey = RequireProperty<SSOKey, "secret">;
-
 export interface SSOAuthIntegration {
   enabled: boolean;
   allowRegistration: boolean;

src/core/server/models/tenant/tenant.ts

@@ -199,7 +199,7 @@ export async function createTenant(
   };
 
   // Insert the Tenant into the database.
-  await collection(mongo).insert(tenant);
+  await collection(mongo).insertOne(tenant);
 
   return tenant;
 }
@@ -306,18 +306,20 @@ export async function createTenantSSOKey(mongo: Db, id: string, now: Date) {
   return result.value || null;
 }
 
-export async function deprecateTenantSSOKey(
+export async function rotateTenantSSOKey(
   mongo: Db,
   id: string,
   kid: string,
-  deprecateAt: Date
+  inactiveAt: Date,
+  now: Date
 ) {
   // Update the tenant.
   const result = await collection(mongo).findOneAndUpdate(
     { id },
     {
       $set: {
-        "auth.integrations.sso.keys.$[keys].deprecateAt": deprecateAt,
+        "auth.integrations.sso.keys.$[keys].inactiveAt": inactiveAt,
+        "auth.integrations.sso.keys.$[keys].rotatedAt": now,
       },
     },
     {

src/core/server/services/migrate/manager.ts

@@ -1,4 +1,5 @@
 import fs from "fs-extra";
+import { Redis } from "ioredis";
 import { Db } from "mongodb";
 import path from "path";
 import now from "performance-now";
@@ -133,7 +134,11 @@ export default class Manager {
     return records.length > 0 ? records[records.length - 1] : null;
   }
 
-  public async executePendingMigrations(mongo: Db, silent = false) {
+  public async executePendingMigrations(
+    mongo: Db,
+    redis: Redis,
+    silent = false
+  ) {
     // Error out if this is ran twice.
     if (this.ran) {
       if (silent) {
@@ -191,6 +196,7 @@ export default class Manager {
 
       if (migration.up) {
         // The migration provides an up method, we should run this per Tenant.
+        // If no tenants are installed, this will essentially be a no-op.
         for await (const tenant of this.tenantCache) {
           log = log.child({ tenantID: tenant.id }, true);
 
@@ -246,5 +252,10 @@ export default class Manager {
       },
       "finished running pending migrations"
     );
+
+    for await (const tenant of this.tenantCache) {
+      // Flush the tenant cache now for each tenant.
+      await this.tenantCache.delete(redis, tenant.id, tenant.domain);
+    }
   }
 }

src/core/server/services/migrate/migrations/1573073491825_sso_tokens.ts

@@ -1,7 +1,7 @@
 import { Db } from "mongodb";
 
 import { SSOKey } from "coral-server/models/settings";
-import { generateSSOKey } from "coral-server/models/tenant";
+import { generateSSOKey, Tenant } from "coral-server/models/tenant";
 import Migration from "coral-server/services/migrate/migration";
 import collections from "coral-server/services/mongodb/collections";
 
@@ -9,7 +9,7 @@ import { GQLTime } from "coral-server/graph/tenant/schema/__generated__/types";
 
 import { MigrationError } from "../error";
 
-interface Tenant {
+interface OldTenant {
   auth: {
     integrations: {
       sso: {
@@ -20,13 +20,21 @@ interface Tenant {
   };
 }
 
+function isOldTenant(tenant: Tenant | OldTenant): tenant is OldTenant {
+  if ((tenant as Tenant).auth.integrations.sso.keys) {
+    return false;
+  }
+
+  return true;
+}
+
 export default class extends Migration {
   public async up(mongo: Db, tenantID: string) {
     // Get the Tenant so we can update it. We don't have to worry about two
     // migration operations conflicting here because we will lock one instance
     // to perform the operations.
     const tenant = await collections
-      .tenants<Tenant>(mongo)
+      .tenants<OldTenant | Tenant>(mongo)
       .findOne({ id: tenantID });
     if (!tenant) {
       throw new MigrationError(tenantID, "could not find tenant", "tenants", [
@@ -34,11 +42,17 @@ export default class extends Migration {
       ]);
     }
 
+    if (!isOldTenant(tenant)) {
+      this.log(tenantID).info("tenant already has the new format for the keys");
+      return;
+    }
+
     // Store the keys in an array.
     const keys: SSOKey[] = [];
 
     // Check to see if a key is set.
     const sso = tenant.auth.integrations.sso;
+
     if (sso.key && sso.keyGeneratedAt) {
       // Create the new SSOKey based on this data.
       const key = generateSSOKey(sso.keyGeneratedAt);
@@ -48,6 +62,13 @@ export default class extends Migration {
 
       // Add this key to the set of keys.
       keys.push(key);
+    } else {
+      throw new MigrationError(
+        tenantID,
+        "expected tenant to have at least one SSO key provided, found none",
+        "tenants",
+        [tenantID]
+      );
     }
 
     // Update the tenant with the new sso keys.

src/core/server/services/migrate/migrations/1573858750460_sso_token_refactor.ts

@@ -0,0 +1,114 @@
+import { DateTime } from "luxon";
+import { Db } from "mongodb";
+
+import { SSOKey } from "coral-server/models/settings";
+import Migration from "coral-server/services/migrate/migration";
+import collections from "coral-server/services/mongodb/collections";
+
+import { MigrationError } from "../error";
+
+interface OldSSOKey {
+  kid: string;
+  secret?: string;
+  createdAt: Date;
+  deprecateAt?: Date;
+  deletedAt?: Date;
+}
+
+function isOldSSOKey(key: SSOKey | OldSSOKey): key is OldSSOKey {
+  if (!key) {
+    return true;
+  }
+
+  if ((key as SSOKey).inactiveAt) {
+    return false;
+  }
+
+  if ((key as SSOKey).rotatedAt) {
+    return false;
+  }
+
+  if (key.secret === "<deleted>") {
+    return false;
+  }
+
+  return true;
+}
+
+interface OldTenant {
+  auth: {
+    integrations: {
+      sso: {
+        keys: OldSSOKey[];
+      };
+    };
+  };
+}
+
+export default class extends Migration {
+  public async down(mongo: Db, id: string) {
+    // Get the Tenant that stores the keys in the old format.
+    const tenant = await collections.tenants(mongo).findOne({ id });
+    if (!tenant) {
+      throw new MigrationError(id, "tenant was not found", "tenants", [id]);
+    }
+
+    // Transform the keys into the new format.
+    const keys: OldSSOKey[] = tenant.auth.integrations.sso.keys.map(
+      (key: OldSSOKey | SSOKey): OldSSOKey =>
+        !isOldSSOKey(key)
+          ? {
+              kid: key.kid,
+              secret: key.secret === "<deleted>" ? undefined : key.secret,
+              createdAt: key.createdAt,
+              deprecateAt: key.inactiveAt,
+            }
+          : key
+    );
+
+    // Update the key to the new format.
+    await collections
+      .tenants<OldTenant>(mongo)
+      .updateOne({ id }, { $set: { "auth.integrations.sso.keys": keys } });
+  }
+
+  public async test(mongo: Db, id: string) {
+    // Get the Tenant that stores the keys in the old format.
+    const tenant = await collections.tenants<OldTenant>(mongo).findOne({ id });
+    if (!tenant) {
+      throw new MigrationError(id, "tenant was not found", "tenants", [id]);
+    }
+
+    if (tenant.auth.integrations.sso.keys.some(key => isOldSSOKey(key))) {
+      throw new MigrationError(id, "old sso key was found", "tenants", [id]);
+    }
+  }
+
+  public async up(mongo: Db, id: string) {
+    // Get the Tenant that stores the keys in the old format.
+    const tenant = await collections.tenants<OldTenant>(mongo).findOne({ id });
+    if (!tenant) {
+      throw new MigrationError(id, "tenant was not found", "tenants", [id]);
+    }
+
+    // Transform the keys into the new format.
+    const keys: SSOKey[] = tenant.auth.integrations.sso.keys.map(
+      (key): SSOKey => ({
+        kid: key.kid,
+        secret: key.secret || "<deleted>",
+        createdAt: key.createdAt,
+        inactiveAt: key.deprecateAt,
+        rotatedAt: key.deprecateAt
+          ? DateTime.fromJSDate(key.deprecateAt)
+              .plus({ month: -1 })
+              .toJSDate()
+          : undefined,
+      })
+    );
+
+    // Update the key to the new format.
+    await collections
+      .tenants(mongo)
+      .updateOne({ id }, { $set: { "auth.integrations.sso.keys": keys } });
+  }
+}

src/core/server/services/tenant/cache/adapter.ts

@@ -9,11 +9,11 @@ export type DeconstructionFn<T> = (tenantID: string, value: T) => Promise<void>;
  * automatically invalidate tenants that have been updated.
  */
 export class TenantCacheAdapter<T> {
-  private cache = new Map<string, T>();
-  private tenantCache: TenantCache;
+  private readonly cache = new Map<string, T>();
+  private readonly tenantCache: TenantCache;
 
+  private readonly deconstructionFn?: DeconstructionFn<T>;
   private unsubscribeFn?: () => void;
-  private deconstructionFn?: DeconstructionFn<T>;
 
   constructor(
     tenantCache: TenantCache,
@@ -26,27 +26,32 @@ export class TenantCacheAdapter<T> {
     this.subscribe();
   }
 
+  private handle = async (tenantID: string) => {
+    // Get the current set value for the item in the cache.
+    const value = this.get(tenantID);
+
+    // Delete the tenant cache item when the tenant changes.
+    this.cache.delete(tenantID);
+
+    if (this.deconstructionFn) {
+      // The deconstruction function is set. We will check that the value
+      // exists, and if it does, we will call the function with the given
+      // identifier, this allows the caller to attach deconstruction
+      // components to the tenant being removed. The only side affect to
+      // note is that by the time that the deconstruction function is
+      // called, the tenant has already been purged from the cache.
+      if (typeof value !== "undefined") {
+        await this.deconstructionFn(tenantID, value);
+      }
+    }
+  };
+
   public subscribe() {
     if (this.tenantCache.cachingEnabled && !this.unsubscribeFn) {
-      this.unsubscribeFn = this.tenantCache.subscribe(async tenant => {
-        // Get the current set value for the item in the cache.
-        const value = this.get(tenant.id);
-
-        // Delete the tenant cache item when the tenant changes.
-        this.cache.delete(tenant.id);
-
-        if (this.deconstructionFn) {
-          // The deconstruction function is set. We will check that the value
-          // exists, and if it does, we will call the function with the given
-          // identifier, this allows the caller to attach deconstruction
-          // components to the tenant being removed. The only side affect to
-          // note is that by the time that the deconstruction function is
-          // called, the tenant has already been purged from the cache.
-          if (typeof value !== "undefined") {
-            await this.deconstructionFn(tenant.id, value);
-          }
-        }
-      });
+      this.unsubscribeFn = this.tenantCache.subscribe(
+        ({ id }) => this.handle(id),
+        id => this.handle(id)
+      );
     }
   }
 

src/core/server/services/tenant/cache/index.ts

@@ -1,8 +1,10 @@
 import DataLoader from "dataloader";
+import { EventEmitter } from "events";
 import { Redis } from "ioredis";
 import { Db } from "mongodb";
 import uuid from "uuid";
 
+import { Omit } from "coral-common/types";
 import { Config } from "coral-server/config";
 import logger from "coral-server/logger";
 import {
@@ -12,38 +14,57 @@ import {
   retrieveManyTenantsByDomain,
   Tenant,
 } from "coral-server/models/tenant";
-import { EventEmitter } from "events";
 
-const TENANT_UPDATE_CHANNEL = "tenant";
+const TENANT_CACHE_CHANNEL = "TENANT_CACHE_CHANNEL";
 
-const EMITTER_EVENT_NAME = "update";
+enum EVENTS {
+  UPDATE = "UPDATE",
+  DELETE = "DELETE",
+}
 
-export type SubscribeCallback = (tenant: Tenant) => void;
+type UpdateSubscribeCallback = (tenant: Tenant) => void;
+type DeleteSubscribeCallback = (tenantID: string, tenantDomain: string) => void;
 
-interface TenantUpdateMessage {
+type Message = UpdateMessage | DeleteMessage;
+
+interface DeleteMessage {
+  event: EVENTS.DELETE;
+  tenantID: string;
+  tenantDomain: string;
+  clientApplicationID: string;
+}
+
+interface UpdateMessage {
+  event: EVENTS.UPDATE;
   tenant: Tenant;
   clientApplicationID: string;
 }
 
+/**
+ * MessageData is a type that is used to select only the data parts of the
+ * message.
+ */
+type MessageData<T extends Message> = Omit<T, "clientApplicationID" | "event">;
+
 // TenantCache provides an interface for retrieving tenant stored in local
 // memory rather than grabbing it from the database every single call.
 export default class TenantCache {
   /**
    * tenantsByID reference the tenants that have been cached/retrieved by ID.
    */
-  private tenantsByID: DataLoader<string, Readonly<Tenant> | null>;
+  private readonly tenantsByID: DataLoader<string, Readonly<Tenant> | null>;
 
   /**
    * tenantsByDomain reference the tenants that have been cached/retrieved by
    * Domain.
    */
-  private tenantsByDomain: DataLoader<string, Readonly<Tenant> | null>;
+  private readonly tenantsByDomain: DataLoader<string, Readonly<Tenant> | null>;
 
   /**
    * tenantCountCache stores all the id's of all the Tenant's that have crossed
    * it.
    */
-  private tenantCountCache = new Set<string>();
+  private readonly tenantCountCache = new Set<string>();
 
   /**
    * primed is true when the cache has already been fully primed.
@@ -55,15 +76,15 @@ export default class TenantCache {
    * generated by this application from being handled as external messages
    * as we should have already processed it.
    */
-  private clientApplicationID = uuid.v4();
+  private readonly clientApplicationID = uuid.v4();
 
-  private mongo: Db;
-  private emitter = new EventEmitter();
+  private readonly mongo: Db;
+  private readonly emitter = new EventEmitter();
 
   /**
    * cachingEnabled is true when tenant caching has been enabled.
    */
-  public cachingEnabled: boolean;
+  public readonly cachingEnabled: boolean;
 
   constructor(mongo: Db, subscriber: Redis, config: Config) {
     this.cachingEnabled = !config.get("disable_tenant_caching");
@@ -124,7 +145,7 @@ export default class TenantCache {
       subscriber.on("message", this.onMessage);
 
       // Subscribe to tenant notifications.
-      subscriber.subscribe(TENANT_UPDATE_CHANNEL);
+      subscriber.subscribe(TENANT_CACHE_CHANNEL);
     }
   }
 
@@ -187,10 +208,14 @@ export default class TenantCache {
       await this.primeAll();
     }
 
+    // Copy the tenant count cache to prevent race conditions related to
+    // clearing during iteration.
+    const cache = new Set(this.tenantCountCache);
+
     // If the tenant's are primed in the cache, then just use the count cache as
     // the iteration source.
     if (this.primed) {
-      for (const tenantID of this.tenantCountCache) {
+      for (const tenantID of cache) {
         const tenant = await this.tenantsByID.load(tenantID);
         if (!tenant) {
           continue;
@@ -210,23 +235,44 @@ export default class TenantCache {
     }
   }
 
+  private onUpdateMessage({ tenant }: MessageData<UpdateMessage>) {
+    // Update the tenant cache.
+    this.tenantsByID.clear(tenant.id).prime(tenant.id, tenant);
+    this.tenantsByDomain.clear(tenant.domain).prime(tenant.domain, tenant);
+    this.tenantCountCache.add(tenant.id);
+
+    // Publish the event for the connected listeners.
+    this.emitter.emit(EVENTS.UPDATE, tenant);
+  }
+
+  private onDeleteMessage({
+    tenantID,
+    tenantDomain,
+  }: MessageData<DeleteMessage>) {
+    // Delete the tenant in the local cache.
+    this.tenantsByID.clear(tenantID);
+    this.tenantsByDomain.clear(tenantDomain);
+    this.tenantCountCache.delete(tenantID);
+
+    // Publish the event for the connected listeners.
+    this.emitter.emit(EVENTS.DELETE, tenantID, tenantDomain);
+  }
+
   /**
    *  onMessage is fired every time the client gets a subscription event.
    */
-  private onMessage = async (
-    channel: string,
-    message: string
-  ): Promise<void> => {
+  private async onMessage(channel: string, data: string): Promise<void> {
     // Only do things when the message is for tenant.
-    if (channel !== TENANT_UPDATE_CHANNEL) {
+    if (channel !== TENANT_CACHE_CHANNEL) {
       return;
     }
 
     try {
-      // Updated tenant come from the messages.
-      const { tenant, clientApplicationID }: TenantUpdateMessage = JSON.parse(
-        message
-      );
+      // Parse the message (which is JSON).
+      const message: Message = JSON.parse(data);
+
+      // Extract some known parameters.
+      const { clientApplicationID } = message;
 
       // Check to see if this was the update issued by this instance.
       if (clientApplicationID === this.clientApplicationID) {
@@ -234,22 +280,26 @@ export default class TenantCache {
         return;
       }
 
-      logger.debug({ tenantID: tenant.id }, "received updated tenant");
+      const log = logger.child({ eventName: message.event }, true);
+      log.debug("received tenant message");
 
-      // Update the tenant cache.
-      this.tenantsByID.clear(tenant.id).prime(tenant.id, tenant);
-      this.tenantsByDomain.clear(tenant.domain).prime(tenant.domain, tenant);
-      this.tenantCountCache.add(tenant.id);
-
-      // Publish the event for the connected listeners.
-      this.emitter.emit(EMITTER_EVENT_NAME, tenant);
+      // Send the message to the correct handler.
+      switch (message.event) {
+        case EVENTS.UPDATE:
+          return this.onUpdateMessage(message);
+        case EVENTS.DELETE:
+          return this.onDeleteMessage(message);
+        default:
+          log.warn("received unknown event");
+          return;
+      }
     } catch (err) {
       logger.error(
         { err },
-        "an error occurred while trying to parse/prime the tenant/tenant cache"
+        "an error occurred while trying to handle a message"
       );
     }
-  };
+  }
 
   public async retrieveByID(id: string): Promise<Readonly<Tenant> | null> {
     return this.tenantsByID.load(id);
@@ -265,17 +315,34 @@ export default class TenantCache {
    * This allows you to subscribe to new Tenant updates. This will also return
    * a function that when called, unsubscribes you from updates.
    *
-   * @param callback the function to be called when there is an updated Tenant.
+   * @param updateCallback the function to be called when there is an updated Tenant.
+   * @param deleteCallback the function to be called when a tenant needs to be purged
    */
-  public subscribe(callback: SubscribeCallback) {
-    this.emitter.on(EMITTER_EVENT_NAME, callback);
+  public subscribe(
+    updateCallback: UpdateSubscribeCallback,
+    deleteCallback: DeleteSubscribeCallback
+  ) {
+    this.emitter.on(EVENTS.UPDATE, updateCallback);
+    this.emitter.on(EVENTS.DELETE, deleteCallback);
 
     // Return the unsubscribe function.
     return () => {
-      this.emitter.removeListener(EMITTER_EVENT_NAME, callback);
+      this.emitter.removeListener(EVENTS.UPDATE, updateCallback);
+      this.emitter.removeListener(EVENTS.DELETE, deleteCallback);
     };
   }
 
+  private async publish(tenantID: string, conn: Redis, message: Message) {
+    const subscribers = await conn.publish(
+      TENANT_CACHE_CHANNEL,
+      JSON.stringify(message)
+    );
+    logger.debug(
+      { tenantID, subscribers, eventName: message.event },
+      "updated tenant in cache"
+    );
+  }
+
   /**
    * update will update the value for Tenant in the local cache and publish
    * a change notification that will be used to keep the other nodes in sync.
@@ -284,28 +351,27 @@ export default class TenantCache {
    * @param tenant the updated Tenant object
    */
   public async update(conn: Redis, tenant: Tenant): Promise<void> {
-    // Update the tenant in the local cache.
-    this.tenantsByID.clear(tenant.id).prime(tenant.id, tenant);
-    this.tenantsByDomain.clear(tenant.domain).prime(tenant.domain, tenant);
-    this.tenantCountCache.add(tenant.id);
+    // Process the tenant update on this node.
+    this.onUpdateMessage({ tenant });
 
     // Notify the other nodes about the tenant change.
-    const message: TenantUpdateMessage = {
+    await this.publish(tenant.id, conn, {
+      event: EVENTS.UPDATE,
       tenant,
       clientApplicationID: this.clientApplicationID,
-    };
+    });
+  }
 
-    const subscribers = await conn.publish(
-      TENANT_UPDATE_CHANNEL,
-      JSON.stringify(message)
-    );
+  public async delete(conn: Redis, tenantID: string, tenantDomain: string) {
+    // Process the tenant update on this node.
+    this.onDeleteMessage({ tenantID, tenantDomain });
 
-    logger.debug(
-      { tenantID: tenant.id, subscribers },
-      "updated tenant in cache"
-    );
-
-    // Publish the event for the connected listeners.
-    this.emitter.emit(EMITTER_EVENT_NAME, tenant);
+    // Notify the other nodes about the tenant change.
+    await this.publish(tenantID, conn, {
+      event: EVENTS.DELETE,
+      tenantID,
+      tenantDomain,
+      clientApplicationID: this.clientApplicationID,
+    });
   }
 }

src/core/server/services/tenant/index.ts

@@ -12,7 +12,7 @@ import {
   createTenant,
   CreateTenantInput,
   createTenantSSOKey,
-  deprecateTenantSSOKey,
+  rotateTenantSSOKey,
   Tenant,
   updateTenant,
 } from "coral-server/models/tenant";
@@ -150,7 +150,7 @@ export async function regenerateSSOKey(
   if (tenant.auth.integrations.sso.keys.length > 0) {
     // Get the old keys that are not deprecated.
     const keysToDeprecate = tenant.auth.integrations.sso.keys.filter(key => {
-      return !key.deletedAt && !key.deprecateAt && key.secret;
+      return !key.rotatedAt;
     });
 
     // Check to see if there are keys to deprecate.
@@ -164,7 +164,7 @@ export async function regenerateSSOKey(
       // Deprecate all the keys that are associated on the tenant that haven't
       // been done.
       for (const key of keysToDeprecate) {
-        await deprecateTenantSSOKey(mongo, tenant.id, key.kid, deprecateAt);
+        await rotateTenantSSOKey(mongo, tenant.id, key.kid, deprecateAt, now);
       }
     }
   }