diff --git a/website/cypress/e2e/create/assistant_reply.cy.ts b/website/cypress/e2e/create/assistant_reply.cy.ts
deleted file mode 100644
index aa72b846..00000000
--- a/website/cypress/e2e/create/assistant_reply.cy.ts
+++ /dev/null
@@ -1,26 +0,0 @@
-import { faker } from "@faker-js/faker";
-
-describe("replying as the assistant", () => {
-  it("completes the current task on submit and on request shows a new task", () => {
-    cy.signInWithEmail("cypress@example.com");
-    cy.visit("/create/assistant_reply");
-
-    cy.get('[data-cy="task-id"').then((taskIdElement) => {
-      const taskId = taskIdElement.text();
-
-      const reply = faker.lorem.sentence();
-      cy.log("reply", reply);
-      cy.get('[data-cy="reply"').type(reply);
-
-      cy.get('[data-cy="submit"]').click();
-
-      cy.get('[data-cy="next-task"]').click();
-
-      cy.get('[data-cy="task-id"').should((taskIdElement) => {
-        expect(taskIdElement.text()).not.to.eq(taskId);
-      });
-    });
-  });
-});
-
-export {};
diff --git a/website/cypress/e2e/create/initial_prompt.cy.ts b/website/cypress/e2e/create/initial_prompt.cy.ts
deleted file mode 100644
index b17f2dd9..00000000
--- a/website/cypress/e2e/create/initial_prompt.cy.ts
+++ /dev/null
@@ -1,26 +0,0 @@
-import { faker } from "@faker-js/faker";
-
-describe("creating initial prompts", () => {
-  it("completes the current task on submit and on request shows a new task", () => {
-    cy.signInWithEmail("cypress@example.com");
-    cy.visit("/create/initial_prompt");
-
-    cy.get('[data-cy="task-id"').then((taskIdElement) => {
-      const taskId = taskIdElement.text();
-
-      const prompt = faker.lorem.sentence();
-      cy.log("prompt", prompt);
-      cy.get('[data-cy="reply"').type(prompt);
-
-      cy.get('[data-cy="submit"]').click();
-
-      cy.get('[data-cy="next-task"]').click();
-
-      cy.get('[data-cy="task-id"').should((taskIdElement) => {
-        expect(taskIdElement.text()).not.to.eq(taskId);
-      });
-    });
-  });
-});
-
-export {};
diff --git a/website/cypress/e2e/create/user_reply.cy.ts b/website/cypress/e2e/create/user_reply.cy.ts
deleted file mode 100644
index c126e9c5..00000000
--- a/website/cypress/e2e/create/user_reply.cy.ts
+++ /dev/null
@@ -1,26 +0,0 @@
-import { faker } from "@faker-js/faker";
-
-describe("replying as the prompter", () => {
-  it("completes the current task on submit and on request shows a new task", () => {
-    cy.signInWithEmail("cypress@example.com");
-    cy.visit("/create/user_reply");
-
-    cy.get('[data-cy="task-id"').then((taskIdElement) => {
-      const taskId = taskIdElement.text();
-
-      const reply = faker.lorem.sentence();
-      cy.log("reply", reply);
-      cy.get('[data-cy="reply"').type(reply);
-
-      cy.get('[data-cy="submit"]').click();
-
-      cy.get('[data-cy="next-task"]').click();
-
-      cy.get('[data-cy="task-id"').should((taskIdElement) => {
-        expect(taskIdElement.text()).not.to.eq(taskId);
-      });
-    });
-  });
-});
-
-export {};
diff --git a/website/cypress/e2e/evaluate/rank_assistant_replies.cy.ts b/website/cypress/e2e/evaluate/rank_assistant_replies.cy.ts
deleted file mode 100644
index 3093bd56..00000000
--- a/website/cypress/e2e/evaluate/rank_assistant_replies.cy.ts
+++ /dev/null
@@ -1,30 +0,0 @@
-describe("ranking prompter replies", () => {
-  it("completes the current task on submit and on request shows a new task", () => {
-    cy.signInWithEmail("cypress@example.com");
-    cy.visit("/evaluate/rank_assistant_replies");
-
-    cy.get('[data-cy="task-id"').then((taskIdElement) => {
-      const taskId = taskIdElement.text();
-
-      // Rank an item using the keyboard so that the submit button is enabled
-      cy.get('button[aria-roledescription="sortable"]')
-        .first()
-        .click()
-        .type("{enter}")
-        .wait(100)
-        .type("{downArrow}")
-        .wait(100)
-        .type("{enter}");
-
-      cy.get('[data-cy="submit"]').click();
-
-      cy.get('[data-cy="next-task"]').click();
-
-      cy.get('[data-cy="task-id"').should((taskIdElement) => {
-        expect(taskIdElement.text()).not.to.eq(taskId);
-      });
-    });
-  });
-});
-
-export {};
diff --git a/website/cypress/e2e/evaluate/rank_initial_prompts.cy.ts b/website/cypress/e2e/evaluate/rank_initial_prompts.cy.ts
deleted file mode 100644
index 8c3f4c9a..00000000
--- a/website/cypress/e2e/evaluate/rank_initial_prompts.cy.ts
+++ /dev/null
@@ -1,30 +0,0 @@
-describe("ranking initial prompts", () => {
-  it("completes the current task on submit and on request shows a new task", () => {
-    cy.signInWithEmail("cypress@example.com");
-    cy.visit("/evaluate/rank_initial_prompts");
-
-    cy.get('[data-cy="task-id"').then((taskIdElement) => {
-      const taskId = taskIdElement.text();
-
-      // Rank an item using the keyboard so that the submit button is enabled
-      cy.get('button[aria-roledescription="sortable"]')
-        .first()
-        .click()
-        .type("{enter}")
-        .wait(100)
-        .type("{downArrow}")
-        .wait(100)
-        .type("{enter}");
-
-      cy.get('[data-cy="submit"]').click();
-
-      cy.get('[data-cy="next-task"]').click();
-
-      cy.get('[data-cy="task-id"').should((taskIdElement) => {
-        expect(taskIdElement.text()).not.to.eq(taskId);
-      });
-    });
-  });
-});
-
-export {};
diff --git a/website/cypress/e2e/evaluate/rank_user_replies.cy.ts b/website/cypress/e2e/evaluate/rank_user_replies.cy.ts
deleted file mode 100644
index 55487f1d..00000000
--- a/website/cypress/e2e/evaluate/rank_user_replies.cy.ts
+++ /dev/null
@@ -1,30 +0,0 @@
-describe("ranking assistant replies", () => {
-  it("completes the current task on submit and on request shows a new task", () => {
-    cy.signInWithEmail("cypress@example.com");
-    cy.visit("/evaluate/rank_user_replies");
-
-    cy.get('[data-cy="task-id"').then((taskIdElement) => {
-      const taskId = taskIdElement.text();
-
-      // Rank an item using the keyboard so that the submit button is enabled
-      cy.get('button[aria-roledescription="sortable"]')
-        .first()
-        .click()
-        .type("{enter}")
-        .wait(100)
-        .type("{downArrow}")
-        .wait(100)
-        .type("{enter}");
-
-      cy.get('[data-cy="submit"]').click();
-
-      cy.get('[data-cy="next-task"]').click();
-
-      cy.get('[data-cy="task-id"').should((taskIdElement) => {
-        expect(taskIdElement.text()).not.to.eq(taskId);
-      });
-    });
-  });
-});
-
-export {};
diff --git a/website/cypress/e2e/tasks/random.cy.ts b/website/cypress/e2e/tasks/random.cy.ts
new file mode 100644
index 00000000..d0d3b44a
--- /dev/null
+++ b/website/cypress/e2e/tasks/random.cy.ts
@@ -0,0 +1,61 @@
+import { faker } from "@faker-js/faker";
+
+describe("handles random tasks", () => {
+  it("completes the current task on submit and on request shows a new task", () => {
+    cy.signInWithEmail("cypress@example.com");
+    cy.visit("/tasks/random");
+
+    // Check all create tasks.
+    cy.get('[data-cy="task"]').then((createTaskElement) => {
+      if (createTaskElement.find('[data-task-type="create-task"]').length) {
+        cy.get('[data-cy="task-id"]').then((taskIdElement) => {
+          const taskId = taskIdElement.text();
+
+          const reply = faker.lorem.sentence();
+          cy.log("reply", reply);
+          cy.get('[data-cy="reply"]').type(reply);
+
+          cy.get('[data-cy="submit"]').click();
+
+          cy.get('[data-cy="next-task"]').click();
+
+          cy.get('[data-cy="task-id]"').should((taskIdElement) => {
+            expect(taskIdElement.text()).not.to.eq(taskId);
+          });
+        });
+      }
+
+      // Check all Evaluate tasks.
+      if (createTaskElement.find('[data-task-type="evaluate-task"]').length) {
+        cy.get('[data-cy="task-id"]').then((taskIdElement) => {
+          const taskId = taskIdElement.text();
+
+          // Rank an item using the keyboard so that the submit button is enabled
+          cy.get('button[aria-roledescription="sortable"]')
+            .first()
+            .click()
+            .type("{enter}")
+            .wait(100)
+            .type("{downArrow}")
+            .wait(100)
+            .type("{enter}");
+
+          cy.get('[data-cy="submit"]').click();
+
+          cy.get('[data-cy="next-task"]').click();
+
+          cy.get('[data-cy="task-id"]').should((taskIdElement) => {
+            expect(taskIdElement.text()).not.to.eq(taskId);
+          });
+        });
+      }
+
+      if (createTaskElement.find('[data-task-type="label-task"]').length) {
+      }
+
+      // TODO(#623): Figure out how to fail the test if none of the checks above pass.
+    });
+  });
+});
+
+export {};