diff --git a/website/src/components/Survey/TaskControls.tsx b/website/src/components/Survey/TaskControls.tsx
index a93889ea..13a2bb4e 100644
--- a/website/src/components/Survey/TaskControls.tsx
+++ b/website/src/components/Survey/TaskControls.tsx
@@ -4,7 +4,7 @@ import { SkipButton } from "src/components/Buttons/Skip";
 import { SubmitButton } from "src/components/Buttons/Submit";
 import { TaskInfo } from "src/components/TaskInfo/TaskInfo";
 
-interface TaskControlsProps {
+export interface TaskControlsProps {
   // we need a task type
   // eslint-disable-next-line  @typescript-eslint/no-explicit-any
   tasks: any[];
diff --git a/website/src/components/Survey/TaskControlsOverridable.tsx b/website/src/components/Survey/TaskControlsOverridable.tsx
new file mode 100644
index 00000000..b5333087
--- /dev/null
+++ b/website/src/components/Survey/TaskControlsOverridable.tsx
@@ -0,0 +1,60 @@
+import {
+  Button,
+  Flex,
+  Modal,
+  ModalBody,
+  ModalCloseButton,
+  ModalContent,
+  ModalFooter,
+  ModalHeader,
+  ModalOverlay,
+  useDisclosure,
+} from "@chakra-ui/react";
+import { TaskControls, TaskControlsProps } from "./TaskControls";
+
+interface TaskControlsOverridableProps extends TaskControlsProps {
+  isValid: boolean;
+  prepareForSubmit: () => void;
+}
+
+export const TaskControlsOverridable = (props: TaskControlsOverridableProps) => {
+  const { isValid, onSubmitResponse, ...rest } = props;
+  const { isOpen: isModalOpen, onOpen: onOpenModal, onClose: onModalClose } = useDisclosure();
+
+  const unchangedResponsePrompt = () => {
+    onOpenModal();
+
+    // Ideally this happens when the user clicks submit, but we can't
+    // reliably wait for it to be executed before submitting the response
+    // without significant refactoring.
+    // As a result, modal will only display once even if the user doesn't proceed
+    props.prepareForSubmit();
+  };
+
+  const onSubmitResponseOverride = () => {
+    onSubmitResponse(props.tasks[0]);
+    onModalClose();
+  };
+
+  return (
+    <>
+      <Modal isOpen={isModalOpen} onClose={onModalClose} isCentered>
+        <ModalOverlay />
+        <ModalContent>
+          <ModalCloseButton />
+          <ModalHeader>Order Unchanged</ModalHeader>
+          <ModalBody>You have not changed the order of the prompts. Are you sure you would like to submit?</ModalBody>
+          <ModalFooter>
+            <Flex justify="center" ml="auto" gap={2}>
+              <Button variant={"ghost"} onClick={onModalClose}>
+                Cancel
+              </Button>
+              <Button onClick={onSubmitResponseOverride}>Submit anyway</Button>
+            </Flex>
+          </ModalFooter>
+        </ModalContent>
+      </Modal>
+      <TaskControls onSubmitResponse={isValid ? props.onSubmitResponse : unchangedResponsePrompt} {...rest} />
+    </>
+  );
+};
diff --git a/website/src/pages/evaluate/rank_assistant_replies.tsx b/website/src/pages/evaluate/rank_assistant_replies.tsx
index 0ed69b09..d20f6364 100644
--- a/website/src/pages/evaluate/rank_assistant_replies.tsx
+++ b/website/src/pages/evaluate/rank_assistant_replies.tsx
@@ -6,7 +6,7 @@ import { LoadingScreen } from "src/components/Loading/LoadingScreen";
 import { Message } from "src/components/Messages";
 import { Sortable } from "src/components/Sortable/Sortable";
 import { SurveyCard } from "src/components/Survey/SurveyCard";
-import { TaskControls } from "src/components/Survey/TaskControls";
+import { TaskControlsOverridable } from "src/components/Survey/TaskControlsOverridable";
 import fetcher from "src/lib/fetcher";
 import poster from "src/lib/poster";
 import useSWRImmutable from "swr/immutable";
@@ -90,7 +90,13 @@ const RankAssistantReplies = () => {
           <Sortable items={replies} onChange={setRanking} className="my-8" />
         </SurveyCard>
 
-        <TaskControls tasks={tasks} onSubmitResponse={submitResponse} onSkip={fetchNextTask} />
+        <TaskControlsOverridable
+          tasks={tasks}
+          isValid={ranking.length == tasks[0].task.replies.length}
+          prepareForSubmit={() => setRanking(tasks[0].task.replies.map((_, idx) => idx))}
+          onSubmitResponse={submitResponse}
+          onSkip={fetchNextTask}
+        />
       </div>
     </>
   );
diff --git a/website/src/pages/evaluate/rank_initial_prompts.tsx b/website/src/pages/evaluate/rank_initial_prompts.tsx
index a9d590ac..a0a48b27 100644
--- a/website/src/pages/evaluate/rank_initial_prompts.tsx
+++ b/website/src/pages/evaluate/rank_initial_prompts.tsx
@@ -4,7 +4,7 @@ import { useEffect, useState } from "react";
 import { LoadingScreen } from "src/components/Loading/LoadingScreen";
 import { Sortable } from "src/components/Sortable/Sortable";
 import { SurveyCard } from "src/components/Survey/SurveyCard";
-import { TaskControls } from "src/components/Survey/TaskControls";
+import { TaskControlsOverridable } from "src/components/Survey/TaskControlsOverridable";
 import fetcher from "src/lib/fetcher";
 import poster from "src/lib/poster";
 import useSWRImmutable from "swr/immutable";
@@ -85,7 +85,13 @@ const RankInitialPrompts = () => {
           <Sortable items={tasks[0].task.prompts} onChange={setRanking} className="my-8" />
         </SurveyCard>
 
-        <TaskControls tasks={tasks} onSubmitResponse={submitResponse} onSkip={fetchNextTask} />
+        <TaskControlsOverridable
+          tasks={tasks}
+          isValid={ranking.length == tasks[0].task.prompts.length}
+          prepareForSubmit={() => setRanking(tasks[0].task.prompts.map((_, idx) => idx))}
+          onSubmitResponse={submitResponse}
+          onSkip={fetchNextTask}
+        />
       </div>
     </>
   );
diff --git a/website/src/pages/evaluate/rank_user_replies.tsx b/website/src/pages/evaluate/rank_user_replies.tsx
index 9a0577cb..09e24fa2 100644
--- a/website/src/pages/evaluate/rank_user_replies.tsx
+++ b/website/src/pages/evaluate/rank_user_replies.tsx
@@ -6,7 +6,7 @@ import { LoadingScreen } from "src/components/Loading/LoadingScreen";
 import { Message } from "src/components/Messages";
 import { Sortable } from "src/components/Sortable/Sortable";
 import { SurveyCard } from "src/components/Survey/SurveyCard";
-import { TaskControls } from "src/components/Survey/TaskControls";
+import { TaskControlsOverridable } from "src/components/Survey/TaskControlsOverridable";
 import fetcher from "src/lib/fetcher";
 import poster from "src/lib/poster";
 import useSWRImmutable from "swr/immutable";
@@ -89,7 +89,13 @@ const RankUserReplies = () => {
           <Sortable items={replies} onChange={setRanking} className="my-8" />
         </SurveyCard>
 
-        <TaskControls tasks={tasks} onSubmitResponse={submitResponse} onSkip={fetchNextTask} />
+        <TaskControlsOverridable
+          tasks={tasks}
+          isValid={ranking.length == tasks[0].task.replies.length}
+          prepareForSubmit={() => setRanking(tasks[0].task.replies.map((_, idx) => idx))}
+          onSubmitResponse={submitResponse}
+          onSkip={fetchNextTask}
+        />
       </div>
     </>
   );