mirror of
https://github.com/wassname/Open-Assistant.git
synced 2026-09-10 11:41:04 +08:00
Show current user rank in leaderboard (#1263)
close #1000 maybe #1178 too * Show current user rank in the leaderboard with +-1 user (only on leaderboard * Extend auto_main script to use random user. * Support colSpan in the DataTable component (I haven't verified colSpan in header yet, leave that until we need it) * Refactor OasstError to include the path and request method.
This commit is contained in:
notmd
+204
-199
@@ -6,12 +6,10 @@ from uuid import uuid4
|
||||
|
||||
import requests
|
||||
import typer
|
||||
from faker import Faker
|
||||
|
||||
app = typer.Typer()
|
||||
|
||||
|
||||
# debug constants
|
||||
USER = {"id": "1234", "display_name": "John Doe", "auth_method": "local"}
|
||||
fake = Faker()
|
||||
|
||||
|
||||
def _random_message_id():
|
||||
@@ -26,19 +24,11 @@ def _render_message(message: dict) -> str:
|
||||
|
||||
|
||||
@app.command()
|
||||
def main(backend_url: str = "http://127.0.0.1:8080", api_key: str = "1234"):
|
||||
def main(
|
||||
backend_url: str = "http://127.0.0.1:8080", api_key: str = "1234", random_users: int = 1, task_per_user: int = 10
|
||||
):
|
||||
"""automates tasks"""
|
||||
|
||||
# make sure dummy user has accepted the terms of service
|
||||
create_user_request = dict(USER)
|
||||
create_user_request["tos_acceptance"] = True
|
||||
response = requests.post(
|
||||
f"{backend_url}/api/v1/frontend_users/", json=create_user_request, headers={"X-API-Key": api_key}
|
||||
)
|
||||
response.raise_for_status()
|
||||
user = response.json()
|
||||
typer.echo(f"user: {user}")
|
||||
|
||||
def _post(path: str, json: dict) -> dict:
|
||||
response = requests.post(f"{backend_url}{path}", json=json, headers={"X-API-Key": api_key})
|
||||
response.raise_for_status()
|
||||
@@ -60,204 +50,219 @@ def main(backend_url: str = "http://127.0.0.1:8080", api_key: str = "1234"):
|
||||
print(shuffled)
|
||||
return ranks
|
||||
|
||||
tasks = [_post("/api/v1/tasks/", {"type": "random", "user": USER})]
|
||||
q = 0
|
||||
while tasks:
|
||||
task = tasks.pop(0)
|
||||
print(task)
|
||||
for i in range(int(random_users)):
|
||||
name = fake.name()
|
||||
USER = {"id": name, "display_name": name, "auth_method": "local"}
|
||||
|
||||
match (task["type"]):
|
||||
case "initial_prompt":
|
||||
typer.echo("Please provide an initial prompt to the assistant.")
|
||||
if task["hint"]:
|
||||
typer.echo(f"Hint: {task['hint']}")
|
||||
# acknowledge task
|
||||
message_id = _random_message_id()
|
||||
_post(f"/api/v1/tasks/{task['id']}/ack", {"message_id": message_id})
|
||||
create_user_request = dict(USER)
|
||||
# make sure dummy user has accepted the terms of service
|
||||
create_user_request["tos_acceptance"] = True
|
||||
response = requests.post(
|
||||
f"{backend_url}/api/v1/frontend_users/", json=create_user_request, headers={"X-API-Key": api_key}
|
||||
)
|
||||
response.raise_for_status()
|
||||
user = response.json()
|
||||
typer.echo(f"user: {user}")
|
||||
q = 0
|
||||
|
||||
prompt = gen_random_text()
|
||||
user_message_id = _random_message_id()
|
||||
# send interaction
|
||||
new_task = _post(
|
||||
"/api/v1/tasks/interaction",
|
||||
{
|
||||
"type": "text_reply_to_message",
|
||||
"message_id": message_id,
|
||||
"task_id": task["id"],
|
||||
"user_message_id": user_message_id,
|
||||
"text": prompt,
|
||||
"user": USER,
|
||||
},
|
||||
)
|
||||
tasks.append(new_task)
|
||||
tasks = [_post("/api/v1/tasks/", {"type": "random", "user": USER})]
|
||||
|
||||
case "label_initial_prompt":
|
||||
typer.echo("Label the following prompt:")
|
||||
typer.echo(task["prompt"])
|
||||
# acknowledge task
|
||||
message_id = _random_message_id()
|
||||
_post(f"/api/v1/tasks/{task['id']}/ack", {"message_id": message_id})
|
||||
while tasks:
|
||||
task = tasks.pop(0)
|
||||
print(task)
|
||||
|
||||
valid_labels = task["valid_labels"]
|
||||
mandatory_labels = task["mandatory_labels"]
|
||||
match (task["type"]):
|
||||
case "initial_prompt":
|
||||
typer.echo("Please provide an initial prompt to the assistant.")
|
||||
if task["hint"]:
|
||||
typer.echo(f"Hint: {task['hint']}")
|
||||
# acknowledge task
|
||||
message_id = _random_message_id()
|
||||
_post(f"/api/v1/tasks/{task['id']}/ack", {"message_id": message_id})
|
||||
|
||||
labels_dict = None
|
||||
if task["mode"] == "simple" and len(valid_labels) == 1:
|
||||
answer = random.choice([True, False])
|
||||
labels_dict = {valid_labels[0]: 1 if answer else 0}
|
||||
else:
|
||||
labels = random.sample(valid_labels, random.randint(1, len(valid_labels)))
|
||||
for l in mandatory_labels:
|
||||
if l not in labels:
|
||||
labels.append(l)
|
||||
labels_dict = {label: random.random() for label in valid_labels}
|
||||
if random.random() < 0.9:
|
||||
labels_dict["spam"] = 0
|
||||
labels_dict["lang_mismatch"] = 0
|
||||
prompt = gen_random_text()
|
||||
user_message_id = _random_message_id()
|
||||
# send interaction
|
||||
new_task = _post(
|
||||
"/api/v1/tasks/interaction",
|
||||
{
|
||||
"type": "text_reply_to_message",
|
||||
"message_id": message_id,
|
||||
"task_id": task["id"],
|
||||
"user_message_id": user_message_id,
|
||||
"text": prompt,
|
||||
"user": USER,
|
||||
},
|
||||
)
|
||||
tasks.append(new_task)
|
||||
|
||||
# send labels
|
||||
new_task = _post(
|
||||
"/api/v1/tasks/interaction",
|
||||
{
|
||||
"type": "text_labels",
|
||||
"message_id": task["message_id"],
|
||||
"task_id": task["id"],
|
||||
"text": task["prompt"],
|
||||
"labels": labels_dict,
|
||||
"user": USER,
|
||||
},
|
||||
)
|
||||
tasks.append(new_task)
|
||||
case "prompter_reply":
|
||||
# acknowledge task
|
||||
message_id = _random_message_id()
|
||||
user_message_id = _random_message_id()
|
||||
_post(f"/api/v1/tasks/{task['id']}/ack", {"message_id": message_id})
|
||||
# send interaction
|
||||
new_task = _post(
|
||||
"/api/v1/tasks/interaction",
|
||||
{
|
||||
"type": "text_reply_to_message",
|
||||
"message_id": message_id,
|
||||
"task_id": task["id"],
|
||||
"user_message_id": user_message_id,
|
||||
"text": gen_random_text(),
|
||||
"user": USER,
|
||||
},
|
||||
)
|
||||
tasks.append(new_task)
|
||||
case "label_initial_prompt":
|
||||
typer.echo("Label the following prompt:")
|
||||
typer.echo(task["prompt"])
|
||||
# acknowledge task
|
||||
message_id = _random_message_id()
|
||||
_post(f"/api/v1/tasks/{task['id']}/ack", {"message_id": message_id})
|
||||
|
||||
case "assistant_reply":
|
||||
# acknowledge task
|
||||
message_id = _random_message_id()
|
||||
user_message_id = _random_message_id()
|
||||
_post(f"/api/v1/tasks/{task['id']}/ack", {"message_id": message_id})
|
||||
# send interaction
|
||||
new_task = _post(
|
||||
"/api/v1/tasks/interaction",
|
||||
{
|
||||
"type": "text_reply_to_message",
|
||||
"message_id": message_id,
|
||||
"task_id": task["id"],
|
||||
"user_message_id": user_message_id,
|
||||
"text": gen_random_text(),
|
||||
"user": USER,
|
||||
},
|
||||
)
|
||||
tasks.append(new_task)
|
||||
valid_labels = task["valid_labels"]
|
||||
mandatory_labels = task["mandatory_labels"]
|
||||
|
||||
case "rank_prompter_replies" | "rank_assistant_replies":
|
||||
# acknowledge task
|
||||
message_id = _random_message_id()
|
||||
user_message_id = _random_message_id()
|
||||
_post(f"/api/v1/tasks/{task['id']}/ack", {"message_id": message_id})
|
||||
# send interaction
|
||||
ranking = gen_random_ranking(task["replies"])
|
||||
print(ranking)
|
||||
new_task = _post(
|
||||
"/api/v1/tasks/interaction",
|
||||
{
|
||||
"type": "message_ranking",
|
||||
"message_id": message_id,
|
||||
"task_id": task["id"],
|
||||
"ranking": ranking,
|
||||
"user": USER,
|
||||
},
|
||||
)
|
||||
tasks.append(new_task)
|
||||
labels_dict = None
|
||||
if task["mode"] == "simple" and len(valid_labels) == 1:
|
||||
answer = random.choice([True, False])
|
||||
labels_dict = {valid_labels[0]: 1 if answer else 0}
|
||||
else:
|
||||
labels = random.sample(valid_labels, random.randint(1, len(valid_labels)))
|
||||
for l in mandatory_labels:
|
||||
if l not in labels:
|
||||
labels.append(l)
|
||||
labels_dict = {label: random.random() for label in valid_labels}
|
||||
if random.random() < 0.9:
|
||||
labels_dict["spam"] = 0
|
||||
labels_dict["lang_mismatch"] = 0
|
||||
|
||||
case "rank_initial_prompts":
|
||||
# acknowledge task
|
||||
message_id = _random_message_id()
|
||||
user_message_id = _random_message_id()
|
||||
_post(f"/api/v1/tasks/{task['id']}/ack", {"message_id": message_id})
|
||||
# send interaction
|
||||
ranking = gen_random_ranking(task["prompots"])
|
||||
new_task = _post(
|
||||
"/api/v1/tasks/interaction",
|
||||
{
|
||||
"type": "message_ranking",
|
||||
"message_id": message_id,
|
||||
"ranking": ranking,
|
||||
"user": USER,
|
||||
},
|
||||
)
|
||||
tasks.append(new_task)
|
||||
# send labels
|
||||
new_task = _post(
|
||||
"/api/v1/tasks/interaction",
|
||||
{
|
||||
"type": "text_labels",
|
||||
"message_id": task["message_id"],
|
||||
"task_id": task["id"],
|
||||
"text": task["prompt"],
|
||||
"labels": labels_dict,
|
||||
"user": USER,
|
||||
},
|
||||
)
|
||||
tasks.append(new_task)
|
||||
case "prompter_reply":
|
||||
# acknowledge task
|
||||
message_id = _random_message_id()
|
||||
user_message_id = _random_message_id()
|
||||
_post(f"/api/v1/tasks/{task['id']}/ack", {"message_id": message_id})
|
||||
# send interaction
|
||||
new_task = _post(
|
||||
"/api/v1/tasks/interaction",
|
||||
{
|
||||
"type": "text_reply_to_message",
|
||||
"message_id": message_id,
|
||||
"task_id": task["id"],
|
||||
"user_message_id": user_message_id,
|
||||
"text": gen_random_text(),
|
||||
"user": USER,
|
||||
},
|
||||
)
|
||||
tasks.append(new_task)
|
||||
|
||||
case "label_prompter_reply" | "label_assistant_reply":
|
||||
# acknowledge task
|
||||
typer.echo("Here is the conversation so far:")
|
||||
for message in task["conversation"]["messages"]:
|
||||
typer.echo(_render_message(message))
|
||||
case "assistant_reply":
|
||||
# acknowledge task
|
||||
message_id = _random_message_id()
|
||||
user_message_id = _random_message_id()
|
||||
_post(f"/api/v1/tasks/{task['id']}/ack", {"message_id": message_id})
|
||||
# send interaction
|
||||
new_task = _post(
|
||||
"/api/v1/tasks/interaction",
|
||||
{
|
||||
"type": "text_reply_to_message",
|
||||
"message_id": message_id,
|
||||
"task_id": task["id"],
|
||||
"user_message_id": user_message_id,
|
||||
"text": gen_random_text(),
|
||||
"user": USER,
|
||||
},
|
||||
)
|
||||
tasks.append(new_task)
|
||||
|
||||
typer.echo("Label the following reply:")
|
||||
typer.echo(task["reply"])
|
||||
message_id = _random_message_id()
|
||||
user_message_id = _random_message_id()
|
||||
_post(f"/api/v1/tasks/{task['id']}/ack", {"message_id": message_id})
|
||||
valid_labels = task["valid_labels"]
|
||||
mandatory_labels = task["mandatory_labels"]
|
||||
case "rank_prompter_replies" | "rank_assistant_replies":
|
||||
# acknowledge task
|
||||
message_id = _random_message_id()
|
||||
user_message_id = _random_message_id()
|
||||
_post(f"/api/v1/tasks/{task['id']}/ack", {"message_id": message_id})
|
||||
# send interaction
|
||||
ranking = gen_random_ranking(task["replies"])
|
||||
print(ranking)
|
||||
new_task = _post(
|
||||
"/api/v1/tasks/interaction",
|
||||
{
|
||||
"type": "message_ranking",
|
||||
"message_id": message_id,
|
||||
"task_id": task["id"],
|
||||
"ranking": ranking,
|
||||
"user": USER,
|
||||
},
|
||||
)
|
||||
tasks.append(new_task)
|
||||
|
||||
labels_dict = None
|
||||
if task["mode"] == "simple" and len(valid_labels) == 1:
|
||||
answer = random.choice([True, False])
|
||||
labels_dict = {valid_labels[0]: 1 if answer else 0}
|
||||
else:
|
||||
labels = random.sample(valid_labels, random.randint(1, len(valid_labels)))
|
||||
for l in mandatory_labels:
|
||||
if l not in labels:
|
||||
labels.append(l)
|
||||
labels_dict = {label: random.random() for label in valid_labels}
|
||||
if random.random() < 0.9:
|
||||
labels_dict["spam"] = 0
|
||||
labels_dict["lang_mismatch"] = 0
|
||||
case "rank_initial_prompts":
|
||||
# acknowledge task
|
||||
message_id = _random_message_id()
|
||||
user_message_id = _random_message_id()
|
||||
_post(f"/api/v1/tasks/{task['id']}/ack", {"message_id": message_id})
|
||||
# send interaction
|
||||
ranking = gen_random_ranking(task["prompots"])
|
||||
new_task = _post(
|
||||
"/api/v1/tasks/interaction",
|
||||
{
|
||||
"type": "message_ranking",
|
||||
"message_id": message_id,
|
||||
"ranking": ranking,
|
||||
"user": USER,
|
||||
},
|
||||
)
|
||||
tasks.append(new_task)
|
||||
|
||||
# send interaction
|
||||
new_task = _post(
|
||||
"/api/v1/tasks/interaction",
|
||||
{
|
||||
"type": "text_labels",
|
||||
"message_id": task["message_id"],
|
||||
"task_id": task["id"],
|
||||
"text": task["reply"],
|
||||
"labels": labels_dict,
|
||||
"user": USER,
|
||||
},
|
||||
)
|
||||
tasks.append(new_task)
|
||||
case "task_done":
|
||||
typer.echo("Task done!")
|
||||
# rerun with new task selected from above cases
|
||||
# add a new task
|
||||
q += 1
|
||||
if q == 10:
|
||||
case "label_prompter_reply" | "label_assistant_reply":
|
||||
# acknowledge task
|
||||
typer.echo("Here is the conversation so far:")
|
||||
for message in task["conversation"]["messages"]:
|
||||
typer.echo(_render_message(message))
|
||||
|
||||
typer.echo("Label the following reply:")
|
||||
typer.echo(task["reply"])
|
||||
message_id = _random_message_id()
|
||||
user_message_id = _random_message_id()
|
||||
_post(f"/api/v1/tasks/{task['id']}/ack", {"message_id": message_id})
|
||||
valid_labels = task["valid_labels"]
|
||||
mandatory_labels = task["mandatory_labels"]
|
||||
|
||||
labels_dict = None
|
||||
if task["mode"] == "simple" and len(valid_labels) == 1:
|
||||
answer = random.choice([True, False])
|
||||
labels_dict = {valid_labels[0]: 1 if answer else 0}
|
||||
else:
|
||||
labels = random.sample(valid_labels, random.randint(1, len(valid_labels)))
|
||||
for l in mandatory_labels:
|
||||
if l not in labels:
|
||||
labels.append(l)
|
||||
labels_dict = {label: random.random() for label in valid_labels}
|
||||
if random.random() < 0.9:
|
||||
labels_dict["spam"] = 0
|
||||
labels_dict["lang_mismatch"] = 0
|
||||
|
||||
# send interaction
|
||||
new_task = _post(
|
||||
"/api/v1/tasks/interaction",
|
||||
{
|
||||
"type": "text_labels",
|
||||
"message_id": task["message_id"],
|
||||
"task_id": task["id"],
|
||||
"text": task["reply"],
|
||||
"labels": labels_dict,
|
||||
"user": USER,
|
||||
},
|
||||
)
|
||||
tasks.append(new_task)
|
||||
case "task_done":
|
||||
typer.echo("Task done!")
|
||||
break
|
||||
tasks = [_post("/api/v1/tasks/", {"type": "random", "user": USER})]
|
||||
#
|
||||
case _:
|
||||
typer.echo(f"Unknown task type {task['type']}")
|
||||
# rerun with new task selected from above cases
|
||||
# rerun with new task selected from above cases
|
||||
# add a new task
|
||||
q += 1
|
||||
if q == task_per_user:
|
||||
typer.echo("Task done!")
|
||||
break
|
||||
tasks = [_post("/api/v1/tasks/", {"type": "random", "user": USER})]
|
||||
#
|
||||
case _:
|
||||
typer.echo(f"Unknown task type {task['type']}")
|
||||
# rerun with new task selected from above cases
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -1,2 +1,3 @@
|
||||
faker==16.6.1
|
||||
requests==2.28.1
|
||||
typer==0.7.0
|
||||
|
||||
Reference in New Issue
Block a user