From 4715a53637720342093f480d412493bdccf18ac3 Mon Sep 17 00:00:00 2001
From: yeungadrian <47532646+yeungadrian@users.noreply.github.com>
Date: Mon, 2 Jan 2023 21:38:54 +0000
Subject: [PATCH] #47- Create API endpoints that return leaderboards (#250)

* Leaderboard endpoints proof of concept
- New leaderboards router
- Two endpoints /api/v1/leaderboards/assistant & /api/v1/leaderboards/prompter
- Function in prompt_repository to calculate user scores

* Separating create vs 'eventual' evaluate routes

* Adding display_name to result

* Missing / in routes

* Updating route to be experimental

* Fixing format for pre-commit
---
 backend/oasst_backend/api/v1/api.py           | 12 ++++++++-
 backend/oasst_backend/api/v1/leaderboards.py  | 25 +++++++++++++++++++
 backend/oasst_backend/prompt_repository.py    | 23 ++++++++++++++++-
 oasst-shared/oasst_shared/schemas/protocol.py | 14 ++++++++++-
 4 files changed, 71 insertions(+), 3 deletions(-)
 create mode 100644 backend/oasst_backend/api/v1/leaderboards.py

diff --git a/backend/oasst_backend/api/v1/api.py b/backend/oasst_backend/api/v1/api.py
index 94975909..a9d09457 100644
--- a/backend/oasst_backend/api/v1/api.py
+++ b/backend/oasst_backend/api/v1/api.py
@@ -1,5 +1,14 @@
 from fastapi import APIRouter
-from oasst_backend.api.v1 import frontend_messages, frontend_users, messages, stats, tasks, text_labels, users
+from oasst_backend.api.v1 import (
+    frontend_messages,
+    frontend_users,
+    leaderboards,
+    messages,
+    stats,
+    tasks,
+    text_labels,
+    users,
+)
 
 api_router = APIRouter()
 api_router.include_router(tasks.router, prefix="/tasks", tags=["tasks"])
@@ -9,3 +18,4 @@ api_router.include_router(frontend_messages.router, prefix="/frontend_messages",
 api_router.include_router(users.router, prefix="/users", tags=["users"])
 api_router.include_router(frontend_users.router, prefix="/frontend_users", tags=["frontend_users"])
 api_router.include_router(stats.router, prefix="/stats", tags=["stats"])
+api_router.include_router(leaderboards.router, prefix="/experimental/leaderboards", tags=["leaderboards"])
diff --git a/backend/oasst_backend/api/v1/leaderboards.py b/backend/oasst_backend/api/v1/leaderboards.py
new file mode 100644
index 00000000..4202edad
--- /dev/null
+++ b/backend/oasst_backend/api/v1/leaderboards.py
@@ -0,0 +1,25 @@
+from fastapi import APIRouter, Depends
+from oasst_backend.api import deps
+from oasst_backend.models import ApiClient
+from oasst_backend.prompt_repository import PromptRepository
+from sqlmodel import Session
+
+router = APIRouter()
+
+
+@router.get("/create/assistant")
+def get_assistant_leaderboard(
+    db: Session = Depends(deps.get_db),
+    api_client: ApiClient = Depends(deps.get_trusted_api_client),
+):
+    pr = PromptRepository(db, api_client, None)
+    return pr.get_user_leaderboard(role="assistant")
+
+
+@router.get("/create/prompter")
+def get_prompter_leaderboard(
+    db: Session = Depends(deps.get_db),
+    api_client: ApiClient = Depends(deps.get_trusted_api_client),
+):
+    pr = PromptRepository(db, api_client, None)
+    return pr.get_user_leaderboard(role="prompter")
diff --git a/backend/oasst_backend/prompt_repository.py b/backend/oasst_backend/prompt_repository.py
index 570d48b8..157e42a7 100644
--- a/backend/oasst_backend/prompt_repository.py
+++ b/backend/oasst_backend/prompt_repository.py
@@ -12,7 +12,7 @@ from oasst_backend.models import ApiClient, Message, MessageReaction, Task, Text
 from oasst_backend.models.payload_column_type import PayloadContainer
 from oasst_shared.exceptions import OasstError, OasstErrorCode
 from oasst_shared.schemas import protocol as protocol_schema
-from oasst_shared.schemas.protocol import SystemStats
+from oasst_shared.schemas.protocol import LeaderboardStats, SystemStats
 from sqlalchemy import update
 from sqlmodel import Session, func
 from starlette.status import HTTP_403_FORBIDDEN, HTTP_404_NOT_FOUND
@@ -705,3 +705,24 @@ class PromptRepository:
             deleted=result.get(True, 0),
             message_trees=result.get(None, 0),
         )
+
+    def get_user_leaderboard(self, role: str) -> LeaderboardStats:
+        """
+        Get leaderboard stats for Messages created,
+        separate leaderboard for prompts & assistants
+
+        """
+        query = (
+            self.db.query(Message.user_id, User.username, User.display_name, func.count(Message.user_id))
+            .join(User, User.id == Message.user_id, isouter=True)
+            .filter(Message.deleted is not True, Message.role == role)
+            .group_by(Message.user_id, User.username, User.display_name)
+            .order_by(func.count(Message.user_id).desc())
+        )
+
+        result = [
+            {"ranking": i, "user_id": j[0], "username": j[1], "display_name": j[2], "score": j[3]}
+            for i, j in enumerate(query.all(), start=1)
+        ]
+
+        return LeaderboardStats(leaderboard=result)
diff --git a/oasst-shared/oasst_shared/schemas/protocol.py b/oasst-shared/oasst_shared/schemas/protocol.py
index d3b2ed6c..652a2c78 100644
--- a/oasst-shared/oasst_shared/schemas/protocol.py
+++ b/oasst-shared/oasst_shared/schemas/protocol.py
@@ -1,6 +1,6 @@
 import enum
 from datetime import datetime
-from typing import Literal, Optional, Union
+from typing import List, Literal, Optional, Union
 from uuid import UUID, uuid4
 
 import pydantic
@@ -281,3 +281,15 @@ class SystemStats(BaseModel):
     active: int = 0
     deleted: int = 0
     message_trees: int = 0
+
+
+class UserScore(BaseModel):
+    ranking: int
+    user_id: UUID
+    username: str
+    display_name: str
+    score: int
+
+
+class LeaderboardStats(BaseModel):
+    leaderboard: List[UserScore]