add polymorphic JSONB payload support

update nullable fields, new auth check

backend/alembic/versions/cd7de470586e_v1_db_structure.py

@@ -72,7 +72,7 @@ def upgrade() -> None:
         sa.Column("id", UUID(as_uuid=True), default=uuid.uuid4, server_default=sa.text("gen_random_uuid()")),
         sa.Column("created_date", sa.DateTime(), nullable=False, server_default=sa.func.current_timestamp()),
         sa.Column("expiry_date", sa.DateTime(), nullable=True),
-        sa.Column("person_id", UUID(as_uuid=True), nullable=False),
+        sa.Column("person_id", UUID(as_uuid=True), nullable=True),
         sa.Column("payload_type", sa.String(200), nullable=False),  # deserialization hint & dbg aid
         sa.Column("payload", JSONB(astext_type=sa.Text()), nullable=False),
         sa.Column("api_client_id", UUID(as_uuid=True), nullable=False),
@@ -88,7 +88,7 @@ def upgrade() -> None:
         sa.Column("parent_id", UUID(as_uuid=True), nullable=True),  # root posts have NULL parent
         sa.Column("thread_id", UUID(as_uuid=True), nullable=False),  # id of thread root
         sa.Column("workpackage_id", UUID(as_uuid=True), nullable=True),  # workpackage id to pass to handler on reply
-        sa.Column("person_id", UUID(as_uuid=True), nullable=False),  # sender (recipients are part of payload)
+        sa.Column("person_id", UUID(as_uuid=True), nullable=True),  # sender (recipients are part of payload)
         sa.Column("api_client_id", UUID(as_uuid=True), nullable=False),
         sa.Column("role", sa.String(128), nullable=False),  # 'assistant', 'user' or something else
         sa.Column("frontend_post_id", sa.String(200), nullable=False),  # unique together with api_client_id

backend/app/api/deps.py

@@ -2,7 +2,7 @@
 from typing import Generator
 
 from app.database import engine
-from app.models import ServiceClient
+from app.models import ApiClient
 from fastapi import HTTPException, Security
 from fastapi.security.api_key import APIKey, APIKeyHeader, APIKeyQuery
 from sqlmodel import Session
@@ -31,20 +31,11 @@ async def get_api_key(
 def api_auth(
     api_key: APIKey,
     db: Session,
-    create: bool = False,
-    read: bool = True,
-    update: bool = False,
-    delete: bool = False,
-) -> ServiceClient:
+) -> ApiClient:
+
     if api_key is not None:
-        api_client = db.query(ServiceClient).filter(ServiceClient.api_key == api_key).first()
-        if api_client is not None:
-            if (
-                (create is False or api_client.can_append)
-                and (read is False or api_client.can_read)
-                and (update is False or api_client.can_write)
-                and (delete is False or api_client.can_delete)
-            ):
-                return api_client
+        api_client = db.query(ApiClient).filter(ApiClient.api_key == api_key).first()
+        if api_client is not None and api_client.enabled:
+            return api_client
 
     raise HTTPException(status_code=HTTP_403_FORBIDDEN, detail="Could not validate credentials")

backend/app/api/v1/labelers.py

@@ -21,7 +21,7 @@ def read_labelers(
     """
     Retrieve labelers.
     """
-    deps.api_auth(api_key, db, read=True)
+    deps.api_auth(api_key, db)
     if limit > 10000:
         raise HTTPException(status_code=HTTP_400_BAD_REQUEST, detail="Bad request")
     labelers = crud.labeler.get_multi(db, begin_id=begin_id, limit=limit)
@@ -38,7 +38,7 @@ def create_labeler(
     """
     Create new labeler.
     """
-    deps.api_auth(api_key, db, create=True)
+    deps.api_auth(api_key, db)
     item = crud.labeler.create(db=db, obj_in=item_in)
     return item
 
@@ -54,7 +54,7 @@ def update_labeler(
     """
     Update a labeler.
     """
-    deps.api_auth(api_key, db, update=True, read=True)
+    deps.api_auth(api_key, db)
     item = crud.labeler.get(db=db, id=id)
     if not item:
         raise HTTPException(status_code=HTTP_404_NOT_FOUND, detail="Item not found")
@@ -72,7 +72,7 @@ def read_labeler_by_username(
     """
     Get labeler by ID.
     """
-    deps.api_auth(api_key, db, read=True)
+    deps.api_auth(api_key, db)
     item = crud.labeler.get_by_discord_username(db=db, discord_username=discord_username)
     if not item:
         raise HTTPException(status_code=404, detail="Item not found")
@@ -89,7 +89,7 @@ def read_labeler(
     """
     Get labeler by ID.
     """
-    deps.api_auth(api_key, db, read=True)
+    deps.api_auth(api_key, db)
     item = crud.labeler.get(db=db, id=id)
     if not item:
         raise HTTPException(status_code=HTTP_404_NOT_FOUND, detail="Item not found")
@@ -106,7 +106,7 @@ def delete_labeler(
     """
     Delete a labeler.
     """
-    deps.api_auth(api_key, db, delete=True)
+    deps.api_auth(api_key, db)
     labeler = crud.labeler.get(db=db, id=id)
     if not labeler:
         raise HTTPException(status_code=HTTP_404_NOT_FOUND, detail="Item not found")

backend/app/api/v1/prompts.py

@@ -21,7 +21,7 @@ def read_prompts(
     """
     Retrieve prompts.
     """
-    deps.api_auth(api_key, db, read=True)
+    deps.api_auth(api_key, db)
     if limit > 10000:
         raise HTTPException(status_code=HTTP_400_BAD_REQUEST, detail="Bad request")
     return crud.prompt.get_multi(db, begin_id=begin_id, limit=limit)
@@ -37,7 +37,7 @@ def create_prompt(
     """
     Create new prompt.
     """
-    deps.api_auth(api_key, db, create=True)
+    deps.api_auth(api_key, db)
     if item_in.labeler_id is None:
         if item_in.discord_username is None:
             raise HTTPException(status_code=HTTP_400_BAD_REQUEST, detail="Bad request")
@@ -66,7 +66,7 @@ def read_prompt(
     """
     Get prompt by ID.
     """
-    deps.api_auth(api_key, db, read=True)
+    deps.api_auth(api_key, db)
     item = crud.prompt.get(db=db, id=id)
     if not item:
         raise HTTPException(status_code=HTTP_404_NOT_FOUND, detail="Item not found")
@@ -83,7 +83,7 @@ def delete_prompt(
     """
     Delete a prompt.
     """
-    deps.api_auth(api_key, db, delete=True)
+    deps.api_auth(api_key, db)
     item = crud.prompt.get(db=db, id=id)
     if not item:
         raise HTTPException(status_code=HTTP_404_NOT_FOUND, detail="Item not found")

backend/app/api/v1/tasks.py

@@ -23,7 +23,7 @@ def request_task(
     """
     Create new task.
     """
-    deps.api_auth(api_key, db, create=True)
+    deps.api_auth(api_key, db)
 
     # TODO: Create a task and store it in the database.
 
@@ -57,7 +57,7 @@ def acknowledge_task(
     """
     The frontend acknowledges a task.
     """
-    deps.api_auth(api_key, db, create=True)
+    deps.api_auth(api_key, db)
 
     match (response.type):
         case "post_created":
@@ -82,7 +82,7 @@ def post_interaction(
     """
     The frontend reports an interaction.
     """
-    deps.api_auth(api_key, db, create=True)
+    deps.api_auth(api_key, db)
 
     response = []
     match (interaction.type):

backend/app/models/payload_column_type.py

@@ -0,0 +1,102 @@
+# -*- coding: utf-8 -*-
+import json
+from typing import Any, Generic, Type, TypeVar
+
+import sqlalchemy.dialects.postgresql as pg
+from fastapi.encoders import jsonable_encoder
+from pydantic import BaseModel, parse_obj_as, validator
+from pydantic.main import ModelMetaclass
+from sqlalchemy.types import TypeDecorator
+
+payload_type_registry = {}
+
+
+P = TypeVar("P", bound=BaseModel)
+
+
+def payload_tpye(cls: Type[P]) -> Type[P]:
+    payload_type_registry[cls.__name__] = cls
+    return cls
+
+
+class PayloadContainer(BaseModel):
+    payload_type: str = ""
+    payload: BaseModel = None
+
+    def __init__(self, **v):
+        p = v["payload"]
+        if isinstance(p, dict):
+            t = v["payload_type"]
+            if t not in payload_type_registry:
+                raise RuntimeError(f"Payload type '{t}' not registered")
+            cls = payload_type_registry[t]
+            v["payload"] = cls(**p)
+        super().__init__(**v)
+
+    @validator("payload", pre=True)
+    def check_payload(cls, v: BaseModel, values: dict[str, Any]) -> BaseModel:
+        values["payload_type"] = type(v).__name__
+        return v
+
+    class Config:
+        orm_mode = True
+
+
+T = TypeVar("T")
+
+
+def payload_column_type(pydantic_type):
+    class PayloadJSONBType(TypeDecorator, Generic[T]):
+        impl = pg.JSONB()
+
+        def __init__(
+            self,
+            json_encoder=json,
+        ):
+            self.json_encoder = json_encoder
+            super(PayloadJSONBType, self).__init__()
+
+        # serialize
+        def bind_processor(self, dialect):
+            impl_processor = self.impl.bind_processor(dialect)
+            dumps = self.json_encoder.dumps
+
+            def process(value: T):
+                if value is not None:
+                    if isinstance(pydantic_type, ModelMetaclass):
+                        # This allows to assign non-InDB models and if they're
+                        # compatible, they're directly parsed into the InDB
+                        # representation, thus hiding the implementation in the
+                        # background. However, the InDB model will still be returned
+                        value_to_dump = pydantic_type.from_orm(value)
+                    else:
+                        value_to_dump = value
+
+                    value = jsonable_encoder(value_to_dump)
+
+                if impl_processor:
+                    return impl_processor(value)
+                else:
+                    return dumps(jsonable_encoder(value_to_dump))
+
+            return process
+
+        # deserialize
+        def result_processor(self, dialect, coltype) -> T:
+            impl_processor = self.impl.result_processor(dialect, coltype)
+
+            def process(value):
+                if impl_processor:
+                    value = impl_processor(value)
+                if value is None:
+                    return None
+                # Explicitly use the generic directly, not type(T)
+                full_obj = parse_obj_as(pydantic_type, value)
+                return full_obj
+
+            return process
+
+        def compare_values(self, x, y):
+            return x == y
+
+    return PayloadJSONBType

backend/app/models/person.py

@@ -10,6 +10,7 @@ from sqlmodel import Field, Index, SQLModel
 
 class Person(SQLModel, table=True):
     __tablename__ = "person"
+    __table_args__ = (Index("ix_person_username", "api_client_id", "username", unique=True),)
 
     id: Optional[UUID] = Field(
         sa_column=sa.Column(
@@ -22,5 +23,3 @@ class Person(SQLModel, table=True):
         sa_column=sa.Column(sa.DateTime(), nullable=False, server_default=sa.func.current_timestamp())
     )
     api_client_id: UUID = Field(foreign_key="api_client.id")
-
-    __table_args__ = (Index("ix_person_username", "api_client_id", "username", unique=True),)

backend/app/models/post.py

@@ -5,9 +5,10 @@ from uuid import UUID, uuid4
 
 import sqlalchemy as sa
 import sqlalchemy.dialects.postgresql as pg
-from pydantic import BaseModel
 from sqlmodel import Field, Index, SQLModel
 
+from .payload_column_type import PayloadContainer, payload_column_type
+
 
 class Post(SQLModel, table=True):
     __tablename__ = "post"
@@ -21,7 +22,7 @@ class Post(SQLModel, table=True):
     parent_id: UUID = Field(nullable=True)
     thread_id: UUID = Field(nullable=False, index=True)
     workpackage_id: UUID = Field(nullable=True, index=True)
-    person_id: UUID = Field(nullable=False, foreign_key="person.id", index=True)
+    person_id: UUID = Field(nullable=True, foreign_key="person.id", index=True)
     role: str = Field(nullable=False, max_length=128)
     api_client_id: UUID = Field(nullable=False, foreign_key="api_client.id")
     frontend_post_id: str = Field(max_length=200, nullable=False)
@@ -29,4 +30,4 @@ class Post(SQLModel, table=True):
         sa_column=sa.Column(sa.DateTime(), nullable=False, server_default=sa.func.current_timestamp())
     )
     payload_type: str = Field(nullable=False, max_length=200)
-    payload: BaseModel = Field(sa_column=sa.Column(pg.JSONB, nullable=False))
+    payload: PayloadContainer = Field(sa_column=sa.Column(payload_column_type(PayloadContainer), nullable=False))

backend/app/models/post_reaction.py

@@ -5,9 +5,10 @@ from uuid import UUID
 
 import sqlalchemy as sa
 import sqlalchemy.dialects.postgresql as pg
-from pydantic import BaseModel
 from sqlmodel import Field, SQLModel
 
+from .payload_column_type import PayloadContainer, payload_column_type
+
 
 class PostReaction(SQLModel, table=True):
     __tablename__ = "post_reaction"
@@ -22,5 +23,5 @@ class PostReaction(SQLModel, table=True):
         sa_column=sa.Column(sa.DateTime(), nullable=False, server_default=sa.func.current_timestamp())
     )
     payload_type: str = Field(nullable=False, max_length=200)
-    payload: BaseModel = Field(sa_column=sa.Column(pg.JSONB, nullable=False))
+    payload: PayloadContainer = Field(sa_column=sa.Column(payload_column_type(PayloadContainer), nullable=False))
     api_client_id: UUID = Field(nullable=False, foreign_key="api_client.id")

backend/app/models/work_package.py

@@ -5,9 +5,10 @@ from uuid import UUID, uuid4
 
 import sqlalchemy as sa
 import sqlalchemy.dialects.postgresql as pg
-from pydantic import BaseModel
 from sqlmodel import Field, SQLModel
 
+from .payload_column_type import PayloadContainer, payload_column_type
+
 
 class WorkPackage(SQLModel, table=True):
     __tablename__ = "work_package"
@@ -21,7 +22,7 @@ class WorkPackage(SQLModel, table=True):
         sa_column=sa.Column(sa.DateTime(), nullable=False, server_default=sa.func.current_timestamp()),
     )
     expiry_date: Optional[datetime] = Field(sa_column=sa.Column(sa.DateTime(), nullable=True))
-    person_id: UUID = Field(nullable=False, foreign_key="person.id", index=True)
+    person_id: UUID = Field(nullable=True, foreign_key="person.id", index=True)
     payload_type: str = Field(nullable=False, max_length=200)
-    payload: BaseModel = Field(sa_column=sa.Column(pg.JSONB, nullable=False))
+    payload: PayloadContainer = Field(sa_column=sa.Column(payload_column_type(PayloadContainer), nullable=False))
     api_client_id: UUID = Field(nullable=False, foreign_key="api_client.id")