Merge pull request #132 from jralduaveuthey/langsmith

added langsmith code

api/.env.example

@@ -9,3 +9,8 @@ CHAT_DB_USER="user"
 CHAT_DB_PASSWORD="we all live in a yellow submarine"
 CHAT_DB_HOST="127.0.0.1"
 CHAT_DB_PORT="3306"
+
+LANGCHAIN_ENDPOINT="https://api.smith.langchain.com"
+LANGCHAIN_API_KEY="ls_XXXXXXXXXXXX" #leave empty to not use langsmith for monitoring
+LANGCHAIN_PROJECT="stampy-chat" #name of your langsmith project
+LANGCHAIN_TRACING_V2="false" #set to "true" only if using langsmith

api/main.py

@@ -1,12 +1,16 @@
 import dataclasses
+import requests
+import datetime
+import uuid
 import json
 import re
+import os
 
 from flask import Flask, jsonify, request, Response, stream_with_context
 from flask_cors import CORS, cross_origin
 
 from stampy_chat import logging
-from stampy_chat.env import PINECONE_INDEX, FLASK_PORT
+from stampy_chat.env import PINECONE_INDEX, FLASK_PORT, LANGCHAIN_API_KEY, LANGCHAIN_PROJECT
 from stampy_chat.settings import Settings
 from stampy_chat.chat import run_query
 from stampy_chat.callbacks import stream_callback
@@ -82,7 +86,6 @@ def chat_simplified(param=''):
 @app.route('/human/<id>', methods=['GET'])
 @cross_origin()
 def human(id):
-    import requests
     r = requests.get(f"https://aisafety.info/questions/{id}")
     logging.info(f"clicked followup '{json.loads(r.text)['data']['title']}': https://stampy.ai/?state={id}")
 
@@ -94,6 +97,30 @@ def human(id):
     # <a href=\"https://stampy.ai/?state=6207&question=What%20is%20%22superintelligence%22%3F\">
     text = re.sub(r'<a href=\\"/\?state=(\d+.*)\\">', r'<a href=\"https://aisafety.info/?state=\1\\">', r.text)
 
+    if LANGCHAIN_API_KEY: #add to langsmith
+        run_id = str(uuid.uuid4())
+        requests.post(
+            "https://api.smith.langchain.com/runs",
+            json={
+                "id": run_id,
+                "name": "aisafety.info/question",
+                "run_type": "chain",
+                "start_time": datetime.datetime.utcnow().isoformat(),
+                "session_name": LANGCHAIN_PROJECT,
+                "inputs": {"text": f"clicked followup '{json.loads(r.text)['data']['title']}': https://stampy.ai/?state={id}"},
+            },
+            headers={"x-api-key": LANGCHAIN_API_KEY},
+        )
+
+        requests.patch(
+            f"https://api.smith.langchain.com/runs/{run_id}",
+            json={
+                "outputs": {"my_output": text},
+                "end_time": datetime.datetime.utcnow().isoformat(),
+            },
+            headers={"x-api-key": LANGCHAIN_API_KEY},
+        )
+
     return Response(text, mimetype='application/json')
 
 # ------------------------------------------------------------------------------

api/src/stampy_chat/chat.py

@@ -1,3 +1,4 @@
+import os
 from typing import Any, Callable, Dict, List
 
 from langchain.chains import LLMChain, OpenAIModerationChain
@@ -12,12 +13,18 @@ from langchain.prompts import (
 from langchain.pydantic_v1 import Extra
 from langchain.schema import BaseMessage, ChatMessage, PromptValue, SystemMessage
 
-from stampy_chat.env import OPENAI_API_KEY, COMPLETIONS_MODEL
+from stampy_chat.env import OPENAI_API_KEY, COMPLETIONS_MODEL, LANGCHAIN_API_KEY, LANGCHAIN_TRACING_V2
 from stampy_chat.settings import Settings
 from stampy_chat.callbacks import StampyCallbackHandler, BroadcastCallbackHandler, LoggerCallbackHandler
 from stampy_chat.followups import StampyChain
 from stampy_chat.citations import make_example_selector
 
+from langsmith import Client
+
+if LANGCHAIN_TRACING_V2 == "true":
+    if not LANGCHAIN_API_KEY:
+        raise Exception("Langsmith tracing is enabled but no api key was provided. Please set LANGCHAIN_API_KEY in the .env file.")
+    client = Client()
 
 class ModerationError(ValueError):
     pass

api/src/stampy_chat/env.py

@@ -47,3 +47,8 @@ DB_CONNECTION_URI = f"mysql+mysqlconnector://{user}:{password}@{host}:{port}/{db
 
 ### Local testing helpers ###
 REMOTE_CHAT_INSTANCE = os.environ.get("REMOTE_CHAT_INSTANCE", "https://chat.stampy.ai:8443")
+
+### Langsmith ###
+LANGCHAIN_API_KEY = os.environ.get("LANGCHAIN_API_KEY")
+LANGCHAIN_PROJECT = os.environ.get("LANGCHAIN_PROJECT")
+LANGCHAIN_TRACING_V2 = os.environ.get("LANGCHAIN_TRACING_V2")