wassname/Unsupervised-Elicitation
1
0
Fork 0
mirror of https://github.com/wassname/Unsupervised-Elicitation.git synced 2026-08-20 12:10:10 +08:00
Code Issues Packages Projects Releases Wiki Activity

breaking refactor

Browse Source
This commit is contained in:
wassname
2025-10-02 16:15:00 +08:00
parent 7d4ba5e3da
commit 48e30269f4
20 changed files with 3722 additions and 642 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+8
View File
@@ -1,3 +1,11 @@
Fork to
- [ ] add moral datasets e.g. daily dilemmas, ETHICS, Machiavelli, moral foundations vignettes
- [ ] refactor to UV and simplify
- [ ] replicate
Original readme
----
## Unsupervised Elicitation of Language Models
We introduce a new unsupervised algorithm for eliciting skills from pretrained language models. This algorithm is competitive with training on human labels on common misconceptions (TruthfulQA), math (GSM8k-verification), and helpfulness reward modeling (Alpaca). Without supervision, we train a helpful chat assistant from the Haiku 3.5 base model that outperforms a similarly trained human-supervised baseline.
core/llm_api/anthropic_llm.py
-210
View File
@@ -1,210 +0,0 @@
import asyncio
import json
import logging
import os
import re
import time
from datetime import datetime
from traceback import format_exc
from typing import Optional, Union
import attrs
from anthropic import AsyncAnthropic
from anthropic.types import ContentBlock as AnthropicContentBlock
from termcolor import cprint
from core.llm_api.base_llm import PRINT_COLORS, LLMResponse, ModelAPIProtocol
from core.llm_api.openai_llm import OAIChatPrompt
ANTHROPIC_MODELS = {
"claude-instant-1",
"claude-2.0",
"claude-v1.3",
"claude-2.1",
"claude-3-opus-20240229",
"claude-3-sonnet-20240229",
"claude-3-haiku-20240307",
"claude-3-5-sonnet-20240620",
"claude-3-5-sonnet-20241022",
"claude-3-5-haiku-20241022"
}
LOGGER = logging.getLogger(__name__)
ACCEPTED_ARGS = [
"model",
"messages",
"max_tokens",
"system",
"max_tokens",
"stop_sequences",
"stream",
"temperature",
"top_p",
"top_k",
]
def extract_system_prompt(messages: list) -> str:
sys_prompt = ""
for message in messages:
if message["role"] == "system":
if sys_prompt:
raise ValueError(
"Multiple system messages found in the prompt. Only one is allowed."
)
sys_prompt = message["content"]
return sys_prompt
def transform_messages(messages: list) -> list:
_messages = []
for message in messages:
role = message["role"]
if role == "system":
continue
content = message["content"]
_messages.append({"role": role, "content": [{"type": "text", "text": content}]})
return _messages
def count_tokens(prompt: str) -> int:
return len(prompt.split())
def price_per_token(model_id: str) -> tuple[float, float]:
"""
Returns the (input token, output token) price for the given model id.
"""
return 0, 0
@attrs.define()
class AnthropicChatModel(ModelAPIProtocol):
num_threads: int
print_prompt_and_response: bool = False
client: AsyncAnthropic = attrs.field(
init=False, default=attrs.Factory(AsyncAnthropic)
)
available_requests: asyncio.BoundedSemaphore = attrs.field(init=False)
def __attrs_post_init__(self):
self.available_requests = asyncio.BoundedSemaphore(int(self.num_threads))
@staticmethod
def _create_prompt_history_file(prompt):
filename = f"{datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f')[:-3]}_prompt.txt"
with open(os.path.join("prompt_history", filename), "w") as f:
json_str = json.dumps(prompt, indent=4)
json_str = json_str.replace("\\n", "\n")
f.write(json_str)
return filename
@staticmethod
def _add_response_to_prompt_file(prompt_file, response):
with open(os.path.join("prompt_history", prompt_file), "a") as f:
f.write("\n\n======RESPONSE======\n\n")
json_str = json.dumps(response.to_dict(), indent=4)
json_str = json_str.replace("\\n", "\n")
f.write(json_str)
async def __call__(
self,
model_ids: list[str],
prompt: Union[str, OAIChatPrompt],
print_prompt_and_response: bool,
max_attempts: int,
**kwargs,
) -> list[LLMResponse]:
start = time.time()
assert (
len(model_ids) == 1
), "Anthropic implementation only supports one model at a time."
model_id = model_ids[0]
max_tokens = kwargs.pop("max_tokens", 2048)
LOGGER.debug(f"Making {model_id} call")
response: Optional[AnthropicContentBlock] = None
duration = None
kwargs = {k: v for k, v in kwargs.items() if k in ACCEPTED_ARGS}
system_prompt = extract_system_prompt(prompt)
raw_prompt = prompt
if "tool" in prompt[0]:
kwargs["tools"] = prompt[0]["tool"]
kwargs["tool_choice"] = prompt[0]["tool_choice"]
prompt = transform_messages(prompt)
# prompt_file = self._create_prompt_history_file([system_prompt] + prompt)
for i in range(max_attempts):
try:
async with self.available_requests:
api_start = time.time()
response = await self.client.messages.create(
messages=prompt,
model=model_id,
system=system_prompt,
max_tokens=max_tokens,
**kwargs,
)
api_duration = time.time() - api_start
except Exception as e:
error_info = f"Exception Type: {type(e).__name__}, Error Details: {str(e)}, Traceback: {format_exc()}"
LOGGER.warn(
f"Encountered API error: {error_info}.\nRetrying now. (Attempt {i})"
)
await asyncio.sleep(1.5**i)
else:
break
if response is None:
raise RuntimeError(
f"Failed to get a response from the API after {max_attempts} attempts."
)
num_context_tokens, num_completion_tokens = (
response.usage.input_tokens,
response.usage.output_tokens,
)
context_token_cost, completion_token_cost = price_per_token(model_id)
cost = (
num_context_tokens * context_token_cost
+ num_completion_tokens * completion_token_cost
)
duration = time.time() - start
LOGGER.debug(f"Completed call to {model_id} in {duration}s")
if "tools" in kwargs:
completion = response.content[0].input
else:
completion = response.content[0].text
llm_response = LLMResponse(
model_id=model_id,
completion=completion,
stop_reason=response.stop_reason,
duration=duration,
api_duration=api_duration,
cost=cost,
)
if self.print_prompt_and_response or print_prompt_and_response:
cprint("Prompt:", "white")
cprint("System: " + system_prompt, PRINT_COLORS["system"])
for message in prompt:
role = message["role"]
content = message["content"]
tag = "Human: " if role == "user" else "Assistant: "
cprint(tag, PRINT_COLORS[role], end="")
cprint(content, PRINT_COLORS[role])
cprint(f"Response ({llm_response.model_id}):", "white")
cprint(
f"{llm_response.completion}", PRINT_COLORS["assistant"], attrs=["bold"]
)
print()
return [{"prompt": raw_prompt, "response": llm_response.to_dict()}]
core/llm_api/usage/usage_anthropic.py
-73
View File
@@ -1,73 +0,0 @@
import concurrent.futures
import os
from typing import Optional
import requests
from core.utils import setup_environment
def can_claude_api_take_n_more_concurrents(n: int) -> bool:
def ping_claude__is_ratelimited() -> Optional[bool]:
data = {
"model": "claude-2.0",
"prompt": "\n\nHuman: Count to 50.\n\nAssistant:",
"max_tokens_to_sample": 1000,
}
headers = {
"content-type": "application/json",
"accept": "application/json",
"anthropic-version": "2023-06-01",
"x-api-key": f"{os.getenv('ANTHROPIC_API_KEY')}",
}
response = requests.post(
"https://api.anthropic.com/v1/complete",
headers=headers,
json=data,
timeout=20,
)
if response.status_code == 200:
return False
elif response.status_code == 429:
return True
else:
response.raise_for_status()
# launch n threads, each of which tries to ping claude
print(f"Checking if claude can currently take {n} more concurrent requests...")
with concurrent.futures.ThreadPoolExecutor(max_workers=n) as executor:
futures = []
for _ in range(n):
futures.append(executor.submit(ping_claude__is_ratelimited))
results = [f.result() for f in futures]
result = not any(results) # if any result is true, is rate limited
print(
f"Claude currently {'can' if result else 'cannot'} take {n} more concurrent requests"
)
return result
def binary_search_for_max_concurrent_claude_requests() -> int:
min_max_new_concurrent_requests = 1
max_max_new_concurrent_requests = 100
while (
min_max_new_concurrent_requests + 5 < max_max_new_concurrent_requests
): # plus five because we don't need it accurate
mid_test_number = (
min_max_new_concurrent_requests + max_max_new_concurrent_requests
) // 2
if can_claude_api_take_n_more_concurrents(mid_test_number):
min_max_new_concurrent_requests = mid_test_number
else:
max_max_new_concurrent_requests = mid_test_number
print(
f"\n\nFinal result: Claude can currently take roughly {min_max_new_concurrent_requests} more concurrent requests.\n"
)
return min_max_new_concurrent_requests
if __name__ == "__main__":
setup_environment()
binary_search_for_max_concurrent_claude_requests()
core/llm_api/usage/usage_openai.py
-91
View File
@@ -1,91 +0,0 @@
import logging
import openai
import requests
from core.utils import setup_environment
logger = logging.getLogger(__name__)
_org_ids = {
"NYU": "org-rRALD2hkdlmLWNVCKk9PG5Xq",
"FAR": "org-AFgHGbU3MeFr5M5QFwrBET31",
"ARG": "org-4L2GWAH28buzKOIhEAb3L5aq",
}
def extract_usage(response):
requests_left = float(response.headers["x-ratelimit-remaining-requests"])
requests_limit = float(response.headers["x-ratelimit-limit-requests"])
request_usage = 1 - (requests_left / requests_limit)
tokens_left = float(response.headers["x-ratelimit-remaining-tokens"])
tokens_limit = float(response.headers["x-ratelimit-limit-tokens"])
token_usage = 1 - (tokens_left / tokens_limit)
overall_usage = max(request_usage, token_usage)
return overall_usage
def get_ratelimit_usage(data, org_id, endpoint):
try:
headers = {
"Content-Type": "application/json",
"Authorization": f"Bearer {openai.api_key}",
"OpenAI-Organization": org_id,
}
response = requests.post(
endpoint,
headers=headers,
json=data,
timeout=20,
)
return extract_usage(response)
except Exception as e:
logger.warning(f"Error fetching ratelimit usage: {e}")
return -1
def fetch_ratelimit_usage(org_id, model_name) -> float:
data = {
"model": model_name,
"messages": [{"role": "user", "content": "Say 1"}],
}
return get_ratelimit_usage(
data, org_id, "https://api.openai.com/v1/chat/completions"
)
def fetch_ratelimit_usage_base(org_id, model_name) -> float:
data = {"model": model_name, "prompt": "a", "max_tokens": 1}
return get_ratelimit_usage(data, org_id, "https://api.openai.com/v1/completions")
def get_current_openai_model_usage() -> None:
models_to_check = [
"gpt-3.5-turbo-instruct",
"gpt-4-1106-preview",
"gpt-4-base",
]
org_names = ["NYU", "FAR", "ARG"]
result_str = (
"\nModel usage: 1 is hitting rate limits, 0 is not in use. -1 is error.\n"
)
for org in org_names:
result_str += f"\n{org}:\n"
for model_name in models_to_check:
if model_name == "gpt-4-base" or model_name == "gpt-3.5-turbo-instruct":
if org == "ARG":
usage = fetch_ratelimit_usage_base(_org_ids[org], model_name)
else:
continue
else:
usage = fetch_ratelimit_usage(_org_ids[org], model_name)
result_str += f"\t{model_name}:\t{usage:.2f}\n"
result_str += "\n"
print(result_str)
if __name__ == "__main__":
setup_environment()
get_current_openai_model_usage()
docs/mjc_research_journal.md
+6
View File
@@ -0,0 +1,6 @@
TODO
- Refactor to UV and simplify
- No Anthropic, just openai compatible API
- No private org code needed
- a non parrelal mode for debugging
- try with moral datasets e.g. daily dilemmas, ETHICS, Machiavelli, moral foundations vignettes
env.yaml
-176
View File
@@ -1,176 +0,0 @@
name: UE
channels:
- conda-forge
- defaults
dependencies:
- _libgcc_mutex=0.1=main
- _openmp_mutex=5.1=1_gnu
- bzip2=1.0.8=h5eee18b_6
- ca-certificates=2024.7.2=h06a4308_0
- expat=2.6.2=h6a678d5_0
- ld_impl_linux-64=2.38=h1181459_1
- libffi=3.4.4=h6a678d5_1
- libgcc-ng=11.2.0=h1234567_1
- libgomp=11.2.0=h1234567_1
- libstdcxx-ng=11.2.0=h1234567_1
- libuuid=1.41.5=h5eee18b_0
- ncurses=6.4=h6a678d5_0
- openssl=3.0.14=h5eee18b_0
- pip=24.2=py312h06a4308_0
- python=3.12.4=h5148396_1
- readline=8.2=h5eee18b_0
- setuptools=72.1.0=py312h06a4308_0
- shortuuid=1.0.13=pyhd8ed1ab_0
- sqlite=3.45.3=h5eee18b_0
- tk=8.6.14=h39e8969_0
- wheel=0.43.0=py312h06a4308_0
- xz=5.4.6=h5eee18b_1
- zlib=1.2.13=h5eee18b_1
- pip:
- adjusttext==1.2.0
- aiohappyeyeballs==2.4.3
- aiohttp==3.10.10
- aiosignal==1.3.1
- alembic==1.13.3
- altair==5.4.1
- annotated-types==0.7.0
- anthropic==0.37.1
- antlr4-python3-runtime==4.9.3
- anyio==4.6.2.post1
- attrs==24.2.0
- beautifulsoup4==4.12.3
- blobfile==3.0.0
- bs4==0.0.2
- cattrs==24.1.2
- certifi==2024.8.30
- cfgv==3.4.0
- charset-normalizer==3.4.0
- click==8.1.7
- contourpy==1.3.0
- cycler==0.12.1
- datasets==3.0.2
- dill==0.3.8
- distlib==0.3.9
- distro==1.9.0
- docker-pycreds==0.4.0
- eval-type-backport==0.2.0
- fastapi==0.100.0
- filelock==3.16.1
- fire==0.7.0
- fonttools==4.54.1
- frozenlist==1.4.1
- fsspec==2024.9.0
- gitdb==4.0.11
- gitpython==3.1.43
- greenlet==3.1.1
- h11==0.14.0
- httpcore==1.0.6
- httptools==0.6.4
- httpx==0.27.2
- huggingface-hub==0.26.1
- hydra-core==1.3.2
- identify==2.6.1
- idna==3.10
- jinja2==3.1.4
- jiter==0.6.1
- joblib==1.4.2
- jsonpath-python==1.0.6
- jsonschema==4.23.0
- jsonschema-specifications==2024.10.1
- kiwisolver==1.4.7
- lxml==5.3.0
- mako==1.3.6
- markdown-it-py==3.0.0
- markupsafe==3.0.2
- matplotlib==3.9.2
- mdurl==0.1.2
- mistralai==1.1.0
- mpmath==1.3.0
- multidict==6.1.0
- multiprocess==0.70.16
- mypy-extensions==1.0.0
- narwhals==1.10.0
- networkx==3.4.2
- nodeenv==1.9.1
- numpy==2.1.2
- nvidia-cublas-cu12==12.4.5.8
- nvidia-cuda-cupti-cu12==12.4.127
- nvidia-cuda-nvrtc-cu12==12.4.127
- nvidia-cuda-runtime-cu12==12.4.127
- nvidia-cudnn-cu12==9.1.0.70
- nvidia-cufft-cu12==11.2.1.3
- nvidia-curand-cu12==10.3.5.147
- nvidia-cusolver-cu12==11.6.1.9
- nvidia-cusparse-cu12==12.3.1.170
- nvidia-nccl-cu12==2.21.5
- nvidia-nvjitlink-cu12==12.4.127
- nvidia-nvtx-cu12==12.4.127
- omegaconf==2.3.0
- openai==0.28.0
- packaging==24.1
- pandas==2.2.3
- pebble==5.0.7
- pillow==10.4.0
- platformdirs==4.3.6
- polars==1.10.0
- pre-commit==4.0.1
- propcache==0.2.0
- protobuf==5.28.2
- psutil==6.1.0
- pyarrow==17.0.0
- pycryptodomex==3.21.0
- pydantic==2.9.2
- pydantic-core==2.23.4
- pygments==2.18.0
- pyparsing==3.2.0
- python-dateutil==2.8.2
- python-dotenv==1.0.1
- pytz==2024.2
- pyyaml==6.0.2
- referencing==0.35.1
- regex==2024.9.11
- replicate==1.0.2
- requests==2.32.3
- rich==13.9.3
- rpds-py==0.20.0
- safetensors==0.4.5
- scikit-learn==1.5.2
- scipy==1.14.1
- seaborn==0.13.2
- sentry-sdk==2.17.0
- setproctitle==1.3.3
- shellingham==1.5.4
- six==1.16.0
- smmap==5.0.1
- sniffio==1.3.1
- soupsieve==2.6
- sqlalchemy==2.0.18
- starlette==0.27.0
- sympy==1.13.1
- tabulate==0.9.0
- tenacity==9.0.0
- termcolor==2.5.0
- threadpoolctl==3.5.0
- tiktoken==0.8.0
- together==1.3.4
- tokenizers==0.20.1
- torch==2.5.1
- tqdm==4.66.5
- transformers==4.46.3
- triton==3.1.0
- trueskill==0.4.5
- typer==0.12.5
- typing-extensions==4.12.2
- typing-inspect==0.9.0
- tzdata==2024.2
- urllib3==2.2.3
- uvicorn==0.22.0
- uvloop==0.21.0
- virtualenv==20.27.0
- wandb==0.18.5
- watchfiles==0.24.0
- websockets==13.1
- xxhash==3.5.0
- yarl==1.16.0
- zstandard==0.23.0
prefix: /home/wenjiaxin/miniconda3/envs/UE
figures/algorithm.png
BIN
View File
Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 131 KiB

figures/claude_performance.png
BIN
View File
Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 191 KiB

figures/llama_performance.png
BIN
View File
Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 232 KiB

nbs/01_main.py
+1
View File
@@ -0,0 +1 @@
# %%
pyproject.toml
+49
View File
@@ -0,0 +1,49 @@
[project]
name = "unsupervised-elicitation-wassname"
version = "0.1.0"
description = "Add your description here"
readme = "README.md"
authors = [
{ name = "wassname", email = "1103714+wassname@users.noreply.github.com" }
]
requires-python = ">=3.10"
dependencies = [
"adjusttext>=1.3.0",
"alembic>=1.16.5",
"altair>=5.5.0",
"anthropic>=0.69.0",
"cattrs>=25.2.0",
"datasets>=4.1.1",
"fastapi==0.100.0",
"fire>=0.7.1",
"hydra-core>=1.3.2",
"matplotlib>=3.10.6",
"mistralai>=1.9.10",
"openai==0.28.0",
"pandas>=2.3.3",
"pebble>=5.1.3",
"polars>=1.33.1",
"pre-commit>=4.3.0",
"pydantic>=2.11.9",
"replicate>=1.0.7",
"scikit-learn>=1.7.2",
"scipy>=1.15.3",
"seaborn>=0.13.2",
"sqlalchemy==2.0.18",
"tabulate>=0.9.0",
"tenacity>=9.1.2",
"termcolor>=3.1.0",
"tiktoken>=0.11.0",
"tqdm>=4.67.1",
"trueskill>=0.4.5",
"typer[all]>=0.19.2",
"uvicorn[standard]==0.22.0",
"wandb>=0.22.1",
]
[project.scripts]
unsupervised-elicitation = "unsupervised_elicitation:main"
[build-system]
requires = ["setuptools>=61"]
build-backend = "setuptools.build_meta"
requirements.txt
-31
View File
@@ -1,31 +0,0 @@
adjustText
alembic
altair
anthropic
cattrs
datasets
fastapi==0.100.0
fire
hydra-core
matplotlib
mistralai
openai==0.28.0
pandas
pebble
polars
pre-commit
pydantic
replicate
scikit-learn
scipy
seaborn
sqlalchemy==2.0.18
tabulate
tenacity
termcolor
tiktoken
tqdm
trueskill
typer[all]
uvicorn[standard]==0.22.0
wandb
core/llm_api/__init__.py → src/__init__.py
View File
src/experiments/ICM.py
+2 -2
View File
@@ -11,7 +11,7 @@ from datasets import load_dataset
import argparse
from core.llm_api.llm import ModelAPI
from core.utils import setup_environment
from unsupervised_elicitation.utils import setup_environment
from src.experiments.ICM_tools import (
propose_consistencyfix,
run_consistencyfix,
@@ -557,4 +557,4 @@ if __name__ == "__main__":
args = get_args()
print("task: ", args.testbed)
random.seed(args.seed)
main(args)
main(args)
src/llm_api/__init__.py
View File
core/llm_api/base_llm.py → src/llm_api/base_llm.py
+1 -1
View File
@@ -5,7 +5,7 @@ from typing import Dict, List, Optional, Protocol
import attrs
import numpy as np
from anthropic import AI_PROMPT, HUMAN_PROMPT
# from anthropic import AI_PROMPT, HUMAN_PROMPT
from pydantic import BaseModel
PRINT_COLORS = {"user": "cyan", "system": "magenta", "assistant": "light_green"}
core/llm_api/llm.py → src/llm_api/llm.py
+15 -57
View File
@@ -9,7 +9,6 @@ from typing import Callable, Literal, Optional, Union
import attrs
from core.llm_api.anthropic_llm import ANTHROPIC_MODELS, AnthropicChatModel
from core.llm_api.base_llm import LLMResponse, ModelAPIProtocol
from core.llm_api.openai_llm import (
BASE_MODELS,
@@ -19,14 +18,13 @@ from core.llm_api.openai_llm import (
OpenAIBaseModel,
OpenAIChatModel,
)
from core.utils import load_secrets
from unsupervised_elicitation.utils import load_secrets
LOGGER = logging.getLogger(__name__)
@attrs.define()
class ModelAPI:
anthropic_num_threads: int = 2 # current redwood limit is 5
openai_fraction_rate_limit: float = attrs.field(
default=0.99, validator=attrs.validators.lt(1)
)
@@ -36,7 +34,6 @@ class ModelAPI:
_openai_base: OpenAIBaseModel = attrs.field(init=False)
_openai_base_arg: OpenAIBaseModel = attrs.field(init=False)
_openai_chat: OpenAIChatModel = attrs.field(init=False)
_anthropic_chat: AnthropicChatModel = attrs.field(init=False)
running_cost: float = attrs.field(init=False, default=0)
model_timings: dict[str, list[float]] = attrs.field(init=False, default={})
@@ -61,10 +58,6 @@ class ModelAPI:
organization=secrets[self.organization],
print_prompt_and_response=self.print_prompt_and_response,
)
self._anthropic_chat = AnthropicChatModel(
num_threads=self.anthropic_num_threads,
print_prompt_and_response=self.print_prompt_and_response,
)
Path("./prompt_history").mkdir(exist_ok=True)
@staticmethod
@@ -130,11 +123,9 @@ class ModelAPI:
Args:
model_ids: The model(s) to call. If multiple models are specified, the output will be sampled from the
cheapest model that has capacity. All models must be from the same class (e.g. OpenAI Base,
OpenAI Chat, or Anthropic Chat). Anthropic chat will error if multiple models are passed in.
Passing in multiple models could speed up the response time if one of the models is overloaded.
OpenAI Chat).
prompt: The prompt to send to the model(s). Type should match what's expected by the model(s).
max_tokens: The maximum number of tokens to request from the API (argument added to
standardize the Anthropic and OpenAI APIs, which have different names for this).
max_tokens: The maximum number of tokens to request from the API
print_prompt_and_response: Whether to print the prompt and response to stdout.
n: The number of completions to request.
max_attempts_per_api_call: Passed to the underlying API call. If the API call fails (e.g. because the
@@ -162,8 +153,6 @@ class ModelAPI:
return self._openai_base
elif model_id in GPT_CHAT_MODELS or "ft:gpt-3.5-turbo" in model_id:
return self._openai_chat
elif model_id in ANTHROPIC_MODELS:
return self._anthropic_chat
raise ValueError(f"Invalid model id: {model_id}")
model_classes = [model_id_to_class(model_id) for model_id in model_ids]
@@ -179,10 +168,7 @@ class ModelAPI:
kwargs.get("max_tokens") if kwargs.get("max_tokens") is not None else 2000
)
model_class = model_classes[0]
if isinstance(model_class, AnthropicChatModel):
kwargs["max_tokens_to_sample"] = max_tokens
else:
kwargs["max_tokens"] = max_tokens
kwargs["max_tokens"] = max_tokens
# Check if current prompt has already been saved in the save file
# If so, directly return previous result
responses = None
@@ -197,32 +183,14 @@ class ModelAPI:
# This is because we may frequently update the parse_fn during development
if responses is None:
num_candidates = num_candidates_per_completion * n
if isinstance(model_class, AnthropicChatModel):
responses = list(
chain.from_iterable(
await asyncio.gather(
*[
model_class(
model_ids,
prompt,
print_prompt_and_response,
max_attempts_per_api_call,
**kwargs,
)
for _ in range(num_candidates)
]
)
)
)
else:
responses = await model_class(
model_ids,
prompt,
print_prompt_and_response,
max_attempts_per_api_call,
n=num_candidates,
**kwargs,
)
responses = await model_class(
model_ids,
prompt,
print_prompt_and_response,
max_attempts_per_api_call,
n=num_candidates,
**kwargs,
)
modified_responses = []
for response in responses:
@@ -257,18 +225,8 @@ class ModelAPI:
async def demo():
model_api = ModelAPI(anthropic_num_threads=2, openai_fraction_rate_limit=0.99)
anthropic_requests = [
model_api(
"claude-3-5-sonnet-20240620",
[
{"role": "system", "content": "You are Claude."},
{"role": "user", "content": "who are you!"},
],
max_tokens=20,
print_prompt_and_response=False,
)
]
model_api = ModelAPI(openai_fraction_rate_limit=0.99)
oai_chat_messages = [
[
{"role": "system", "content": "You are gpt-3.5-turbo."},
@@ -293,7 +251,7 @@ async def demo():
)
for message in oai_chat_messages
]
answer = await asyncio.gather(*anthropic_requests, *oai_chat_requests)
answer = await asyncio.gather(*oai_chat_requests)
for responses in answer:
for i in responses:
core/llm_api/openai_llm.py → src/llm_api/openai_llm.py
-1
View File
@@ -22,7 +22,6 @@ from core.llm_api.base_llm import (
PRINT_COLORS,
LLMResponse,
ModelAPIProtocol,
messages_to_single_prompt,
)
OAIChatPrompt = list[dict[str, str]]
core/utils.py → src/utils.py
View File
uv.lock
Generated
+3640
View File
File diff suppressed because it is too large Load Diff