mirror of
https://github.com/wassname/openrouter-python-sdk-retry-errors.git
synced 2026-07-31 12:30:30 +08:00
Compare commits
4
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
7e29e408af | ||
|
Christine ChenandGitHub
|
30bc131b97 | ||
|
Christine Chen
|
1b0118c365 | ||
|
Christine Chen
|
cda73e5ebf |
@@ -0,0 +1,86 @@
|
||||
name: Auto-merge Speakeasy PR
|
||||
|
||||
# Speakeasy mode:pr opens PRs (speakeasy-sdk-regen-*) and labels them
|
||||
# patch/minor/major. Merge automatically so sdk_publish.yaml can run.
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [labeled]
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: write
|
||||
|
||||
concurrency:
|
||||
group: auto-merge-speakeasy
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
auto-merge:
|
||||
if: |
|
||||
github.event.sender.login == 'github-actions[bot]' &&
|
||||
github.event.pull_request.user.login == 'github-actions[bot]' &&
|
||||
startsWith(github.event.pull_request.head.ref, 'speakeasy-sdk-regen-') &&
|
||||
contains(github.event.pull_request.title, '🐝 Update SDK') &&
|
||||
contains(fromJSON('["patch", "minor", "major"]'), github.event.label.name)
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Close superseded Speakeasy PRs
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
CURRENT_PR="${{ github.event.pull_request.number }}"
|
||||
|
||||
PRIOR_JSON=$(gh pr list \
|
||||
--repo "${{ github.repository }}" \
|
||||
--state open \
|
||||
--search "head:speakeasy-sdk-regen-" \
|
||||
--limit 500 \
|
||||
--json number \
|
||||
| jq -r --arg cur "$CURRENT_PR" \
|
||||
'[.[].number | select(tostring != $cur)] | .[]')
|
||||
|
||||
if [ -z "$PRIOR_JSON" ]; then
|
||||
echo "No superseded Speakeasy PRs to close"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
mapfile -t PRIOR <<< "$PRIOR_JSON"
|
||||
|
||||
echo "Closing ${#PRIOR[@]} superseded Speakeasy PR(s)"
|
||||
for N in "${PRIOR[@]}"; do
|
||||
gh pr close "$N" --repo "${{ github.repository }}" --delete-branch \
|
||||
--comment "Superseded by newer Speakeasy SDK generation PR #${CURRENT_PR}" \
|
||||
|| echo "::warning::Failed to close PR #$N (continuing)"
|
||||
sleep 1
|
||||
done
|
||||
|
||||
- name: Auto-merge Speakeasy PR
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
PR_NUM="${{ github.event.pull_request.number }}"
|
||||
REPO="${{ github.repository }}"
|
||||
|
||||
STATE=$(gh pr view "$PR_NUM" --repo "$REPO" --json state --jq .state)
|
||||
if [ "$STATE" != "OPEN" ]; then
|
||||
echo "PR #$PR_NUM is $STATE — skipping merge"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Prefer GitHub auto-merge when required checks exist; otherwise
|
||||
# squash-merge directly (same pattern as sdk-release-prs.yaml).
|
||||
AUTO_MERGE_NOOP_PATTERN='is in clean status|protected branch rules|Branch does not have required protected branch rules'
|
||||
if gh pr merge "$PR_NUM" --repo "$REPO" --squash --auto --delete-branch 2> /tmp/gh-merge.err; then
|
||||
echo "Auto-merge enabled for Speakeasy PR #$PR_NUM"
|
||||
elif grep -qiE "$AUTO_MERGE_NOOP_PATTERN" /tmp/gh-merge.err; then
|
||||
echo "Auto-merge not applicable — merging PR #$PR_NUM directly"
|
||||
cat /tmp/gh-merge.err >&2
|
||||
gh pr merge "$PR_NUM" --repo "$REPO" --squash --delete-branch
|
||||
else
|
||||
echo "::error::Failed to enable auto-merge on Speakeasy PR #$PR_NUM"
|
||||
cat /tmp/gh-merge.err >&2
|
||||
exit 1
|
||||
fi
|
||||
+399
-554
@@ -3644,350 +3644,6 @@ components:
|
||||
oneOf:
|
||||
- $ref: '#/components/schemas/ContainerAutoEnvironment'
|
||||
- $ref: '#/components/schemas/ContainerReferenceEnvironment'
|
||||
BenchmarkPricing:
|
||||
description: OpenRouter pricing per token for this model. Null if pricing is unavailable.
|
||||
example:
|
||||
completion: '0.000015'
|
||||
prompt: '0.000003'
|
||||
nullable: true
|
||||
properties:
|
||||
completion:
|
||||
description: Cost per output token (USD, decimal string).
|
||||
example: '0.000015'
|
||||
type: string
|
||||
prompt:
|
||||
description: Cost per input token (USD, decimal string).
|
||||
example: '0.000003'
|
||||
type: string
|
||||
required:
|
||||
- prompt
|
||||
- completion
|
||||
type: object
|
||||
BenchmarksAAItem:
|
||||
example:
|
||||
aa_name: GPT-4o
|
||||
agentic_index: 58.3
|
||||
coding_index: 65.8
|
||||
intelligence_index: 71.2
|
||||
model_permaslug: openai/gpt-4o
|
||||
pricing:
|
||||
completion: '0.00001'
|
||||
prompt: '0.0000025'
|
||||
properties:
|
||||
aa_name:
|
||||
description: Model name as listed on Artificial Analysis.
|
||||
example: GPT-4o
|
||||
type: string
|
||||
agentic_index:
|
||||
description: Artificial Analysis Agentic Index composite score. Higher is better.
|
||||
example: 58.3
|
||||
format: double
|
||||
nullable: true
|
||||
type: number
|
||||
coding_index:
|
||||
description: Artificial Analysis Coding Index composite score. Higher is better.
|
||||
example: 65.8
|
||||
format: double
|
||||
nullable: true
|
||||
type: number
|
||||
intelligence_index:
|
||||
description: Artificial Analysis Intelligence Index composite score. Higher is better.
|
||||
example: 71.2
|
||||
format: double
|
||||
nullable: true
|
||||
type: number
|
||||
model_permaslug:
|
||||
description: Stable OpenRouter model identifier.
|
||||
example: openai/gpt-4o
|
||||
type: string
|
||||
pricing:
|
||||
$ref: '#/components/schemas/BenchmarkPricing'
|
||||
required:
|
||||
- model_permaslug
|
||||
- aa_name
|
||||
- intelligence_index
|
||||
- coding_index
|
||||
- agentic_index
|
||||
- pricing
|
||||
type: object
|
||||
BenchmarksAAMeta:
|
||||
example:
|
||||
as_of: '2026-06-03T12:00:00Z'
|
||||
citation: 'Source: Artificial Analysis (artificialanalysis.ai) via OpenRouter (openrouter.ai/rankings).'
|
||||
model_count: 50
|
||||
source: artificial-analysis
|
||||
source_url: https://artificialanalysis.ai
|
||||
version: v1
|
||||
properties:
|
||||
as_of:
|
||||
description: ISO-8601 timestamp of when this data was last updated.
|
||||
example: '2026-06-03T12:00:00Z'
|
||||
type: string
|
||||
citation:
|
||||
description: Required attribution when republishing this data.
|
||||
example: 'Source: Artificial Analysis (artificialanalysis.ai) via OpenRouter (openrouter.ai/rankings).'
|
||||
type: string
|
||||
model_count:
|
||||
description: Number of unique models in the response.
|
||||
type: integer
|
||||
source:
|
||||
description: Data source identifier.
|
||||
enum:
|
||||
- artificial-analysis
|
||||
type: string
|
||||
source_url:
|
||||
description: URL of the upstream data source.
|
||||
enum:
|
||||
- https://artificialanalysis.ai
|
||||
type: string
|
||||
version:
|
||||
description: Dataset version.
|
||||
enum:
|
||||
- v1
|
||||
type: string
|
||||
required:
|
||||
- as_of
|
||||
- version
|
||||
- source
|
||||
- source_url
|
||||
- citation
|
||||
- model_count
|
||||
type: object
|
||||
BenchmarksAAResponse:
|
||||
example:
|
||||
data:
|
||||
- aa_name: GPT-4o
|
||||
agentic_index: 58.3
|
||||
coding_index: 65.8
|
||||
intelligence_index: 71.2
|
||||
model_permaslug: openai/gpt-4o
|
||||
pricing:
|
||||
completion: '0.00001'
|
||||
prompt: '0.0000025'
|
||||
meta:
|
||||
as_of: '2026-06-03T12:00:00Z'
|
||||
citation: 'Source: Artificial Analysis (artificialanalysis.ai) via OpenRouter (openrouter.ai/rankings).'
|
||||
model_count: 1
|
||||
source: artificial-analysis
|
||||
source_url: https://artificialanalysis.ai
|
||||
version: v1
|
||||
properties:
|
||||
data:
|
||||
items:
|
||||
$ref: '#/components/schemas/BenchmarksAAItem'
|
||||
type: array
|
||||
meta:
|
||||
$ref: '#/components/schemas/BenchmarksAAMeta'
|
||||
required:
|
||||
- data
|
||||
- meta
|
||||
type: object
|
||||
BenchmarksDAItem:
|
||||
example:
|
||||
arena: models
|
||||
avg_generation_time_ms: 3200
|
||||
category: codecategories
|
||||
display_name: Claude Sonnet 4
|
||||
elo: 1423
|
||||
model_permaslug: anthropic/claude-sonnet-4
|
||||
pricing:
|
||||
completion: '0.000015'
|
||||
prompt: '0.000003'
|
||||
tournament_stats:
|
||||
first_place: 12
|
||||
fourth_place: 2
|
||||
second_place: 8
|
||||
third_place: 5
|
||||
total: 27
|
||||
win_rate: 72
|
||||
properties:
|
||||
arena:
|
||||
description: Arena this ranking belongs to.
|
||||
example: models
|
||||
type: string
|
||||
avg_generation_time_ms:
|
||||
description: Average generation time in milliseconds.
|
||||
example: 3200
|
||||
format: double
|
||||
nullable: true
|
||||
type: number
|
||||
category:
|
||||
description: Category within the arena.
|
||||
example: codecategories
|
||||
type: string
|
||||
display_name:
|
||||
description: Human-readable model name from Design Arena.
|
||||
example: Claude Sonnet 4
|
||||
type: string
|
||||
elo:
|
||||
description: ELO rating from head-to-head arena battles.
|
||||
example: 1423
|
||||
format: double
|
||||
type: number
|
||||
model_permaslug:
|
||||
description: >-
|
||||
Stable OpenRouter model identifier when the model is on OpenRouter; otherwise the upstream Design Arena
|
||||
model id. Use pricing != null to detect OpenRouter-mapped models.
|
||||
example: anthropic/claude-sonnet-4
|
||||
type: string
|
||||
pricing:
|
||||
$ref: '#/components/schemas/BenchmarkPricing'
|
||||
tournament_stats:
|
||||
description: Placement distribution from tournament matches.
|
||||
properties:
|
||||
first_place:
|
||||
nullable: true
|
||||
type: integer
|
||||
fourth_place:
|
||||
nullable: true
|
||||
type: integer
|
||||
second_place:
|
||||
nullable: true
|
||||
type: integer
|
||||
third_place:
|
||||
nullable: true
|
||||
type: integer
|
||||
total:
|
||||
nullable: true
|
||||
type: integer
|
||||
required:
|
||||
- first_place
|
||||
- second_place
|
||||
- third_place
|
||||
- fourth_place
|
||||
- total
|
||||
type: object
|
||||
win_rate:
|
||||
description: Win rate as a percentage (0–100).
|
||||
example: 72
|
||||
format: double
|
||||
type: number
|
||||
required:
|
||||
- model_permaslug
|
||||
- display_name
|
||||
- arena
|
||||
- category
|
||||
- elo
|
||||
- win_rate
|
||||
- avg_generation_time_ms
|
||||
- tournament_stats
|
||||
- pricing
|
||||
type: object
|
||||
BenchmarksDAMeta:
|
||||
example:
|
||||
arena: models
|
||||
as_of: '2026-06-03T12:00:00Z'
|
||||
category: null
|
||||
citation: 'Source: Design Arena (www.designarena.ai) via OpenRouter (openrouter.ai/rankings).'
|
||||
elo_bounds:
|
||||
max: 1600
|
||||
min: 900
|
||||
model_count: 50
|
||||
source: design-arena
|
||||
source_url: https://www.designarena.ai
|
||||
version: v1
|
||||
properties:
|
||||
arena:
|
||||
description: The arena filter applied.
|
||||
type: string
|
||||
as_of:
|
||||
description: ISO-8601 timestamp of when this data was generated.
|
||||
example: '2026-06-03T12:00:00Z'
|
||||
type: string
|
||||
category:
|
||||
description: The category filter applied, or null if showing all.
|
||||
nullable: true
|
||||
type: string
|
||||
citation:
|
||||
description: Required attribution when republishing this data.
|
||||
example: 'Source: Design Arena (www.designarena.ai) via OpenRouter (openrouter.ai/rankings).'
|
||||
type: string
|
||||
elo_bounds:
|
||||
description: ELO range across all returned models for normalization.
|
||||
properties:
|
||||
max:
|
||||
description: Maximum ELO in the result set.
|
||||
format: double
|
||||
type: number
|
||||
min:
|
||||
description: Minimum ELO in the result set.
|
||||
format: double
|
||||
type: number
|
||||
required:
|
||||
- min
|
||||
- max
|
||||
type: object
|
||||
model_count:
|
||||
description: Number of unique models in the response.
|
||||
type: integer
|
||||
source:
|
||||
description: Data source identifier.
|
||||
enum:
|
||||
- design-arena
|
||||
type: string
|
||||
source_url:
|
||||
description: URL of the upstream data source.
|
||||
enum:
|
||||
- https://www.designarena.ai
|
||||
type: string
|
||||
version:
|
||||
description: Dataset version.
|
||||
enum:
|
||||
- v1
|
||||
type: string
|
||||
required:
|
||||
- as_of
|
||||
- version
|
||||
- source
|
||||
- source_url
|
||||
- citation
|
||||
- model_count
|
||||
- arena
|
||||
- category
|
||||
- elo_bounds
|
||||
type: object
|
||||
BenchmarksDAResponse:
|
||||
example:
|
||||
data:
|
||||
- arena: models
|
||||
avg_generation_time_ms: 3200
|
||||
category: codecategories
|
||||
display_name: Claude Sonnet 4
|
||||
elo: 1423
|
||||
model_permaslug: anthropic/claude-sonnet-4
|
||||
pricing:
|
||||
completion: '0.000015'
|
||||
prompt: '0.000003'
|
||||
tournament_stats:
|
||||
first_place: 12
|
||||
fourth_place: 2
|
||||
second_place: 8
|
||||
third_place: 5
|
||||
total: 27
|
||||
win_rate: 72
|
||||
meta:
|
||||
arena: models
|
||||
as_of: '2026-06-03T12:00:00Z'
|
||||
category: null
|
||||
citation: 'Source: Design Arena (www.designarena.ai) via OpenRouter (openrouter.ai/rankings).'
|
||||
elo_bounds:
|
||||
max: 1600
|
||||
min: 900
|
||||
model_count: 1
|
||||
source: design-arena
|
||||
source_url: https://www.designarena.ai
|
||||
version: v1
|
||||
properties:
|
||||
data:
|
||||
items:
|
||||
$ref: '#/components/schemas/BenchmarksDAItem'
|
||||
type: array
|
||||
meta:
|
||||
$ref: '#/components/schemas/BenchmarksDAMeta'
|
||||
required:
|
||||
- data
|
||||
- meta
|
||||
type: object
|
||||
BigNumberUnion:
|
||||
description: Price per million prompt tokens
|
||||
example: 1000
|
||||
@@ -20562,6 +20218,263 @@ components:
|
||||
- code
|
||||
- message
|
||||
type: object
|
||||
UnifiedBenchmarkPricing:
|
||||
description: OpenRouter pricing per token for this model. Null if pricing is unavailable.
|
||||
example:
|
||||
completion: '0.000015'
|
||||
prompt: '0.000003'
|
||||
nullable: true
|
||||
properties:
|
||||
completion:
|
||||
description: Cost per output token (USD, decimal string).
|
||||
example: '0.000015'
|
||||
type: string
|
||||
prompt:
|
||||
description: Cost per input token (USD, decimal string).
|
||||
example: '0.000003'
|
||||
type: string
|
||||
required:
|
||||
- prompt
|
||||
- completion
|
||||
type: object
|
||||
UnifiedBenchmarksAAItem:
|
||||
example:
|
||||
agentic_index: 58.3
|
||||
coding_index: 65.8
|
||||
display_name: GPT-4o
|
||||
intelligence_index: 71.2
|
||||
model_permaslug: openai/gpt-4o
|
||||
pricing:
|
||||
completion: '0.00001'
|
||||
prompt: '0.0000025'
|
||||
source: artificial-analysis
|
||||
properties:
|
||||
agentic_index:
|
||||
description: Artificial Analysis Agentic Index composite score. Higher is better.
|
||||
example: 58.3
|
||||
format: double
|
||||
nullable: true
|
||||
type: number
|
||||
coding_index:
|
||||
description: Artificial Analysis Coding Index composite score. Higher is better.
|
||||
example: 65.8
|
||||
format: double
|
||||
nullable: true
|
||||
type: number
|
||||
display_name:
|
||||
description: Model name as listed on Artificial Analysis.
|
||||
example: GPT-4o
|
||||
type: string
|
||||
intelligence_index:
|
||||
description: Artificial Analysis Intelligence Index composite score. Higher is better.
|
||||
example: 71.2
|
||||
format: double
|
||||
nullable: true
|
||||
type: number
|
||||
model_permaslug:
|
||||
description: Stable OpenRouter model identifier.
|
||||
example: openai/gpt-4o
|
||||
type: string
|
||||
pricing:
|
||||
$ref: '#/components/schemas/UnifiedBenchmarkPricing'
|
||||
source:
|
||||
description: Benchmark source discriminator.
|
||||
enum:
|
||||
- artificial-analysis
|
||||
type: string
|
||||
required:
|
||||
- source
|
||||
- model_permaslug
|
||||
- display_name
|
||||
- intelligence_index
|
||||
- coding_index
|
||||
- agentic_index
|
||||
- pricing
|
||||
type: object
|
||||
UnifiedBenchmarksDAItem:
|
||||
example:
|
||||
arena: models
|
||||
avg_generation_time_ms: 3200
|
||||
category: codecategories
|
||||
display_name: Claude Sonnet 4
|
||||
elo: 1423
|
||||
model_permaslug: anthropic/claude-sonnet-4
|
||||
pricing:
|
||||
completion: '0.000015'
|
||||
prompt: '0.000003'
|
||||
source: design-arena
|
||||
tournament_stats:
|
||||
first_place: 12
|
||||
fourth_place: 2
|
||||
second_place: 8
|
||||
third_place: 5
|
||||
total: 27
|
||||
win_rate: 72
|
||||
properties:
|
||||
arena:
|
||||
description: Arena this ranking belongs to.
|
||||
example: models
|
||||
type: string
|
||||
avg_generation_time_ms:
|
||||
description: Average generation time in milliseconds.
|
||||
example: 3200
|
||||
format: double
|
||||
nullable: true
|
||||
type: number
|
||||
category:
|
||||
description: Category within the arena.
|
||||
example: codecategories
|
||||
type: string
|
||||
display_name:
|
||||
description: Human-readable model name from Design Arena.
|
||||
example: Claude Sonnet 4
|
||||
type: string
|
||||
elo:
|
||||
description: ELO rating from head-to-head arena battles.
|
||||
example: 1423
|
||||
format: double
|
||||
type: number
|
||||
model_permaslug:
|
||||
description: Stable OpenRouter model identifier when mapped; otherwise the upstream Design Arena model id.
|
||||
example: anthropic/claude-sonnet-4
|
||||
type: string
|
||||
pricing:
|
||||
$ref: '#/components/schemas/UnifiedBenchmarkPricing'
|
||||
source:
|
||||
description: Benchmark source discriminator.
|
||||
enum:
|
||||
- design-arena
|
||||
type: string
|
||||
tournament_stats:
|
||||
description: Placement distribution from tournament matches.
|
||||
properties:
|
||||
first_place:
|
||||
nullable: true
|
||||
type: integer
|
||||
fourth_place:
|
||||
nullable: true
|
||||
type: integer
|
||||
second_place:
|
||||
nullable: true
|
||||
type: integer
|
||||
third_place:
|
||||
nullable: true
|
||||
type: integer
|
||||
total:
|
||||
nullable: true
|
||||
type: integer
|
||||
required:
|
||||
- first_place
|
||||
- second_place
|
||||
- third_place
|
||||
- fourth_place
|
||||
- total
|
||||
type: object
|
||||
win_rate:
|
||||
description: Win rate as a percentage (0–100).
|
||||
example: 72
|
||||
format: double
|
||||
type: number
|
||||
required:
|
||||
- source
|
||||
- model_permaslug
|
||||
- display_name
|
||||
- arena
|
||||
- category
|
||||
- elo
|
||||
- win_rate
|
||||
- avg_generation_time_ms
|
||||
- tournament_stats
|
||||
- pricing
|
||||
type: object
|
||||
UnifiedBenchmarksMeta:
|
||||
example:
|
||||
as_of: '2026-06-03T12:00:00Z'
|
||||
citation: 'Source: Artificial Analysis (artificialanalysis.ai) via OpenRouter (openrouter.ai/rankings).'
|
||||
model_count: 50
|
||||
source: artificial-analysis
|
||||
source_url: https://artificialanalysis.ai
|
||||
task_type: null
|
||||
version: v1
|
||||
properties:
|
||||
as_of:
|
||||
description: ISO-8601 timestamp of when this data was last updated.
|
||||
example: '2026-06-03T12:00:00Z'
|
||||
type: string
|
||||
citation:
|
||||
description: Required attribution when republishing this data.
|
||||
example: 'Source: Artificial Analysis (artificialanalysis.ai) via OpenRouter (openrouter.ai/rankings).'
|
||||
type: string
|
||||
model_count:
|
||||
description: Number of unique models in the response.
|
||||
type: integer
|
||||
source:
|
||||
description: The source filter applied.
|
||||
enum:
|
||||
- artificial-analysis
|
||||
- design-arena
|
||||
example: artificial-analysis
|
||||
type: string
|
||||
source_url:
|
||||
description: URL of the upstream data source.
|
||||
example: https://artificialanalysis.ai
|
||||
type: string
|
||||
task_type:
|
||||
description: The task_type filter applied, or null if showing all.
|
||||
nullable: true
|
||||
type: string
|
||||
version:
|
||||
description: Dataset version.
|
||||
enum:
|
||||
- v1
|
||||
type: string
|
||||
required:
|
||||
- as_of
|
||||
- version
|
||||
- source
|
||||
- source_url
|
||||
- citation
|
||||
- model_count
|
||||
- task_type
|
||||
type: object
|
||||
UnifiedBenchmarksResponse:
|
||||
example:
|
||||
data:
|
||||
- agentic_index: 58.3
|
||||
coding_index: 65.8
|
||||
display_name: GPT-4o
|
||||
intelligence_index: 71.2
|
||||
model_permaslug: openai/gpt-4o
|
||||
pricing:
|
||||
completion: '0.00001'
|
||||
prompt: '0.0000025'
|
||||
source: artificial-analysis
|
||||
meta:
|
||||
as_of: '2026-06-03T12:00:00Z'
|
||||
citation: 'Source: Artificial Analysis (artificialanalysis.ai) via OpenRouter (openrouter.ai/rankings).'
|
||||
model_count: 1
|
||||
source: artificial-analysis
|
||||
source_url: https://artificialanalysis.ai
|
||||
task_type: null
|
||||
version: v1
|
||||
properties:
|
||||
data:
|
||||
items:
|
||||
discriminator:
|
||||
mapping:
|
||||
artificial-analysis: '#/components/schemas/UnifiedBenchmarksAAItem'
|
||||
design-arena: '#/components/schemas/UnifiedBenchmarksDAItem'
|
||||
propertyName: source
|
||||
oneOf:
|
||||
- $ref: '#/components/schemas/UnifiedBenchmarksAAItem'
|
||||
- $ref: '#/components/schemas/UnifiedBenchmarksDAItem'
|
||||
type: array
|
||||
meta:
|
||||
$ref: '#/components/schemas/UnifiedBenchmarksMeta'
|
||||
required:
|
||||
- data
|
||||
- meta
|
||||
type: object
|
||||
UnprocessableEntityResponse:
|
||||
description: Unprocessable Entity - Semantic validation failure
|
||||
example:
|
||||
@@ -23168,6 +23081,146 @@ paths:
|
||||
tags:
|
||||
- OAuth
|
||||
x-speakeasy-name-override: createAuthCode
|
||||
/benchmarks:
|
||||
get:
|
||||
description: >-
|
||||
Unified benchmark endpoint that aggregates scores from multiple benchmark sources (Artificial Analysis, Design
|
||||
Arena). Filter by source to reproduce the exact shapes from the legacy per-source endpoints, or use task_type to
|
||||
find models suited for specific workloads. Authenticate with any valid OpenRouter API key. Rate-limited to 30
|
||||
requests/minute per key and 500 requests/day per account.
|
||||
operationId: getBenchmarks
|
||||
parameters:
|
||||
- description: Benchmark source to query. Determines the shape of the returned items.
|
||||
in: query
|
||||
name: source
|
||||
required: true
|
||||
schema:
|
||||
description: Benchmark source to query. Determines the shape of the returned items.
|
||||
enum:
|
||||
- artificial-analysis
|
||||
- design-arena
|
||||
example: artificial-analysis
|
||||
type: string
|
||||
- description: >-
|
||||
Filter results by task type. For Artificial Analysis, maps to the corresponding index. For Design Arena,
|
||||
maps to the matching category.
|
||||
in: query
|
||||
name: task_type
|
||||
required: false
|
||||
schema:
|
||||
description: >-
|
||||
Filter results by task type. For Artificial Analysis, maps to the corresponding index. For Design Arena,
|
||||
maps to the matching category.
|
||||
enum:
|
||||
- coding
|
||||
- intelligence
|
||||
- agentic
|
||||
example: coding
|
||||
type: string
|
||||
- description: 'Design Arena only: arena to query. Defaults to `models` when source is `design-arena`.'
|
||||
in: query
|
||||
name: arena
|
||||
required: false
|
||||
schema:
|
||||
description: 'Design Arena only: arena to query. Defaults to `models` when source is `design-arena`.'
|
||||
enum:
|
||||
- models
|
||||
- builders
|
||||
- agents
|
||||
example: models
|
||||
type: string
|
||||
- description: >-
|
||||
Design Arena only: category within the arena (e.g. `codecategories`, `uicomponent`, `gamedev`, `3d`,
|
||||
`dataviz`, `image`, `video`, `svg`). When omitted, returns all categories.
|
||||
in: query
|
||||
name: category
|
||||
required: false
|
||||
schema:
|
||||
description: >-
|
||||
Design Arena only: category within the arena (e.g. `codecategories`, `uicomponent`, `gamedev`, `3d`,
|
||||
`dataviz`, `image`, `video`, `svg`). When omitted, returns all categories.
|
||||
example: codecategories
|
||||
type: string
|
||||
- description: Max results to return (1–100, default 50).
|
||||
in: query
|
||||
name: max_results
|
||||
required: false
|
||||
schema:
|
||||
default: 50
|
||||
description: Max results to return (1–100, default 50).
|
||||
example: 20
|
||||
maximum: 100
|
||||
minimum: 1
|
||||
type: integer
|
||||
responses:
|
||||
'200':
|
||||
content:
|
||||
application/json:
|
||||
example:
|
||||
data:
|
||||
- agentic_index: 58.3
|
||||
coding_index: 65.8
|
||||
display_name: GPT-4o
|
||||
intelligence_index: 71.2
|
||||
model_permaslug: openai/gpt-4o
|
||||
pricing:
|
||||
completion: '0.00001'
|
||||
prompt: '0.0000025'
|
||||
source: artificial-analysis
|
||||
meta:
|
||||
as_of: '2026-06-03T12:00:00Z'
|
||||
citation: 'Source: Artificial Analysis (artificialanalysis.ai) via OpenRouter (openrouter.ai/rankings).'
|
||||
model_count: 1
|
||||
source: artificial-analysis
|
||||
source_url: https://artificialanalysis.ai
|
||||
task_type: null
|
||||
version: v1
|
||||
schema:
|
||||
$ref: '#/components/schemas/UnifiedBenchmarksResponse'
|
||||
description: Benchmark results filtered by the specified source and optional task type.
|
||||
'400':
|
||||
content:
|
||||
application/json:
|
||||
example:
|
||||
error:
|
||||
code: 400
|
||||
message: Invalid request parameters
|
||||
schema:
|
||||
$ref: '#/components/schemas/BadRequestResponse'
|
||||
description: Bad Request - Invalid request parameters or malformed input
|
||||
'401':
|
||||
content:
|
||||
application/json:
|
||||
example:
|
||||
error:
|
||||
code: 401
|
||||
message: Missing Authentication header
|
||||
schema:
|
||||
$ref: '#/components/schemas/UnauthorizedResponse'
|
||||
description: Unauthorized - Authentication required or invalid credentials
|
||||
'429':
|
||||
content:
|
||||
application/json:
|
||||
example:
|
||||
error:
|
||||
code: 429
|
||||
message: Rate limit exceeded
|
||||
schema:
|
||||
$ref: '#/components/schemas/TooManyRequestsResponse'
|
||||
description: Too Many Requests - Rate limit exceeded
|
||||
'500':
|
||||
content:
|
||||
application/json:
|
||||
example:
|
||||
error:
|
||||
code: 500
|
||||
message: Internal Server Error
|
||||
schema:
|
||||
$ref: '#/components/schemas/InternalServerResponse'
|
||||
description: Internal Server Error - Unexpected server error
|
||||
summary: List Benchmarks
|
||||
tags:
|
||||
- Benchmarks
|
||||
/byok:
|
||||
get:
|
||||
description: >-
|
||||
@@ -24247,216 +24300,6 @@ paths:
|
||||
outputs:
|
||||
results: $.data
|
||||
type: offsetLimit
|
||||
/datasets/benchmarks/artificial-analysis:
|
||||
get:
|
||||
description: >-
|
||||
Returns composite index scores (Intelligence, Coding, Agentic) from Artificial Analysis for LLM models. Includes
|
||||
OpenRouter pricing per model. Authenticate with any valid OpenRouter API key. Rate-limited to 30 requests/minute
|
||||
per key and 500 requests/day per account.
|
||||
operationId: getBenchmarksArtificialAnalysis
|
||||
parameters:
|
||||
- description: Max results to return (1–100, default 50).
|
||||
in: query
|
||||
name: max_results
|
||||
required: false
|
||||
schema:
|
||||
default: 50
|
||||
description: Max results to return (1–100, default 50).
|
||||
example: 20
|
||||
maximum: 100
|
||||
minimum: 1
|
||||
type: integer
|
||||
responses:
|
||||
'200':
|
||||
content:
|
||||
application/json:
|
||||
example:
|
||||
data:
|
||||
- aa_name: GPT-4o
|
||||
agentic_index: 58.3
|
||||
coding_index: 65.8
|
||||
intelligence_index: 71.2
|
||||
model_permaslug: openai/gpt-4o
|
||||
pricing:
|
||||
completion: '0.00001'
|
||||
prompt: '0.0000025'
|
||||
meta:
|
||||
as_of: '2026-06-03T12:00:00Z'
|
||||
citation: 'Source: Artificial Analysis (artificialanalysis.ai) via OpenRouter (openrouter.ai/rankings).'
|
||||
model_count: 1
|
||||
source: artificial-analysis
|
||||
source_url: https://artificialanalysis.ai
|
||||
version: v1
|
||||
schema:
|
||||
$ref: '#/components/schemas/BenchmarksAAResponse'
|
||||
description: Artificial Analysis composite index scores with pricing and attribution metadata.
|
||||
'400':
|
||||
content:
|
||||
application/json:
|
||||
example:
|
||||
error:
|
||||
code: 400
|
||||
message: Invalid request parameters
|
||||
schema:
|
||||
$ref: '#/components/schemas/BadRequestResponse'
|
||||
description: Bad Request - Invalid request parameters or malformed input
|
||||
'401':
|
||||
content:
|
||||
application/json:
|
||||
example:
|
||||
error:
|
||||
code: 401
|
||||
message: Missing Authentication header
|
||||
schema:
|
||||
$ref: '#/components/schemas/UnauthorizedResponse'
|
||||
description: Unauthorized - Authentication required or invalid credentials
|
||||
'429':
|
||||
content:
|
||||
application/json:
|
||||
example:
|
||||
error:
|
||||
code: 429
|
||||
message: Rate limit exceeded
|
||||
schema:
|
||||
$ref: '#/components/schemas/TooManyRequestsResponse'
|
||||
description: Too Many Requests - Rate limit exceeded
|
||||
'500':
|
||||
content:
|
||||
application/json:
|
||||
example:
|
||||
error:
|
||||
code: 500
|
||||
message: Internal Server Error
|
||||
schema:
|
||||
$ref: '#/components/schemas/InternalServerResponse'
|
||||
description: Internal Server Error - Unexpected server error
|
||||
summary: Artificial Analysis Benchmark Indices
|
||||
tags:
|
||||
- Datasets
|
||||
/datasets/benchmarks/design-arena:
|
||||
get:
|
||||
description: >-
|
||||
Returns ELO ratings from head-to-head arena battles on Design Arena. Filterable by arena
|
||||
(models/builders/agents) and category. Includes OpenRouter pricing per model. Authenticate with any valid
|
||||
OpenRouter API key. Rate-limited to 30 requests/minute per key and 500 requests/day per account.
|
||||
operationId: getBenchmarksDesignArena
|
||||
parameters:
|
||||
- description: Arena to query. Defaults to `models`.
|
||||
in: query
|
||||
name: arena
|
||||
required: false
|
||||
schema:
|
||||
default: models
|
||||
description: Arena to query. Defaults to `models`.
|
||||
enum:
|
||||
- models
|
||||
- builders
|
||||
- agents
|
||||
example: models
|
||||
type: string
|
||||
- description: >-
|
||||
Category within the arena (e.g. `codecategories`, `uicomponent`, `gamedev`, `3d`, `dataviz`, `image`,
|
||||
`video`, `svg`). When omitted, returns all categories.
|
||||
in: query
|
||||
name: category
|
||||
required: false
|
||||
schema:
|
||||
description: >-
|
||||
Category within the arena (e.g. `codecategories`, `uicomponent`, `gamedev`, `3d`, `dataviz`, `image`,
|
||||
`video`, `svg`). When omitted, returns all categories.
|
||||
example: codecategories
|
||||
type: string
|
||||
- description: 'Max results to return: per category when no category filter is applied (1–100, default 50).'
|
||||
in: query
|
||||
name: max_results
|
||||
required: false
|
||||
schema:
|
||||
default: 50
|
||||
description: 'Max results to return: per category when no category filter is applied (1–100, default 50).'
|
||||
example: 20
|
||||
maximum: 100
|
||||
minimum: 1
|
||||
type: integer
|
||||
responses:
|
||||
'200':
|
||||
content:
|
||||
application/json:
|
||||
example:
|
||||
data:
|
||||
- arena: models
|
||||
avg_generation_time_ms: 3200
|
||||
category: codecategories
|
||||
display_name: Claude Sonnet 4
|
||||
elo: 1423
|
||||
model_permaslug: anthropic/claude-sonnet-4
|
||||
pricing:
|
||||
completion: '0.000015'
|
||||
prompt: '0.000003'
|
||||
tournament_stats:
|
||||
first_place: 12
|
||||
fourth_place: 2
|
||||
second_place: 8
|
||||
third_place: 5
|
||||
total: 27
|
||||
win_rate: 72
|
||||
meta:
|
||||
arena: models
|
||||
as_of: '2026-06-03T12:00:00Z'
|
||||
category: null
|
||||
citation: 'Source: Design Arena (www.designarena.ai) via OpenRouter (openrouter.ai/rankings).'
|
||||
elo_bounds:
|
||||
max: 1600
|
||||
min: 900
|
||||
model_count: 1
|
||||
source: design-arena
|
||||
source_url: https://www.designarena.ai
|
||||
version: v1
|
||||
schema:
|
||||
$ref: '#/components/schemas/BenchmarksDAResponse'
|
||||
description: Design Arena ELO rankings with pricing and attribution metadata.
|
||||
'400':
|
||||
content:
|
||||
application/json:
|
||||
example:
|
||||
error:
|
||||
code: 400
|
||||
message: Invalid request parameters
|
||||
schema:
|
||||
$ref: '#/components/schemas/BadRequestResponse'
|
||||
description: Bad Request - Invalid request parameters or malformed input
|
||||
'401':
|
||||
content:
|
||||
application/json:
|
||||
example:
|
||||
error:
|
||||
code: 401
|
||||
message: Missing Authentication header
|
||||
schema:
|
||||
$ref: '#/components/schemas/UnauthorizedResponse'
|
||||
description: Unauthorized - Authentication required or invalid credentials
|
||||
'429':
|
||||
content:
|
||||
application/json:
|
||||
example:
|
||||
error:
|
||||
code: 429
|
||||
message: Rate limit exceeded
|
||||
schema:
|
||||
$ref: '#/components/schemas/TooManyRequestsResponse'
|
||||
description: Too Many Requests - Rate limit exceeded
|
||||
'500':
|
||||
content:
|
||||
application/json:
|
||||
example:
|
||||
error:
|
||||
code: 500
|
||||
message: Internal Server Error
|
||||
schema:
|
||||
$ref: '#/components/schemas/InternalServerResponse'
|
||||
description: Internal Server Error - Unexpected server error
|
||||
summary: Design Arena Benchmark Rankings
|
||||
tags:
|
||||
- Datasets
|
||||
/datasets/rankings-daily:
|
||||
get:
|
||||
description: |-
|
||||
@@ -32775,6 +32618,8 @@ tags:
|
||||
name: Anthropic Messages
|
||||
- description: BYOK endpoints
|
||||
name: BYOK
|
||||
- description: Benchmarks endpoints
|
||||
name: Benchmarks
|
||||
- description: Chat completion endpoints
|
||||
name: Chat
|
||||
- description: Credit management endpoints
|
||||
|
||||
Reference in New Issue
Block a user