From 7e29e408afba6116f78ad0b471d0f5830bf395fe Mon Sep 17 00:00:00 2001 From: robert-j-y <212159665+robert-j-y@users.noreply.github.com> Date: Thu, 18 Jun 2026 10:20:41 -0700 Subject: [PATCH] chore: update OpenAPI spec from monorepo (#333) Co-authored-by: OpenRouter SDK Bot --- .speakeasy/in.openapi.yaml | 953 ++++++++++++++++--------------------- 1 file changed, 399 insertions(+), 554 deletions(-) diff --git a/.speakeasy/in.openapi.yaml b/.speakeasy/in.openapi.yaml index 07dc908..8b443ff 100644 --- a/.speakeasy/in.openapi.yaml +++ b/.speakeasy/in.openapi.yaml @@ -3644,350 +3644,6 @@ components: oneOf: - $ref: '#/components/schemas/ContainerAutoEnvironment' - $ref: '#/components/schemas/ContainerReferenceEnvironment' - BenchmarkPricing: - description: OpenRouter pricing per token for this model. Null if pricing is unavailable. - example: - completion: '0.000015' - prompt: '0.000003' - nullable: true - properties: - completion: - description: Cost per output token (USD, decimal string). - example: '0.000015' - type: string - prompt: - description: Cost per input token (USD, decimal string). - example: '0.000003' - type: string - required: - - prompt - - completion - type: object - BenchmarksAAItem: - example: - aa_name: GPT-4o - agentic_index: 58.3 - coding_index: 65.8 - intelligence_index: 71.2 - model_permaslug: openai/gpt-4o - pricing: - completion: '0.00001' - prompt: '0.0000025' - properties: - aa_name: - description: Model name as listed on Artificial Analysis. - example: GPT-4o - type: string - agentic_index: - description: Artificial Analysis Agentic Index composite score. Higher is better. - example: 58.3 - format: double - nullable: true - type: number - coding_index: - description: Artificial Analysis Coding Index composite score. Higher is better. - example: 65.8 - format: double - nullable: true - type: number - intelligence_index: - description: Artificial Analysis Intelligence Index composite score. Higher is better. - example: 71.2 - format: double - nullable: true - type: number - model_permaslug: - description: Stable OpenRouter model identifier. - example: openai/gpt-4o - type: string - pricing: - $ref: '#/components/schemas/BenchmarkPricing' - required: - - model_permaslug - - aa_name - - intelligence_index - - coding_index - - agentic_index - - pricing - type: object - BenchmarksAAMeta: - example: - as_of: '2026-06-03T12:00:00Z' - citation: 'Source: Artificial Analysis (artificialanalysis.ai) via OpenRouter (openrouter.ai/rankings).' - model_count: 50 - source: artificial-analysis - source_url: https://artificialanalysis.ai - version: v1 - properties: - as_of: - description: ISO-8601 timestamp of when this data was last updated. - example: '2026-06-03T12:00:00Z' - type: string - citation: - description: Required attribution when republishing this data. - example: 'Source: Artificial Analysis (artificialanalysis.ai) via OpenRouter (openrouter.ai/rankings).' - type: string - model_count: - description: Number of unique models in the response. - type: integer - source: - description: Data source identifier. - enum: - - artificial-analysis - type: string - source_url: - description: URL of the upstream data source. - enum: - - https://artificialanalysis.ai - type: string - version: - description: Dataset version. - enum: - - v1 - type: string - required: - - as_of - - version - - source - - source_url - - citation - - model_count - type: object - BenchmarksAAResponse: - example: - data: - - aa_name: GPT-4o - agentic_index: 58.3 - coding_index: 65.8 - intelligence_index: 71.2 - model_permaslug: openai/gpt-4o - pricing: - completion: '0.00001' - prompt: '0.0000025' - meta: - as_of: '2026-06-03T12:00:00Z' - citation: 'Source: Artificial Analysis (artificialanalysis.ai) via OpenRouter (openrouter.ai/rankings).' - model_count: 1 - source: artificial-analysis - source_url: https://artificialanalysis.ai - version: v1 - properties: - data: - items: - $ref: '#/components/schemas/BenchmarksAAItem' - type: array - meta: - $ref: '#/components/schemas/BenchmarksAAMeta' - required: - - data - - meta - type: object - BenchmarksDAItem: - example: - arena: models - avg_generation_time_ms: 3200 - category: codecategories - display_name: Claude Sonnet 4 - elo: 1423 - model_permaslug: anthropic/claude-sonnet-4 - pricing: - completion: '0.000015' - prompt: '0.000003' - tournament_stats: - first_place: 12 - fourth_place: 2 - second_place: 8 - third_place: 5 - total: 27 - win_rate: 72 - properties: - arena: - description: Arena this ranking belongs to. - example: models - type: string - avg_generation_time_ms: - description: Average generation time in milliseconds. - example: 3200 - format: double - nullable: true - type: number - category: - description: Category within the arena. - example: codecategories - type: string - display_name: - description: Human-readable model name from Design Arena. - example: Claude Sonnet 4 - type: string - elo: - description: ELO rating from head-to-head arena battles. - example: 1423 - format: double - type: number - model_permaslug: - description: >- - Stable OpenRouter model identifier when the model is on OpenRouter; otherwise the upstream Design Arena - model id. Use pricing != null to detect OpenRouter-mapped models. - example: anthropic/claude-sonnet-4 - type: string - pricing: - $ref: '#/components/schemas/BenchmarkPricing' - tournament_stats: - description: Placement distribution from tournament matches. - properties: - first_place: - nullable: true - type: integer - fourth_place: - nullable: true - type: integer - second_place: - nullable: true - type: integer - third_place: - nullable: true - type: integer - total: - nullable: true - type: integer - required: - - first_place - - second_place - - third_place - - fourth_place - - total - type: object - win_rate: - description: Win rate as a percentage (0–100). - example: 72 - format: double - type: number - required: - - model_permaslug - - display_name - - arena - - category - - elo - - win_rate - - avg_generation_time_ms - - tournament_stats - - pricing - type: object - BenchmarksDAMeta: - example: - arena: models - as_of: '2026-06-03T12:00:00Z' - category: null - citation: 'Source: Design Arena (www.designarena.ai) via OpenRouter (openrouter.ai/rankings).' - elo_bounds: - max: 1600 - min: 900 - model_count: 50 - source: design-arena - source_url: https://www.designarena.ai - version: v1 - properties: - arena: - description: The arena filter applied. - type: string - as_of: - description: ISO-8601 timestamp of when this data was generated. - example: '2026-06-03T12:00:00Z' - type: string - category: - description: The category filter applied, or null if showing all. - nullable: true - type: string - citation: - description: Required attribution when republishing this data. - example: 'Source: Design Arena (www.designarena.ai) via OpenRouter (openrouter.ai/rankings).' - type: string - elo_bounds: - description: ELO range across all returned models for normalization. - properties: - max: - description: Maximum ELO in the result set. - format: double - type: number - min: - description: Minimum ELO in the result set. - format: double - type: number - required: - - min - - max - type: object - model_count: - description: Number of unique models in the response. - type: integer - source: - description: Data source identifier. - enum: - - design-arena - type: string - source_url: - description: URL of the upstream data source. - enum: - - https://www.designarena.ai - type: string - version: - description: Dataset version. - enum: - - v1 - type: string - required: - - as_of - - version - - source - - source_url - - citation - - model_count - - arena - - category - - elo_bounds - type: object - BenchmarksDAResponse: - example: - data: - - arena: models - avg_generation_time_ms: 3200 - category: codecategories - display_name: Claude Sonnet 4 - elo: 1423 - model_permaslug: anthropic/claude-sonnet-4 - pricing: - completion: '0.000015' - prompt: '0.000003' - tournament_stats: - first_place: 12 - fourth_place: 2 - second_place: 8 - third_place: 5 - total: 27 - win_rate: 72 - meta: - arena: models - as_of: '2026-06-03T12:00:00Z' - category: null - citation: 'Source: Design Arena (www.designarena.ai) via OpenRouter (openrouter.ai/rankings).' - elo_bounds: - max: 1600 - min: 900 - model_count: 1 - source: design-arena - source_url: https://www.designarena.ai - version: v1 - properties: - data: - items: - $ref: '#/components/schemas/BenchmarksDAItem' - type: array - meta: - $ref: '#/components/schemas/BenchmarksDAMeta' - required: - - data - - meta - type: object BigNumberUnion: description: Price per million prompt tokens example: 1000 @@ -20562,6 +20218,263 @@ components: - code - message type: object + UnifiedBenchmarkPricing: + description: OpenRouter pricing per token for this model. Null if pricing is unavailable. + example: + completion: '0.000015' + prompt: '0.000003' + nullable: true + properties: + completion: + description: Cost per output token (USD, decimal string). + example: '0.000015' + type: string + prompt: + description: Cost per input token (USD, decimal string). + example: '0.000003' + type: string + required: + - prompt + - completion + type: object + UnifiedBenchmarksAAItem: + example: + agentic_index: 58.3 + coding_index: 65.8 + display_name: GPT-4o + intelligence_index: 71.2 + model_permaslug: openai/gpt-4o + pricing: + completion: '0.00001' + prompt: '0.0000025' + source: artificial-analysis + properties: + agentic_index: + description: Artificial Analysis Agentic Index composite score. Higher is better. + example: 58.3 + format: double + nullable: true + type: number + coding_index: + description: Artificial Analysis Coding Index composite score. Higher is better. + example: 65.8 + format: double + nullable: true + type: number + display_name: + description: Model name as listed on Artificial Analysis. + example: GPT-4o + type: string + intelligence_index: + description: Artificial Analysis Intelligence Index composite score. Higher is better. + example: 71.2 + format: double + nullable: true + type: number + model_permaslug: + description: Stable OpenRouter model identifier. + example: openai/gpt-4o + type: string + pricing: + $ref: '#/components/schemas/UnifiedBenchmarkPricing' + source: + description: Benchmark source discriminator. + enum: + - artificial-analysis + type: string + required: + - source + - model_permaslug + - display_name + - intelligence_index + - coding_index + - agentic_index + - pricing + type: object + UnifiedBenchmarksDAItem: + example: + arena: models + avg_generation_time_ms: 3200 + category: codecategories + display_name: Claude Sonnet 4 + elo: 1423 + model_permaslug: anthropic/claude-sonnet-4 + pricing: + completion: '0.000015' + prompt: '0.000003' + source: design-arena + tournament_stats: + first_place: 12 + fourth_place: 2 + second_place: 8 + third_place: 5 + total: 27 + win_rate: 72 + properties: + arena: + description: Arena this ranking belongs to. + example: models + type: string + avg_generation_time_ms: + description: Average generation time in milliseconds. + example: 3200 + format: double + nullable: true + type: number + category: + description: Category within the arena. + example: codecategories + type: string + display_name: + description: Human-readable model name from Design Arena. + example: Claude Sonnet 4 + type: string + elo: + description: ELO rating from head-to-head arena battles. + example: 1423 + format: double + type: number + model_permaslug: + description: Stable OpenRouter model identifier when mapped; otherwise the upstream Design Arena model id. + example: anthropic/claude-sonnet-4 + type: string + pricing: + $ref: '#/components/schemas/UnifiedBenchmarkPricing' + source: + description: Benchmark source discriminator. + enum: + - design-arena + type: string + tournament_stats: + description: Placement distribution from tournament matches. + properties: + first_place: + nullable: true + type: integer + fourth_place: + nullable: true + type: integer + second_place: + nullable: true + type: integer + third_place: + nullable: true + type: integer + total: + nullable: true + type: integer + required: + - first_place + - second_place + - third_place + - fourth_place + - total + type: object + win_rate: + description: Win rate as a percentage (0–100). + example: 72 + format: double + type: number + required: + - source + - model_permaslug + - display_name + - arena + - category + - elo + - win_rate + - avg_generation_time_ms + - tournament_stats + - pricing + type: object + UnifiedBenchmarksMeta: + example: + as_of: '2026-06-03T12:00:00Z' + citation: 'Source: Artificial Analysis (artificialanalysis.ai) via OpenRouter (openrouter.ai/rankings).' + model_count: 50 + source: artificial-analysis + source_url: https://artificialanalysis.ai + task_type: null + version: v1 + properties: + as_of: + description: ISO-8601 timestamp of when this data was last updated. + example: '2026-06-03T12:00:00Z' + type: string + citation: + description: Required attribution when republishing this data. + example: 'Source: Artificial Analysis (artificialanalysis.ai) via OpenRouter (openrouter.ai/rankings).' + type: string + model_count: + description: Number of unique models in the response. + type: integer + source: + description: The source filter applied. + enum: + - artificial-analysis + - design-arena + example: artificial-analysis + type: string + source_url: + description: URL of the upstream data source. + example: https://artificialanalysis.ai + type: string + task_type: + description: The task_type filter applied, or null if showing all. + nullable: true + type: string + version: + description: Dataset version. + enum: + - v1 + type: string + required: + - as_of + - version + - source + - source_url + - citation + - model_count + - task_type + type: object + UnifiedBenchmarksResponse: + example: + data: + - agentic_index: 58.3 + coding_index: 65.8 + display_name: GPT-4o + intelligence_index: 71.2 + model_permaslug: openai/gpt-4o + pricing: + completion: '0.00001' + prompt: '0.0000025' + source: artificial-analysis + meta: + as_of: '2026-06-03T12:00:00Z' + citation: 'Source: Artificial Analysis (artificialanalysis.ai) via OpenRouter (openrouter.ai/rankings).' + model_count: 1 + source: artificial-analysis + source_url: https://artificialanalysis.ai + task_type: null + version: v1 + properties: + data: + items: + discriminator: + mapping: + artificial-analysis: '#/components/schemas/UnifiedBenchmarksAAItem' + design-arena: '#/components/schemas/UnifiedBenchmarksDAItem' + propertyName: source + oneOf: + - $ref: '#/components/schemas/UnifiedBenchmarksAAItem' + - $ref: '#/components/schemas/UnifiedBenchmarksDAItem' + type: array + meta: + $ref: '#/components/schemas/UnifiedBenchmarksMeta' + required: + - data + - meta + type: object UnprocessableEntityResponse: description: Unprocessable Entity - Semantic validation failure example: @@ -23168,6 +23081,146 @@ paths: tags: - OAuth x-speakeasy-name-override: createAuthCode + /benchmarks: + get: + description: >- + Unified benchmark endpoint that aggregates scores from multiple benchmark sources (Artificial Analysis, Design + Arena). Filter by source to reproduce the exact shapes from the legacy per-source endpoints, or use task_type to + find models suited for specific workloads. Authenticate with any valid OpenRouter API key. Rate-limited to 30 + requests/minute per key and 500 requests/day per account. + operationId: getBenchmarks + parameters: + - description: Benchmark source to query. Determines the shape of the returned items. + in: query + name: source + required: true + schema: + description: Benchmark source to query. Determines the shape of the returned items. + enum: + - artificial-analysis + - design-arena + example: artificial-analysis + type: string + - description: >- + Filter results by task type. For Artificial Analysis, maps to the corresponding index. For Design Arena, + maps to the matching category. + in: query + name: task_type + required: false + schema: + description: >- + Filter results by task type. For Artificial Analysis, maps to the corresponding index. For Design Arena, + maps to the matching category. + enum: + - coding + - intelligence + - agentic + example: coding + type: string + - description: 'Design Arena only: arena to query. Defaults to `models` when source is `design-arena`.' + in: query + name: arena + required: false + schema: + description: 'Design Arena only: arena to query. Defaults to `models` when source is `design-arena`.' + enum: + - models + - builders + - agents + example: models + type: string + - description: >- + Design Arena only: category within the arena (e.g. `codecategories`, `uicomponent`, `gamedev`, `3d`, + `dataviz`, `image`, `video`, `svg`). When omitted, returns all categories. + in: query + name: category + required: false + schema: + description: >- + Design Arena only: category within the arena (e.g. `codecategories`, `uicomponent`, `gamedev`, `3d`, + `dataviz`, `image`, `video`, `svg`). When omitted, returns all categories. + example: codecategories + type: string + - description: Max results to return (1–100, default 50). + in: query + name: max_results + required: false + schema: + default: 50 + description: Max results to return (1–100, default 50). + example: 20 + maximum: 100 + minimum: 1 + type: integer + responses: + '200': + content: + application/json: + example: + data: + - agentic_index: 58.3 + coding_index: 65.8 + display_name: GPT-4o + intelligence_index: 71.2 + model_permaslug: openai/gpt-4o + pricing: + completion: '0.00001' + prompt: '0.0000025' + source: artificial-analysis + meta: + as_of: '2026-06-03T12:00:00Z' + citation: 'Source: Artificial Analysis (artificialanalysis.ai) via OpenRouter (openrouter.ai/rankings).' + model_count: 1 + source: artificial-analysis + source_url: https://artificialanalysis.ai + task_type: null + version: v1 + schema: + $ref: '#/components/schemas/UnifiedBenchmarksResponse' + description: Benchmark results filtered by the specified source and optional task type. + '400': + content: + application/json: + example: + error: + code: 400 + message: Invalid request parameters + schema: + $ref: '#/components/schemas/BadRequestResponse' + description: Bad Request - Invalid request parameters or malformed input + '401': + content: + application/json: + example: + error: + code: 401 + message: Missing Authentication header + schema: + $ref: '#/components/schemas/UnauthorizedResponse' + description: Unauthorized - Authentication required or invalid credentials + '429': + content: + application/json: + example: + error: + code: 429 + message: Rate limit exceeded + schema: + $ref: '#/components/schemas/TooManyRequestsResponse' + description: Too Many Requests - Rate limit exceeded + '500': + content: + application/json: + example: + error: + code: 500 + message: Internal Server Error + schema: + $ref: '#/components/schemas/InternalServerResponse' + description: Internal Server Error - Unexpected server error + summary: List Benchmarks + tags: + - Benchmarks /byok: get: description: >- @@ -24247,216 +24300,6 @@ paths: outputs: results: $.data type: offsetLimit - /datasets/benchmarks/artificial-analysis: - get: - description: >- - Returns composite index scores (Intelligence, Coding, Agentic) from Artificial Analysis for LLM models. Includes - OpenRouter pricing per model. Authenticate with any valid OpenRouter API key. Rate-limited to 30 requests/minute - per key and 500 requests/day per account. - operationId: getBenchmarksArtificialAnalysis - parameters: - - description: Max results to return (1–100, default 50). - in: query - name: max_results - required: false - schema: - default: 50 - description: Max results to return (1–100, default 50). - example: 20 - maximum: 100 - minimum: 1 - type: integer - responses: - '200': - content: - application/json: - example: - data: - - aa_name: GPT-4o - agentic_index: 58.3 - coding_index: 65.8 - intelligence_index: 71.2 - model_permaslug: openai/gpt-4o - pricing: - completion: '0.00001' - prompt: '0.0000025' - meta: - as_of: '2026-06-03T12:00:00Z' - citation: 'Source: Artificial Analysis (artificialanalysis.ai) via OpenRouter (openrouter.ai/rankings).' - model_count: 1 - source: artificial-analysis - source_url: https://artificialanalysis.ai - version: v1 - schema: - $ref: '#/components/schemas/BenchmarksAAResponse' - description: Artificial Analysis composite index scores with pricing and attribution metadata. - '400': - content: - application/json: - example: - error: - code: 400 - message: Invalid request parameters - schema: - $ref: '#/components/schemas/BadRequestResponse' - description: Bad Request - Invalid request parameters or malformed input - '401': - content: - application/json: - example: - error: - code: 401 - message: Missing Authentication header - schema: - $ref: '#/components/schemas/UnauthorizedResponse' - description: Unauthorized - Authentication required or invalid credentials - '429': - content: - application/json: - example: - error: - code: 429 - message: Rate limit exceeded - schema: - $ref: '#/components/schemas/TooManyRequestsResponse' - description: Too Many Requests - Rate limit exceeded - '500': - content: - application/json: - example: - error: - code: 500 - message: Internal Server Error - schema: - $ref: '#/components/schemas/InternalServerResponse' - description: Internal Server Error - Unexpected server error - summary: Artificial Analysis Benchmark Indices - tags: - - Datasets - /datasets/benchmarks/design-arena: - get: - description: >- - Returns ELO ratings from head-to-head arena battles on Design Arena. Filterable by arena - (models/builders/agents) and category. Includes OpenRouter pricing per model. Authenticate with any valid - OpenRouter API key. Rate-limited to 30 requests/minute per key and 500 requests/day per account. - operationId: getBenchmarksDesignArena - parameters: - - description: Arena to query. Defaults to `models`. - in: query - name: arena - required: false - schema: - default: models - description: Arena to query. Defaults to `models`. - enum: - - models - - builders - - agents - example: models - type: string - - description: >- - Category within the arena (e.g. `codecategories`, `uicomponent`, `gamedev`, `3d`, `dataviz`, `image`, - `video`, `svg`). When omitted, returns all categories. - in: query - name: category - required: false - schema: - description: >- - Category within the arena (e.g. `codecategories`, `uicomponent`, `gamedev`, `3d`, `dataviz`, `image`, - `video`, `svg`). When omitted, returns all categories. - example: codecategories - type: string - - description: 'Max results to return: per category when no category filter is applied (1–100, default 50).' - in: query - name: max_results - required: false - schema: - default: 50 - description: 'Max results to return: per category when no category filter is applied (1–100, default 50).' - example: 20 - maximum: 100 - minimum: 1 - type: integer - responses: - '200': - content: - application/json: - example: - data: - - arena: models - avg_generation_time_ms: 3200 - category: codecategories - display_name: Claude Sonnet 4 - elo: 1423 - model_permaslug: anthropic/claude-sonnet-4 - pricing: - completion: '0.000015' - prompt: '0.000003' - tournament_stats: - first_place: 12 - fourth_place: 2 - second_place: 8 - third_place: 5 - total: 27 - win_rate: 72 - meta: - arena: models - as_of: '2026-06-03T12:00:00Z' - category: null - citation: 'Source: Design Arena (www.designarena.ai) via OpenRouter (openrouter.ai/rankings).' - elo_bounds: - max: 1600 - min: 900 - model_count: 1 - source: design-arena - source_url: https://www.designarena.ai - version: v1 - schema: - $ref: '#/components/schemas/BenchmarksDAResponse' - description: Design Arena ELO rankings with pricing and attribution metadata. - '400': - content: - application/json: - example: - error: - code: 400 - message: Invalid request parameters - schema: - $ref: '#/components/schemas/BadRequestResponse' - description: Bad Request - Invalid request parameters or malformed input - '401': - content: - application/json: - example: - error: - code: 401 - message: Missing Authentication header - schema: - $ref: '#/components/schemas/UnauthorizedResponse' - description: Unauthorized - Authentication required or invalid credentials - '429': - content: - application/json: - example: - error: - code: 429 - message: Rate limit exceeded - schema: - $ref: '#/components/schemas/TooManyRequestsResponse' - description: Too Many Requests - Rate limit exceeded - '500': - content: - application/json: - example: - error: - code: 500 - message: Internal Server Error - schema: - $ref: '#/components/schemas/InternalServerResponse' - description: Internal Server Error - Unexpected server error - summary: Design Arena Benchmark Rankings - tags: - - Datasets /datasets/rankings-daily: get: description: |- @@ -32775,6 +32618,8 @@ tags: name: Anthropic Messages - description: BYOK endpoints name: BYOK + - description: Benchmarks endpoints + name: Benchmarks - description: Chat completion endpoints name: Chat - description: Credit management endpoints