{ "openapi": "3.1.1", "info": { "title": "Model Behavior Regression Explorer API", "version": "1.0.0", "summary": "Compare eval results of two model versions: accuracy, regressions, McNemar p-value, flaky items.", "description": "Every endpoint performs the same calculation as the human page and returns inputs, result, units, assumptions, sources, warnings and computedAt. MCP endpoint: /mcp." }, "servers": [ { "url": "https://modelbehaviorregressionexplorer.usefulweb.site" } ], "paths": { "/api/v1/calculate": { "post": { "operationId": "calculate", "summary": "Compare eval results of two model versions: accuracy, regressions, McNemar p-value, flaky items.", "description": "Paste eval results for two model versions on the same items (CSV or JSONL with id, input, expected, output_a, output_b and optional category). Each output is scored with your metric – exact, normalised exact, contains, numeric tolerance or regex – then accuracy is compared per version and category, regressions and improvements are listed, an exact McNemar test tells whether the difference is more than noise, and flaky (inconsistent repeated runs) or ambiguous items are flagged.\n\nMethodology:\n- Metrics: exact (trimmed), normalised exact (Unicode NFKC, lower case, punctuation/symbols and the articles a/an/the removed, whitespace collapsed), contains (normalised expected appears as whole words in the normalised output), numeric (last number in the output vs the expected number, absolute tolerance; thousands separators removed), regex (expected is a case-insensitive regular expression).\n- Repeated ids are repeated runs: an item counts as correct for a version when the majority of its runs are correct; items whose runs disagree are flaky and excluded from the McNemar test.\n- Accuracy = correct / scored items per version. Regression = A correct, B wrong; improvement = A wrong, B correct.\n- Exact McNemar test: two-sided binomial test of the discordant pairs (b = only A correct, c = only B correct) against p = 0.5: p = min(1, 2·Σ_{i≤min(b,c)} C(b+c, i)/2^(b+c)). Significant at α = 0.05.\n- Ambiguous: empty expected value, unparsable regex/number, or a near miss that only fails because of formatting (e.g. would pass under the normalised or contains metric).", "tags": [ "ai-infra" ], "x-mcp-tool": "model_behavior_regression_explorer", "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ModelBehaviorRegressionExplorerInput" }, "examples": { "example1": { "summary": "Prompt v1 vs v2 on 12 QA/math items (CSV)", "value": { "labelA": "prompt-v1", "labelB": "prompt-v2", "data": "id,category,input,expected,output_a,output_b\nq1,geo,Capital of France?,Paris,Paris,Paris.\nq2,geo,Capital of Australia?,Canberra,Sydney,Canberra\nq3,geo,Longest river in Europe?,Volga,The Volga,Danube\nq4,geo,Capital of Canada?,Ottawa,Ottawa,Ottawa\nq5,geo,Highest mountain in Africa?,Kilimanjaro,Kilimanjaro,Mount Kilimanjaro\nm1,math,12*7,84,84,84\nm2,math,15% of 240,36,36,36.0\nm3,math,sqrt(144),12,12,11\nm4,math,2^10,1024,1024,1024\nm5,math,100/8,12.5,12,12.5\nm6,math,17+26,43,43,43\nm7,math,9*9,81,81,81\nm7,math,9*9,81,81,18" } }, "example2": { "summary": "Numeric metric with JSONL", "value": { "metric": "numeric", "tolerance": 0.5, "data": "{\"id\":\"a\",\"category\":\"unit-conversion\",\"expected\":\"26.2\",\"output_a\":\"About 26.2 miles\",\"output_b\":\"42.195 km is 26.2 miles\"}\n{\"id\":\"b\",\"category\":\"unit-conversion\",\"expected\":\"100\",\"output_a\":\"212 °F is 100 °C\",\"output_b\":\"It is 100\"}\n{\"id\":\"c\",\"category\":\"arithmetic\",\"expected\":\"1,250\",\"output_a\":\"Total: 1,250\",\"output_b\":\"Total: 1,205\"}\n{\"id\":\"d\",\"category\":\"arithmetic\",\"expected\":\"7\",\"output_a\":\"Answer: 8\",\"output_b\":\"Answer: 7\"}" } } } } } }, "responses": { "200": { "description": "Calculation result envelope.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ModelBehaviorRegressionExplorerResult" } } } }, "400": { "description": "Malformed JSON body.", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } }, "422": { "description": "Invalid input (see errors[]).", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } }, "429": { "description": "Rate limit exceeded (see Retry-After).", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } } } } }, "/api/v1/tool": { "get": { "operationId": "getToolDefinition", "summary": "Inputs, output schema and examples for this tool", "responses": { "200": { "description": "Tool definition.", "content": { "application/json": { "schema": { "type": "object" } } } } } } } }, "components": { "schemas": { "Problem": { "type": "object", "description": "RFC 9457 problem details. 422 responses include per-field `errors`.", "required": [ "type", "title", "status", "detail" ], "properties": { "type": { "type": "string" }, "title": { "type": "string" }, "status": { "type": "integer" }, "detail": { "type": "string" }, "code": { "type": "string" }, "errors": { "type": "array", "items": { "type": "object", "required": [ "field", "message" ], "properties": { "field": { "type": "string" }, "message": { "type": "string" } } } } } }, "ModelBehaviorRegressionExplorerInput": { "type": "object", "additionalProperties": false, "properties": { "data": { "title": "Eval results (CSV or JSONL)", "description": "Columns: id, input, expected, output_a, output_b, category (optional). Repeated ids = repeated runs.", "type": "string", "maxLength": 5000000 }, "metric": { "title": "Scoring metric", "type": "string", "enum": [ "exact", "normalized", "contains", "numeric", "regex" ], "default": "normalized" }, "tolerance": { "title": "Numeric tolerance (absolute)", "type": "number", "minimum": 0, "default": 0.01 }, "labelA": { "title": "Name of version A", "type": "string", "maxLength": 60, "default": "A" }, "labelB": { "title": "Name of version B", "type": "string", "maxLength": 60, "default": "B" }, "maxListed": { "title": "Maximum items listed per table", "type": "integer", "minimum": 1, "maximum": 500, "default": 50 } }, "required": [ "data" ] }, "ModelBehaviorRegressionExplorerResult": { "type": "object", "additionalProperties": false, "required": [ "schemaVersion", "tool", "computedAt", "inputs", "summary", "result", "units", "assumptions", "sources", "warnings", "presentation", "links" ], "properties": { "schemaVersion": { "type": "string", "const": "1.0" }, "tool": { "type": "object", "additionalProperties": false, "required": [ "id", "name", "version" ], "properties": { "id": { "type": "string" }, "name": { "type": "string" }, "version": { "type": "string" } } }, "computedAt": { "type": "string", "format": "date-time" }, "inputs": { "type": "object", "description": "Normalised inputs actually used, including applied defaults." }, "summary": { "type": "string" }, "result": { "type": "object", "additionalProperties": false, "required": [ "format", "metric", "items", "accuracyA", "accuracyB", "delta", "contingency", "mcnemarP", "significant", "categories", "regressions", "improvements", "flaky", "ambiguous" ], "properties": { "format": { "type": "string", "enum": [ "csv", "jsonl", "json" ] }, "metric": { "type": "string" }, "items": { "type": "integer", "description": "Distinct item ids." }, "accuracyA": { "type": [ "number", "null" ] }, "accuracyB": { "type": [ "number", "null" ] }, "delta": { "type": [ "number", "null" ], "description": "accuracyB − accuracyA (percentage points)." }, "contingency": { "type": "object", "additionalProperties": false, "required": [ "bothCorrect", "onlyA", "onlyB", "bothWrong", "excluded" ], "properties": { "bothCorrect": { "type": "integer" }, "onlyA": { "type": "integer" }, "onlyB": { "type": "integer" }, "bothWrong": { "type": "integer" }, "excluded": { "type": "integer" } } }, "mcnemarP": { "type": "number" }, "significant": { "type": "boolean" }, "categories": { "type": "array", "items": { "type": "object", "additionalProperties": false, "required": [ "category", "n", "accuracyA", "accuracyB", "delta", "regressions", "improvements" ], "properties": { "category": { "type": "string" }, "n": { "type": "integer" }, "accuracyA": { "type": [ "number", "null" ] }, "accuracyB": { "type": [ "number", "null" ] }, "delta": { "type": [ "number", "null" ] }, "regressions": { "type": "integer" }, "improvements": { "type": "integer" } } } }, "regressions": { "type": "array", "items": { "type": "object", "additionalProperties": false, "required": [ "id", "category", "input", "expected", "outputA", "outputB" ], "properties": { "id": { "type": "string" }, "category": { "type": "string" }, "input": { "type": "string" }, "expected": { "type": "string" }, "outputA": { "type": [ "string", "null" ] }, "outputB": { "type": [ "string", "null" ] } } } }, "improvements": { "type": "array", "items": { "type": "object", "additionalProperties": false, "required": [ "id", "category", "input", "expected", "outputA", "outputB" ], "properties": { "id": { "type": "string" }, "category": { "type": "string" }, "input": { "type": "string" }, "expected": { "type": "string" }, "outputA": { "type": [ "string", "null" ] }, "outputB": { "type": [ "string", "null" ] } } } }, "flaky": { "type": "array", "items": { "type": "object", "additionalProperties": false, "required": [ "id", "runsA", "runsB", "note" ], "properties": { "id": { "type": "string" }, "runsA": { "type": "integer" }, "runsB": { "type": "integer" }, "note": { "type": "string" } } } }, "ambiguous": { "type": "array", "items": { "type": "object", "additionalProperties": false, "required": [ "id", "reason" ], "properties": { "id": { "type": "string" }, "reason": { "type": "string" } } } } } }, "units": { "type": "object", "additionalProperties": { "type": "string" } }, "assumptions": { "type": "array", "items": { "type": "string" } }, "sources": { "type": "array", "items": { "type": "object", "additionalProperties": false, "required": [ "title" ], "properties": { "title": { "type": "string" }, "url": { "type": "string" }, "publisher": { "type": "string" }, "retrievedAt": { "type": "string" }, "note": { "type": "string" } } } }, "warnings": { "type": "array", "items": { "type": "string" } }, "presentation": { "type": "array", "items": { "type": "object" }, "description": "Display blocks used by the human page (same data)." }, "links": { "type": "object", "additionalProperties": false, "required": [ "human", "api", "openapi", "methodology" ], "properties": { "human": { "type": "string", "format": "uri" }, "api": { "type": "string", "format": "uri" }, "openapi": { "type": "string", "format": "uri" }, "methodology": { "type": "string", "format": "uri" } } } } } } } }