{ "openapi": "3.0.3", "info": { "title": "AI Flow - Evaluation API", "description": "Evaluate AI/chat applications for quality using precision, recall, and groundedness metrics. The Evaluation API allows you to test your AI systems against ground truth data without modifying your codebase.\n\n## Key Features\n\n- **External Service Evaluation**: Test any chat/AI application via API calls\n- **Semantic Metrics**: Precision, recall, and groundedness scores using advanced LLM evaluation\n- **Intermediate Step Evaluation**: Optionally evaluate individual workflow steps\n- **Question Collections**: Organize test cases into reusable collections\n- **Tracing Support**: Get detailed execution traces for debugging\n\n## Architecture\n\n```\n┌─────────────┐ ┌──────────────────┐\n│ AI-Flow │ POST /your/api │ Your Chat App │\n│ Eval API │ ─────────────────► │ (Your Server) │\n│ │ │ │\n│ │ ◄───────────────── │ │\n│ │ Response with │ │\n│ │ aiflow_tracing │ │\n└─────────────┘ └──────────────────┘\n │\n │ Calculate precision/recall\n │ using LLM evaluation\n ▼\n┌─────────────┐\n│ Results │\n│ Dashboard │\n└─────────────┘\n```\n\n## Authentication\n\nAll API requests require an API key. You can generate an API key in the AI Flow dashboard under **Settings → API Keys**.\n\nInclude your API key in the request headers:\n```\nAuthorization: Bearer YOUR_API_KEY\n```\nor\n```\nX-API-Key: YOUR_API_KEY\n```\n\n## Quick Start\n\n1. Create a question collection with ground truth answers\n2. Modify your API to return `aiflow_tracing` when the `X-AIFlow-Eval-Run: true` header is present\n3. Start an evaluation run against your service\n4. Retrieve results with precision, recall, and groundedness scores", "version": "1.0.0", "contact": { "name": "AI Flow Support", "email": "support@ai-flow.eu", "url": "https://www.ai-flow.eu" }, "license": { "name": "Proprietary", "url": "https://www.ai-flow.eu/terms" } }, "servers": [ { "url": "https://www.ai-flow.eu", "description": "Production server" }, { "url": "http://localhost:3000", "description": "Local development server" } ], "tags": [ { "name": "Collections", "description": "Manage question collections for organizing test cases" }, { "name": "Questions", "description": "Manage individual questions with ground truth answers" }, { "name": "Services", "description": "Register and manage external services to evaluate" }, { "name": "Evaluation", "description": "Run evaluations and retrieve results" }, { "name": "Feedback", "description": "Collect and manage user feedback on AI/chat responses" } ], "paths": { "/api/V1/eval/collections": { "get": { "tags": ["Collections"], "summary": "List all question collections", "description": "Retrieve all question collections for the authenticated user. Collections are automatically created when you add questions with a `collectionId`.", "operationId": "listCollections", "security": [{ "bearerAuth": [] }, { "apiKeyHeader": [] }], "responses": { "200": { "description": "List of collections", "content": { "application/json": { "schema": { "type": "object", "properties": { "collections": { "type": "array", "items": { "$ref": "#/components/schemas/Collection" } } } }, "example": { "collections": [ { "collectionId": "customer-support-v1", "name": "customer-support-v1", "questionCount": 50, "lastModified": "2024-01-15T10:30:00Z" }, { "collectionId": "product-qa", "name": "product-qa", "questionCount": 25, "lastModified": "2024-01-10T14:20:00Z" } ] } } } }, "401": { "description": "Unauthorized - invalid or missing API key", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ErrorResponse" } } } }, "403": { "description": "Forbidden - insufficient permissions", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/PermissionErrorResponse" } } } } } } }, "/api/V1/eval/questions": { "post": { "tags": ["Questions"], "summary": "Add questions to a collection", "description": "Add one or more questions with ground truth answers to a collection. If the collection doesn't exist, it will be created automatically.", "operationId": "addQuestions", "security": [{ "bearerAuth": [] }, { "apiKeyHeader": [] }], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/AddQuestionsRequest" }, "examples": { "simple": { "summary": "Simple questions", "value": { "collectionId": "customer-support-v1", "questions": [ { "question": "How do I reset my password?", "groundTruth": "Click Forgot Password on the login page, enter your email, and follow the reset link." }, { "question": "What are your business hours?", "groundTruth": "We are open Monday-Friday 9am-5pm EST." } ] } }, "withStepGroundTruths": { "summary": "With intermediate step ground truths", "value": { "collectionId": "rag-evaluation", "questions": [ { "question": "What is the refund policy?", "groundTruth": "Full refunds are available within 30 days of purchase.", "stepGroundTruths": { "Document Retrieval": "Refund policy document, Terms and conditions section 5", "Answer Generation": "30-day full refund policy with original receipt" }, "metadata": { "category": "billing", "difficulty": "easy" } } ] } } } } } }, "responses": { "200": { "description": "Questions added successfully", "content": { "application/json": { "schema": { "type": "object", "properties": { "addedCount": { "type": "integer", "description": "Number of questions added" }, "questionIds": { "type": "array", "items": { "type": "string" }, "description": "IDs of the created questions" } } }, "example": { "addedCount": 2, "questionIds": ["q1abc123", "q2def456"] } } } }, "400": { "description": "Bad request - missing or invalid parameters", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ErrorResponse" }, "example": { "error": "questions array is required" } } } }, "401": { "$ref": "#/components/responses/Unauthorized" }, "403": { "$ref": "#/components/responses/Forbidden" } } }, "get": { "tags": ["Questions"], "summary": "Get questions from a collection", "description": "Retrieve all questions, optionally filtered by collection ID.", "operationId": "getQuestions", "security": [{ "bearerAuth": [] }, { "apiKeyHeader": [] }], "parameters": [ { "name": "collectionId", "in": "query", "description": "Filter questions by collection ID", "required": false, "schema": { "type": "string" }, "example": "customer-support-v1" } ], "responses": { "200": { "description": "List of questions", "content": { "application/json": { "schema": { "type": "object", "properties": { "questions": { "type": "array", "items": { "$ref": "#/components/schemas/Question" } } } }, "example": { "questions": [ { "_id": "q1abc123", "question": "How do I reset my password?", "groundTruth": "Click Forgot Password on the login page, enter your email, and follow the reset link.", "collectionId": "customer-support-v1", "questionCollectionTag": "customer-support-v1", "stepGroundTruths": {}, "metadata": {} } ] } } } }, "401": { "$ref": "#/components/responses/Unauthorized" }, "403": { "$ref": "#/components/responses/Forbidden" } } } }, "/api/V1/eval/questions/{questionId}": { "put": { "tags": ["Questions"], "summary": "Update a question", "description": "Update an existing question's content, ground truth, or metadata.", "operationId": "updateQuestion", "security": [{ "bearerAuth": [] }, { "apiKeyHeader": [] }], "parameters": [ { "name": "questionId", "in": "path", "required": true, "description": "The ID of the question to update", "schema": { "type": "string" } } ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/UpdateQuestionRequest" }, "example": { "question": "Updated question text?", "groundTruth": "Updated expected answer." } } } }, "responses": { "200": { "description": "Question updated successfully", "content": { "application/json": { "schema": { "type": "object", "properties": { "success": { "type": "boolean" }, "questionId": { "type": "string" } } }, "example": { "success": true, "questionId": "q1abc123" } } } }, "404": { "description": "Question not found", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ErrorResponse" }, "example": { "error": "Question not found" } } } }, "401": { "$ref": "#/components/responses/Unauthorized" }, "403": { "$ref": "#/components/responses/Forbidden" } } }, "delete": { "tags": ["Questions"], "summary": "Delete a question", "description": "Permanently delete a question from a collection.", "operationId": "deleteQuestion", "security": [{ "bearerAuth": [] }, { "apiKeyHeader": [] }], "parameters": [ { "name": "questionId", "in": "path", "required": true, "description": "The ID of the question to delete", "schema": { "type": "string" } } ], "responses": { "200": { "description": "Question deleted successfully", "content": { "application/json": { "schema": { "type": "object", "properties": { "success": { "type": "boolean" } } }, "example": { "success": true } } } }, "404": { "description": "Question not found", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ErrorResponse" }, "example": { "error": "Question not found" } } } }, "401": { "$ref": "#/components/responses/Unauthorized" }, "403": { "$ref": "#/components/responses/Forbidden" } } } }, "/api/V1/eval/services": { "post": { "tags": ["Services"], "summary": "Register an external service", "description": "Register an external chat/AI service for evaluation. This allows you to save service configuration for reuse across multiple evaluation runs.", "operationId": "createService", "security": [{ "bearerAuth": [] }, { "apiKeyHeader": [] }], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/CreateServiceRequest" }, "example": { "name": "Production Chat API", "description": "Main customer-facing chat service", "url": "https://api.myapp.com/chat", "authType": "bearer", "authToken": "your-service-api-key", "requestTimeout": 30000, "concurrency": 5 } } } }, "responses": { "200": { "description": "Service registered successfully", "content": { "application/json": { "schema": { "type": "object", "properties": { "serviceId": { "type": "string" }, "name": { "type": "string" } } }, "example": { "serviceId": "svc123abc", "name": "Production Chat API" } } } }, "400": { "description": "Bad request - missing required fields", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ErrorResponse" }, "example": { "error": "name and url are required" } } } }, "401": { "$ref": "#/components/responses/Unauthorized" }, "403": { "$ref": "#/components/responses/Forbidden" } } }, "get": { "tags": ["Services"], "summary": "List registered services", "description": "Retrieve all external services registered for evaluation.", "operationId": "listServices", "security": [{ "bearerAuth": [] }, { "apiKeyHeader": [] }], "responses": { "200": { "description": "List of services", "content": { "application/json": { "schema": { "type": "object", "properties": { "services": { "type": "array", "items": { "$ref": "#/components/schemas/Service" } } } }, "example": { "services": [ { "_id": "svc123abc", "name": "Production Chat API", "description": "Main customer-facing chat service", "url": "https://api.myapp.com/chat", "authType": "bearer", "requestTimeout": 30000, "concurrency": 5 } ] } } } }, "401": { "$ref": "#/components/responses/Unauthorized" }, "403": { "$ref": "#/components/responses/Forbidden" } } } }, "/api/V1/eval/run": { "post": { "tags": ["Evaluation"], "summary": "Start an evaluation run", "description": "Start an evaluation run against a question collection. You can either use a registered service (by ID) or provide an ad-hoc service URL.\n\n### How It Works\n\n1. AI Flow sends each question to your external service\n2. Your service returns responses with optional `aiflow_tracing` data\n3. AI Flow calculates precision, recall, and groundedness metrics\n4. Results are stored and can be retrieved via the GET endpoint\n\n### Your Service Requirements\n\nWhen AI Flow calls your service, it includes the header `X-AIFlow-Eval-Run: true`. Your service should:\n\n1. Process the request normally\n2. Include `aiflow_tracing` in the response with the final answer\n\n```json\n{\n \"response\": \"Your normal response\",\n \"aiflow_tracing\": {\n \"final\": \"The answer text for evaluation\",\n \"fullContext\": \"Optional: context for groundedness check\"\n }\n}\n```", "operationId": "startEvaluationRun", "security": [{ "bearerAuth": [] }, { "apiKeyHeader": [] }], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/StartEvaluationRequest" }, "examples": { "withServiceId": { "summary": "Using registered service", "value": { "collectionId": "customer-support-v1", "externalServiceId": "svc123abc", "runOptions": { "name": "Production Test - Jan 2024", "evaluateIntermediateSteps": false } } }, "withAdHocUrl": { "summary": "Using ad-hoc service URL", "value": { "collectionId": "customer-support-v1", "externalServiceUrl": "https://staging.myapp.com/api/chat", "externalServiceAuth": { "type": "bearer", "token": "staging-api-key" }, "runOptions": { "name": "Staging Test", "comment": "Testing new model deployment" } } } } } } }, "responses": { "200": { "description": "Evaluation started successfully", "content": { "application/json": { "schema": { "type": "object", "properties": { "status": { "type": "string", "enum": ["started", "running", "completed"] }, "runId": { "type": "string", "description": "Use this ID to check status and retrieve results" } } }, "example": { "status": "started", "runId": "run_abc123xyz" } } } }, "400": { "description": "Bad request", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ErrorResponse" }, "examples": { "missingCollection": { "value": { "error": "collectionId is required" } }, "missingService": { "value": { "error": "Either externalServiceId or externalServiceUrl must be provided" } } } } } }, "404": { "description": "Collection or service not found", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ErrorResponse" }, "examples": { "noQuestions": { "value": { "error": "No questions found in collection" } }, "serviceNotFound": { "value": { "error": "External service not found" } } } } } }, "401": { "$ref": "#/components/responses/Unauthorized" }, "403": { "$ref": "#/components/responses/Forbidden" } } } }, "/api/V1/eval/run/{runId}": { "get": { "tags": ["Evaluation"], "summary": "Get evaluation run results", "description": "Retrieve the status and results of an evaluation run. Results include precision, recall, and groundedness scores for each question.", "operationId": "getEvaluationRun", "security": [{ "bearerAuth": [] }, { "apiKeyHeader": [] }], "parameters": [ { "name": "runId", "in": "path", "required": true, "description": "The ID of the evaluation run", "schema": { "type": "string" } } ], "responses": { "200": { "description": "Evaluation run details and results", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/EvaluationRunResult" }, "example": { "status": "completed", "runId": "run_abc123xyz", "name": "Production Test - Jan 2024", "createdAt": "2024-01-15T10:30:00Z", "stats": { "avgPrecision": 0.89, "avgRecall": 0.92, "avgGroundedness": 0.95 }, "results": [ { "questionId": "q1abc123", "finalOutput": "To reset your password, go to the login page and click 'Forgot Password'...", "finalPrecision": 0.95, "finalRecall": 0.88, "stepEvaluations": [] } ] } } } }, "404": { "description": "Run not found", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ErrorResponse" }, "example": { "error": "Run not found" } } } }, "401": { "$ref": "#/components/responses/Unauthorized" }, "403": { "$ref": "#/components/responses/Forbidden" } } } }, "/api/V1/feedback": { "get": { "tags": ["Feedback"], "summary": "List user feedbacks", "description": "Retrieve all user feedbacks for your workflows. Filter by workflow, rating, or use pagination.", "operationId": "listFeedbacks", "security": [{ "bearerAuth": [] }, { "apiKeyHeader": [] }], "parameters": [ { "name": "workflowId", "in": "query", "description": "Filter feedbacks by workflow ID", "schema": { "type": "string" } }, { "name": "rating", "in": "query", "description": "Filter by rating value", "schema": { "type": "string", "enum": ["positive", "negative"] } }, { "name": "limit", "in": "query", "description": "Maximum number of feedbacks to return (default: 100)", "schema": { "type": "integer", "default": 100, "maximum": 1000 } }, { "name": "skip", "in": "query", "description": "Number of feedbacks to skip for pagination", "schema": { "type": "integer", "default": 0 } } ], "responses": { "200": { "description": "List of feedbacks", "content": { "application/json": { "schema": { "type": "object", "properties": { "feedbacks": { "type": "array", "items": { "$ref": "#/components/schemas/Feedback" } }, "total": { "type": "integer", "description": "Total number of feedbacks matching the filter" } } }, "example": { "feedbacks": [ { "_id": "fb123abc", "workflowId": "wf456def", "messageId": "msg789xyz", "rating": "positive", "comment": "Very helpful response!", "userMessage": "How do I reset my password?", "assistantMessage": "To reset your password, go to Settings > Security...", "createdAt": "2024-01-15T10:30:00Z" } ], "total": 1 } } } }, "401": { "$ref": "#/components/responses/Unauthorized" }, "403": { "$ref": "#/components/responses/Forbidden" } } }, "post": { "tags": ["Feedback"], "summary": "Submit user feedback", "description": "Submit feedback for a specific chat message. This is typically called from your chat interface when users rate responses as helpful or unhelpful.\n\n### Integration Example\n\nAdd thumbs up/down buttons to your chat UI and call this endpoint when users click them:\n\n```javascript\nawait fetch('https://www.ai-flow.eu/api/V1/feedback', {\n method: 'POST',\n headers: {\n 'Authorization': 'Bearer YOUR_API_KEY',\n 'Content-Type': 'application/json'\n },\n body: JSON.stringify({\n workflowId: 'your-workflow-id',\n messageId: 'unique-message-id',\n rating: 'positive', // or 'negative'\n comment: 'Optional user comment',\n userMessage: 'What the user asked',\n assistantMessage: 'What the AI responded'\n })\n});\n```", "operationId": "submitFeedback", "security": [{ "bearerAuth": [] }, { "apiKeyHeader": [] }], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/CreateFeedbackRequest" }, "example": { "workflowId": "wf456def", "messageId": "msg789xyz", "rating": "positive", "comment": "Very helpful and accurate response!", "userMessage": "How do I reset my password?", "assistantMessage": "To reset your password, go to Settings > Security > Change Password..." } } } }, "responses": { "200": { "description": "Feedback submitted successfully", "content": { "application/json": { "schema": { "type": "object", "properties": { "success": { "type": "boolean" }, "feedbackId": { "type": "string" } } }, "example": { "success": true, "feedbackId": "fb123abc" } } } }, "400": { "description": "Bad request - missing required fields", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ErrorResponse" }, "example": { "error": "messageId and rating are required" } } } }, "401": { "$ref": "#/components/responses/Unauthorized" }, "403": { "$ref": "#/components/responses/Forbidden" } } } }, "/api/V1/feedback/{feedbackId}": { "get": { "tags": ["Feedback"], "summary": "Get a specific feedback", "description": "Retrieve details of a specific feedback entry by its ID.", "operationId": "getFeedback", "security": [{ "bearerAuth": [] }, { "apiKeyHeader": [] }], "parameters": [ { "name": "feedbackId", "in": "path", "required": true, "description": "The ID of the feedback to retrieve", "schema": { "type": "string" } } ], "responses": { "200": { "description": "Feedback details", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Feedback" }, "example": { "_id": "fb123abc", "workflowId": "wf456def", "messageId": "msg789xyz", "rating": "positive", "comment": "Very helpful response!", "userMessage": "How do I reset my password?", "assistantMessage": "To reset your password, go to Settings > Security...", "createdAt": "2024-01-15T10:30:00Z" } } } }, "404": { "description": "Feedback not found", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ErrorResponse" }, "example": { "error": "Feedback not found" } } } }, "401": { "$ref": "#/components/responses/Unauthorized" }, "403": { "$ref": "#/components/responses/Forbidden" } } }, "delete": { "tags": ["Feedback"], "summary": "Delete a feedback", "description": "Permanently delete a feedback entry.", "operationId": "deleteFeedback", "security": [{ "bearerAuth": [] }, { "apiKeyHeader": [] }], "parameters": [ { "name": "feedbackId", "in": "path", "required": true, "description": "The ID of the feedback to delete", "schema": { "type": "string" } } ], "responses": { "200": { "description": "Feedback deleted successfully", "content": { "application/json": { "schema": { "type": "object", "properties": { "success": { "type": "boolean" } } }, "example": { "success": true } } } }, "404": { "description": "Feedback not found", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ErrorResponse" }, "example": { "error": "Feedback not found" } } } }, "401": { "$ref": "#/components/responses/Unauthorized" }, "403": { "$ref": "#/components/responses/Forbidden" } } } } }, "components": { "securitySchemes": { "bearerAuth": { "type": "http", "scheme": "bearer", "description": "API key as Bearer token: `Authorization: Bearer YOUR_API_KEY`" }, "apiKeyHeader": { "type": "apiKey", "in": "header", "name": "X-API-Key", "description": "API key in custom header: `X-API-Key: YOUR_API_KEY`" } }, "responses": { "Unauthorized": { "description": "Unauthorized - invalid or missing API key", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ErrorResponse" }, "example": { "error": "Invalid API key" } } } }, "Forbidden": { "description": "Forbidden - insufficient permissions", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/PermissionErrorResponse" }, "example": { "error": "Insufficient permissions", "message": "Your plan does not include access to the Evaluation API. Please upgrade to Plus or higher." } } } } }, "schemas": { "Collection": { "type": "object", "properties": { "collectionId": { "type": "string", "description": "Unique identifier for the collection" }, "name": { "type": "string", "description": "Display name of the collection" }, "questionCount": { "type": "integer", "description": "Number of questions in the collection" }, "lastModified": { "type": "string", "format": "date-time", "description": "When the collection was last modified" } } }, "Question": { "type": "object", "properties": { "_id": { "type": "string", "description": "Unique question identifier" }, "question": { "type": "string", "description": "The test question" }, "groundTruth": { "type": "string", "description": "The expected/correct answer" }, "collectionId": { "type": "string", "description": "Collection this question belongs to" }, "questionCollectionTag": { "type": "string", "description": "Alternative collection identifier" }, "stepGroundTruths": { "type": "object", "additionalProperties": { "type": "string" }, "description": "Expected outputs for intermediate workflow steps" }, "metadata": { "type": "object", "description": "Custom metadata for categorization" } } }, "AddQuestionsRequest": { "type": "object", "required": ["questions"], "properties": { "collectionId": { "type": "string", "description": "Collection to add questions to. Created if doesn't exist." }, "questions": { "type": "array", "items": { "type": "object", "required": ["question", "groundTruth"], "properties": { "question": { "type": "string", "description": "The test question" }, "groundTruth": { "type": "string", "description": "The expected/correct answer" }, "stepGroundTruths": { "type": "object", "additionalProperties": { "type": "string" }, "description": "Expected outputs for intermediate steps (keyed by step name)" }, "metadata": { "type": "object", "description": "Custom metadata (category, difficulty, etc.)" } } }, "minItems": 1 } } }, "UpdateQuestionRequest": { "type": "object", "properties": { "question": { "type": "string" }, "groundTruth": { "type": "string" }, "stepGroundTruths": { "type": "object", "additionalProperties": { "type": "string" } }, "metadata": { "type": "object" }, "collectionId": { "type": "string" } } }, "Service": { "type": "object", "properties": { "_id": { "type": "string" }, "name": { "type": "string" }, "description": { "type": "string" }, "url": { "type": "string", "format": "uri" }, "authType": { "type": "string", "enum": ["none", "bearer", "apiKey", "basic"] }, "requestTimeout": { "type": "integer", "description": "Request timeout in milliseconds" }, "concurrency": { "type": "integer", "description": "Maximum concurrent requests" } } }, "CreateServiceRequest": { "type": "object", "required": ["name", "url"], "properties": { "name": { "type": "string", "description": "Display name for the service" }, "description": { "type": "string", "description": "Optional description" }, "url": { "type": "string", "format": "uri", "description": "The endpoint URL to call" }, "authType": { "type": "string", "enum": ["none", "bearer", "apiKey", "basic"], "default": "none", "description": "Authentication type" }, "authToken": { "type": "string", "description": "Auth token (for bearer/apiKey types)" }, "authSourceId": { "type": "string", "description": "ID of a saved AI Flow source for authentication" }, "requestTimeout": { "type": "integer", "default": 30000, "description": "Request timeout in milliseconds" }, "concurrency": { "type": "integer", "default": 5, "description": "Maximum concurrent requests" } } }, "StartEvaluationRequest": { "type": "object", "required": ["collectionId"], "properties": { "collectionId": { "type": "string", "description": "ID of the question collection to evaluate" }, "externalServiceId": { "type": "string", "description": "ID of a registered external service (use this OR externalServiceUrl)" }, "externalServiceUrl": { "type": "string", "format": "uri", "description": "Ad-hoc service URL (use this OR externalServiceId)" }, "externalServiceAuth": { "type": "object", "description": "Authentication for ad-hoc service", "properties": { "type": { "type": "string", "enum": ["none", "bearer", "apiKey", "basic"] }, "token": { "type": "string" } } }, "runOptions": { "type": "object", "properties": { "name": { "type": "string", "description": "Name for this evaluation run" }, "comment": { "type": "string", "description": "Optional comment or notes" }, "evaluateIntermediateSteps": { "type": "boolean", "default": false, "description": "Whether to evaluate intermediate workflow steps" }, "timeout": { "type": "integer", "description": "Request timeout override" }, "concurrency": { "type": "integer", "description": "Concurrency override" } } } } }, "EvaluationRunResult": { "type": "object", "properties": { "status": { "type": "string", "enum": ["started", "running", "completed", "failed"] }, "runId": { "type": "string" }, "name": { "type": "string" }, "createdAt": { "type": "string", "format": "date-time" }, "stats": { "type": "object", "properties": { "avgPrecision": { "type": "number", "minimum": 0, "maximum": 1, "description": "Average precision score (0-1)" }, "avgRecall": { "type": "number", "minimum": 0, "maximum": 1, "description": "Average recall score (0-1)" }, "avgGroundedness": { "type": "number", "minimum": 0, "maximum": 1, "description": "Average groundedness score (0-1)" } } }, "results": { "type": "array", "items": { "$ref": "#/components/schemas/QuestionResult" } } } }, "QuestionResult": { "type": "object", "properties": { "questionId": { "type": "string" }, "finalOutput": { "type": "string", "description": "The response from the evaluated service" }, "finalPrecision": { "type": "number", "minimum": 0, "maximum": 1, "description": "Precision score for this question" }, "finalRecall": { "type": "number", "minimum": 0, "maximum": 1, "description": "Recall score for this question" }, "stepEvaluations": { "type": "array", "items": { "type": "object", "properties": { "stepName": { "type": "string" }, "precision": { "type": "number" }, "recall": { "type": "number" } } }, "description": "Evaluations for intermediate steps (if enabled)" } } }, "Feedback": { "type": "object", "properties": { "_id": { "type": "string", "description": "Unique feedback identifier" }, "workflowId": { "type": "string", "description": "ID of the workflow/agent that generated the response" }, "messageId": { "type": "string", "description": "Unique identifier for the chat message being rated" }, "rating": { "type": "string", "enum": ["positive", "negative"], "description": "User's rating of the response" }, "comment": { "type": "string", "description": "Optional user comment explaining the rating" }, "userMessage": { "type": "string", "description": "The user's original message/question" }, "assistantMessage": { "type": "string", "description": "The AI/assistant's response that was rated" }, "metadata": { "type": "object", "description": "Optional custom metadata" }, "createdAt": { "type": "string", "format": "date-time", "description": "When the feedback was submitted" } } }, "CreateFeedbackRequest": { "type": "object", "required": ["messageId", "rating"], "properties": { "workflowId": { "type": "string", "description": "ID of the workflow/agent (optional, for organization)" }, "messageId": { "type": "string", "description": "Unique identifier for the chat message being rated. This should be a unique ID you generate for each message in your chat interface." }, "rating": { "type": "string", "enum": ["positive", "negative"], "description": "User's rating - typically mapped to thumbs up/down icons" }, "comment": { "type": "string", "description": "Optional comment from the user explaining their rating" }, "userMessage": { "type": "string", "description": "The user's original message/question (stored for context)" }, "assistantMessage": { "type": "string", "description": "The AI response that was rated (stored for analysis)" }, "metadata": { "type": "object", "description": "Optional custom metadata (e.g., model version, session ID)" } } }, "ErrorResponse": { "type": "object", "properties": { "error": { "type": "string", "description": "Error message" }, "details": { "type": "string", "description": "Additional error details" } } }, "PermissionErrorResponse": { "type": "object", "properties": { "error": { "type": "string" }, "message": { "type": "string", "description": "Human-readable explanation" } } } } } }