{ "openapi": "3.1.0", "info": { "title": "Mabyduck API", "version": "", "description": "API for managing subjective studies on Mabyduck." }, "paths": { "/organizations/{organization_id}/billing/{billing_id}/events/": { "get": { "operationId": "organizations_billing_events_list", "description": "Returns billing events for an organization billing account, newest first. Only organization owners and billing managers can access this endpoint. API requests require an organization API key created by one of those roles.", "summary": "List organization billing events", "parameters": [ { "in": "path", "name": "billing_id", "schema": { "type": "string", "pattern": "^[a-zA-Z0-9]+$" }, "required": true }, { "in": "path", "name": "organization_id", "schema": { "type": "string", "pattern": "^[a-zA-Z0-9]+$" }, "required": true } ], "tags": [ "Organizations" ], "security": [ { "ApiKeyAuth": [] }, { "ApiKeyAuth": [] } ], "responses": { "200": { "content": { "application/json": { "schema": { "type": "array", "items": { "$ref": "#/components/schemas/OrganizationBillingEvent" } } } }, "description": "" } } } }, "/projects/{project_id}/billing/events/": { "get": { "operationId": "billing_events_list", "description": "Returns a list of billing events belonging to the given project, newest first. Only accessible with an API key created by a billing manager.", "summary": "List billing events", "parameters": [ { "in": "path", "name": "project_id", "schema": { "type": "string", "pattern": "^[a-zA-Z0-9]+$" }, "required": true } ], "tags": [ "Billing" ], "security": [ { "ApiKeyAuth": [] }, { "ApiKeyAuth": [] } ], "responses": { "200": { "content": { "application/json": { "schema": { "type": "array", "items": { "$ref": "#/components/schemas/BillingEvent" } } } }, "description": "" } } } }, "/projects/{project_id}/datasets/": { "get": { "operationId": "datasets_list", "description": "Returns a list of datasets belonging to the given project.", "summary": "List all datasets", "parameters": [ { "in": "path", "name": "project_id", "schema": { "type": "string", "pattern": "^[a-zA-Z0-9]+$" }, "required": true } ], "tags": [ "Datasets" ], "security": [ { "ApiKeyAuth": [] }, { "ApiKeyAuth": [] } ], "responses": { "200": { "content": { "application/json": { "schema": { "type": "array", "items": { "$ref": "#/components/schemas/Dataset" } } } }, "description": "" } } }, "post": { "operationId": "datasets_create", "description": "Creates a new dataset and returns a signed URL for uploading the dataset file.", "summary": "Create a dataset", "parameters": [ { "in": "path", "name": "project_id", "schema": { "type": "string", "pattern": "^[a-zA-Z0-9]+$" }, "required": true } ], "tags": [ "Datasets" ], "requestBody": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/PublicDatasetCreate" }, "examples": { "Request": { "value": { "name": "Dataset", "filename": "dataset.zip" } } } }, "application/x-www-form-urlencoded": { "schema": { "$ref": "#/components/schemas/PublicDatasetCreate" } }, "multipart/form-data": { "schema": { "$ref": "#/components/schemas/PublicDatasetCreate" } } }, "required": true }, "security": [ { "ApiKeyAuth": [] }, { "ApiKeyAuth": [] } ], "responses": { "201": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/PublicDatasetCreate" }, "examples": { "Response": { "value": { "id": "{dataset_id}", "name": "Dataset", "upload_url": "https://mabyduck-temp.s3.nl-ams.scw.cloud/{signed-upload-url}", "status_url": "https://api.mabyduck.com/projects/{project_id}/datasets/{dataset_id}/status/" } } } } }, "description": "" } }, "x-codeSamples": [ { "lang": "curl", "label": "cURL", "source": "# Create an empty dataset\ncurl -X POST \\\n 'https://api.mabyduck.com/projects/{project_id}/datasets/' \\\n -H 'Authorization: Api-Key ' \\\n -H 'Content-Type: application/json' \\\n -d '{\"name\": \"Dataset\", \"filename\": \"dataset.zip\"}'\n\n# Upload dataset file to the returned URL\ncurl --upload-file 'dataset.zip' {upload_url}" }, { "lang": "Python", "label": "Python", "source": "import requests\n\nurl = f\"https://api.mabyduck.com/projects/{project_id}/datasets/\"\npayload = {\"name\": \"Dataset\", \"filename\": \"dataset.zip\"}\nheaders = {\n \"Content-Type\": \"application/json\",\n \"Authorization\": \"Api-Key \"\n}\n\n# Create an empty dataset\nresponse = requests.post(url, json=payload, headers=headers)\ndata = response.json()\n\n# Upload dataset file\nwith open('dataset.zip', 'rb') as f:\n response = requests.put(data['upload_url'], data=f)" } ] } }, "/projects/{project_id}/datasets/{dataset_id}/": { "get": { "operationId": "datasets_retrieve", "description": "Returns detailed information for a single dataset identified by `dataset_id`.", "summary": "Retrieve a dataset", "parameters": [ { "in": "path", "name": "dataset_id", "schema": { "type": "string", "pattern": "^[a-zA-Z0-9]+$" }, "required": true }, { "in": "path", "name": "project_id", "schema": { "type": "string", "pattern": "^[a-zA-Z0-9]+$" }, "required": true } ], "tags": [ "Datasets" ], "security": [ { "ApiKeyAuth": [] }, { "ApiKeyAuth": [] } ], "responses": { "200": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/DatasetDetail" } } }, "description": "" } } } }, "/projects/{project_id}/datasets/{dataset_id}/status/": { "get": { "operationId": "datasets_status_retrieve", "description": "Returns the current status of a dataset, including any error messages if processing failed.", "summary": "Get dataset status", "parameters": [ { "in": "path", "name": "dataset_id", "schema": { "type": "string", "pattern": "^[a-zA-Z0-9]+$" }, "required": true }, { "in": "path", "name": "project_id", "schema": { "type": "string", "pattern": "^[a-zA-Z0-9]+$" }, "required": true } ], "tags": [ "Datasets" ], "security": [ { "ApiKeyAuth": [] }, { "ApiKeyAuth": [] } ], "responses": { "200": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/DatasetStatus" } } }, "description": "" } } } }, "/projects/{project_id}/experiments/": { "get": { "operationId": "experiments_list", "description": "Returns a list of experiments belonging to the given project.", "summary": "List all experiments", "parameters": [ { "in": "path", "name": "project_id", "schema": { "type": "string", "pattern": "^[a-zA-Z0-9]+$" }, "required": true } ], "tags": [ "Experiments" ], "security": [ { "ApiKeyAuth": [] }, { "ApiKeyAuth": [] } ], "responses": { "200": { "content": { "application/json": { "schema": { "type": "array", "items": { "$ref": "#/components/schemas/Experiment" } } } }, "description": "" } } }, "post": { "operationId": "experiments_create", "description": "Creates a new draft experiment with the specified configuration.", "summary": "Create an experiment", "parameters": [ { "in": "path", "name": "project_id", "schema": { "type": "string", "pattern": "^[a-zA-Z0-9]+$" }, "required": true } ], "tags": [ "Experiments" ], "requestBody": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/CreateExperiment" } }, "application/x-www-form-urlencoded": { "schema": { "$ref": "#/components/schemas/CreateExperiment" } }, "multipart/form-data": { "schema": { "$ref": "#/components/schemas/CreateExperiment" } } }, "required": true }, "security": [ { "ApiKeyAuth": [] }, { "ApiKeyAuth": [] } ], "responses": { "201": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Experiment" } } }, "description": "" } } } }, "/projects/{project_id}/experiments/{experiment_id}/": { "get": { "operationId": "experiments_retrieve", "description": "Returns detailed information for a single experiment identified by `experiment_id`.", "summary": "Retrieve an experiment", "parameters": [ { "in": "path", "name": "experiment_id", "schema": { "type": "string", "pattern": "^[a-zA-Z0-9]+$" }, "required": true }, { "in": "path", "name": "project_id", "schema": { "type": "string", "pattern": "^[a-zA-Z0-9]+$" }, "required": true } ], "tags": [ "Experiments" ], "security": [ { "ApiKeyAuth": [] }, { "ApiKeyAuth": [] } ], "responses": { "200": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ExperimentDetail" } } }, "description": "" } } } }, "/projects/{project_id}/experiments/{experiment_id}/jobs/": { "get": { "operationId": "experiments_jobs_list", "description": "Returns a list of jobs belonging to the given experiment.", "summary": "List all jobs", "parameters": [ { "in": "path", "name": "experiment_id", "schema": { "type": "string", "pattern": "^[a-zA-Z0-9]+$" }, "required": true }, { "in": "path", "name": "project_id", "schema": { "type": "string", "pattern": "^[a-zA-Z0-9]+$" }, "required": true } ], "tags": [ "Experiments" ], "security": [ { "ApiKeyAuth": [] }, { "ApiKeyAuth": [] } ], "responses": { "200": { "content": { "application/json": { "schema": { "type": "array", "items": { "$ref": "#/components/schemas/Job" } } } }, "description": "" } } }, "post": { "operationId": "experiments_jobs_create", "description": "Creates a new draft job with the specified configuration.", "summary": "Create a job", "parameters": [ { "in": "path", "name": "experiment_id", "schema": { "type": "string", "pattern": "^[a-zA-Z0-9]+$" }, "required": true }, { "in": "path", "name": "project_id", "schema": { "type": "string", "pattern": "^[a-zA-Z0-9]+$" }, "required": true } ], "tags": [ "Experiments" ], "requestBody": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/CreateJob" } }, "application/x-www-form-urlencoded": { "schema": { "$ref": "#/components/schemas/CreateJob" } }, "multipart/form-data": { "schema": { "$ref": "#/components/schemas/CreateJob" } } }, "required": true }, "security": [ { "ApiKeyAuth": [] }, { "ApiKeyAuth": [] } ], "responses": { "201": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/JobCreateResponse" } } }, "description": "" } } } }, "/projects/{project_id}/experiments/{experiment_id}/jobs/{job_id}/": { "get": { "operationId": "experiments_jobs_retrieve", "description": "Returns detailed information for a single job identified by `job_id`.", "summary": "Retrieve a job", "parameters": [ { "in": "path", "name": "experiment_id", "schema": { "type": "string", "pattern": "^[a-zA-Z0-9]+$" }, "required": true }, { "in": "path", "name": "job_id", "schema": { "type": "string", "pattern": "^[a-zA-Z0-9]+$" }, "required": true }, { "in": "path", "name": "project_id", "schema": { "type": "string", "pattern": "^[a-zA-Z0-9]+$" }, "required": true } ], "tags": [ "Experiments" ], "security": [ { "ApiKeyAuth": [] }, { "ApiKeyAuth": [] } ], "responses": { "200": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/JobDetail" } } }, "description": "" } } } }, "/projects/{project_id}/experiments/{experiment_id}/jobs/{job_id}/costs/": { "get": { "operationId": "experiments_jobs_costs_retrieve", "description": "Returns cost breakdown information for a specific job.", "summary": "Get job cost breakdown", "parameters": [ { "in": "path", "name": "experiment_id", "schema": { "type": "string", "pattern": "^[a-zA-Z0-9]+$" }, "required": true }, { "in": "path", "name": "job_id", "schema": { "type": "string", "pattern": "^[a-zA-Z0-9]+$" }, "required": true }, { "in": "path", "name": "project_id", "schema": { "type": "string", "pattern": "^[a-zA-Z0-9]+$" }, "required": true } ], "tags": [ "Experiments" ], "security": [ { "ApiKeyAuth": [] }, { "ApiKeyAuth": [] } ], "responses": { "200": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/JobCosts" } } }, "description": "" } } } }, "/projects/{project_id}/experiments/{experiment_id}/jobs/{job_id}/launch/": { "post": { "operationId": "experiments_jobs_launch_create", "description": "Launches the specified job. Requires a valid job costs token.", "summary": "Launch a job", "parameters": [ { "in": "path", "name": "experiment_id", "schema": { "type": "string", "pattern": "^[a-zA-Z0-9]+$" }, "required": true }, { "in": "path", "name": "job_id", "schema": { "type": "string", "pattern": "^[a-zA-Z0-9]+$" }, "required": true }, { "in": "path", "name": "project_id", "schema": { "type": "string", "pattern": "^[a-zA-Z0-9]+$" }, "required": true } ], "tags": [ "Experiments" ], "requestBody": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/JobLaunch" } }, "application/x-www-form-urlencoded": { "schema": { "$ref": "#/components/schemas/JobLaunch" } }, "multipart/form-data": { "schema": { "$ref": "#/components/schemas/JobLaunch" } } }, "required": true }, "security": [ { "ApiKeyAuth": [] }, { "ApiKeyAuth": [] } ], "responses": { "200": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/JobLaunch" } } }, "description": "" } } } }, "/projects/{project_id}/experiments/{experiment_id}/rater-pools/": { "get": { "operationId": "experiments_rater_pools_list", "description": "Returns a list of rater pools available for a given experiment.", "summary": "List rater pools", "parameters": [ { "in": "path", "name": "experiment_id", "schema": { "type": "string", "pattern": "^[a-zA-Z0-9]+$" }, "required": true }, { "in": "query", "name": "kind", "schema": { "type": "integer", "enum": [ 0, 100, 400, 500, 600, 700 ] }, "description": "Filter by rater pool kind." }, { "in": "path", "name": "project_id", "schema": { "type": "string", "pattern": "^[a-zA-Z0-9]+$" }, "required": true } ], "tags": [ "Experiments" ], "security": [ { "ApiKeyAuth": [] }, { "ApiKeyAuth": [] } ], "responses": { "200": { "content": { "application/json": { "schema": { "type": "array", "items": { "$ref": "#/components/schemas/RaterPool" } } } }, "description": "" } } } }, "/projects/{project_id}/experiments/{experiment_id}/raters/": { "get": { "operationId": "experiments_raters_list", "description": "Returns a list of raters of a given experiment.", "summary": "List all raters", "parameters": [ { "in": "path", "name": "experiment_id", "schema": { "type": "string", "pattern": "^[a-zA-Z0-9]+$" }, "required": true }, { "in": "path", "name": "project_id", "schema": { "type": "string", "pattern": "^[a-zA-Z0-9]+$" }, "required": true } ], "tags": [ "Experiments" ], "security": [ { "ApiKeyAuth": [] }, { "ApiKeyAuth": [] } ], "responses": { "200": { "content": { "application/json": { "schema": { "type": "array", "items": { "$ref": "#/components/schemas/Rater" } } } }, "description": "" } } } }, "/projects/{project_id}/experiments/{experiment_id}/ratings/": { "get": { "operationId": "experiments_ratings_list", "description": "Returns a list of ratings belonging to the given experiment.", "summary": "List all ratings", "parameters": [ { "in": "path", "name": "experiment_id", "schema": { "type": "string", "pattern": "^[a-zA-Z0-9]+$" }, "required": true }, { "in": "path", "name": "project_id", "schema": { "type": "string", "pattern": "^[a-zA-Z0-9]+$" }, "required": true }, { "in": "query", "name": "training", "schema": { "type": "string", "enum": [ "false", "true" ] }, "description": "Whether to include training data." } ], "tags": [ "Experiments" ], "security": [ { "ApiKeyAuth": [] }, { "ApiKeyAuth": [] } ], "responses": { "200": { "content": { "application/json": { "schema": { "type": "array", "items": { "$ref": "#/components/schemas/RatingDetail" } } } }, "description": "" } } } }, "/projects/{project_id}/experiments/{experiment_id}/results/": { "get": { "operationId": "experiments_results_list", "description": "Returns a list of metrics derived from the results of an experiment.", "summary": "List all results", "parameters": [ { "in": "path", "name": "experiment_id", "schema": { "type": "string", "pattern": "^[a-zA-Z0-9]+$" }, "required": true }, { "in": "query", "name": "metric", "schema": { "type": "string", "enum": [ "bayesian_elo", "elo", "mean", "percent_approved", "percent_chosen", "stats", "win_loss" ] }, "description": "Filter by this type of metric." }, { "in": "path", "name": "project_id", "schema": { "type": "string", "pattern": "^[a-zA-Z0-9]+$" }, "required": true }, { "in": "query", "name": "source", "schema": { "type": "string" }, "description": "An optional comma-separated list of job or experiment IDs." } ], "tags": [ "Experiments" ], "security": [ { "ApiKeyAuth": [] }, { "ApiKeyAuth": [] } ], "responses": { "200": { "content": { "application/json": { "schema": { "type": "array", "items": { "$ref": "#/components/schemas/Results" } } } }, "description": "" } } } }, "/projects/{project_id}/experiments/{experiment_id}/sessions/": { "get": { "operationId": "experiments_sessions_list", "description": "Returns a list of sessions belonging to the given experiment.", "summary": "List all sessions", "parameters": [ { "in": "path", "name": "experiment_id", "schema": { "type": "string", "pattern": "^[a-zA-Z0-9]+$" }, "required": true }, { "in": "path", "name": "project_id", "schema": { "type": "string", "pattern": "^[a-zA-Z0-9]+$" }, "required": true }, { "in": "query", "name": "training", "schema": { "type": "string", "enum": [ "false", "true" ] }, "description": "Whether to include training data." } ], "tags": [ "Experiments" ], "security": [ { "ApiKeyAuth": [] }, { "ApiKeyAuth": [] } ], "responses": { "200": { "content": { "application/json": { "schema": { "type": "array", "items": { "$ref": "#/components/schemas/Session" } } } }, "description": "" } } } }, "/projects/{project_id}/experiments/{experiment_id}/slates/": { "get": { "operationId": "experiments_slates_list", "description": "Returns a list of slates and ratings belonging to the given experiment.", "summary": "List all slates", "parameters": [ { "in": "path", "name": "experiment_id", "schema": { "type": "string", "pattern": "^[a-zA-Z0-9]+$" }, "required": true }, { "in": "path", "name": "project_id", "schema": { "type": "string", "pattern": "^[a-zA-Z0-9]+$" }, "required": true }, { "in": "query", "name": "training", "schema": { "type": "string", "enum": [ "false", "true" ] }, "description": "Whether to include training data." } ], "tags": [ "Experiments" ], "security": [ { "ApiKeyAuth": [] }, { "ApiKeyAuth": [] } ], "responses": { "200": { "content": { "application/json": { "schema": { "type": "array", "items": { "$ref": "#/components/schemas/Slate" } } } }, "description": "" } } } }, "/projects/{project_id}/metrics/": { "get": { "operationId": "metrics_list", "description": "Returns a list of metrics corresponding to various sources.", "summary": "List metrics", "parameters": [ { "in": "query", "name": "metric", "schema": { "type": "string", "enum": [ "bayesian_elo", "elo", "mean", "percent_approved", "percent_chosen", "stats", "win_loss" ] }, "description": "Filter by this type of metric." }, { "in": "path", "name": "project_id", "schema": { "type": "string", "pattern": "^[a-zA-Z0-9]+$" }, "required": true }, { "in": "query", "name": "source", "schema": { "type": "string" }, "description": "A comma-separated list of job, experiment, or leaderboard IDs." } ], "tags": [ "Metrics" ], "security": [ { "ApiKeyAuth": [] }, { "ApiKeyAuth": [] } ], "responses": { "200": { "content": { "application/json": { "schema": { "type": "array", "items": { "$ref": "#/components/schemas/Metric" } } } }, "description": "" } } } }, "/projects/{project_id}/rubrics/": { "get": { "operationId": "rubrics_list", "description": "Returns a list of rubrics belonging to the given project.", "summary": "List all rubrics", "parameters": [ { "in": "path", "name": "project_id", "schema": { "type": "string", "pattern": "^[a-zA-Z0-9]+$" }, "required": true } ], "tags": [ "Rubrics" ], "security": [ { "ApiKeyAuth": [] }, { "ApiKeyAuth": [] } ], "responses": { "200": { "content": { "application/json": { "schema": { "type": "array", "items": { "$ref": "#/components/schemas/Rubric" } } } }, "description": "" } } } }, "/projects/{project_id}/rubrics/{rubric_id}/": { "get": { "operationId": "rubrics_retrieve", "description": "Returns data of all leaderboards belonging to a rubric.", "summary": "Retrieve a rubric", "parameters": [ { "in": "path", "name": "project_id", "schema": { "type": "string", "pattern": "^[a-zA-Z0-9]+$" }, "required": true }, { "in": "path", "name": "rubric_id", "schema": { "type": "string", "pattern": "^[a-zA-Z0-9]+$" }, "required": true } ], "tags": [ "Rubrics" ], "security": [ { "ApiKeyAuth": [] }, { "ApiKeyAuth": [] } ], "responses": { "200": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/RubricDetail" } } }, "description": "" } } } }, "/projects/{project_id}/rubrics/{rubric_id}/leaderboards/": { "get": { "operationId": "rubrics_leaderboards_list", "description": "Lists all leaderboards belonging to the given rubric.", "summary": "List all leaderboards", "parameters": [ { "in": "path", "name": "project_id", "schema": { "type": "string", "pattern": "^[a-zA-Z0-9]+$" }, "required": true }, { "in": "path", "name": "rubric_id", "schema": { "type": "string", "pattern": "^[a-zA-Z0-9]+$" }, "required": true } ], "tags": [ "Rubrics" ], "security": [ { "ApiKeyAuth": [] }, { "ApiKeyAuth": [] } ], "responses": { "200": { "content": { "application/json": { "schema": { "type": "array", "items": { "$ref": "#/components/schemas/LeaderboardDetail" } } } }, "description": "" } } } }, "/projects/{project_id}/rubrics/{rubric_id}/leaderboards/{leaderboard_id}/": { "get": { "operationId": "rubrics_leaderboards_retrieve", "description": "Returns only the data of a specific leaderboard.", "summary": "Retrieve a leaderboard", "parameters": [ { "in": "path", "name": "leaderboard_id", "schema": { "type": "string", "pattern": "^[a-zA-Z0-9]+$" }, "required": true }, { "in": "path", "name": "project_id", "schema": { "type": "string", "pattern": "^[a-zA-Z0-9]+$" }, "required": true }, { "in": "path", "name": "rubric_id", "schema": { "type": "string", "pattern": "^[a-zA-Z0-9]+$" }, "required": true } ], "tags": [ "Rubrics" ], "security": [ { "ApiKeyAuth": [] }, { "ApiKeyAuth": [] } ], "responses": { "200": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/LeaderboardDetail" } } }, "description": "" } } } }, "/projects/{project_id}/webhooks/": { "get": { "operationId": "webhooks_list", "description": "Returns a list of webhooks belonging to the given project.", "summary": "List all webhooks", "parameters": [ { "in": "path", "name": "project_id", "schema": { "type": "string", "pattern": "^[a-zA-Z0-9]+$" }, "required": true } ], "tags": [ "Webhooks" ], "security": [ { "ApiKeyAuth": [] }, { "ApiKeyAuth": [] } ], "responses": { "200": { "content": { "application/json": { "schema": { "type": "array", "items": { "$ref": "#/components/schemas/Webhook" } } } }, "description": "" } } }, "post": { "operationId": "webhooks_create", "description": "Creates a new webhook subscription. The project's webhook secret will be auto-created if one does not already exist.", "summary": "Create a webhook", "parameters": [ { "in": "path", "name": "project_id", "schema": { "type": "string", "pattern": "^[a-zA-Z0-9]+$" }, "required": true } ], "tags": [ "Webhooks" ], "requestBody": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/WebhookCreate" } }, "application/x-www-form-urlencoded": { "schema": { "$ref": "#/components/schemas/WebhookCreate" } }, "multipart/form-data": { "schema": { "$ref": "#/components/schemas/WebhookCreate" } } }, "required": true }, "security": [ { "ApiKeyAuth": [] }, { "ApiKeyAuth": [] } ], "responses": { "201": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Webhook" } } }, "description": "" } } } }, "/projects/{project_id}/webhooks/{webhook_id}/": { "get": { "operationId": "webhooks_retrieve", "description": "Returns detailed information for a single webhook.", "summary": "Retrieve a webhook", "parameters": [ { "in": "path", "name": "project_id", "schema": { "type": "string", "pattern": "^[a-zA-Z0-9]+$" }, "required": true }, { "in": "path", "name": "webhook_id", "schema": { "type": "string", "pattern": "^[a-zA-Z0-9]+$" }, "required": true } ], "tags": [ "Webhooks" ], "security": [ { "ApiKeyAuth": [] }, { "ApiKeyAuth": [] } ], "responses": { "200": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Webhook" } } }, "description": "" } } }, "patch": { "operationId": "webhooks_partial_update", "description": "Updates one or more fields on an existing webhook.", "summary": "Update a webhook", "parameters": [ { "in": "path", "name": "project_id", "schema": { "type": "string", "pattern": "^[a-zA-Z0-9]+$" }, "required": true }, { "in": "path", "name": "webhook_id", "schema": { "type": "string", "pattern": "^[a-zA-Z0-9]+$" }, "required": true } ], "tags": [ "Webhooks" ], "requestBody": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/PatchedWebhookUpdate" } }, "application/x-www-form-urlencoded": { "schema": { "$ref": "#/components/schemas/PatchedWebhookUpdate" } }, "multipart/form-data": { "schema": { "$ref": "#/components/schemas/PatchedWebhookUpdate" } } } }, "security": [ { "ApiKeyAuth": [] }, { "ApiKeyAuth": [] } ], "responses": { "200": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Webhook" } } }, "description": "" } } }, "delete": { "operationId": "webhooks_destroy", "description": "Permanently deletes a webhook subscription.", "summary": "Delete a webhook", "parameters": [ { "in": "path", "name": "project_id", "schema": { "type": "string", "pattern": "^[a-zA-Z0-9]+$" }, "required": true }, { "in": "path", "name": "webhook_id", "schema": { "type": "string", "pattern": "^[a-zA-Z0-9]+$" }, "required": true } ], "tags": [ "Webhooks" ], "security": [ { "ApiKeyAuth": [] }, { "ApiKeyAuth": [] } ], "responses": { "204": { "description": "No response body" } } } }, "/projects/{project_id}/webhooks/events/": { "get": { "operationId": "webhooks_events_list", "description": "Returns the list of event types that webhooks can subscribe to. Each event type follows the format `resource.action` (e.g., `dataset.status_changed`, `job.completed`).", "summary": "List available webhook events", "parameters": [ { "in": "path", "name": "project_id", "schema": { "type": "string", "pattern": "^[a-zA-Z0-9]+$" }, "required": true } ], "tags": [ "Webhooks" ], "security": [ { "ApiKeyAuth": [] }, { "ApiKeyAuth": [] } ], "responses": { "200": { "content": { "application/json": { "schema": { "type": "array", "items": { "type": "array", "items": { "type": "string", "enum": [ "dataset.status_changed", "job.completed", "session.completed" ] } } } } }, "description": "" } } } }, "/projects/{project_id}/webhooks/secrets/": { "get": { "operationId": "webhooks_secrets_list", "description": "Returns all webhook signing secrets for this project, newest first. The most recent secret is automatically assigned to new webhooks. Use the secret value to verify webhook payload signatures using HMAC-SHA256.", "summary": "List webhook secrets", "parameters": [ { "in": "path", "name": "project_id", "schema": { "type": "string", "pattern": "^[a-zA-Z0-9]+$" }, "required": true } ], "tags": [ "Webhooks" ], "security": [ { "ApiKeyAuth": [] }, { "ApiKeyAuth": [] } ], "responses": { "200": { "content": { "application/json": { "schema": { "type": "array", "items": { "$ref": "#/components/schemas/WebhookSecret" } } } }, "description": "" } } }, "post": { "operationId": "webhooks_secrets_create", "description": "Generates a new webhook signing secret for this project. New webhooks will automatically use this secret for signing payloads. Existing webhooks continue using the secret they were created with.", "summary": "Create a new webhook secret", "parameters": [ { "in": "path", "name": "project_id", "schema": { "type": "string", "pattern": "^[a-zA-Z0-9]+$" }, "required": true } ], "tags": [ "Webhooks" ], "security": [ { "ApiKeyAuth": [] }, { "ApiKeyAuth": [] } ], "responses": { "201": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/WebhookSecret" } } }, "description": "" } } } }, "/projects/{project_id}/webhooks/secrets/{secret_id}/": { "get": { "operationId": "webhooks_secrets_retrieve", "description": "Returns a single webhook secret by ID, including the secret value and the IDs of all webhooks using it.", "summary": "Retrieve a webhook secret", "parameters": [ { "in": "path", "name": "project_id", "schema": { "type": "string", "pattern": "^[a-zA-Z0-9]+$" }, "required": true }, { "in": "path", "name": "secret_id", "schema": { "type": "string", "pattern": "^[a-zA-Z0-9]+$" }, "required": true } ], "tags": [ "Webhooks" ], "security": [ { "ApiKeyAuth": [] }, { "ApiKeyAuth": [] } ], "responses": { "200": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/WebhookSecret" } } }, "description": "" } } }, "delete": { "operationId": "webhooks_secrets_destroy", "description": "Permanently deletes a webhook secret. This operation will fail if any webhooks are still using this secret. To delete a secret, first reassign or remove all webhooks that reference it.", "summary": "Delete a webhook secret", "parameters": [ { "in": "path", "name": "project_id", "schema": { "type": "string", "pattern": "^[a-zA-Z0-9]+$" }, "required": true }, { "in": "path", "name": "secret_id", "schema": { "type": "string", "pattern": "^[a-zA-Z0-9]+$" }, "required": true } ], "tags": [ "Webhooks" ], "security": [ { "ApiKeyAuth": [] }, { "ApiKeyAuth": [] } ], "responses": { "204": { "description": "No response body" } } } }, "/projects/{project_id}/workflows/create/": { "post": { "operationId": "workflows_create_create", "description": "Creates a new draft job with the specified configuration.", "summary": "Create a job", "parameters": [ { "in": "path", "name": "project_id", "schema": { "type": "string", "pattern": "^[a-zA-Z0-9]+$" }, "required": true } ], "tags": [ "Workflows" ], "requestBody": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/WorkflowCreate" }, "examples": { "Request": { "value": { "experiment": { "name": "My experiment", "type": "acr_video", "language": "en", "title": "Video quality assessment", "question": "How would you rate the quality of this video?", "description": "", "introduction": "", "config": { "acrVideo": { "hasAudio": false, "categories": [ { "score": 1, "label": "Bad" }, { "score": 2, "label": "Poor" }, { "score": 3, "label": "Fair" }, { "score": 4, "label": "Good" }, { "score": 5, "label": "Excellent" } ], "fullscreen": false, "allowMobile": false, "minPlayDuration": 1000.0 } } }, "job": { "launch": false, "rater_pool_id": "b22bo3b", "max_repetitions": 1, "min_rest_time": 0, "strategy": "custom" }, "sessions": [ [ { "stimuli": [ "https://mabyduck.s3.nl-ams.scw.cloud/ducks/h264.mp4" ] }, { "stimuli": [ "https://mabyduck.s3.nl-ams.scw.cloud/zebras/h265.mp4" ] } ] ] } } } }, "application/x-www-form-urlencoded": { "schema": { "$ref": "#/components/schemas/WorkflowCreate" } }, "multipart/form-data": { "schema": { "$ref": "#/components/schemas/WorkflowCreate" } } }, "required": true }, "security": [ { "ApiKeyAuth": [] }, { "ApiKeyAuth": [] } ], "responses": { "201": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Workflow" }, "examples": { "Response": { "value": { "dataset": { "id": "d1shw0u", "name": "External dataset", "date_created": "2026-03-18T22:28:34.237961Z", "date_last_used": "2026-03-18T22:28:34.237974Z", "is_deleted": false, "status": "draft" }, "experiment": { "id": "x1ty1ni", "name": "My experiment", "type": "acr_video", "language": "en", "dataset_ids": [ "d1shw0u" ], "training_dataset_ids": [], "job_ids": [ "j1xh94y" ], "hero_media_url": null, "hero_caption": null, "date_created": "2026-03-18T22:28:34.349141Z", "date_updated": "2026-03-18T22:28:34.349148Z", "config": { "acrVideo": { "hasAudio": false, "categories": [ { "score": 1, "label": "Bad" }, { "score": 2, "label": "Poor" }, { "score": 3, "label": "Fair" }, { "score": 4, "label": "Good" }, { "score": 5, "label": "Excellent" } ], "fullscreen": false, "allowMobile": false, "minPlayDuration": 1000.0 } } }, "job": { "id": "j1xh94y", "experiment_id": "x1ty1ni", "num_sessions": 1, "num_comparisons": 2, "num_training": 0, "date_created": "2026-03-18T22:28:34.478841Z", "date_updated": "2026-03-18T22:28:34.478847Z", "date_launched": null, "rater_pool": { "id": "b22bo3b", "kind": 100, "name": "Project members", "label": "" }, "status": "draft", "config": { "strategy": { "type": "custom", "id": "c2pzig6" } } } } } } } }, "description": "" } }, "x-codeSamples": [ { "lang": "Python", "label": "Python", "source": "import requests\n\nurl = f\"https://api.mabyduck.com/projects/{project_id}/workflows/create/\"\nheaders = {\n \"Content-Type\": \"application/json\",\n \"Authorization\": \"Api-Key \"\n}\npayload = {\n \"experiment\": {\n \"name\": \"My experiment\",\n \"type\": \"acr_video\",\n \"language\": \"en\",\n \"title\": \"Video quality assessment\",\n \"question\": \"How would you rate the quality of this video?\",\n \"description\": \"\",\n \"introduction\": \"\",\n \"config\": {\n \"acrVideo\": {\n \"hasAudio\": False,\n \"categories\": [\n {\"score\": 1, \"label\": \"Bad\"},\n {\"score\": 2, \"label\": \"Poor\"},\n {\"score\": 3, \"label\": \"Fair\"},\n {\"score\": 4, \"label\": \"Good\"},\n {\"score\": 5, \"label\": \"Excellent\"},\n ],\n \"fullscreen\": False,\n \"allowMobile\": False,\n \"minPlayDuration\": 1000.0\n }\n }\n },\n \"job\": {\n \"launch\": False,\n \"rater_pool_id\": \"b22bo3b\",\n \"max_repetitions\": 1,\n \"min_rest_time\": 0,\n \"strategy\": \"custom\"\n },\n \"sessions\": [\n [\n {\n \"stimuli\": [\n \"https://mabyduck-datasets-development.s3.nl-ams.scw.cloud/ducks/h264%40crf%3D48.mp4\",\n ]\n },\n {\n \"stimuli\": [\n \"https://mabyduck-datasets-development.s3.nl-ams.scw.cloud/ducks/reference.mp4\",\n ]\n },\n ]\n ]\n}\n\n# Create an experiment and a job with a custom sampling strategy\nresponse = requests.post(url, json=payload, headers=headers)\ndata = response.json()\n" } ] } } }, "components": { "schemas": { "BillingEvent": { "type": "object", "properties": { "kind": { "$ref": "#/components/schemas/BillingEventKind" }, "amount_credits": { "type": "string", "format": "decimal", "pattern": "^-?\\d{0,12}(?:\\.\\d{0,2})?$", "readOnly": true }, "remaining_credits": { "type": "string", "format": "decimal", "pattern": "^-?\\d{0,12}(?:\\.\\d{0,2})?$", "readOnly": true }, "currency": { "type": "string", "readOnly": true }, "date": { "type": "string", "format": "date-time", "readOnly": true }, "note": { "type": "string", "maxLength": 255 }, "creator": { "allOf": [ { "$ref": "#/components/schemas/Creator" } ], "readOnly": true }, "api_key": { "allOf": [ { "$ref": "#/components/schemas/ProjectAPIKey" } ], "readOnly": true } }, "required": [ "amount_credits", "api_key", "creator", "currency", "date", "kind", "remaining_credits" ] }, "BillingEventKind": { "enum": [ "manual_adjustment", "invoice_paid", "pre_paid_credit_purchase", "credit_expiration", "billing_change", "credit_usage", "currency_change", "refund" ], "type": "string", "description": "* `manual_adjustment` - Manual adjustment\n* `invoice_paid` - Invoice paid\n* `pre_paid_credit_purchase` - Credit purchase\n* `credit_expiration` - Credit expiration\n* `billing_change` - Billing change\n* `credit_usage` - Credit usage\n* `currency_change` - Currency change\n* `refund` - Refund" }, "BillingEventProject": { "type": "object", "description": "Serialize project attribution for an organization billing event.", "properties": { "id": { "type": "string", "readOnly": true }, "name": { "type": "string", "maxLength": 128 } }, "required": [ "id" ] }, "CreateExperiment": { "type": "object", "description": "Serializer for creating draft experiments via API.", "properties": { "name": { "type": "string", "description": "A description identifying the experiment.", "maxLength": 128 }, "type": { "allOf": [ { "$ref": "#/components/schemas/CreateExperimentTypeEnum" } ], "description": "Type of the experiment (e.g., \"mushra\", \"pairwise_image\")" }, "language": { "allOf": [ { "$ref": "#/components/schemas/LanguageEnum" } ], "description": "The language used for the experiment's interface.\n\n* `ar` - Arabic\n* `de` - German\n* `en` - English\n* `es` - Spanish\n* `fr` - French\n* `hi` - Hindi\n* `it` - Italian\n* `ja` - Japanese\n* `ko` - Korean\n* `pl` - Polish\n* `pt` - Portuguese\n* `vi` - Vietnamese\n* `zh-hans` - Chinese (simplified)" }, "datasets": { "type": "array", "items": { "type": "string" }, "description": "List of dataset IDs to use in this experiment" }, "training_datasets": { "type": "array", "items": { "type": "string" }, "description": "List of training dataset IDs (optional)" }, "golden_datasets": { "type": "array", "items": { "type": "string" }, "description": "List of golden dataset IDs (optional)" }, "title": { "type": [ "string", "null" ], "description": "A short title displayed at the top of the experiment screen.", "maxLength": 128 }, "question": { "type": [ "string", "null" ], "description": "A question or instructions displayed at the top of the experiment screen.", "maxLength": 512 }, "description": { "type": [ "string", "null" ], "description": "An optional longer explanation of how to evaluate the data." }, "introduction": { "type": [ "string", "null" ], "description": "A brief introduction explaining how to participate in the experiment. Leave empty to use default introduction.", "maxLength": 2048 }, "hero_media_url": { "type": "string", "description": "URL of a custom image or video to replace the default screenshot.", "maxLength": 1024 }, "hero_caption": { "type": [ "string", "null" ], "description": "Caption displayed below the custom hero image/video.", "maxLength": 256 }, "config": { "$ref": "#/components/schemas/ExperimentConfig" } }, "required": [ "datasets", "name", "type" ] }, "CreateExperimentTypeEnum": { "enum": [ "acr_audio", "acr_video", "acr_image", "multi_acr_embed", "multi_acr_video", "binary_audio", "binary_image", "binary_video", "inpainting_image", "mushra", "pairwise_audio", "pairwise_image", "pairwise_video", "survey_audio", "survey_embed", "survey_video", "survey_image", "pre_screen_audio", "pre_screen_image", "rank_audio" ], "type": "string" }, "CreateJob": { "type": "object", "description": "Serializer for creating draft jobs via API.", "properties": { "rater_pool": { "type": "string", "description": "ID of the rater pool to use for this job" }, "rater_pool_id": { "type": "string", "description": "ID of the rater pool to use for this job" }, "num_sessions": { "type": "integer", "maximum": 10000, "minimum": 1, "description": "Number of sessions (times the task is performed)" }, "num_comparisons": { "type": "integer", "minimum": 1, "description": "Number of comparisons per session" }, "num_training": { "type": "integer", "minimum": 0, "default": 0, "description": "Number of training comparisons" }, "num_golden": { "type": "integer", "minimum": 0, "default": 0, "description": "Number of golden questions" }, "max_repetitions": { "type": "integer", "maximum": 2147483647, "minimum": 0, "title": "Maximum number of repetitions", "description": "How often an individual rater is allowed to repeat the task." }, "min_rest_time": { "type": "integer", "maximum": 2147483647, "minimum": 0, "title": "Minimum rest time", "description": "How many minutes should pass before a rater can repeat the task." }, "strategy": { "default": "randomized", "description": "Strategy for presenting stimuli. Either a strategy type string (e.g., \"randomized\") or a full configuration object (e.g., {\"type\": \"randomized\", \"reference\": \"exclude\"})." }, "strategy_id": { "type": "string", "description": "ID of a custom sampling strategy to use" }, "training_strategy_id": { "type": "string", "description": "ID of a custom sampling strategy to use" }, "note": { "type": [ "string", "null" ], "description": "An optional note for this job. Not visible to raters." }, "hourly_rate_gbp": { "type": "string", "format": "decimal", "pattern": "^-?\\d{0,2}(?:\\.\\d{0,2})?$", "description": "Hourly rate in GBP to pay raters." } }, "required": [ "num_comparisons", "num_sessions" ] }, "Creator": { "type": "object", "properties": { "id": { "type": "string", "readOnly": true }, "email": { "type": "string", "format": "email", "title": "Email address", "maxLength": 254 }, "name": { "type": "string", "title": "Full name", "maxLength": 256 } }, "required": [ "email", "id", "name" ] }, "Dataset": { "type": "object", "properties": { "id": { "type": "string" }, "name": { "type": "string", "description": "A short name identifying the dataset.", "maxLength": 64 }, "date_created": { "type": "string", "format": "date-time", "readOnly": true }, "date_last_used": { "type": "string", "format": "date-time", "readOnly": true, "title": "Last used" }, "is_deleted": { "type": "boolean", "description": "The files in this dataset have been deleted." }, "status": { "allOf": [ { "$ref": "#/components/schemas/DatasetLifecycleStatus" } ], "description": "Current status of the dataset.\n\n* `draft` - Draft\n* `enqueued` - Enqueued\n* `processing` - Processing\n* `ready` - Ready\n* `error` - Error" } }, "required": [ "date_created", "date_last_used", "id", "name" ] }, "DatasetDetail": { "type": "object", "description": "Serializer for the Dataset.meta field.", "properties": { "id": { "type": "string" }, "name": { "type": "string", "description": "A short name identifying the dataset.", "maxLength": 64 }, "date_created": { "type": "string", "format": "date-time", "readOnly": true }, "date_last_used": { "type": "string", "format": "date-time", "readOnly": true, "title": "Last used" }, "is_deleted": { "type": "boolean", "description": "The files in this dataset have been deleted." }, "status": { "allOf": [ { "$ref": "#/components/schemas/DatasetLifecycleStatus" } ], "description": "Current status of the dataset.\n\n* `draft` - Draft\n* `enqueued` - Enqueued\n* `processing` - Processing\n* `ready` - Ready\n* `error` - Error" }, "config": {}, "stimuli": { "type": "array", "items": { "$ref": "#/components/schemas/Stimulus" }, "readOnly": true } }, "required": [ "date_created", "date_last_used", "id", "name", "stimuli" ] }, "DatasetLifecycleStatus": { "enum": [ "draft", "enqueued", "processing", "ready", "error" ], "type": "string", "description": "* `draft` - Draft\n* `enqueued` - Enqueued\n* `processing` - Processing\n* `ready` - Ready\n* `error` - Error" }, "DatasetStatus": { "type": "object", "description": "Serializer for dataset status information.", "properties": { "status": { "allOf": [ { "$ref": "#/components/schemas/DatasetLifecycleStatus" } ], "readOnly": true, "description": "Current status of the dataset.\n\n* `draft` - Draft\n* `enqueued` - Enqueued\n* `processing` - Processing\n* `ready` - Ready\n* `error` - Error" }, "error": { "type": [ "string", "null" ], "readOnly": true, "description": "Error message if processing failed." } }, "required": [ "error", "status" ] }, "EventsEnum": { "enum": [ "dataset.status_changed", "job.completed", "session.completed" ], "type": "string", "description": "* `dataset.status_changed` - Dataset status changed\n* `job.completed` - Job completed\n* `session.completed` - Session completed" }, "Experiment": { "type": "object", "properties": { "id": { "type": "string" }, "name": { "type": "string", "description": "A description identifying the experiment.", "maxLength": 128 }, "type": { "type": "string" }, "language": { "allOf": [ { "$ref": "#/components/schemas/LanguageEnum" } ], "description": "The language used for the experiment's interface.\n\n* `ar` - Arabic\n* `de` - German\n* `en` - English\n* `es` - Spanish\n* `fr` - French\n* `hi` - Hindi\n* `it` - Italian\n* `ja` - Japanese\n* `ko` - Korean\n* `pl` - Polish\n* `pt` - Portuguese\n* `vi` - Vietnamese\n* `zh-hans` - Chinese (simplified)" }, "dataset_ids": { "type": "array", "items": { "type": "string" }, "readOnly": true }, "training_dataset_ids": { "type": "array", "items": { "type": "string" }, "readOnly": true }, "golden_dataset_ids": { "type": "array", "items": { "type": "string" }, "readOnly": true }, "job_ids": { "type": "array", "items": { "type": "string" }, "readOnly": true }, "hero_media_url": { "type": [ "string", "null" ], "readOnly": true }, "hero_caption": { "type": [ "string", "null" ], "description": "Caption displayed below the custom hero image/video.", "maxLength": 256 }, "date_created": { "type": "string", "format": "date-time", "readOnly": true }, "date_updated": { "type": "string", "format": "date-time", "readOnly": true, "title": "Last updated" }, "config": {} }, "required": [ "dataset_ids", "date_created", "date_updated", "golden_dataset_ids", "hero_media_url", "id", "job_ids", "name", "training_dataset_ids", "type" ] }, "ExperimentConfig": { "type": [ "object", "null" ], "additionalProperties": false, "properties": { "displayHealth": { "type": "boolean", "description": "Whether to expose a rater's performance to the rater." }, "intro": { "type": "object", "additionalProperties": false, "properties": { "maxWidth": { "type": "integer", "minimum": 0, "description": "Maximum width of the introduction text in pixels." }, "hero": { "type": "object", "additionalProperties": false, "properties": { "fullscreen": { "type": "boolean" } } } } }, "acrAudio": { "type": "object", "additionalProperties": false, "required": [ "categories" ], "properties": { "allowMobile": { "type": "boolean" }, "hiddenReference": { "type": "boolean", "description": "Whether to include a hidden reference among the stimuli to rate." }, "showReference": { "type": "boolean", "description": "Whether to show a reference stimulus." }, "showWaveform": { "type": "string", "enum": [ "stimulus", "reference" ], "description": "Determines how the waveform of the condition is rendered." }, "minPlayDuration": { "type": "number", "minimum": 0, "maximum": 60000, "description": "Force raters to watch or listen to at least this many milliseconds." }, "categories": { "type": "array", "minItems": 2, "maxItems": 10, "uniqueItems": true, "items": { "type": "object", "additionalProperties": false, "required": [ "label", "score" ], "properties": { "score": { "type": "integer" }, "label": { "type": "string" }, "description": { "type": "string", "maxLength": 32 } } } } } }, "acrImage": { "type": "object", "additionalProperties": false, "required": [ "categories" ], "properties": { "allowCropRefresh": { "type": "boolean", "description": "Allow raters to request different random crops." }, "allowMobile": { "type": "boolean" }, "scaleToFit": { "type": "boolean" }, "responseType": { "type": "string", "enum": [ "discrete", "continuous" ] }, "showReference": { "type": "boolean", "description": "Whether to show a reference image alongside the test sample." }, "mode": { "type": "string", "enum": [ "side-by-side", "flip-between" ], "description": "Controls how the reference and test sample are presented." }, "categories": { "type": "array", "minItems": 2, "maxItems": 10, "uniqueItems": true, "items": { "type": "object", "additionalProperties": false, "required": [ "label", "score" ], "properties": { "score": { "type": "integer" }, "label": { "type": "string" }, "description": { "type": "string", "maxLength": 32 } } } }, "hiddenReference": { "type": "boolean" }, "maxCropSize": { "type": "integer", "minimum": 32, "description": "Raters will view image crops instead of the full image." }, "minViewDuration": { "type": "number", "minimum": 0, "maximum": 5000, "description": "Force raters to view the image/video for at least this many milliseconds." } } }, "acrVideo": { "type": "object", "additionalProperties": false, "required": [ "categories" ], "properties": { "allowMobile": { "type": "boolean" }, "categories": { "type": "array", "minItems": 2, "maxItems": 10, "uniqueItems": true, "items": { "type": "object", "additionalProperties": false, "required": [ "label", "score" ], "properties": { "score": { "type": "integer" }, "label": { "type": "string" }, "description": { "type": "string", "maxLength": 32 } } } }, "hasAudio": { "type": "boolean" }, "hiddenReference": { "type": "boolean", "description": "Whether to include a hidden reference among the stimuli to rate." }, "showReference": { "type": "boolean", "description": "Whether to show a reference alongside the test sample." }, "mode": { "type": "string", "enum": [ "side-by-side", "flip-between" ], "description": "Controls how the reference and test sample are presented." }, "videoPlayer": { "type": "string", "enum": [ "sequential", "parallel" ] }, "fullscreen": { "type": "boolean", "description": "Require raters to view the video in fullscreen mode." }, "allowSeeking": { "type": "boolean", "description": "Whether raters can seek through the video." }, "minPlayDuration": { "type": "number", "minimum": 0, "maximum": 60000, "description": "Force raters to watch or listen to at least this many milliseconds." } } }, "multiAcrVideo": { "type": "object", "additionalProperties": false, "required": [ "questions", "categories" ], "properties": { "questions": { "type": "array", "minItems": 1, "maxItems": 10, "uniqueItems": true, "items": { "type": "object", "additionalProperties": false, "required": [ "statement", "dimension" ], "properties": { "statement": { "type": "string" }, "dimension": { "type": "string", "maxLength": 32 } } } }, "categories": { "type": "array", "minItems": 2, "maxItems": 10, "uniqueItems": true, "items": { "type": "object", "additionalProperties": false, "required": [ "label", "score" ], "properties": { "score": { "type": "integer" }, "label": { "type": "string" }, "description": { "type": "string", "maxLength": 32 } } } }, "hasAudio": { "type": "boolean" }, "fullscreen": { "type": "boolean", "description": "Require raters to view the video in fullscreen mode." }, "minPlayDuration": { "type": "number", "minimum": 0, "maximum": 60000, "description": "Force raters to watch or listen to at least this many milliseconds." }, "allowSeeking": { "type": "boolean", "description": "Whether raters can seek through the video." } } }, "multiAcrEmbed": { "type": "object", "additionalProperties": false, "required": [ "questions", "categories" ], "properties": { "questions": { "type": "array", "minItems": 1, "maxItems": 10, "uniqueItems": true, "items": { "type": "object", "additionalProperties": false, "required": [ "statement", "dimension" ], "properties": { "statement": { "type": "string" }, "dimension": { "type": "string", "maxLength": 32 } } } }, "categories": { "type": "array", "minItems": 2, "maxItems": 10, "uniqueItems": true, "items": { "type": "object", "additionalProperties": false, "required": [ "label", "score" ], "properties": { "score": { "type": "integer" }, "label": { "type": "string" }, "description": { "type": "string", "maxLength": 32 } } } }, "hasAudio": { "type": "boolean" }, "iframeSize": { "type": [ "object", "null" ], "additionalProperties": false, "required": [ "width", "height" ], "properties": { "width": { "type": "integer" }, "height": { "type": "integer" } } }, "interactionDuration": { "type": [ "object", "null" ], "additionalProperties": false, "required": [ "min", "max" ], "description": "The minimum and maximum time a rater is expected to spend interacting with each embedded stimulus, in milliseconds. Used to estimate session completion time.", "properties": { "min": { "type": "number", "minimum": 10000, "maximum": 1200000 }, "max": { "type": "number", "minimum": 10000, "maximum": 1200000 } } } } }, "binaryAudio": { "type": "object", "additionalProperties": false, "required": [ "labels" ], "properties": { "minPlayDuration": { "type": "number", "minimum": 0, "maximum": 60000, "description": "Force raters to watch or listen to at least this many milliseconds." }, "labels": { "type": "object", "required": [ "real", "fake" ], "properties": { "real": { "type": "string" }, "fake": { "type": "string" } } }, "referenceProbability": { "type": "number", "minimum": 0, "maximum": 1, "description": "Probability of showing the reference." }, "giveFeedback": { "type": "boolean", "description": "Play a sound to indicate to raters if a decision was correct." } } }, "binaryImage": { "type": "object", "additionalProperties": false, "required": [ "labels" ], "properties": { "allowCropRefresh": { "type": "boolean", "description": "Allow raters to request different random crops." }, "maxCropSize": { "type": "integer", "minimum": 32, "description": "Raters will view image crops instead of the full image." }, "labels": { "type": "object", "required": [ "real", "fake" ], "properties": { "real": { "type": "string" }, "fake": { "type": "string" } } }, "minViewDuration": { "type": "number", "minimum": 0, "maximum": 5000, "description": "Force raters to view the image/video for at least this many milliseconds." }, "referenceProbability": { "type": "number", "minimum": 0, "maximum": 1, "description": "Probability of showing the reference." }, "giveFeedback": { "type": "boolean", "description": "Play a sound to indicate to raters if a decision was correct." } } }, "binaryVideo": { "type": "object", "additionalProperties": false, "required": [ "labels" ], "properties": { "hasAudio": { "type": "boolean" }, "labels": { "type": "object", "required": [ "real", "fake" ], "properties": { "real": { "type": "string" }, "fake": { "type": "string" } } }, "icons": { "type": "object", "properties": { "real": { "type": [ "string", "null" ] }, "fake": { "type": [ "string", "null" ] } } }, "minPlayDuration": { "type": "number", "minimum": 0, "maximum": 60000, "description": "Force raters to watch or listen to at least this many milliseconds." }, "giveFeedback": { "type": "boolean", "description": "Play a sound to indicate to raters if a decision was correct." }, "referenceProbability": { "type": "number", "minimum": 0, "maximum": 1, "description": "Probability of showing the reference." }, "allowSeeking": { "type": "boolean", "description": "Whether raters can seek through the video." }, "allowReplay": { "type": "boolean", "description": "Whether raters can play the video more than once." }, "screenshotDescription": { "type": "string" } } }, "inpaintingImage": { "type": "object", "additionalProperties": false, "properties": { "minViewDuration": { "type": "number", "minimum": 0, "description": "Minimum time user must view image before clicking (milliseconds)." } } }, "mushra": { "type": "object", "additionalProperties": false, "properties": { "hiddenReference": { "type": "boolean", "description": "Whether to include a hidden reference among the stimuli to rate." }, "showReference": { "type": "boolean", "description": "Whether to show the reference stimulus." }, "minPlayDuration": { "type": "number", "minimum": 0, "maximum": 60000, "description": "Force raters to watch or listen to at least this many milliseconds." }, "fadeTime": { "type": "number", "minimum": 0, "maximum": 1000, "description": "Fade-out and fade-in audio for this many milliseconds." }, "labels": { "type": "array", "minItems": 2, "maxItems": 10, "items": { "type": "string" } }, "anchors": { "type": "array", "items": { "type": "object", "additionalProperties": false, "required": [ "modifiers" ], "properties": { "label": { "type": "string" }, "expectedScore": { "type": "array", "items": { "type": "number" }, "minItems": 2, "maxItems": 2, "description": "A range of admissible scores for a given stimulus." }, "modifiers": { "type": "object", "additionalProperties": false, "required": [ "lowPass" ], "properties": { "lowPass": { "type": "number" } } } } }, "description": "Anchors automatically generated based on reference." }, "showWaveform": { "type": "string", "enum": [ "stimulus", "reference", "stylized", "none" ], "description": "Visualize waveform of the stimulus, only the reference, or none." }, "showHints": { "type": "boolean", "description": "Controls hints above references and anchors during training sessions." } } }, "pairwiseAudio": { "type": "object", "additionalProperties": false, "properties": { "allowTie": { "type": "boolean" }, "categories": { "type": "array", "minItems": 2, "maxItems": 10, "uniqueItems": true, "items": { "type": "object", "additionalProperties": false, "required": [ "label", "score" ], "properties": { "score": { "type": "integer" }, "label": { "type": "string" }, "description": { "type": "string", "maxLength": 32 } } } }, "dynamicColors": { "type": "boolean" }, "hiddenReference": { "type": "boolean" }, "labels": { "type": "object", "properties": { "audio1": { "type": "string" }, "audio2": { "type": "string" } } }, "showDimensionLabels": { "type": "boolean", "description": "Whether to display dimension labels on questions." }, "minPlayDuration": { "type": "number", "minimum": 0, "maximum": 60000, "description": "Force raters to watch or listen to at least this many milliseconds." }, "questions": { "type": "array", "minItems": 0, "maxItems": 10, "uniqueItems": true, "items": { "type": "object", "additionalProperties": false, "required": [ "statement", "dimension" ], "properties": { "statement": { "type": "string" }, "dimension": { "type": "string", "maxLength": 32 }, "tieLabel": { "type": "string", "maxLength": 128 }, "survey": { "type": "array", "minItems": 1, "maxItems": 10, "uniqueItems": true, "items": { "type": "object", "additionalProperties": false, "required": [ "kind", "question", "dimension" ], "properties": { "minSelect": { "type": "integer", "minimum": 1 }, "maxSelect": { "type": "integer", "minimum": 1 }, "kind": { "type": "string", "enum": [ "textarea", "radio", "checkbox", "highlight", "discrete_slider", "continuous_slider" ], "description": "Question type: textarea (open-ended), checkbox (multiple choices), radio (single choice), slider (range input)." }, "question": { "type": "string" }, "choices": { "type": "array", "minItems": 1, "maxItems": 10, "uniqueItems": true, "items": { "type": "object", "additionalProperties": false, "required": [ "label" ], "properties": { "label": { "type": "string" }, "description": { "type": "string", "maxLength": 32 } } } }, "dimension": { "type": "string", "maxLength": 32 }, "categories": { "type": "array", "minItems": 2, "maxItems": 10, "uniqueItems": true, "items": { "type": "object", "additionalProperties": false, "required": [ "label", "score" ], "properties": { "score": { "type": "number" }, "label": { "type": "string" } } } }, "responseType": { "type": "string", "enum": [ "discrete", "continuous" ], "description": "Type of slider response. Discrete uses integer steps, continuous allows any value." } }, "allOf": [ { "if": { "properties": { "kind": { "const": "textarea" } } }, "then": { "not": { "anyOf": [ { "required": [ "choices" ] }, { "required": [ "minSelect" ] }, { "required": [ "maxSelect" ] }, { "required": [ "categories" ] }, { "required": [ "responseType" ] } ] } } }, { "if": { "properties": { "kind": { "const": "radio" } } }, "then": { "required": [ "choices" ], "not": { "anyOf": [ { "required": [ "minSelect" ] }, { "required": [ "maxSelect" ] }, { "required": [ "categories" ] }, { "required": [ "responseType" ] } ] } } }, { "if": { "properties": { "kind": { "const": "checkbox" } } }, "then": { "required": [ "choices" ], "not": { "anyOf": [ { "required": [ "categories" ] }, { "required": [ "responseType" ] } ] } } }, { "if": { "properties": { "kind": { "const": "highlight" } } }, "then": { "not": { "anyOf": [ { "required": [ "choices" ] }, { "required": [ "minSelect" ] }, { "required": [ "maxSelect" ] }, { "required": [ "categories" ] }, { "required": [ "responseType" ] } ] } } }, { "if": { "properties": { "kind": { "const": "discrete_slider" } } }, "then": { "required": [ "categories", "responseType" ], "properties": { "responseType": { "const": "discrete" } }, "not": { "anyOf": [ { "required": [ "choices" ] }, { "required": [ "minSelect" ] }, { "required": [ "maxSelect" ] } ] } } }, { "if": { "properties": { "kind": { "const": "continuous_slider" } } }, "then": { "required": [ "categories", "responseType" ], "properties": { "responseType": { "const": "continuous" } }, "not": { "anyOf": [ { "required": [ "choices" ] }, { "required": [ "minSelect" ] }, { "required": [ "maxSelect" ] } ] } } } ] } } } } }, "responseType": { "type": "string", "enum": [ "binary", "discrete", "continuous" ], "description": "Type of response expected from the rater." }, "showReference": { "type": "boolean" }, "showWaveform": { "type": "string", "enum": [ "default", "stylized", "none" ], "description": "Visualize the waveform, a stylized version of it, or hide it." }, "tieLabel": { "type": "string", "maxLength": 128 }, "tieScore": { "type": "number" } } }, "pairwiseImage": { "type": "object", "additionalProperties": false, "properties": { "mode": { "type": "string", "enum": [ "side-by-side", "flip-between", "flip-between-reference" ] }, "allowPan": { "type": "boolean" }, "allowTie": { "type": "boolean" }, "allowZoom": { "type": "boolean" }, "allowCropRefresh": { "type": "boolean" }, "hiddenReference": { "type": "boolean" }, "scaleToFit": { "type": "boolean" }, "showReference": { "type": "boolean" }, "maskDuration": { "type": "integer", "minimum": 0, "maximum": 1000 }, "maxCropSize": { "type": "integer", "minimum": 32 }, "imageRendering": { "type": "string", "enum": [ "auto", "smooth", "crisp-edges", "pixelated" ] }, "zoomLevel": { "type": "number", "minimum": 0.1, "maximum": 10, "description": "The initial zoom level represented as a decimal (e.g, 2 = 200%)." }, "zoomLevels": { "type": "array", "minItems": 1, "maxItems": 100, "items": { "type": "number", "minimum": 0.25, "maximum": 10 } } } }, "pairwiseVideo": { "type": "object", "additionalProperties": false, "properties": { "allowTie": { "type": "boolean" }, "allowCropRefresh": { "type": "boolean" }, "categories": { "type": "array", "minItems": 2, "maxItems": 10, "uniqueItems": true, "items": { "type": "object", "additionalProperties": false, "required": [ "label", "score" ], "properties": { "score": { "type": "integer" }, "label": { "type": "string" }, "description": { "type": "string", "maxLength": 32 } } } }, "dynamicColors": { "type": "boolean" }, "hasAudio": { "type": "boolean" }, "hiddenReference": { "type": "boolean" }, "labels": { "type": "object", "properties": { "video1": { "type": "string" }, "video2": { "type": "string" } } }, "showDimensionLabels": { "type": "boolean", "description": "Whether to display dimension labels on questions." }, "loopVideo": { "type": "boolean" }, "maskDuration": { "type": "integer", "minimum": 0, "maximum": 1000 }, "maxCropSize": { "type": "integer", "minimum": 32 }, "minPlayDuration": { "type": "number", "minimum": 0, "maximum": 60000, "description": "Force raters to watch or listen to at least this many milliseconds." }, "minViewDuration": { "type": "number", "minimum": 0, "maximum": 5000, "description": "Force raters to view the image/video for at least this many milliseconds." }, "minDisplayDuration": { "type": "integer", "minimum": 0, "maximum": 5000, "description": "Minimum time a video must be displayed before switching is allowed." }, "mode": { "type": "string", "enum": [ "side-by-side", "flip-between", "flip-between-reference" ] }, "tieLabel": { "type": "string", "maxLength": 128 }, "tieScore": { "type": "number" }, "questions": { "type": "array", "minItems": 0, "maxItems": 10, "uniqueItems": true, "items": { "type": "object", "additionalProperties": false, "required": [ "statement", "dimension" ], "properties": { "statement": { "type": "string" }, "dimension": { "type": "string", "maxLength": 32 }, "tieLabel": { "type": "string", "maxLength": 128 }, "survey": { "type": "array", "minItems": 1, "maxItems": 10, "uniqueItems": true, "items": { "type": "object", "additionalProperties": false, "required": [ "kind", "question", "dimension" ], "properties": { "minSelect": { "type": "integer", "minimum": 1 }, "maxSelect": { "type": "integer", "minimum": 1 }, "kind": { "type": "string", "enum": [ "textarea", "radio", "checkbox", "highlight", "discrete_slider", "continuous_slider" ], "description": "Question type: textarea (open-ended), checkbox (multiple choices), radio (single choice), slider (range input)." }, "question": { "type": "string" }, "choices": { "type": "array", "minItems": 1, "maxItems": 10, "uniqueItems": true, "items": { "type": "object", "additionalProperties": false, "required": [ "label" ], "properties": { "label": { "type": "string" }, "description": { "type": "string", "maxLength": 32 } } } }, "dimension": { "type": "string", "maxLength": 32 }, "categories": { "type": "array", "minItems": 2, "maxItems": 10, "uniqueItems": true, "items": { "type": "object", "additionalProperties": false, "required": [ "label", "score" ], "properties": { "score": { "type": "number" }, "label": { "type": "string" } } } }, "responseType": { "type": "string", "enum": [ "discrete", "continuous" ], "description": "Type of slider response. Discrete uses integer steps, continuous allows any value." } }, "allOf": [ { "if": { "properties": { "kind": { "const": "textarea" } } }, "then": { "not": { "anyOf": [ { "required": [ "choices" ] }, { "required": [ "minSelect" ] }, { "required": [ "maxSelect" ] }, { "required": [ "categories" ] }, { "required": [ "responseType" ] } ] } } }, { "if": { "properties": { "kind": { "const": "radio" } } }, "then": { "required": [ "choices" ], "not": { "anyOf": [ { "required": [ "minSelect" ] }, { "required": [ "maxSelect" ] }, { "required": [ "categories" ] }, { "required": [ "responseType" ] } ] } } }, { "if": { "properties": { "kind": { "const": "checkbox" } } }, "then": { "required": [ "choices" ], "not": { "anyOf": [ { "required": [ "categories" ] }, { "required": [ "responseType" ] } ] } } }, { "if": { "properties": { "kind": { "const": "highlight" } } }, "then": { "not": { "anyOf": [ { "required": [ "choices" ] }, { "required": [ "minSelect" ] }, { "required": [ "maxSelect" ] }, { "required": [ "categories" ] }, { "required": [ "responseType" ] } ] } } }, { "if": { "properties": { "kind": { "const": "discrete_slider" } } }, "then": { "required": [ "categories", "responseType" ], "properties": { "responseType": { "const": "discrete" } }, "not": { "anyOf": [ { "required": [ "choices" ] }, { "required": [ "minSelect" ] }, { "required": [ "maxSelect" ] } ] } } }, { "if": { "properties": { "kind": { "const": "continuous_slider" } } }, "then": { "required": [ "categories", "responseType" ], "properties": { "responseType": { "const": "continuous" } }, "not": { "anyOf": [ { "required": [ "choices" ] }, { "required": [ "minSelect" ] }, { "required": [ "maxSelect" ] } ] } } } ] } } } } }, "responseType": { "type": "string", "enum": [ "binary", "discrete", "continuous" ], "description": "Type of response expected from the rater." }, "scaleToFit": { "type": "boolean" }, "showReference": { "type": "boolean" }, "videoPlayer": { "type": "string", "enum": [ "sequential", "parallel" ] }, "videoSize": { "type": "object", "additionalProperties": false, "properties": { "minWidth": { "type": "integer", "minimum": 0, "maximum": 1920 }, "maxWidth": { "type": "integer", "minimum": 120 }, "minHeight": { "type": "integer", "minimum": 0, "maximum": 1080 }, "maxHeight": { "type": "integer", "minimum": 120 } } }, "scaleReference": { "type": "integer", "minimum": 50, "maximum": 100, "description": "Scale the reference video to this percentage of its original size." }, "allowSeeking": { "type": "boolean", "description": "Whether raters can seek through the video." } } }, "surveyAudio": { "type": "object", "additionalProperties": false, "required": [ "survey" ], "properties": { "survey": { "type": "array", "minItems": 1, "maxItems": 10, "uniqueItems": true, "items": { "type": "object", "additionalProperties": false, "required": [ "kind", "question", "dimension" ], "properties": { "minSelect": { "type": "integer", "minimum": 1 }, "maxSelect": { "type": "integer", "minimum": 1 }, "kind": { "type": "string", "enum": [ "textarea", "radio", "checkbox", "highlight", "discrete_slider", "continuous_slider" ], "description": "Question type: textarea (open-ended), checkbox (multiple choices), radio (single choice), slider (range input)." }, "question": { "type": "string" }, "choices": { "type": "array", "minItems": 1, "maxItems": 10, "uniqueItems": true, "items": { "type": "object", "additionalProperties": false, "required": [ "label" ], "properties": { "label": { "type": "string" }, "description": { "type": "string", "maxLength": 32 } } } }, "dimension": { "type": "string", "maxLength": 32 }, "categories": { "type": "array", "minItems": 2, "maxItems": 10, "uniqueItems": true, "items": { "type": "object", "additionalProperties": false, "required": [ "label", "score" ], "properties": { "score": { "type": "number" }, "label": { "type": "string" } } } }, "responseType": { "type": "string", "enum": [ "discrete", "continuous" ], "description": "Type of slider response. Discrete uses integer steps, continuous allows any value." } }, "allOf": [ { "if": { "properties": { "kind": { "const": "textarea" } } }, "then": { "not": { "anyOf": [ { "required": [ "choices" ] }, { "required": [ "minSelect" ] }, { "required": [ "maxSelect" ] }, { "required": [ "categories" ] }, { "required": [ "responseType" ] } ] } } }, { "if": { "properties": { "kind": { "const": "radio" } } }, "then": { "required": [ "choices" ], "not": { "anyOf": [ { "required": [ "minSelect" ] }, { "required": [ "maxSelect" ] }, { "required": [ "categories" ] }, { "required": [ "responseType" ] } ] } } }, { "if": { "properties": { "kind": { "const": "checkbox" } } }, "then": { "required": [ "choices" ], "not": { "anyOf": [ { "required": [ "categories" ] }, { "required": [ "responseType" ] } ] } } }, { "if": { "properties": { "kind": { "const": "highlight" } } }, "then": { "not": { "anyOf": [ { "required": [ "choices" ] }, { "required": [ "minSelect" ] }, { "required": [ "maxSelect" ] }, { "required": [ "categories" ] }, { "required": [ "responseType" ] } ] } } }, { "if": { "properties": { "kind": { "const": "discrete_slider" } } }, "then": { "required": [ "categories", "responseType" ], "properties": { "responseType": { "const": "discrete" } }, "not": { "anyOf": [ { "required": [ "choices" ] }, { "required": [ "minSelect" ] }, { "required": [ "maxSelect" ] } ] } } }, { "if": { "properties": { "kind": { "const": "continuous_slider" } } }, "then": { "required": [ "categories", "responseType" ], "properties": { "responseType": { "const": "continuous" } }, "not": { "anyOf": [ { "required": [ "choices" ] }, { "required": [ "minSelect" ] }, { "required": [ "maxSelect" ] } ] } } } ] } }, "hiddenReference": { "type": "boolean", "description": "Whether to include a hidden reference among the stimuli to rate." }, "showReference": { "type": "boolean", "description": "Whether to show a reference stimulus." }, "showWaveform": { "type": "string", "enum": [ "stimulus", "reference" ], "description": "Determines how the waveform of the condition is rendered." }, "minPlayDuration": { "type": "number", "minimum": 0, "maximum": 60000, "description": "Force raters to watch or listen to at least this many milliseconds." }, "showDimensionLabels": { "type": "boolean", "description": "Whether to display dimension labels on survey questions." } } }, "surveyEmbed": { "type": "object", "additionalProperties": false, "required": [ "survey" ], "properties": { "survey": { "type": "array", "minItems": 1, "maxItems": 10, "uniqueItems": true, "items": { "type": "object", "additionalProperties": false, "required": [ "kind", "question", "dimension" ], "properties": { "minSelect": { "type": "integer", "minimum": 1 }, "maxSelect": { "type": "integer", "minimum": 1 }, "kind": { "type": "string", "enum": [ "textarea", "radio", "checkbox", "highlight", "discrete_slider", "continuous_slider" ], "description": "Question type: textarea (open-ended), checkbox (multiple choices), radio (single choice), slider (range input)." }, "question": { "type": "string" }, "choices": { "type": "array", "minItems": 1, "maxItems": 10, "uniqueItems": true, "items": { "type": "object", "additionalProperties": false, "required": [ "label" ], "properties": { "label": { "type": "string" }, "description": { "type": "string", "maxLength": 32 } } } }, "dimension": { "type": "string", "maxLength": 32 }, "categories": { "type": "array", "minItems": 2, "maxItems": 10, "uniqueItems": true, "items": { "type": "object", "additionalProperties": false, "required": [ "label", "score" ], "properties": { "score": { "type": "number" }, "label": { "type": "string" } } } }, "responseType": { "type": "string", "enum": [ "discrete", "continuous" ], "description": "Type of slider response. Discrete uses integer steps, continuous allows any value." } }, "allOf": [ { "if": { "properties": { "kind": { "const": "textarea" } } }, "then": { "not": { "anyOf": [ { "required": [ "choices" ] }, { "required": [ "minSelect" ] }, { "required": [ "maxSelect" ] }, { "required": [ "categories" ] }, { "required": [ "responseType" ] } ] } } }, { "if": { "properties": { "kind": { "const": "radio" } } }, "then": { "required": [ "choices" ], "not": { "anyOf": [ { "required": [ "minSelect" ] }, { "required": [ "maxSelect" ] }, { "required": [ "categories" ] }, { "required": [ "responseType" ] } ] } } }, { "if": { "properties": { "kind": { "const": "checkbox" } } }, "then": { "required": [ "choices" ], "not": { "anyOf": [ { "required": [ "categories" ] }, { "required": [ "responseType" ] } ] } } }, { "if": { "properties": { "kind": { "const": "highlight" } } }, "then": { "not": { "anyOf": [ { "required": [ "choices" ] }, { "required": [ "minSelect" ] }, { "required": [ "maxSelect" ] }, { "required": [ "categories" ] }, { "required": [ "responseType" ] } ] } } }, { "if": { "properties": { "kind": { "const": "discrete_slider" } } }, "then": { "required": [ "categories", "responseType" ], "properties": { "responseType": { "const": "discrete" } }, "not": { "anyOf": [ { "required": [ "choices" ] }, { "required": [ "minSelect" ] }, { "required": [ "maxSelect" ] } ] } } }, { "if": { "properties": { "kind": { "const": "continuous_slider" } } }, "then": { "required": [ "categories", "responseType" ], "properties": { "responseType": { "const": "continuous" } }, "not": { "anyOf": [ { "required": [ "choices" ] }, { "required": [ "minSelect" ] }, { "required": [ "maxSelect" ] } ] } } } ] } }, "hasAudio": { "type": "boolean" }, "iframeSize": { "type": [ "object", "null" ], "additionalProperties": false, "required": [ "width", "height" ], "properties": { "width": { "type": "integer" }, "height": { "type": "integer" } } }, "interactionDuration": { "type": [ "object", "null" ], "additionalProperties": false, "required": [ "min", "max" ], "description": "The minimum and maximum time a rater is expected to spend interacting with each embedded stimulus, in milliseconds. Used to estimate session completion time.", "properties": { "min": { "type": "number", "minimum": 10000, "maximum": 1200000 }, "max": { "type": "number", "minimum": 10000, "maximum": 1200000 } } }, "showDimensionLabels": { "type": "boolean", "description": "Whether to display dimension labels on survey questions." } } }, "surveyVideo": { "type": "object", "additionalProperties": false, "required": [ "survey" ], "properties": { "allowMobile": { "type": "boolean" }, "survey": { "type": "array", "minItems": 1, "maxItems": 10, "uniqueItems": true, "items": { "type": "object", "additionalProperties": false, "required": [ "kind", "question", "dimension" ], "properties": { "minSelect": { "type": "integer", "minimum": 1 }, "maxSelect": { "type": "integer", "minimum": 1 }, "kind": { "type": "string", "enum": [ "textarea", "radio", "checkbox", "highlight", "discrete_slider", "continuous_slider" ], "description": "Question type: textarea (open-ended), checkbox (multiple choices), radio (single choice), slider (range input)." }, "question": { "type": "string" }, "choices": { "type": "array", "minItems": 1, "maxItems": 10, "uniqueItems": true, "items": { "type": "object", "additionalProperties": false, "required": [ "label" ], "properties": { "label": { "type": "string" }, "description": { "type": "string", "maxLength": 32 } } } }, "dimension": { "type": "string", "maxLength": 32 }, "categories": { "type": "array", "minItems": 2, "maxItems": 10, "uniqueItems": true, "items": { "type": "object", "additionalProperties": false, "required": [ "label", "score" ], "properties": { "score": { "type": "number" }, "label": { "type": "string" } } } }, "responseType": { "type": "string", "enum": [ "discrete", "continuous" ], "description": "Type of slider response. Discrete uses integer steps, continuous allows any value." } }, "allOf": [ { "if": { "properties": { "kind": { "const": "textarea" } } }, "then": { "not": { "anyOf": [ { "required": [ "choices" ] }, { "required": [ "minSelect" ] }, { "required": [ "maxSelect" ] }, { "required": [ "categories" ] }, { "required": [ "responseType" ] } ] } } }, { "if": { "properties": { "kind": { "const": "radio" } } }, "then": { "required": [ "choices" ], "not": { "anyOf": [ { "required": [ "minSelect" ] }, { "required": [ "maxSelect" ] }, { "required": [ "categories" ] }, { "required": [ "responseType" ] } ] } } }, { "if": { "properties": { "kind": { "const": "checkbox" } } }, "then": { "required": [ "choices" ], "not": { "anyOf": [ { "required": [ "categories" ] }, { "required": [ "responseType" ] } ] } } }, { "if": { "properties": { "kind": { "const": "highlight" } } }, "then": { "not": { "anyOf": [ { "required": [ "choices" ] }, { "required": [ "minSelect" ] }, { "required": [ "maxSelect" ] }, { "required": [ "categories" ] }, { "required": [ "responseType" ] } ] } } }, { "if": { "properties": { "kind": { "const": "discrete_slider" } } }, "then": { "required": [ "categories", "responseType" ], "properties": { "responseType": { "const": "discrete" } }, "not": { "anyOf": [ { "required": [ "choices" ] }, { "required": [ "minSelect" ] }, { "required": [ "maxSelect" ] } ] } } }, { "if": { "properties": { "kind": { "const": "continuous_slider" } } }, "then": { "required": [ "categories", "responseType" ], "properties": { "responseType": { "const": "continuous" } }, "not": { "anyOf": [ { "required": [ "choices" ] }, { "required": [ "minSelect" ] }, { "required": [ "maxSelect" ] } ] } } } ] } }, "hasAudio": { "type": "boolean" }, "contentSizing": { "type": "string", "enum": [ "crop", "scale_to_fit_content", "scale_to_fit_survey" ] }, "videoSize": { "type": "object", "additionalProperties": false, "properties": { "minWidth": { "type": "integer", "minimum": 0, "maximum": 1920 }, "maxWidth": { "type": "integer", "minimum": 120 }, "minHeight": { "type": "integer", "minimum": 0, "maximum": 1080 }, "maxHeight": { "type": "integer", "minimum": 120 } } }, "fullscreen": { "type": "boolean", "description": "Require raters to view the video in fullscreen mode." }, "minPlayDuration": { "type": "number", "minimum": 0, "maximum": 60000, "description": "Force raters to watch or listen to at least this many milliseconds." }, "allowSeeking": { "type": "boolean", "description": "Whether raters can seek through the video." }, "showDimensionLabels": { "type": "boolean", "description": "Whether to display dimension labels on survey questions." } } }, "surveyImage": { "type": "object", "additionalProperties": false, "required": [ "survey" ], "properties": { "survey": { "type": "array", "minItems": 1, "maxItems": 10, "uniqueItems": true, "items": { "type": "object", "additionalProperties": false, "required": [ "kind", "question", "dimension" ], "properties": { "minSelect": { "type": "integer", "minimum": 1 }, "maxSelect": { "type": "integer", "minimum": 1 }, "kind": { "type": "string", "enum": [ "textarea", "radio", "checkbox", "highlight", "discrete_slider", "continuous_slider" ], "description": "Question type: textarea (open-ended), checkbox (multiple choices), radio (single choice), slider (range input)." }, "question": { "type": "string" }, "choices": { "type": "array", "minItems": 1, "maxItems": 10, "uniqueItems": true, "items": { "type": "object", "additionalProperties": false, "required": [ "label" ], "properties": { "label": { "type": "string" }, "description": { "type": "string", "maxLength": 32 } } } }, "dimension": { "type": "string", "maxLength": 32 }, "categories": { "type": "array", "minItems": 2, "maxItems": 10, "uniqueItems": true, "items": { "type": "object", "additionalProperties": false, "required": [ "label", "score" ], "properties": { "score": { "type": "number" }, "label": { "type": "string" } } } }, "responseType": { "type": "string", "enum": [ "discrete", "continuous" ], "description": "Type of slider response. Discrete uses integer steps, continuous allows any value." } }, "allOf": [ { "if": { "properties": { "kind": { "const": "textarea" } } }, "then": { "not": { "anyOf": [ { "required": [ "choices" ] }, { "required": [ "minSelect" ] }, { "required": [ "maxSelect" ] }, { "required": [ "categories" ] }, { "required": [ "responseType" ] } ] } } }, { "if": { "properties": { "kind": { "const": "radio" } } }, "then": { "required": [ "choices" ], "not": { "anyOf": [ { "required": [ "minSelect" ] }, { "required": [ "maxSelect" ] }, { "required": [ "categories" ] }, { "required": [ "responseType" ] } ] } } }, { "if": { "properties": { "kind": { "const": "checkbox" } } }, "then": { "required": [ "choices" ], "not": { "anyOf": [ { "required": [ "categories" ] }, { "required": [ "responseType" ] } ] } } }, { "if": { "properties": { "kind": { "const": "highlight" } } }, "then": { "not": { "anyOf": [ { "required": [ "choices" ] }, { "required": [ "minSelect" ] }, { "required": [ "maxSelect" ] }, { "required": [ "categories" ] }, { "required": [ "responseType" ] } ] } } }, { "if": { "properties": { "kind": { "const": "discrete_slider" } } }, "then": { "required": [ "categories", "responseType" ], "properties": { "responseType": { "const": "discrete" } }, "not": { "anyOf": [ { "required": [ "choices" ] }, { "required": [ "minSelect" ] }, { "required": [ "maxSelect" ] } ] } } }, { "if": { "properties": { "kind": { "const": "continuous_slider" } } }, "then": { "required": [ "categories", "responseType" ], "properties": { "responseType": { "const": "continuous" } }, "not": { "anyOf": [ { "required": [ "choices" ] }, { "required": [ "minSelect" ] }, { "required": [ "maxSelect" ] } ] } } } ] } }, "allowCropRefresh": { "type": "boolean", "description": "Allow raters to request different random crops." }, "allowMobile": { "type": "boolean" }, "maxCropSize": { "type": "integer", "minimum": 32, "description": "Raters will view image crops instead of the full image." }, "minViewDuration": { "type": "number", "minimum": 0, "maximum": 5000, "description": "Force raters to view the image/video for at least this many milliseconds." }, "showDimensionLabels": { "type": "boolean", "description": "Whether to display dimension labels on survey questions." } } }, "preScreenAudio": {}, "preScreenImage": {}, "rankAudio": { "type": "object", "additionalProperties": false, "properties": { "hiddenReference": { "type": "boolean" }, "minPlayDuration": { "type": "number", "minimum": 0, "maximum": 60000, "description": "Force raters to watch or listen to at least this many milliseconds." }, "showReference": { "type": "boolean" }, "showWaveform": { "type": "string", "enum": [ "default", "stylized", "none" ], "description": "Visualize the waveform, a stylized version of it, or hide it." } } } } }, "ExperimentDetail": { "type": "object", "properties": { "id": { "type": "string" }, "name": { "type": "string", "description": "A description identifying the experiment.", "maxLength": 128 }, "type": { "type": "string" }, "language": { "allOf": [ { "$ref": "#/components/schemas/LanguageEnum" } ], "description": "The language used for the experiment's interface.\n\n* `ar` - Arabic\n* `de` - German\n* `en` - English\n* `es` - Spanish\n* `fr` - French\n* `hi` - Hindi\n* `it` - Italian\n* `ja` - Japanese\n* `ko` - Korean\n* `pl` - Polish\n* `pt` - Portuguese\n* `vi` - Vietnamese\n* `zh-hans` - Chinese (simplified)" }, "datasets": { "type": "array", "items": { "$ref": "#/components/schemas/Dataset" }, "readOnly": true }, "training_datasets": { "type": "array", "items": { "$ref": "#/components/schemas/Dataset" }, "readOnly": true }, "golden_datasets": { "type": "array", "items": { "$ref": "#/components/schemas/Dataset" }, "readOnly": true }, "jobs": { "type": "array", "items": { "$ref": "#/components/schemas/Job" }, "readOnly": true }, "title": { "type": [ "string", "null" ], "description": "A short title displayed at the top of the experiment screen.", "maxLength": 128 }, "question": { "type": [ "string", "null" ], "description": "A question or instructions displayed at the top of the experiment screen.", "maxLength": 512 }, "description": { "type": [ "string", "null" ], "description": "An optional longer explanation of how to evaluate the data." }, "introduction": { "type": [ "string", "null" ], "description": "A brief introduction explaining how to participate in the experiment. Leave empty to use default introduction.", "maxLength": 2048 }, "hero_media_url": { "type": [ "string", "null" ], "readOnly": true }, "hero_caption": { "type": [ "string", "null" ], "description": "Caption displayed below the custom hero image/video.", "maxLength": 256 }, "date_created": { "type": "string", "format": "date-time", "readOnly": true }, "date_updated": { "type": "string", "format": "date-time", "readOnly": true, "title": "Last updated" }, "config": {} }, "required": [ "datasets", "date_created", "date_updated", "golden_datasets", "hero_media_url", "id", "jobs", "name", "training_datasets", "type" ] }, "ExplicitContentTypesEnum": { "enum": [ "sexual_or_adult_content", "nudity", "graphic_violence_or_gore", "hate_or_harassment", "strong_or_offensive_language", "illegal_activities", "drugs_or_alcohol", "other" ], "type": "string", "description": "* `sexual_or_adult_content` - Sexual or adult content\n* `nudity` - Nudity\n* `graphic_violence_or_gore` - Graphic violence or gore\n* `hate_or_harassment` - Hate or harassment\n* `strong_or_offensive_language` - Strong or offensive language\n* `illegal_activities` - Illegal activities\n* `drugs_or_alcohol` - Drugs or alcohol\n* `other` - Other" }, "Job": { "type": "object", "properties": { "id": { "type": "string" }, "experiment_id": { "type": "string", "readOnly": true }, "experiment_job_id": { "type": [ "integer", "null" ], "maximum": 2147483647, "minimum": 0, "description": "The ID of the job in the experiment." }, "num_sessions": { "type": "integer", "maximum": 10000, "minimum": 0, "title": "Number of sessions", "description": "How often the task is performed." }, "num_comparisons": { "type": "integer", "maximum": 1000, "minimum": 1, "title": "Number of comparisons per rater", "description": "The maximum number of groups of stimuli evaluated by each rater." }, "num_training": { "type": "integer", "maximum": 2147483647, "minimum": 0, "title": "Number of training comparisons", "description": "Set to zero to disable training." }, "date_created": { "type": "string", "format": "date-time", "readOnly": true }, "date_updated": { "type": "string", "format": "date-time", "readOnly": true, "title": "Last updated" }, "date_launched": { "type": [ "string", "null" ], "format": "date-time" }, "rater_pool": { "allOf": [ { "$ref": "#/components/schemas/RaterPool" } ], "readOnly": true }, "status": { "type": "string", "readOnly": true }, "config": { "description": "Configures stimulus selection strategy." } }, "required": [ "date_created", "date_updated", "experiment_id", "id", "num_comparisons", "num_sessions", "rater_pool", "status" ] }, "JobCosts": { "type": "object", "properties": { "num_sessions": { "type": "integer" }, "cost": { "type": "number", "format": "double" }, "cost_per_additional_session": { "type": "number", "format": "double" }, "currency": { "type": "string" }, "token": { "type": "string", "description": "Time-limited token to confirm cost calculation." }, "token_expires_in": { "type": "integer", "description": "Token expiry time in seconds." }, "token_expires_at": { "type": "string", "format": "date-time", "description": "Token expiry timestamp (ISO 8601)." } }, "required": [ "cost", "cost_per_additional_session", "currency", "num_sessions", "token", "token_expires_at", "token_expires_in" ] }, "JobCreateResponse": { "type": "object", "properties": { "id": { "type": "string" }, "experiment_id": { "type": "string", "readOnly": true }, "experiment_job_id": { "type": [ "integer", "null" ], "maximum": 2147483647, "minimum": 0, "description": "The ID of the job in the experiment." }, "num_sessions": { "type": "integer", "maximum": 10000, "minimum": 0, "title": "Number of sessions", "description": "How often the task is performed." }, "num_comparisons": { "type": "integer", "maximum": 1000, "minimum": 1, "title": "Number of comparisons per rater", "description": "The maximum number of groups of stimuli evaluated by each rater." }, "num_training": { "type": "integer", "maximum": 2147483647, "minimum": 0, "title": "Number of training comparisons", "description": "Set to zero to disable training." }, "date_created": { "type": "string", "format": "date-time", "readOnly": true }, "date_updated": { "type": "string", "format": "date-time", "readOnly": true, "title": "Last updated" }, "date_launched": { "type": [ "string", "null" ], "format": "date-time" }, "rater_pool": { "allOf": [ { "$ref": "#/components/schemas/RaterPool" } ], "readOnly": true }, "status": { "type": "string", "readOnly": true }, "config": { "description": "Configures stimulus selection strategy." }, "warnings": { "type": "array", "items": { "type": "string" }, "readOnly": true, "description": "Warning messages about the job configuration, if any." } }, "required": [ "date_created", "date_updated", "experiment_id", "id", "num_comparisons", "num_sessions", "rater_pool", "status", "warnings" ] }, "JobDetail": { "type": "object", "properties": { "id": { "type": "string" }, "experiment_id": { "type": "string", "readOnly": true }, "experiment_job_id": { "type": [ "integer", "null" ], "maximum": 2147483647, "minimum": 0, "description": "The ID of the job in the experiment." }, "num_sessions": { "type": "integer", "maximum": 10000, "minimum": 0, "title": "Number of sessions", "description": "How often the task is performed." }, "num_comparisons": { "type": "integer", "maximum": 1000, "minimum": 1, "title": "Number of comparisons per rater", "description": "The maximum number of groups of stimuli evaluated by each rater." }, "num_training": { "type": "integer", "maximum": 2147483647, "minimum": 0, "title": "Number of training comparisons", "description": "Set to zero to disable training." }, "date_created": { "type": "string", "format": "date-time", "readOnly": true }, "date_updated": { "type": "string", "format": "date-time", "readOnly": true, "title": "Last updated" }, "date_launched": { "type": [ "string", "null" ], "format": "date-time" }, "rater_pool": { "allOf": [ { "$ref": "#/components/schemas/RaterPool" } ], "readOnly": true }, "status": { "type": "string", "readOnly": true }, "config": { "description": "Configures stimulus selection strategy." }, "max_repetitions": { "type": "integer", "maximum": 2147483647, "minimum": 0, "title": "Maximum number of repetitions", "description": "How often an individual rater is allowed to repeat the task." }, "min_rest_time": { "type": "integer", "maximum": 2147483647, "minimum": 0, "title": "Minimum rest time", "description": "How many minutes should pass before a rater can repeat the task." } }, "required": [ "date_created", "date_updated", "experiment_id", "id", "num_comparisons", "num_sessions", "rater_pool", "status" ] }, "JobLaunch": { "type": "object", "properties": { "token": { "type": "string", "description": "Time-limited token to confirm job launch." } }, "required": [ "token" ] }, "LanguageEnum": { "enum": [ "ar", "de", "en", "es", "fr", "hi", "it", "ja", "ko", "pl", "pt", "vi", "zh-hans" ], "type": "string", "description": "* `ar` - Arabic\n* `de` - German\n* `en` - English\n* `es` - Spanish\n* `fr` - French\n* `hi` - Hindi\n* `it` - Italian\n* `ja` - Japanese\n* `ko` - Korean\n* `pl` - Polish\n* `pt` - Portuguese\n* `vi` - Vietnamese\n* `zh-hans` - Chinese (simplified)" }, "Leaderboard": { "type": "object", "properties": { "id": { "type": "string" }, "name": { "type": "string", "description": "A descriptive name for the leaderboard.", "maxLength": 100 }, "date_updated": { "type": "string", "format": "date-time", "readOnly": true, "title": "Last updated" } }, "required": [ "date_updated", "id", "name" ] }, "LeaderboardDetail": { "type": "object", "properties": { "id": { "type": "string" }, "name": { "type": "string", "description": "A descriptive name for the leaderboard.", "maxLength": 100 }, "description": { "type": "string", "description": "Optional description of the leaderboard." }, "date_created": { "type": "string", "format": "date-time", "readOnly": true }, "date_updated": { "type": "string", "format": "date-time", "readOnly": true, "title": "Last updated" }, "include_patterns": { "type": "string", "title": "Include conditions", "description": "Comma-separated list of conditions to include. Supports wildcards (*). Leave empty to include all methods." }, "exclude_patterns": { "type": "string", "title": "Exclude conditions", "description": "Comma-separated list of conditions to exclude. Supports wildcards (*). Applied after include patterns." }, "entries": { "type": "array", "items": { "type": "object", "additionalProperties": {} }, "readOnly": true } }, "required": [ "date_created", "date_updated", "entries", "id", "name" ] }, "Metric": { "type": "object", "properties": { "kind": { "$ref": "#/components/schemas/MetricKind" }, "dimension": { "type": [ "string", "null" ], "description": "Used by experiments evaluating multiple criteria simultaneously.", "maxLength": 32 }, "source": { "type": [ "string", "null" ], "description": "Return the source of the metric (job or experiment).", "readOnly": true }, "scores": { "type": "array", "items": {}, "description": "Return the scores for each method.", "readOnly": true } }, "required": [ "kind", "scores", "source" ] }, "MetricKind": { "enum": [ "percent_approved", "bayesian_elo", "elo", "mean", "win_loss", "stats", "percent_chosen" ], "type": "string", "description": "* `percent_approved` - Approval rate\n* `bayesian_elo` - Elo\n* `elo` - Elo (vanilla)\n* `mean` - MOS\n* `win_loss` - Win/loss\n* `stats` - Statistics\n* `percent_chosen` - Percent chosen" }, "OrganizationBillingEvent": { "type": "object", "description": "Serialize billing events with optional organization project attribution.", "properties": { "kind": { "$ref": "#/components/schemas/BillingEventKind" }, "amount_credits": { "type": "string", "format": "decimal", "pattern": "^-?\\d{0,12}(?:\\.\\d{0,2})?$", "readOnly": true }, "remaining_credits": { "type": "string", "format": "decimal", "pattern": "^-?\\d{0,12}(?:\\.\\d{0,2})?$", "readOnly": true }, "currency": { "type": "string", "readOnly": true }, "date": { "type": "string", "format": "date-time", "readOnly": true }, "note": { "type": "string", "maxLength": 255 }, "creator": { "allOf": [ { "$ref": "#/components/schemas/Creator" } ], "readOnly": true }, "api_key": { "allOf": [ { "$ref": "#/components/schemas/ProjectAPIKey" } ], "readOnly": true }, "project": { "oneOf": [ { "$ref": "#/components/schemas/BillingEventProject" }, { "type": "null" } ], "readOnly": true } }, "required": [ "amount_credits", "api_key", "creator", "currency", "date", "kind", "project", "remaining_credits" ] }, "PatchedWebhookUpdate": { "type": "object", "description": "Serializer for updating a Webhook instance.\nAllows partial updates of the target URL, events, status, and secret ID.", "properties": { "target_url": { "type": "string", "format": "uri", "maxLength": 2048 }, "events": { "type": "array", "items": { "$ref": "#/components/schemas/EventsEnum" }, "description": "List of event types to subscribe to (e.g., [\"dataset.status_changed\", \"job.completed\"])." }, "status": { "$ref": "#/components/schemas/StatusC7dEnum" }, "secret_id": { "type": "string", "description": "ID of the secret to use for this webhook." } } }, "ProjectAPIKey": { "type": "object", "properties": { "name": { "type": "string", "description": "A name for the API key.", "maxLength": 50 }, "creator": { "allOf": [ { "$ref": "#/components/schemas/Creator" } ], "readOnly": true } }, "required": [ "creator" ] }, "PublicDatasetCreate": { "type": "object", "description": "Serializer for creating a new dataset with single file (used by public API).", "properties": { "id": { "type": "string", "readOnly": true }, "name": { "type": "string", "description": "A short name identifying the dataset.", "maxLength": 64 }, "filename": { "type": "string", "writeOnly": true, "description": "The name of the zip file that will be uploaded." }, "upload_url": { "type": "string", "format": "uri", "readOnly": true }, "retention_days": { "type": [ "integer", "null" ], "maximum": 2147483647, "minimum": -1, "description": "Number of days to retain this dataset. Use -1 for unlimited retention. Empty means use project default." }, "secrets": { "type": "array", "items": { "type": "object", "additionalProperties": {} }, "writeOnly": true, "description": "List of secrets associated with the dataset" }, "config": {}, "contains_pii": { "type": "boolean", "default": false }, "contains_sensitive_content": { "type": "boolean", "default": false }, "contains_explicit_content": { "type": "boolean", "default": false }, "explicit_content_types": { "type": "array", "items": { "$ref": "#/components/schemas/ExplicitContentTypesEnum" } } }, "required": [ "filename", "id", "name", "upload_url" ] }, "Rater": { "type": "object", "properties": { "id": { "type": "string" }, "user": { "$ref": "#/components/schemas/User" }, "meta": {} }, "required": [ "id", "user" ] }, "RaterPool": { "type": "object", "properties": { "id": { "type": "string" }, "kind": { "allOf": [ { "$ref": "#/components/schemas/RaterPoolKindEnum" } ], "description": "The type of rater pool.\n\n* `0` - Anyone\n* `100` - Project members\n* `400` - Crowd-sourced raters\n* `500` - AI raters\n* `600` - Rater pool\n* `700` - Flock raters", "minimum": -2147483648, "maximum": 2147483647 }, "name": { "type": "string" }, "label": { "type": "string", "description": "Optional name for the rater pool displayed to our users.", "maxLength": 48 }, "size": { "type": [ "integer", "null" ], "description": "Return the size of the rater pool, if known.\n\nFor `RATER_LIST` pools, this is the number of members. For other pool types,\nthe size is unknown and `None` unless manually specified in the config.", "readOnly": true } }, "required": [ "id", "name", "size" ] }, "RaterPoolKindEnum": { "enum": [ 0, 100, 400, 500, 600, 700 ], "type": "integer", "description": "* `0` - Anyone\n* `100` - Project members\n* `400` - Crowd-sourced raters\n* `500` - AI raters\n* `600` - Rater pool\n* `700` - Flock raters" }, "Rating": { "type": "object", "properties": { "score": { "type": [ "number", "null" ], "format": "double" }, "stimulus_id": { "type": "integer", "readOnly": true }, "stimulus": { "type": "string" }, "stimulus_fullpath": { "type": "string" }, "condition": { "type": "string" }, "dimension": { "type": [ "string", "null" ], "maxLength": 32 }, "meta": { "description": "For example, expected score range." }, "is_correct": { "type": [ "boolean", "null" ], "description": "Checks if score is within expectation.\n\nFirst checks for explicit correctness in `meta['isCorrect']`.\nFalls back to comparing score within bounds given in `meta['expectedScores']` (inclusive).\n\nReturns:\n `True` if score is correct, `False` if incorrect, or `None` if neither.", "readOnly": true } }, "required": [ "condition", "is_correct", "stimulus", "stimulus_fullpath", "stimulus_id" ] }, "RatingDetail": { "type": "object", "properties": { "score": { "type": [ "number", "null" ], "format": "double" }, "rater_id": { "type": [ "string", "null" ], "readOnly": true }, "slate_id": { "type": "integer", "readOnly": true }, "session_id": { "type": "string" }, "job_id": { "type": "string" }, "stimulus_id": { "type": "integer", "readOnly": true }, "stimulus": { "type": "string" }, "stimulus_fullpath": { "type": "string" }, "dimension": { "type": [ "string", "null" ], "maxLength": 32 }, "is_training": { "type": "boolean" }, "meta": { "description": "For example, expected score range." }, "is_correct": { "type": [ "boolean", "null" ], "description": "Checks if score is within expectation.\n\nFirst checks for explicit correctness in `meta['isCorrect']`.\nFalls back to comparing score within bounds given in `meta['expectedScores']` (inclusive).\n\nReturns:\n `True` if score is correct, `False` if incorrect, or `None` if neither.", "readOnly": true } }, "required": [ "is_correct", "is_training", "job_id", "rater_id", "session_id", "slate_id", "stimulus", "stimulus_fullpath", "stimulus_id" ] }, "Results": { "type": "object", "description": "Add options used by the front-end to render results.", "properties": { "kind": { "$ref": "#/components/schemas/MetricKind" }, "dimension": { "type": [ "string", "null" ], "description": "Used by experiments evaluating multiple criteria simultaneously.", "maxLength": 32 }, "source": { "type": [ "string", "null" ], "description": "Return the source of the metric (job or experiment).", "readOnly": true }, "scores": { "type": "array", "items": {}, "description": "Return the scores for each method.", "readOnly": true }, "options": { "type": "object", "additionalProperties": {}, "description": "Front-end options.", "readOnly": true } }, "required": [ "kind", "options", "scores", "source" ] }, "Rubric": { "type": "object", "properties": { "id": { "type": "string" }, "leaderboards": { "type": "array", "items": { "$ref": "#/components/schemas/Leaderboard" }, "readOnly": true }, "name": { "type": "string", "description": "A descriptive name for the rubric.", "maxLength": 100 }, "description": { "type": "string", "description": "Optional description of the rubric." }, "date_created": { "type": "string", "format": "date-time", "readOnly": true }, "date_updated": { "type": "string", "format": "date-time", "readOnly": true, "title": "Last updated" } }, "required": [ "date_created", "date_updated", "id", "leaderboards", "name" ] }, "RubricDetail": { "type": "object", "properties": { "id": { "type": "string" }, "leaderboards": { "type": "array", "items": { "$ref": "#/components/schemas/LeaderboardDetail" }, "readOnly": true }, "name": { "type": "string", "description": "A descriptive name for the rubric.", "maxLength": 100 }, "description": { "type": "string", "description": "Optional description of the rubric." }, "date_created": { "type": "string", "format": "date-time", "readOnly": true }, "date_updated": { "type": "string", "format": "date-time", "readOnly": true, "title": "Last updated" } }, "required": [ "date_created", "date_updated", "id", "leaderboards", "name" ] }, "Session": { "type": "object", "properties": { "id": { "type": "string" }, "job_id": { "type": "string" }, "job_session_id": { "type": [ "integer", "null" ], "maximum": 2147483647, "minimum": 0, "description": "The ID of the session in the job." }, "experiment_id": { "type": "string" }, "date_created": { "type": "string", "format": "date-time", "readOnly": true }, "date_updated": { "type": "string", "format": "date-time", "readOnly": true, "title": "Last updated" }, "date_completed": { "type": [ "string", "null" ], "format": "date-time", "title": "Completion date" }, "template_id": { "type": [ "integer", "null" ], "readOnly": true }, "meta": {}, "rater": { "$ref": "#/components/schemas/Rater" }, "health": { "type": [ "number", "null" ], "format": "double", "readOnly": true } }, "required": [ "date_created", "date_updated", "experiment_id", "health", "id", "job_id", "rater", "template_id" ] }, "SimpleDataset": { "type": "object", "description": "Used in workflows to configure datasets.", "properties": { "contains_pii": { "type": "boolean", "default": false }, "contains_sensitive_content": { "type": "boolean", "default": false }, "contains_explicit_content": { "type": "boolean", "default": false }, "explicit_content_types": { "type": "array", "items": { "$ref": "#/components/schemas/ExplicitContentTypesEnum" } }, "name": { "type": "string", "maxLength": 64 } } }, "Slate": { "type": "object", "properties": { "id": { "type": "integer", "readOnly": true }, "rater_id": { "type": [ "string", "null" ], "readOnly": true }, "session_id": { "type": "string" }, "job_id": { "type": "string" }, "ratings": { "type": "array", "items": { "$ref": "#/components/schemas/Rating" }, "readOnly": true }, "date_updated": { "type": "string", "format": "date-time", "readOnly": true, "title": "Last updated" }, "meta": { "description": "For example, the time it took to complete the slate." }, "is_training": { "type": "boolean" }, "is_golden": { "type": [ "boolean", "null" ], "description": "True if slate corresponds to a golden question." } }, "required": [ "date_updated", "id", "is_training", "job_id", "rater_id", "ratings", "session_id" ] }, "StatusC7dEnum": { "enum": [ "active", "inactive", "failed" ], "type": "string", "description": "* `active` - Active\n* `inactive` - Inactive\n* `failed` - Failed" }, "Stimulus": { "type": "object", "properties": { "id": { "type": "integer", "readOnly": true }, "filepath": { "type": "string" }, "fullpath": { "type": "string", "description": "Complete path within storage bucket.", "readOnly": true }, "modifiers": { "description": "Parameters of signal processors modifying the stimulus." }, "group_id": { "type": [ "integer", "null" ], "readOnly": true } }, "required": [ "filepath", "fullpath", "group_id", "id" ] }, "User": { "type": "object", "properties": { "id": { "type": "string" }, "name": { "type": "string", "title": "Full name", "maxLength": 256 } }, "required": [ "id", "name" ] }, "Webhook": { "type": "object", "description": "Serializer for the Webhook model.", "properties": { "id": { "type": "string", "readOnly": true }, "target_url": { "type": "string", "format": "uri", "maxLength": 2048 }, "events": { "type": "array", "items": { "$ref": "#/components/schemas/EventsEnum" }, "description": "List of event types to subscribe to (e.g., [\"dataset.status_changed\", \"job.completed\"])." }, "is_active": { "type": "boolean", "description": "Deprecated. Use status instead. This field will be sunsetted in version 1.0.", "readOnly": true, "deprecated": true }, "status": { "allOf": [ { "$ref": "#/components/schemas/StatusC7dEnum" } ], "readOnly": true }, "errors": { "readOnly": true }, "secret_id": { "type": "string", "readOnly": true }, "date_created": { "type": "string", "format": "date-time", "readOnly": true }, "date_updated": { "type": "string", "format": "date-time", "readOnly": true } }, "required": [ "date_created", "date_updated", "errors", "events", "id", "is_active", "secret_id", "status", "target_url" ] }, "WebhookCreate": { "type": "object", "description": "Serializer for creating a new Webhook instance.", "properties": { "target_url": { "type": "string", "format": "uri", "maxLength": 2048 }, "events": { "type": "array", "items": { "$ref": "#/components/schemas/EventsEnum" }, "description": "List of event types to subscribe to (e.g., [\"dataset.status_changed\", \"job.completed\"])." }, "secret_id": { "type": "string", "description": "ID of the secret to use. If omitted, the most recent project secret is used. If the project has no secrets, a new one will be created automatically." } }, "required": [ "events", "target_url" ] }, "WebhookSecret": { "type": "object", "description": "Serializer for WebhookSecret model, including related Webhook IDs.", "properties": { "id": { "type": "string", "readOnly": true }, "value": { "type": "string", "readOnly": true }, "webhook_ids": { "type": "array", "items": { "type": "string" }, "description": "Get the list of webhook IDs associated with this secret.\n\nArgs:\n obj (WebhookSecret): The WebhookSecret instance.\nReturns:\n list[str]: List of webhook IDs associated with the secret.", "readOnly": true }, "date_created": { "type": "string", "format": "date-time", "readOnly": true } }, "required": [ "date_created", "id", "value", "webhook_ids" ] }, "Workflow": { "type": "object", "description": "Serialized shape returned after creating a workflow (experiment + job + datasets).", "properties": { "dataset": { "allOf": [ { "$ref": "#/components/schemas/Dataset" } ], "readOnly": true }, "experiment": { "allOf": [ { "$ref": "#/components/schemas/Experiment" } ], "readOnly": true }, "job": { "allOf": [ { "$ref": "#/components/schemas/Job" } ], "readOnly": true }, "training_dataset": { "oneOf": [ { "$ref": "#/components/schemas/Dataset" }, { "type": "null" } ], "readOnly": true } }, "required": [ "dataset", "experiment", "job", "training_dataset" ] }, "WorkflowCreate": { "type": "object", "description": "Serializer for creating a draft experiment and job via API in one request.", "properties": { "experiment": { "$ref": "#/components/schemas/WorkflowsCreateExperiment" }, "job": { "$ref": "#/components/schemas/WorkflowsCreateJob" }, "dataset": { "$ref": "#/components/schemas/SimpleDataset" }, "training_dataset": { "$ref": "#/components/schemas/SimpleDataset" }, "sessions": { "type": "array", "items": { "type": "array", "items": {}, "minItems": 1 }, "minItems": 1 }, "session_url": { "type": "string" }, "training_sessions": { "type": "array", "items": { "type": "array", "items": {} } }, "training_session_url": { "type": "string" } }, "required": [ "experiment", "job" ] }, "WorkflowsCreateExperiment": { "type": "object", "description": "Serializer for creating draft experiments via API with minimal fields.", "properties": { "name": { "type": "string", "description": "A description identifying the experiment.", "maxLength": 128 }, "title": { "type": [ "string", "null" ], "description": "A short title displayed at the top of the experiment screen.", "maxLength": 128 }, "type": { "allOf": [ { "$ref": "#/components/schemas/WorkflowsCreateExperimentTypeEnum" } ], "description": "Type of the experiment (e.g., \"mushra\", \"pairwise_image\")\n\n* `acr_audio` - acr_audio\n* `acr_video` - acr_video\n* `acr_image` - acr_image\n* `multi_acr_embed` - multi_acr_embed\n* `multi_acr_video` - multi_acr_video\n* `binary_audio` - binary_audio\n* `binary_image` - binary_image\n* `binary_video` - binary_video\n* `inpainting_image` - inpainting_image\n* `mushra` - mushra\n* `pairwise_audio` - pairwise_audio\n* `pairwise_image` - pairwise_image\n* `pairwise_video` - pairwise_video\n* `survey_audio` - survey_audio\n* `survey_embed` - survey_embed\n* `survey_video` - survey_video\n* `survey_image` - survey_image\n* `pre_screen_audio` - pre_screen_audio\n* `pre_screen_image` - pre_screen_image\n* `rank_audio` - rank_audio" }, "language": { "allOf": [ { "$ref": "#/components/schemas/LanguageEnum" } ], "description": "The language used for the experiment's interface.\n\n* `ar` - Arabic\n* `de` - German\n* `en` - English\n* `es` - Spanish\n* `fr` - French\n* `hi` - Hindi\n* `it` - Italian\n* `ja` - Japanese\n* `ko` - Korean\n* `pl` - Polish\n* `pt` - Portuguese\n* `vi` - Vietnamese\n* `zh-hans` - Chinese (simplified)" }, "question": { "type": [ "string", "null" ], "description": "A question or instructions displayed at the top of the experiment screen.", "maxLength": 512 }, "description": { "type": [ "string", "null" ], "description": "An optional longer explanation of how to evaluate the data." }, "introduction": { "type": [ "string", "null" ], "description": "A brief introduction explaining how to participate in the experiment. Leave empty to use default introduction.", "maxLength": 2048 }, "hero_media_url": { "type": "string", "description": "URL of a custom image or video to replace the default screenshot.", "maxLength": 1024 }, "hero_caption": { "type": [ "string", "null" ], "description": "Caption displayed below the custom hero image/video.", "maxLength": 256 }, "config": { "$ref": "#/components/schemas/ExperimentConfig" } }, "required": [ "name", "type" ] }, "WorkflowsCreateExperimentTypeEnum": { "enum": [ "acr_audio", "acr_video", "acr_image", "multi_acr_embed", "multi_acr_video", "binary_audio", "binary_image", "binary_video", "inpainting_image", "mushra", "pairwise_audio", "pairwise_image", "pairwise_video", "survey_audio", "survey_embed", "survey_video", "survey_image", "pre_screen_audio", "pre_screen_image", "rank_audio" ], "type": "string", "description": "* `acr_audio` - acr_audio\n* `acr_video` - acr_video\n* `acr_image` - acr_image\n* `multi_acr_embed` - multi_acr_embed\n* `multi_acr_video` - multi_acr_video\n* `binary_audio` - binary_audio\n* `binary_image` - binary_image\n* `binary_video` - binary_video\n* `inpainting_image` - inpainting_image\n* `mushra` - mushra\n* `pairwise_audio` - pairwise_audio\n* `pairwise_image` - pairwise_image\n* `pairwise_video` - pairwise_video\n* `survey_audio` - survey_audio\n* `survey_embed` - survey_embed\n* `survey_video` - survey_video\n* `survey_image` - survey_image\n* `pre_screen_audio` - pre_screen_audio\n* `pre_screen_image` - pre_screen_image\n* `rank_audio` - rank_audio" }, "WorkflowsCreateJob": { "type": "object", "description": "Serializer for creating draft jobs via API with minimal fields.", "properties": { "rater_pool_id": { "type": "string", "description": "Id of the rater pool to use for this job" }, "rater_pool": { "type": "string", "description": "Id of the rater pool to use for this job" }, "max_repetitions": { "type": "integer", "maximum": 2147483647, "minimum": 0, "title": "Maximum number of repetitions", "description": "How often an individual rater is allowed to repeat the task." }, "min_rest_time": { "type": "integer", "maximum": 2147483647, "minimum": 0, "title": "Minimum rest time", "description": "How many minutes should pass before a rater can repeat the task." }, "num_comparisons": { "type": "integer", "maximum": 1000, "minimum": 1, "title": "Number of comparisons per rater", "description": "The maximum number of groups of stimuli evaluated by each rater." }, "num_sessions": { "type": "integer", "maximum": 10000, "minimum": 0, "title": "Number of sessions", "description": "How often the task is performed." }, "strategy": { "default": "randomized", "description": "Strategy for presenting stimuli. Either a strategy type string (e.g., \"randomized\") or a full configuration object (e.g., {\"type\": \"randomized\", \"reference\": \"exclude\"})." }, "launch": { "type": "boolean", "default": false, "description": "Whether to launch the job immediately after creation." }, "hourly_rate_gbp": { "type": "string", "format": "decimal", "pattern": "^-?\\d{0,2}(?:\\.\\d{0,2})?$", "description": "Hourly rate in GBP to pay raters." } } } }, "securitySchemes": { "ApiKeyAuth": { "type": "apiKey", "in": "header", "name": "Authorization", "description": "API key authentication. Send your key in the `Authorization` header, prefixed with `Api-Key`, e.g. `Api-Key `." }, "cookieAuth": { "type": "apiKey", "in": "cookie", "name": "sessionid" } } }, "servers": [ { "url": "https://api.mabyduck.com" } ], "tags": [ { "name": "Datasets", "description": "Operations related to datasets." }, { "name": "Experiments", "description": "Operations related to experiments." }, { "name": "Rubrics", "description": "Operations related to rubrics and leaderboards." } ] }