// 20241229125428 // https://api.datafuel.dev/openapi.json { "openapi": "3.1.0", "info": { "title": "DataFuel API", "summary": "DataFuel API is a simple API that allows you to turn website into LLM-ready data", "description": "DataFuel API is a simple API that allows you to turn website into LLM-ready data", "version": "0.0.2" }, "servers": [ { "url": "https://api.datafuel.dev" } ], "paths": { "/ping": { "get": { "summary": "Hello", "operationId": "hello_ping_get", "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { } } } } } } }, "/": { "get": { "summary": "Root", "operationId": "root__get", "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { } } } } } } }, "/scrape": { "post": { "summary": "Scrape", "operationId": "scrape_scrape_post", "requestBody": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ScrapeRequest" } } }, "required": true }, "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ScrapeResponse" } } } }, "400": { "description": "Bad Request" }, "401": { "description": "Unauthorized" }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } }, "security": [ { "HTTPBearer": [ ] } ] } }, "/crawl": { "post": { "summary": "Crawl", "operationId": "crawl_crawl_post", "requestBody": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/CrawlRequest" } } }, "required": true }, "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ScrapeResponse" } } } }, "400": { "description": "Bad Request" }, "401": { "description": "Unauthorized" }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } }, "security": [ { "HTTPBearer": [ ] } ] } }, "/scrape_with_login": { "post": { "summary": "Scrape With Login", "operationId": "scrape_with_login_scrape_with_login_post", "requestBody": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ScrapeWithLoginRequest" } } }, "required": true }, "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ScrapeResponse" } } } }, "400": { "description": "Bad Request" }, "401": { "description": "Unauthorized" }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } }, "security": [ { "HTTPBearer": [ ] } ] } }, "/list_scrapes": { "get": { "summary": "Get Scrapes Metadata", "operationId": "get_scrapes_metadata_list_scrapes_get", "security": [ { "HTTPBearer": [ ] } ], "parameters": [ { "name": "job_id", "in": "query", "required": false, "schema": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "description": "Optional job ID to filter scrapes", "title": "Job Id" }, "description": "Optional job ID to filter scrapes" }, { "name": "markdown", "in": "query", "required": false, "schema": { "type": "boolean", "description": "If true, return markdown data on top of the signed_url", "default": false, "title": "Markdown" }, "description": "If true, return markdown data on top of the signed_url" }, { "name": "ai_response", "in": "query", "required": false, "schema": { "type": "boolean", "description": "If true, return AI response data on top of the signed_url", "default": false, "title": "Ai Response" }, "description": "If true, return AI response data on top of the signed_url" }, { "name": "url", "in": "query", "required": false, "schema": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "description": "If provided, filter scrapes by the given URL", "title": "Url" }, "description": "If provided, filter scrapes by the given URL" } ], "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "type": "array", "items": { "$ref": "#/components/schemas/ScrapeMetadata" }, "title": "Response Get Scrapes Metadata List Scrapes Get" } } } }, "401": { "description": "Unauthorized" }, "404": { "description": "Not Found" }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } } }, "/me": { "get": { "summary": "Get User Info", "operationId": "get_user_info_me_get", "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/UserInfo" } } } }, "401": { "description": "Unauthorized" } }, "security": [ { "HTTPBearer": [ ] } ] } }, "/list_jobs": { "get": { "summary": "Get Job Ids", "operationId": "get_job_ids_list_jobs_get", "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "items": { "$ref": "#/components/schemas/JobId" }, "type": "array", "title": "Response Get Job Ids List Jobs Get" } } } }, "401": { "description": "Unauthorized" }, "404": { "description": "No jobs found" } }, "security": [ { "HTTPBearer": [ ] } ] } }, "/webhook/subscribe": { "post": { "summary": "Subscribe Webhook", "operationId": "subscribe_webhook_webhook_subscribe_post", "requestBody": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/WebhookSubscription" } } }, "required": true }, "responses": { "201": { "description": "Successful Response", "content": { "application/json": { "schema": { } } } }, "400": { "description": "Invalid webhook URL" }, "401": { "description": "Unauthorized" }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } }, "security": [ { "HTTPBearer": [ ] } ] } }, "/webhook/unsubscribe": { "delete": { "summary": "Unsubscribe Webhook", "operationId": "unsubscribe_webhook_webhook_unsubscribe_delete", "security": [ { "HTTPBearer": [ ] } ], "parameters": [ { "name": "webhook_url", "in": "query", "required": true, "schema": { "type": "string", "description": "URL to unsubscribe from webhook notifications", "title": "Webhook Url" }, "description": "URL to unsubscribe from webhook notifications" } ], "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { } } } }, "401": { "description": "Unauthorized" }, "404": { "description": "Webhook URL not found" }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } } }, "/webhook": { "post": { "summary": "Webhook Handler", "operationId": "webhook_handler_webhook_post", "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { } } } } } } } }, "components": { "schemas": { "CrawlRequest": { "properties": { "url": { "type": "string", "title": "Url" }, "ai_prompt": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Ai Prompt" }, "json_schema": { "anyOf": [ { "type": "object" }, { "type": "null" } ], "title": "Json Schema", "description": "Optional schema definition for structured data extraction. \nFormat should follow OpenAI's function calling schema format (https://platform.openai.com/docs/guides/structured-outputs).\n\nExample types:\n- string: \"type\": \"string\"\n- integer: \"type\": \"integer\"\n- number: \"type\": \"number\"\n- boolean: \"type\": \"boolean\"\n- array: \"type\": \"array\", \"items\": {\"type\": \"string\"}\n- object: \"type\": \"object\", \"properties\": {...}", "example": { "description": "Schema for capturing product information", "name": "Product Schema", "schema": { "properties": { "product_url": { "description": "The URL of the specific product", "type": "string" }, "product_name": { "description": "The name of the specific product", "type": "string" }, "price": { "description": "The price of the product", "type": "number" }, "product_images": { "description": "List of product image URLs", "items": { "properties": { "url": { "description": "URL of the product image", "type": "string" } }, "required": [ "url" ], "type": "object" }, "type": "array" } }, "required": [ "product_url", "product_name", "price", "product_images" ], "type": "object" } } }, "javascript_scenario": { "anyOf": [ { "items": { "type": "object" }, "type": "array" }, { "type": "null" } ], "title": "Javascript Scenario" }, "depth": { "type": "integer", "title": "Depth", "description": "The depth of the crawl 1 depth mean only the first level of links will be scraped like https://example.com/page1 and https://example.com/page2", "default": 1, "example": 1 }, "limit": { "type": "integer", "title": "Limit", "description": "The maximum number of pages to scrape", "default": 1, "example": 1 }, "exclusion_pattern": { "type": "string", "title": "Exclusion Pattern", "description": "Regex pattern to exclude specific URLs (e.g., 'https://.*datafuel\\.dev/blog/.*' to exclude blog pages)", "default": "", "example": "https://.*datafuel\\.dev/blog/.*" }, "excluded_links": { "type": "string", "title": "Excluded Links", "description": "Comma-separated list of URLs to exclude from crawling", "default": "", "example": "https://www.datafuel.dev/pricing,https://www.datafuel.dev/blog" } }, "type": "object", "required": [ "url" ], "title": "CrawlRequest" }, "HTTPValidationError": { "properties": { "detail": { "items": { "$ref": "#/components/schemas/ValidationError" }, "type": "array", "title": "Detail" } }, "type": "object", "title": "HTTPValidationError" }, "JobId": { "properties": { "id": { "type": "string", "title": "Id", "description": "Unique job ID associated with the API key", "example": "f47ac10b-58cc-4372-a567-0e02b2c3d479" } }, "type": "object", "required": [ "id" ], "title": "JobId" }, "JobStatus": { "type": "string", "enum": [ "pending", "finished" ], "title": "JobStatus" }, "ScrapeMetadata": { "properties": { "job_id": { "type": "string", "title": "Job Id", "description": "The unique identifier for the scraping job that can contain multiple scrapes", "example": "f47ac10b-58cc-4372-a567-0e02b2c3d479" }, "scrape_id": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Scrape Id", "description": "The unique identifier for the each URL scrape", "example": "550e8400-e29b-41d4-a716-446655440000" }, "scrape_status": { "$ref": "#/components/schemas/ScrapeStatus" }, "scrape_url": { "type": "string", "title": "Scrape Url", "description": "The URL that was scraped", "example": "https://example.com" }, "scrape_timestamp": { "type": "string", "title": "Scrape Timestamp", "description": "The timestamp of when the scrape was created", "example": "2024-03-15T10:30:00Z" }, "signed_url": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Signed Url", "description": "Pre-signed URL containing the scraping results in JSON format with structure:\n{\n \"ai_response\": \"AI-generated analysis or structured data\",\n \"markdown\": \"Content formatted in markdown\",\n \"html\": \"Raw HTML content from the scraped page\"\n}", "example": "https://docyjgyvimrauivbukcp.supabase.co/storage/v1/object/sign/scrapes-data/..." }, "ai_response": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Ai Response", "description": "AI-generated analysis or structured data" }, "markdown": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Markdown", "description": "Content formatted in markdown" }, "job_status": { "$ref": "#/components/schemas/JobStatus" } }, "type": "object", "required": [ "job_id", "scrape_status", "scrape_url", "scrape_timestamp", "job_status" ], "title": "ScrapeMetadata", "example": { "job_id": "f47ac10b-58cc-4372-a567-0e02b2c3d479", "job_status": "finished", "scrape_id": "550e8400-e29b-41d4-a716-446655440000", "scrape_status": "success", "scrape_timestamp": "2024-03-15T10:30:00Z", "scrape_url": "https://example.com", "signed_url": "https://docyjgyvimrauivbukcp.supabase.co/storage/v1/object/sign/scrapes-data/..." }, "signed_url_response_format": { "description": "The signed_url returns a JSON with the following structure:", "schema": { "ai_response": "AI-generated analysis or structured data", "html": "Raw HTML content from the scraped page", "markdown": "Content formatted in markdown" } } }, "ScrapeRequest": { "properties": { "url": { "type": "string", "title": "Url" }, "ai_prompt": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Ai Prompt" }, "json_schema": { "anyOf": [ { "type": "object" }, { "type": "null" } ], "title": "Json Schema", "description": "Optional schema definition for structured data extraction. \nFormat should follow OpenAI's function calling schema format (https://platform.openai.com/docs/guides/structured-outputs).\n\nExample types:\n- string: \"type\": \"string\"\n- integer: \"type\": \"integer\"\n- number: \"type\": \"number\"\n- boolean: \"type\": \"boolean\"\n- array: \"type\": \"array\", \"items\": {\"type\": \"string\"}\n- object: \"type\": \"object\", \"properties\": {...}", "example": { "description": "Schema for capturing product information", "name": "Product Schema", "schema": { "properties": { "product_url": { "description": "The URL of the specific product", "type": "string" }, "product_name": { "description": "The name of the specific product", "type": "string" }, "price": { "description": "The price of the product", "type": "number" }, "product_images": { "description": "List of product image URLs", "items": { "properties": { "url": { "description": "URL of the product image", "type": "string" } }, "required": [ "url" ], "type": "object" }, "type": "array" } }, "required": [ "product_url", "product_name", "price", "product_images" ], "type": "object" } } }, "javascript_scenario": { "anyOf": [ { "items": { "type": "object" }, "type": "array" }, { "type": "null" } ], "title": "Javascript Scenario" } }, "type": "object", "required": [ "url" ], "title": "ScrapeRequest" }, "ScrapeResponse": { "properties": { "job_id": { "type": "string", "title": "Job Id", "description": "The identifier for the scraping job", "example": "f47ac10b-58cc-4372-a567-0e02b2c3d479" } }, "type": "object", "required": [ "job_id" ], "title": "ScrapeResponse" }, "ScrapeStatus": { "type": "string", "enum": [ "pending", "failed", "success" ], "title": "ScrapeStatus" }, "ScrapeWithLoginRequest": { "properties": { "url": { "type": "string", "title": "Url" }, "ai_prompt": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Ai Prompt" }, "json_schema": { "anyOf": [ { "type": "object" }, { "type": "null" } ], "title": "Json Schema", "description": "Optional schema definition for structured data extraction. \nFormat should follow OpenAI's function calling schema format (https://platform.openai.com/docs/guides/structured-outputs).\n\nExample types:\n- string: \"type\": \"string\"\n- integer: \"type\": \"integer\"\n- number: \"type\": \"number\"\n- boolean: \"type\": \"boolean\"\n- array: \"type\": \"array\", \"items\": {\"type\": \"string\"}\n- object: \"type\": \"object\", \"properties\": {...}", "example": { "description": "Schema for capturing product information", "name": "Product Schema", "schema": { "properties": { "product_url": { "description": "The URL of the specific product", "type": "string" }, "product_name": { "description": "The name of the specific product", "type": "string" }, "price": { "description": "The price of the product", "type": "number" }, "product_images": { "description": "List of product image URLs", "items": { "properties": { "url": { "description": "URL of the product image", "type": "string" } }, "required": [ "url" ], "type": "object" }, "type": "array" } }, "required": [ "product_url", "product_name", "price", "product_images" ], "type": "object" } } }, "javascript_scenario": { "anyOf": [ { "items": { "type": "object" }, "type": "array" }, { "type": "null" } ], "title": "Javascript Scenario" }, "username": { "type": "string", "title": "Username" }, "password": { "type": "string", "title": "Password" }, "login_url": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Login Url" } }, "type": "object", "required": [ "url", "username", "password" ], "title": "ScrapeWithLoginRequest" }, "UserInfo": { "properties": { "user_id": { "type": "string", "title": "User Id" }, "is_valid": { "type": "boolean", "title": "Is Valid", "default": true } }, "type": "object", "required": [ "user_id" ], "title": "UserInfo" }, "ValidationError": { "properties": { "loc": { "items": { "anyOf": [ { "type": "string" }, { "type": "integer" } ] }, "type": "array", "title": "Location" }, "msg": { "type": "string", "title": "Message" }, "type": { "type": "string", "title": "Error Type" } }, "type": "object", "required": [ "loc", "msg", "type" ], "title": "ValidationError" }, "WebhookSubscription": { "properties": { "webhook_url": { "type": "string", "title": "Webhook Url", "description": "URL to receive webhook notifications", "example": "https://api.example.com/webhook" } }, "type": "object", "required": [ "webhook_url" ], "title": "WebhookSubscription" } }, "securitySchemes": { "HTTPBearer": { "type": "http", "scheme": "bearer" } } } }