Request to crawl from a base URL and return a list of discovered URLs with their associated data. You can specify the crawl depth and limit the number of pages to crawl.
{"docsConfig":{"$schema":"https://mintlify.com/docs.json","theme":"mint","name":"DataFuel","colors":{"primary":"#0069ff","light":"#e6f0ff","dark":"#0052cc"},"favicon":"/Lucide Fuel (3).png","logo":{"light":"https://mintcdn.com/bezier/qb9ycHhMyN-s_tYE/datafuellight2.png?fit=max\u0026auto=format\u0026n=qb9ycHhMyN-s_tYE\u0026q=85\u0026s=7c2479a48219201a11d73535277d31ea","dark":"https://mintcdn.com/bezier/qb9ycHhMyN-s_tYE/datafuellight2.png?fit=max\u0026auto=format\u0026n=qb9ycHhMyN-s_tYE\u0026q=85\u0026s=7c2479a48219201a11d73535277d31ea","href":"https://app.datafuel.dev"},"appearance":{"default":"light","strict":true},"navbar":{"links":[{"label":"Support","href":"mailto:sacha@datafuel.dev"}],"primary":{"type":"button","label":"Website","href":"https://app.datafuel.dev"}},"footer":{"socials":{"x":"https://x.com/dumay_sacha","linkedin":"https://www.linkedin.com/in/dumaysacha/"}},"navigation":{"pages":[]}},"deployment":{"deploymentMetadataValue":{"subdomain":"bezier","hasAudioNarration":false,"actualSubdomain":"bezier","customDomains":["docs.datafuel.dev"],"gitSource":{"type":"github","owner":"racinger","repo":"scrapewebappdoc","deployBranch":"main","contentDirectory":"","isPrivate":true,"mountPath":"","label":""},"chroma":{"liveDeploymentHistoryId":"6a21a62232881c474e321212","collectionId":"a3fdf723-1bcf-454f-9978-96c888963881","schemaVersion":"v2"},"entitlements":{},"buildId":"6ae91f0af4c9034c2a6d02aa:success","clientVersion":"0.0.4076","searchFilterCounts":[{"language":"default","tag":"Documentation","count":5},{"language":"default","tag":"Endpoints","count":4},{"language":"default","tag":"Get Started","count":1}]},"assistantConfig":{},"basePath":"","hasChatPermissions":false,"hasWebMcpPermissions":false,"toggles":[{"name":"kill-isr","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"agent-mintlify-slack","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"agent-integrations-settings","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"rbac-expansion","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"automations-pr-grouping","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"custom-prefix","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"analytics-v7","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"user-flows-analytics","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"no-publish-flow","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"workos-onboarding","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"llms-txt-v2","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"snapshot-serving-live","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"stripTemperatureMiddleware","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"support-platform-pylon-zendesk","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"snapshot-serving","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"instant-rollback","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"granular-notifications-server","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"granular-notifications","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"automations-fable","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"fixed-ai-action-pricing","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"automation-resolve-merge-conflict","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"scoped-publishing","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"mintql","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"page-level-analytics","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"editor-version-history-v2","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"billing-page-v2","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"full-page-search","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"workspace-projects","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"assistant-v5","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"fork-pr-previews","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"revalidation-jwt","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"workos-sso-scim","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"hobby-abuse-scan","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"dashboard-inbox","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"workos-signup","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"turbopuffer-dual-write","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"workos-enabled","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"approve-review-flow","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"sso-setup-deadline-banner","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"turbopuffer-serve","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"live-agents","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"comment-mode","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false}],"googleFontsInlined":false},"customCss":[],"page":{"pagePath":"api-reference/endpoint/crawl","slug":"/api-reference/endpoint/crawl","pageMetadata":{"title":"Crawl","description":"Request to crawl from a base URL and return a list of discovered URLs with their associated data. You can specify the crawl depth and limit the number of pages to crawl.","openapi":"post /crawl","href":"/api-reference/endpoint/crawl"},"description":{"compiledSource":"\"use strict\";\nconst {jsx: _jsx} = arguments[0];\nconst {useMDXComponents: _provideComponents} = arguments[0];\nfunction _createMdxContent(props) {\n const _components = {\n p: \"p\",\n ..._provideComponents(),\n ...props.components\n };\n return _jsx(_components.p, {\n children: \"Request to crawl from a base URL and return a list of discovered URLs with their associated data. You can specify the crawl depth and limit the number of pages to crawl.\"\n });\n}\nfunction MDXContent(props = {}) {\n const {wrapper: MDXLayout} = {\n ..._provideComponents(),\n ...props.components\n };\n return MDXLayout ? _jsx(MDXLayout, {\n ...props,\n children: _jsx(_createMdxContent, {\n ...props\n })\n }) : _createMdxContent(props);\n}\nreturn {\n default: MDXContent\n};\n","frontmatter":{},"scope":{}},"pageType":"default","mdx":{"mdxSource":{"compiledSource":"\"use strict\";const{Fragment:_Fragment,jsx:_jsx}=arguments[0];const{useMDXComponents:_provideComponents}=arguments[0];function _createMdxContent(props){return _jsx(_Fragment,{})}function MDXContent(props={}){const{wrapper:MDXLayout}={..._provideComponents(),...props.components};return MDXLayout?_jsx(MDXLayout,{...props,children:_jsx(_createMdxContent,{...props})}):_createMdxContent(props)}return{default:MDXContent}","frontmatter":{},"scope":{"config":{},"pageMetadata":{"title":"Crawl","description":"Request to crawl from a base URL and return a list of discovered URLs with their associated data. You can specify the crawl depth and limit the number of pages to crawl.","openapi":"post /crawl","href":"/api-reference/endpoint/crawl"}}},"mdxSourceWithNoJs":null},"mdxExtracts":{"tableOfContents":[],"codeExamples":{}},"jsLinks":[],"subdomain":"bezier","siteName":"DataFuel","title":"Crawl - DataFuel","navMaps":{"firstHrefInVersion":{},"firstHrefInLanguage":{}},"lastModified":"2026-01-07T00:53:20.515Z","originalFileLocation":"api-reference/endpoint/crawl.mdx"},"headHtml":"\u003cscript data-mint-head>(function(){var d=document.documentElement,p=\"light\";if(!p){try{p=localStorage.getItem(\"isDarkMode\")}catch(e){}if(p==='true')p='dark';else if(p==='false')p='light';if(p!=='light'\u0026\u0026p!=='dark'\u0026\u0026p!=='system')p=\"light\"}var r=p==='system'?(matchMedia(\"(prefers-color-scheme: dark)\").matches?'dark':'light'):p;d.classList.remove(r==='dark'?'light':'dark');d.classList.add(r);d.setAttribute(\"data-theme-preference\",p)})();\u003c/script>\u003cmeta data-mint-head name=\"description\" content=\"Request to crawl from a base URL and return a list of discovered URLs with their associated data. You can specify the crawl depth and limit the number of pages to crawl.\">\u003cmeta data-mint-head name=\"application-name\" content=\"DataFuel\">\u003cmeta data-mint-head name=\"generator\" content=\"Mintlify\">\u003clink data-mint-head rel=\"icon\" type=\"image/png\" sizes=\"192x192\" href=\"/mintlify-assets/_mintlify/favicons/bezier/Rd3R1-5VI8Q8P2NU/_generated/favicon/android-chrome-192x192.png\" media=\"(prefers-color-scheme: light)\">\u003clink data-mint-head rel=\"apple-touch-icon\" type=\"image/png\" sizes=\"180x180\" href=\"/mintlify-assets/_mintlify/favicons/bezier/Rd3R1-5VI8Q8P2NU/_generated/favicon/apple-touch-icon.png\">\u003clink data-mint-head rel=\"icon\" type=\"image/png\" sizes=\"16x16\" href=\"/mintlify-assets/_mintlify/favicons/bezier/Rd3R1-5VI8Q8P2NU/_generated/favicon/favicon-16x16.png\" media=\"(prefers-color-scheme: light)\">\u003clink data-mint-head rel=\"icon\" type=\"image/png\" sizes=\"32x32\" href=\"/mintlify-assets/_mintlify/favicons/bezier/Rd3R1-5VI8Q8P2NU/_generated/favicon/favicon-32x32.png\" media=\"(prefers-color-scheme: light)\">\u003clink data-mint-head rel=\"shortcut icon\" type=\"image/x-icon\" href=\"/mintlify-assets/_mintlify/favicons/bezier/Rd3R1-5VI8Q8P2NU/_generated/favicon/favicon.ico\" media=\"(prefers-color-scheme: light)\">\u003clink data-mint-head rel=\"icon\" type=\"image/png\" sizes=\"192x192\" href=\"/mintlify-assets/_mintlify/favicons/bezier/Rd3R1-5VI8Q8P2NU/_generated/favicon-dark/android-chrome-192x192.png\" media=\"(prefers-color-scheme: dark)\">\u003clink data-mint-head rel=\"icon\" type=\"image/png\" sizes=\"16x16\" href=\"/mintlify-assets/_mintlify/favicons/bezier/Rd3R1-5VI8Q8P2NU/_generated/favicon-dark/favicon-16x16.png\" media=\"(prefers-color-scheme: dark)\">\u003clink data-mint-head rel=\"icon\" type=\"image/png\" sizes=\"32x32\" href=\"/mintlify-assets/_mintlify/favicons/bezier/Rd3R1-5VI8Q8P2NU/_generated/favicon-dark/favicon-32x32.png\" media=\"(prefers-color-scheme: dark)\">\u003clink data-mint-head rel=\"shortcut icon\" type=\"image/x-icon\" href=\"/mintlify-assets/_mintlify/favicons/bezier/Rd3R1-5VI8Q8P2NU/_generated/favicon-dark/favicon.ico\" media=\"(prefers-color-scheme: dark)\">\u003cmeta data-mint-head property=\"og:title\" content=\"Crawl - DataFuel\">\u003cmeta data-mint-head property=\"og:description\" content=\"Request to crawl from a base URL and return a list of discovered URLs with their associated data. You can specify the crawl depth and limit the number of pages to crawl.\">\u003cmeta data-mint-head property=\"og:site_name\" content=\"DataFuel\">\u003cmeta data-mint-head property=\"og:url\" content=\"https://docs.datafuel.dev/api-reference/endpoint/crawl\">\u003cmeta data-mint-head property=\"og:type\" content=\"website\">\u003cmeta data-mint-head property=\"og:image\" content=\"https://bezier.mintlify.site/_mintlify/api/og/v2?division=Endpoints\u0026amp;title=Crawl\u0026amp;description=Request+to+crawl+from+a+base+URL+and+return+a+list+of+discovered+URLs+with+their+associated+data.+You+can+specify+the+crawl+depth+and+limit+the+number+of+pages+\u0026amp;theme=90737a5b8bb48718b802f711\u0026amp;signature=fe59295f702501a75046e8b4b06d7ed612b1da64b3a242a8233945f6271346dd\">\u003cmeta data-mint-head property=\"og:image:width\" content=\"1200\">\u003cmeta data-mint-head property=\"og:image:height\" content=\"630\">\u003cmeta data-mint-head name=\"twitter:card\" content=\"summary_large_image\">\u003cmeta data-mint-head name=\"twitter:title\" content=\"Crawl - DataFuel\">\u003cmeta data-mint-head name=\"twitter:description\" content=\"Request to crawl from a base URL and return a list of discovered URLs with their associated data. You can specify the crawl depth and limit the number of pages to crawl.\">\u003cmeta data-mint-head name=\"twitter:image\" content=\"https://bezier.mintlify.site/_mintlify/api/og/v2?division=Endpoints\u0026amp;title=Crawl\u0026amp;description=Request+to+crawl+from+a+base+URL+and+return+a+list+of+discovered+URLs+with+their+associated+data.+You+can+specify+the+crawl+depth+and+limit+the+number+of+pages+\u0026amp;theme=90737a5b8bb48718b802f711\u0026amp;signature=fe59295f702501a75046e8b4b06d7ed612b1da64b3a242a8233945f6271346dd\">\u003cmeta data-mint-head name=\"twitter:image:width\" content=\"1200\">\u003cmeta data-mint-head name=\"twitter:image:height\" content=\"630\">\u003cmeta data-mint-head name=\"msapplication-config\" content=\"/mintlify-assets/_mintlify/favicons/bezier/Rd3R1-5VI8Q8P2NU/_generated/favicon/browserconfig.xml\">\u003cmeta data-mint-head name=\"apple-mobile-web-app-title\" content=\"DataFuel\">\u003cmeta data-mint-head name=\"msapplication-TileColor\" content=\"#0069ff\">\u003clink data-mint-head rel=\"canonical\" href=\"https://docs.datafuel.dev/api-reference/endpoint/crawl\">\u003clink data-mint-head rel=\"alternate\" type=\"application/xml\" href=\"/sitemap.xml\">\u003clink data-mint-head rel=\"alternate\" type=\"text/markdown\" href=\"/api-reference/endpoint/crawl.md\">\u003cscript data-mint-head type=\"application/ld+json\">{\"@context\":\"https://schema.org\",\"@graph\":[{\"@type\":\"Organization\",\"@id\":\"https://docs.datafuel.dev/#organization\",\"name\":\"DataFuel\",\"url\":\"https://docs.datafuel.dev\",\"logo\":{\"@type\":\"ImageObject\",\"url\":\"https://mintcdn.com/bezier/qb9ycHhMyN-s_tYE/datafuellight2.png?fit=max\u0026auto=format\u0026n=qb9ycHhMyN-s_tYE\u0026q=85\u0026s=7c2479a48219201a11d73535277d31ea\"}},{\"@type\":\"WebSite\",\"@id\":\"https://docs.datafuel.dev#website\",\"name\":\"DataFuel\",\"url\":\"https://docs.datafuel.dev\",\"publisher\":{\"@id\":\"https://docs.datafuel.dev/#organization\"}},{\"@type\":\"WebPage\",\"@id\":\"https://docs.datafuel.dev/api-reference/endpoint/crawl#webpage\",\"url\":\"https://docs.datafuel.dev/api-reference/endpoint/crawl\",\"name\":\"Crawl\",\"description\":\"Request to crawl from a base URL and return a list of discovered URLs with their associated data. You can specify the crawl depth and limit the number of pages to crawl.\",\"dateModified\":\"2026-01-07T00:53:20.515Z\",\"isPartOf\":{\"@id\":\"https://docs.datafuel.dev#website\"},\"breadcrumb\":{\"@id\":\"https://docs.datafuel.dev/api-reference/endpoint/crawl#breadcrumb\"}},{\"@type\":\"BreadcrumbList\",\"@id\":\"https://docs.datafuel.dev/api-reference/endpoint/crawl#breadcrumb\",\"itemListElement\":[{\"@type\":\"ListItem\",\"position\":1,\"name\":\"Endpoints\",\"item\":\"https://docs.datafuel.dev/api-reference/endpoint/scrape\"},{\"@type\":\"ListItem\",\"position\":2,\"name\":\"Crawl\",\"item\":\"https://docs.datafuel.dev/api-reference/endpoint/crawl\"}]},{\"@type\":[\"TechArticle\",\"APIReference\"],\"@id\":\"https://docs.datafuel.dev/api-reference/endpoint/crawl#apireference\",\"headline\":\"Crawl\",\"name\":\"Crawl\",\"description\":\"Request to crawl from a base URL and return a list of discovered URLs with their associated data. You can specify the crawl depth and limit the number of pages to crawl.\",\"url\":\"https://docs.datafuel.dev/api-reference/endpoint/crawl\",\"mainEntityOfPage\":{\"@id\":\"https://docs.datafuel.dev/api-reference/endpoint/crawl#webpage\"},\"dateModified\":\"2026-01-07T00:53:20.515Z\",\"publisher\":{\"@id\":\"https://docs.datafuel.dev/#organization\"},\"isPartOf\":{\"@id\":\"https://docs.datafuel.dev#website\"}}]}\u003c/script>\u003cscript data-mint-head type=\"application/ld+json\">{\"@context\":\"https://schema.org\",\"@type\":\"WebSite\",\"name\":\"DataFuel\",\"creator\":{\"@type\":\"Organization\",\"name\":\"Mintlify\",\"url\":\"https://mintlify.com\"}}\u003c/script>","siteName":"DataFuel","nav":{"global":null,"anchors":[{"anchor":"Documentation","icon":"book-open","groups":[{"group":"Get Started","pages":[{"title":"Introduction","href":"/introduction"}]},{"group":"Endpoints","pages":[{"title":"Scrape","openapi":"post /scrape","href":"/api-reference/endpoint/scrape"},{"title":"Scrape With Login","description":"Scrapes web content from authenticated pages by handling login processes before accessing the target URL. Supports authentication form-based login","openapi":"post /scrape_with_login","href":"/api-reference/endpoint/scrape-with-login"},{"title":"Crawl","openapi":"post /crawl","href":"/api-reference/endpoint/crawl"},{"title":"Get Scrapes","description":"Returns a list of all scraping results ordered by timestamp (most recent first). Each result contains details about the scraped URL, including its status and access URL for the scraped content.","openapi":"get /list_scrapes","href":"/api-reference/endpoint/get-scrapes"}]}]},{"anchor":"Support","href":"mailto:sacha@datafuel.dev","icon":"envelope"}]},"basePath":"","title":"Crawl - DataFuel","buildId":"dpl_8LJbrAiZvr1K98WQNZXCyY5AvT34","deploymentId":"dpl_8LJbrAiZvr1K98WQNZXCyY5AvT34","dynamics":["b0f84f53ff"],"chunks":["Page-BUDYNlhG.js","ApiExamples-Ct_Lv36y.js","utils-ChLwp8m0.js","_Uint8Array-BVIqJ90l.js","isObjectLike-C8vmzPEN.js","isArray-Dcu1dQ5t.js","cloneDeep-izs3Hzxb.js","utils-DRktVTDc.js","toString-BX-iyh8G.js","isSymbol-CSH9U2Gm.js","isEqual-BfaiBbcl.js","usePlaygroundInputsStore-Cn2PYw9e.js","shallow-rkyymjXQ.js","snippetPresets-ClQ6XrKD.js","style-to-object-BY3JEAdk.js","v4-DIgz8zy6.js","getApiPlaygroundDisplay-BB9vkXtE.js","CodeSnippet-Bm5RU7m1.js","CodeSnippets-BgU9Deuz.js","getPastedJson-BD-6NNum.js","icon-8Z6tud2e.js","MethodPill-Tuzr4g3V.js","DuplicateIcon-0ceWFYhN.js","createIcon-FBXow_ST.js","DynamicMarkdown-CD8713Yy.js","addUniqueKeysToSchema-CA2pEs4T.js","SectionHeading-7zgzQbH-.js","axios-vUo_bRtp.js"]}
