Scrape URL

curl --request POST \
  --url https://scrapebadger.com/v1/web/scrape \
  --header 'Content-Type: application/json' \
  --header 'x-api-key: <api-key>' \
  --data '
{
  "url": "<string>",
  "engine": "auto",
  "format": "html",
  "render_js": false,
  "wait_for": "<string>",
  "wait_timeout": 30000,
  "wait_after_load": 123,
  "js_scenario": [
    {}
  ],
  "session_id": "<string>",
  "retry_count": 3,
  "retry_on_block": true,
  "country": "<string>",
  "custom_headers": {},
  "screenshot": false,
  "video": false,
  "anti_bot": false,
  "escalate": false,
  "max_cost": 123,
  "ai_extract": false,
  "ai_prompt": "<string>"
}
'

import requests

url = "https://scrapebadger.com/v1/web/scrape"

payload = {
    "url": "<string>",
    "engine": "auto",
    "format": "html",
    "render_js": False,
    "wait_for": "<string>",
    "wait_timeout": 30000,
    "wait_after_load": 123,
    "js_scenario": [{}],
    "session_id": "<string>",
    "retry_count": 3,
    "retry_on_block": True,
    "country": "<string>",
    "custom_headers": {},
    "screenshot": False,
    "video": False,
    "anti_bot": False,
    "escalate": False,
    "max_cost": 123,
    "ai_extract": False,
    "ai_prompt": "<string>"
}
headers = {
    "x-api-key": "<api-key>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {'x-api-key': '<api-key>', 'Content-Type': 'application/json'},
  body: JSON.stringify({
    url: '<string>',
    engine: 'auto',
    format: 'html',
    render_js: false,
    wait_for: '<string>',
    wait_timeout: 30000,
    wait_after_load: 123,
    js_scenario: [{}],
    session_id: '<string>',
    retry_count: 3,
    retry_on_block: true,
    country: '<string>',
    custom_headers: {},
    screenshot: false,
    video: false,
    anti_bot: false,
    escalate: false,
    max_cost: 123,
    ai_extract: false,
    ai_prompt: '<string>'
  })
};

fetch('https://scrapebadger.com/v1/web/scrape', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://scrapebadger.com/v1/web/scrape",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'url' => '<string>',
    'engine' => 'auto',
    'format' => 'html',
    'render_js' => false,
    'wait_for' => '<string>',
    'wait_timeout' => 30000,
    'wait_after_load' => 123,
    'js_scenario' => [
        [
                
        ]
    ],
    'session_id' => '<string>',
    'retry_count' => 3,
    'retry_on_block' => true,
    'country' => '<string>',
    'custom_headers' => [
        
    ],
    'screenshot' => false,
    'video' => false,
    'anti_bot' => false,
    'escalate' => false,
    'max_cost' => 123,
    'ai_extract' => false,
    'ai_prompt' => '<string>'
  ]),
  CURLOPT_HTTPHEADER => [
    "Content-Type: application/json",
    "x-api-key: <api-key>"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://scrapebadger.com/v1/web/scrape"

	payload := strings.NewReader("{\n  \"url\": \"<string>\",\n  \"engine\": \"auto\",\n  \"format\": \"html\",\n  \"render_js\": false,\n  \"wait_for\": \"<string>\",\n  \"wait_timeout\": 30000,\n  \"wait_after_load\": 123,\n  \"js_scenario\": [\n    {}\n  ],\n  \"session_id\": \"<string>\",\n  \"retry_count\": 3,\n  \"retry_on_block\": true,\n  \"country\": \"<string>\",\n  \"custom_headers\": {},\n  \"screenshot\": false,\n  \"video\": false,\n  \"anti_bot\": false,\n  \"escalate\": false,\n  \"max_cost\": 123,\n  \"ai_extract\": false,\n  \"ai_prompt\": \"<string>\"\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("x-api-key", "<api-key>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://scrapebadger.com/v1/web/scrape")
  .header("x-api-key", "<api-key>")
  .header("Content-Type", "application/json")
  .body("{\n  \"url\": \"<string>\",\n  \"engine\": \"auto\",\n  \"format\": \"html\",\n  \"render_js\": false,\n  \"wait_for\": \"<string>\",\n  \"wait_timeout\": 30000,\n  \"wait_after_load\": 123,\n  \"js_scenario\": [\n    {}\n  ],\n  \"session_id\": \"<string>\",\n  \"retry_count\": 3,\n  \"retry_on_block\": true,\n  \"country\": \"<string>\",\n  \"custom_headers\": {},\n  \"screenshot\": false,\n  \"video\": false,\n  \"anti_bot\": false,\n  \"escalate\": false,\n  \"max_cost\": 123,\n  \"ai_extract\": false,\n  \"ai_prompt\": \"<string>\"\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://scrapebadger.com/v1/web/scrape")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["x-api-key"] = '<api-key>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"url\": \"<string>\",\n  \"engine\": \"auto\",\n  \"format\": \"html\",\n  \"render_js\": false,\n  \"wait_for\": \"<string>\",\n  \"wait_timeout\": 30000,\n  \"wait_after_load\": 123,\n  \"js_scenario\": [\n    {}\n  ],\n  \"session_id\": \"<string>\",\n  \"retry_count\": 3,\n  \"retry_on_block\": true,\n  \"country\": \"<string>\",\n  \"custom_headers\": {},\n  \"screenshot\": false,\n  \"video\": false,\n  \"anti_bot\": false,\n  \"escalate\": false,\n  \"max_cost\": 123,\n  \"ai_extract\": false,\n  \"ai_prompt\": \"<string>\"\n}"

response = http.request(request)
puts response.read_body

{
  "success": true,
  "url": "https://scrapebadger.com",
  "status_code": 200,
  "content": "# Example Domain\n\nThis domain is for use in illustrative examples...",
  "format": "markdown",
  "engine_used": "http",
  "credits_used": 1,
  "duration_ms": 342,
  "retries_used": 0,
  "content_length": 1256,
  "screenshot_url": null,
  "video_url": null,
  "headers": {
    "content-type": "text/html; charset=UTF-8"
  },
  "blocking_detected": false,
  "blocking_details": null,
  "antibot_systems": [],
  "captcha_systems": [],
  "anti_bot_solved": false,
  "solver_used": null,
  "ai_extraction": null,
  "ai_model": null,
  "ai_error": null
}

{
  "success": true,
  "url": "https://scrapebadger.com/products",
  "status_code": 200,
  "content": "# Products\n\n...",
  "format": "markdown",
  "engine_used": "http",
  "credits_used": 3,
  "duration_ms": 2150,
  "retries_used": 0,
  "content_length": 8432,
  "screenshot_url": null,
  "video_url": null,
  "headers": {},
  "blocking_detected": false,
  "blocking_details": null,
  "antibot_systems": [],
  "captcha_systems": [],
  "anti_bot_solved": false,
  "solver_used": null,
  "ai_extraction": [
    { "name": "Widget Pro", "price": "$29.99", "rating": 4.5 },
    { "name": "Widget Basic", "price": "$9.99", "rating": 4.2 }
  ],
  "ai_model": "gpt-4o-mini",
  "ai_error": null
}

{
  "success": false,
  "url": "https://protected-site.com",
  "status_code": 403,
  "content": null,
  "format": "markdown",
  "engine_used": "http",
  "credits_used": 0,
  "duration_ms": 5230,
  "retries_used": 3,
  "content_length": 0,
  "screenshot_url": null,
  "video_url": null,
  "headers": {},
  "blocking_detected": true,
  "blocking_details": {
    "is_blocked": true,
    "block_type": "cloudflare",
    "confidence": 0.95,
    "details": "Cloudflare challenge page detected"
  },
  "antibot_systems": [
    { "system": "cloudflare_turnstile", "confidence": 0.95, "details": null }
  ],
  "captcha_systems": [],
  "anti_bot_solved": false,
  "solver_used": null,
  "ai_extraction": null,
  "ai_model": null,
  "ai_error": null
}

POST

web

scrape

Scrape URL

curl --request POST \
  --url https://scrapebadger.com/v1/web/scrape \
  --header 'Content-Type: application/json' \
  --header 'x-api-key: <api-key>' \
  --data '
{
  "url": "<string>",
  "engine": "auto",
  "format": "html",
  "render_js": false,
  "wait_for": "<string>",
  "wait_timeout": 30000,
  "wait_after_load": 123,
  "js_scenario": [
    {}
  ],
  "session_id": "<string>",
  "retry_count": 3,
  "retry_on_block": true,
  "country": "<string>",
  "custom_headers": {},
  "screenshot": false,
  "video": false,
  "anti_bot": false,
  "escalate": false,
  "max_cost": 123,
  "ai_extract": false,
  "ai_prompt": "<string>"
}
'

import requests

url = "https://scrapebadger.com/v1/web/scrape"

payload = {
    "url": "<string>",
    "engine": "auto",
    "format": "html",
    "render_js": False,
    "wait_for": "<string>",
    "wait_timeout": 30000,
    "wait_after_load": 123,
    "js_scenario": [{}],
    "session_id": "<string>",
    "retry_count": 3,
    "retry_on_block": True,
    "country": "<string>",
    "custom_headers": {},
    "screenshot": False,
    "video": False,
    "anti_bot": False,
    "escalate": False,
    "max_cost": 123,
    "ai_extract": False,
    "ai_prompt": "<string>"
}
headers = {
    "x-api-key": "<api-key>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {'x-api-key': '<api-key>', 'Content-Type': 'application/json'},
  body: JSON.stringify({
    url: '<string>',
    engine: 'auto',
    format: 'html',
    render_js: false,
    wait_for: '<string>',
    wait_timeout: 30000,
    wait_after_load: 123,
    js_scenario: [{}],
    session_id: '<string>',
    retry_count: 3,
    retry_on_block: true,
    country: '<string>',
    custom_headers: {},
    screenshot: false,
    video: false,
    anti_bot: false,
    escalate: false,
    max_cost: 123,
    ai_extract: false,
    ai_prompt: '<string>'
  })
};

fetch('https://scrapebadger.com/v1/web/scrape', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://scrapebadger.com/v1/web/scrape",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'url' => '<string>',
    'engine' => 'auto',
    'format' => 'html',
    'render_js' => false,
    'wait_for' => '<string>',
    'wait_timeout' => 30000,
    'wait_after_load' => 123,
    'js_scenario' => [
        [
                
        ]
    ],
    'session_id' => '<string>',
    'retry_count' => 3,
    'retry_on_block' => true,
    'country' => '<string>',
    'custom_headers' => [
        
    ],
    'screenshot' => false,
    'video' => false,
    'anti_bot' => false,
    'escalate' => false,
    'max_cost' => 123,
    'ai_extract' => false,
    'ai_prompt' => '<string>'
  ]),
  CURLOPT_HTTPHEADER => [
    "Content-Type: application/json",
    "x-api-key: <api-key>"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://scrapebadger.com/v1/web/scrape"

	payload := strings.NewReader("{\n  \"url\": \"<string>\",\n  \"engine\": \"auto\",\n  \"format\": \"html\",\n  \"render_js\": false,\n  \"wait_for\": \"<string>\",\n  \"wait_timeout\": 30000,\n  \"wait_after_load\": 123,\n  \"js_scenario\": [\n    {}\n  ],\n  \"session_id\": \"<string>\",\n  \"retry_count\": 3,\n  \"retry_on_block\": true,\n  \"country\": \"<string>\",\n  \"custom_headers\": {},\n  \"screenshot\": false,\n  \"video\": false,\n  \"anti_bot\": false,\n  \"escalate\": false,\n  \"max_cost\": 123,\n  \"ai_extract\": false,\n  \"ai_prompt\": \"<string>\"\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("x-api-key", "<api-key>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://scrapebadger.com/v1/web/scrape")
  .header("x-api-key", "<api-key>")
  .header("Content-Type", "application/json")
  .body("{\n  \"url\": \"<string>\",\n  \"engine\": \"auto\",\n  \"format\": \"html\",\n  \"render_js\": false,\n  \"wait_for\": \"<string>\",\n  \"wait_timeout\": 30000,\n  \"wait_after_load\": 123,\n  \"js_scenario\": [\n    {}\n  ],\n  \"session_id\": \"<string>\",\n  \"retry_count\": 3,\n  \"retry_on_block\": true,\n  \"country\": \"<string>\",\n  \"custom_headers\": {},\n  \"screenshot\": false,\n  \"video\": false,\n  \"anti_bot\": false,\n  \"escalate\": false,\n  \"max_cost\": 123,\n  \"ai_extract\": false,\n  \"ai_prompt\": \"<string>\"\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://scrapebadger.com/v1/web/scrape")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["x-api-key"] = '<api-key>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"url\": \"<string>\",\n  \"engine\": \"auto\",\n  \"format\": \"html\",\n  \"render_js\": false,\n  \"wait_for\": \"<string>\",\n  \"wait_timeout\": 30000,\n  \"wait_after_load\": 123,\n  \"js_scenario\": [\n    {}\n  ],\n  \"session_id\": \"<string>\",\n  \"retry_count\": 3,\n  \"retry_on_block\": true,\n  \"country\": \"<string>\",\n  \"custom_headers\": {},\n  \"screenshot\": false,\n  \"video\": false,\n  \"anti_bot\": false,\n  \"escalate\": false,\n  \"max_cost\": 123,\n  \"ai_extract\": false,\n  \"ai_prompt\": \"<string>\"\n}"

response = http.request(request)
puts response.read_body

{
  "success": true,
  "url": "https://scrapebadger.com",
  "status_code": 200,
  "content": "# Example Domain\n\nThis domain is for use in illustrative examples...",
  "format": "markdown",
  "engine_used": "http",
  "credits_used": 1,
  "duration_ms": 342,
  "retries_used": 0,
  "content_length": 1256,
  "screenshot_url": null,
  "video_url": null,
  "headers": {
    "content-type": "text/html; charset=UTF-8"
  },
  "blocking_detected": false,
  "blocking_details": null,
  "antibot_systems": [],
  "captcha_systems": [],
  "anti_bot_solved": false,
  "solver_used": null,
  "ai_extraction": null,
  "ai_model": null,
  "ai_error": null
}

{
  "success": true,
  "url": "https://scrapebadger.com/products",
  "status_code": 200,
  "content": "# Products\n\n...",
  "format": "markdown",
  "engine_used": "http",
  "credits_used": 3,
  "duration_ms": 2150,
  "retries_used": 0,
  "content_length": 8432,
  "screenshot_url": null,
  "video_url": null,
  "headers": {},
  "blocking_detected": false,
  "blocking_details": null,
  "antibot_systems": [],
  "captcha_systems": [],
  "anti_bot_solved": false,
  "solver_used": null,
  "ai_extraction": [
    { "name": "Widget Pro", "price": "$29.99", "rating": 4.5 },
    { "name": "Widget Basic", "price": "$9.99", "rating": 4.2 }
  ],
  "ai_model": "gpt-4o-mini",
  "ai_error": null
}

{
  "success": false,
  "url": "https://protected-site.com",
  "status_code": 403,
  "content": null,
  "format": "markdown",
  "engine_used": "http",
  "credits_used": 0,
  "duration_ms": 5230,
  "retries_used": 3,
  "content_length": 0,
  "screenshot_url": null,
  "video_url": null,
  "headers": {},
  "blocking_detected": true,
  "blocking_details": {
    "is_blocked": true,
    "block_type": "cloudflare",
    "confidence": 0.95,
    "details": "Cloudflare challenge page detected"
  },
  "antibot_systems": [
    { "system": "cloudflare_turnstile", "confidence": 0.95, "details": null }
  ],
  "captcha_systems": [],
  "anti_bot_solved": false,
  "solver_used": null,
  "ai_extraction": null,
  "ai_model": null,
  "ai_error": null
}

Request Body

string

required

The URL to scrape. Must be a valid HTTP or HTTPS URL. Private IPs and cloud metadata endpoints are blocked for security.

string

default:"auto"

Scraping engine tier to use. ScrapeBadger automatically selects the best approach.

Value	Description	Cost
`auto`	Automatically picks the best engine for the target site (recommended)	From 1 credit
`browser`	Force headless browser with full JavaScript rendering	5 credits

In auto mode, simple pages use fast HTTP (1 credit) and JavaScript-heavy pages use a browser (5 credits). When escalate is enabled, the system can further escalate to premium browsers (10 credits) for heavily protected sites.

string

default:"html"

Output format for the scraped content. Applies to text responses only — a binary target (image, PDF, archive) ignores it and is returned untouched.

html — Raw HTML of the page
markdown — Converted to clean Markdown
text — Plain text with HTML tags stripped

These are the only three values. There is no raw format — to get an unwrapped body, use raw_content.

boolean

default:false

Force JavaScript rendering before extracting content. Automatically switches to the browser engine. Use this for single-page applications or pages that load content dynamically.

string

CSS selector or XPath expression to wait for before extracting content. Only works with browser engines (browser, patchright, windows_chrome). If render_js is false and this is set, JS rendering is forced automatically.

{ "wait_for": "#main-content" }

{ "wait_for": "//div[@class='results']" }

integer

default:30000

Maximum time in milliseconds to wait for the wait_for selector to appear. Range: 1000 – 120000.

integer

Additional milliseconds to wait after the page has finished loading, before extracting content. Useful for pages with animations or delayed rendering. Only works with browser engines. Range: 0 – 30000.

array

A list of browser actions to perform before extracting content. Forces the browser engine. Each action is an object with a type and action-specific parameters.Supported actions:

Type	Parameters	Description
`click`	`selector`	Click an element
`fill`	`selector`, `value`	Fill an input field
`scroll`	`direction`, `amount`	Scroll the page
`wait`	`milliseconds`	Wait for a duration

{
  "js_scenario": [
    { "type": "click", "selector": "#load-more" },
    { "type": "wait", "milliseconds": 2000 },
    { "type": "scroll", "direction": "down", "amount": 500 }
  ]
}

string

A unique identifier to persist cookies, fingerprint, and browser storage across multiple requests. Use the same session_id to maintain login state or continue a browsing session.

{ "session_id": "my-shopping-session" }

integer

default:3

Maximum number of retry attempts when a blocking page is detected. Retries are free — you only pay for the final successful engine. Range: 0 – 10.

boolean

default:true

Whether to automatically retry when a blocking page is detected. Set to false to get the blocked response immediately.

string

ISO 3166-1 alpha-2 country code for proxy geo-targeting. Routes the request through a proxy in the specified country.

{ "country": "US" }

Common values: US, GB, DE, FR, JP, BR, AU.

object

Additional HTTP headers to include in the request to the target URL. Accepts a key-value object.

{
  "custom_headers": {
    "Accept-Language": "en-US",
    "Referer": "https://google.com"
  }
}

boolean

default:false

Capture a full-page screenshot (PNG). Forces the browser engine. Returned as base64 in the screenshot_url response field.

boolean

default:false

Record a video of the browser session (animated GIF). Forces the browser engine. Returned as base64 in the video_url response field. Adds +3 credits. Useful for debugging, visual verification, or monitoring how a page loads.

boolean

default:false

Attempt to bypass detected anti-bot protection using registered solvers. Adds +5 credits to the request cost when a solver is invoked. Only triggered when blocking is actually detected.

boolean

default:false

Allow automatic escalation to more powerful engines when the initial engine is blocked.Escalation path: curl_cffi → browser → windows_chromeYou only pay for the engine that succeeds — costs are not cumulative. Without this flag, only the selected engine is tried.

integer

Maximum credits to spend on this request. The request fails with a 400 error if the estimated cost would exceed this budget. Useful for controlling costs when using escalate or anti_bot. Minimum: 1.

boolean

default:false

Return the body directly as the HTTP response instead of wrapping it in JSON. Metadata comes back in X-Scrape-* response headers.Use it for two things: skipping the JSON encode/decode on large HTML payloads (saves 300–1000 ms on 1 MB+ responses), and downloading binary files without the ~33% base64 overhead. See Binary files and raw bodies.Cannot be combined with ai_extract, screenshot or video — those need the JSON envelope, so the request falls back to it automatically.

boolean

default:false

Run AI-powered extraction on the scraped content using the instruction in ai_prompt. Adds +2 credits to the request cost. The scrape result is still returned even if AI extraction fails.

string

Natural language instruction for AI data extraction. Required when ai_extract is true. Maximum 2000 characters.

{
  "ai_extract": true,
  "ai_prompt": "Extract all product names and prices as a JSON array"
}

Response

boolean

Whether the scrape completed successfully. false when all retries are exhausted and the page is still blocked.

string

The final URL after any redirects.

integer

HTTP status code from the target URL.

string

The scraped content in the requested format. null when success is false, and null for binary targets — those come back in content_base64.

string

Base64-encoded response body, returned instead of content when the target serves a binary payload. Only present when is_binary is true. Bodies above 25 MB are not base64-encoded — re-request those with raw_content: true.

boolean

Whether the target returned a binary (non-text) body.

string

The target’s response Content-Type, normalised to the bare media type (e.g. image/jpeg).

string

The output format used: html, markdown, or text.

string

The engine tier that produced the final result.

integer

Total credits charged for this request, including engine cost, solver, and AI extraction.

integer

Total request processing time in milliseconds.

integer

Number of retry attempts performed. 0 if the first attempt succeeded.

integer

Size of the returned content in bytes.

string

Base64-encoded PNG screenshot of the page. Only present when screenshot: true was requested.

string

Base64-encoded animated GIF of the browser session. Only present when video: true was requested.

object

HTTP response headers from the target URL.

boolean

Whether a blocking page was detected during scraping.

object

Details about the detected blocking page. Only present when blocking_detected is true.

Show child attributes

boolean

Whether the page is confirmed as a blocking page.

string

Type of block detected (e.g., cloudflare, datadome, akamai, kasada).

number

Confidence score from 0.0 to 1.0.

string

Human-readable description of the block.

array

List of anti-bot systems detected on the page.

Show child attributes

string

System name (e.g., cloudflare_turnstile, datadome, akamai, kasada, amazon_waf).

number

Confidence score from 0.0 to 1.0.

string

Additional detection details.

array

List of CAPTCHA systems detected on the page.

Show child attributes

string

System name (e.g., recaptcha_v2, recaptcha_v3, hcaptcha, geetest).

number

Confidence score from 0.0 to 1.0.

string

Additional detection details.

boolean

Whether the anti-bot solver successfully bypassed the protection.

string

Name of the solver that successfully bypassed the block. null if no solver was used.

object | string | array

Structured data extracted by the LLM based on ai_prompt. The shape depends on your prompt. null when ai_extract is false or extraction failed.

string

The LLM model used for extraction (e.g., gpt-4o-mini). null when AI extraction was not used.

string

Error message if AI extraction failed. The scrape result is still returned. null on success.

Binary files and raw bodies

/v1/web/scrape handles binary targets — images, PDFs, archives, fonts, audio and video — as well as HTML. The bytes are returned exactly as the origin sent them; nothing is decoded, parsed or re-encoded on the way through. This works through the same anti-bot machinery as a page scrape, so a file behind Cloudflare, DataDome or Imperva is fetched with the same engine, proxy tier and session you would use for its parent page. Pass the same session_id you used to scrape the page the file was linked from and the file download reuses that session’s cookies and fingerprint. There are two ways to get the bytes. raw_content: true — the body itself. No base64 overhead. Best for large files and for piping straight to disk.

curl -X POST "https://scrapebadger.com/v1/web/scrape" \
  -H "X-API-Key: YOUR_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "url": "https://example.com/photos/item.jpg",
    "raw_content": true,
    "session_id": "gallery-session"
  }' \
  --output item.jpg

import requests

response = requests.post(
    "https://scrapebadger.com/v1/web/scrape",
    headers={"X-API-Key": "YOUR_API_KEY"},
    json={
        "url": "https://example.com/photos/item.jpg",
        "raw_content": True,
        "session_id": "gallery-session",
    },
)

with open("item.jpg", "wb") as f:
    f.write(response.content)          # exact bytes — do NOT use response.text

print(response.headers["Content-Type"])
print(response.headers["X-Credits-Used"])

const response = await fetch("https://scrapebadger.com/v1/web/scrape", {
  method: "POST",
  headers: {
    "X-API-Key": "YOUR_API_KEY",
    "Content-Type": "application/json",
  },
  body: JSON.stringify({
    url: "https://example.com/photos/item.jpg",
    raw_content: true,
    session_id: "gallery-session",
  }),
});

const buffer = Buffer.from(await response.arrayBuffer());

Read the response as bytes, not text. response.text in Python or response.text() in JavaScript will decode the payload as a string and corrupt it. Use response.content / response.arrayBuffer().

JSON with content_base64 — one request, metadata included. Use this when you also want credits_used, engine_used or the protection detections alongside the file.

Python

import base64
import requests

data = requests.post(
    "https://scrapebadger.com/v1/web/scrape",
    headers={"X-API-Key": "YOUR_API_KEY"},
    json={"url": "https://example.com/report.pdf"},
).json()

if data["is_binary"]:
    with open("report.pdf", "wb") as f:
        f.write(base64.b64decode(data["content_base64"]))
    print(data["content_type"])   # application/pdf
else:
    print(data["content"])        # a text target — HTML/markdown/text

Binary responses served with raw_content always carry Content-Disposition: attachment and X-Content-Type-Options: nosniff, and their Content-Type is restricted to a known-safe set — scraped bytes are never labelled in a way that would let a browser execute them. Bodies larger than 25 MB are not base64-encoded into JSON. Those return content_base64: null with a detail telling you to re-request using raw_content: true, which returns them without the base64 expansion. Very large files are still held in memory end to end, so treat a few hundred MB as the practical ceiling for a single request.

Examples

Basic scrape

curl -X POST "https://scrapebadger.com/v1/web/scrape" \
  -H "x-api-key: YOUR_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{"url": "https://scrapebadger.com", "format": "markdown"}'

import requests

response = requests.post(
    "https://scrapebadger.com/v1/web/scrape",
    headers={"x-api-key": "YOUR_API_KEY"},
    json={"url": "https://scrapebadger.com", "format": "markdown"}
)
print(response.json()["content"])

const res = await fetch("https://scrapebadger.com/v1/web/scrape", {
  method: "POST",
  headers: { "x-api-key": "YOUR_API_KEY", "Content-Type": "application/json" },
  body: JSON.stringify({ url: "https://scrapebadger.com", format: "markdown" })
});
const data = await res.json();
console.log(data.content);

JavaScript rendering with wait

{
  "url": "https://scrapebadger.com/spa-page",
  "format": "html",
  "render_js": true,
  "wait_for": "#dynamic-content",
  "wait_timeout": 10000
}

AI extraction

{
  "url": "https://scrapebadger.com/products",
  "format": "markdown",
  "ai_extract": true,
  "ai_prompt": "Extract all product names, prices, and ratings as a JSON array of objects with keys: name, price, rating"
}

Full anti-bot bypass with budget

{
  "url": "https://heavily-protected-site.com",
  "format": "markdown",
  "escalate": true,
  "anti_bot": true,
  "max_cost": 20,
  "country": "US"
}

Browser automation scenario

{
  "url": "https://scrapebadger.com/infinite-scroll",
  "format": "text",
  "js_scenario": [
    { "type": "scroll", "direction": "down", "amount": 1000 },
    { "type": "wait", "milliseconds": 2000 },
    { "type": "scroll", "direction": "down", "amount": 1000 },
    { "type": "wait", "milliseconds": 2000 }
  ]
}

Error Responses

Status	Description
`400`	Invalid URL, cost exceeds `max_cost`, or requested engine not available
`402`	Insufficient credits
`422`	Blocking detected after all retries exhausted (`success: false`, `blocking_details` populated)
`429`	Rate limit exceeded
`500`	Unexpected server error

{
  "success": true,
  "url": "https://scrapebadger.com",
  "status_code": 200,
  "content": "# Example Domain\n\nThis domain is for use in illustrative examples...",
  "format": "markdown",
  "engine_used": "http",
  "credits_used": 1,
  "duration_ms": 342,
  "retries_used": 0,
  "content_length": 1256,
  "screenshot_url": null,
  "video_url": null,
  "headers": {
    "content-type": "text/html; charset=UTF-8"
  },
  "blocking_detected": false,
  "blocking_details": null,
  "antibot_systems": [],
  "captcha_systems": [],
  "anti_bot_solved": false,
  "solver_used": null,
  "ai_extraction": null,
  "ai_model": null,
  "ai_error": null
}

{
  "success": true,
  "url": "https://scrapebadger.com/products",
  "status_code": 200,
  "content": "# Products\n\n...",
  "format": "markdown",
  "engine_used": "http",
  "credits_used": 3,
  "duration_ms": 2150,
  "retries_used": 0,
  "content_length": 8432,
  "screenshot_url": null,
  "video_url": null,
  "headers": {},
  "blocking_detected": false,
  "blocking_details": null,
  "antibot_systems": [],
  "captcha_systems": [],
  "anti_bot_solved": false,
  "solver_used": null,
  "ai_extraction": [
    { "name": "Widget Pro", "price": "$29.99", "rating": 4.5 },
    { "name": "Widget Basic", "price": "$9.99", "rating": 4.2 }
  ],
  "ai_model": "gpt-4o-mini",
  "ai_error": null
}

{
  "success": false,
  "url": "https://protected-site.com",
  "status_code": 403,
  "content": null,
  "format": "markdown",
  "engine_used": "http",
  "credits_used": 0,
  "duration_ms": 5230,
  "retries_used": 3,
  "content_length": 0,
  "screenshot_url": null,
  "video_url": null,
  "headers": {},
  "blocking_detected": true,
  "blocking_details": {
    "is_blocked": true,
    "block_type": "cloudflare",
    "confidence": 0.95,
    "details": "Cloudflare challenge page detected"
  },
  "antibot_systems": [
    { "system": "cloudflare_turnstile", "confidence": 0.95, "details": null }
  ],
  "captcha_systems": [],
  "anti_bot_solved": false,
  "solver_used": null,
  "ai_extraction": null,
  "ai_model": null,
  "ai_error": null
}

Authorizations

x-api-key

string

header

required

Body

application/json

url

string

required

The URL to scrape. Must be a valid HTTP or HTTPS URL.

engine

enum<string>

default:auto

Scraping engine tier to use.

Available options:

auto,

browser

format

enum<string>

default:html

Output format for the scraped content.

Available options:

html,

markdown,

text

render_js

boolean

default:false

Force JavaScript rendering.

wait_for

string

CSS selector or XPath to wait for before extracting.

wait_timeout

integer

default:30000

Max wait time in ms for wait_for selector.

wait_after_load

integer

Additional ms to wait after page load.

js_scenario

object[]

Browser actions to perform before extracting.

session_id

string

Persist cookies and state across requests.

retry_count

integer

default:3

Max retry attempts on blocking detection.

retry_on_block

boolean

default:true

Auto-retry on blocking page detection.

country

string

ISO 3166-1 alpha-2 country code for proxy geo-targeting.

custom_headers

object

Additional HTTP headers for the target request.

screenshot

boolean

default:false

Capture a full-page PNG screenshot.

video

boolean

default:false

Record browser session as animated GIF (+3 credits).

anti_bot

boolean

default:false

Attempt anti-bot bypass when blocking detected.

escalate

boolean

default:false

Allow auto-escalation to stronger engines.

max_cost

integer

Maximum credits budget for this request.

ai_extract

boolean

default:false

Run AI extraction on scraped content.

ai_prompt

string

Natural language instruction for AI extraction.

Response

200 - application/json

Successful scrape

success

boolean

url

string

status_code

integer

content

string

format

string

engine_used

string

credits_used

integer

duration_ms

integer

retries_used

integer

content_length

integer

screenshot_url

string | null

video_url

string | null

headers

object

blocking_detected

boolean

blocking_details

object | null

antibot_systems

array

captcha_systems

array

anti_bot_solved

boolean

solver_used

string | null

ai_extraction

unknown

ai_model

string | null

ai_error

string | null

Overview

API Reference

Scrape URL

Request Body

Response

Binary files and raw bodies

Examples

Basic scrape

JavaScript rendering with wait

AI extraction

Full anti-bot bypass with budget

Browser automation scenario

Error Responses

Authorizations

Body

Response

​Request Body

​Response

​Binary files and raw bodies

​Examples

​Basic scrape

​JavaScript rendering with wait

​AI extraction

​Full anti-bot bypass with budget

​Browser automation scenario

​Error Responses

Authorizations

Body

Response

Request Body

Response

Binary files and raw bodies

Examples

Basic scrape

JavaScript rendering with wait

AI extraction

Full anti-bot bypass with budget

Browser automation scenario

Error Responses