Modern sites often block plain curl. Using a real browser engine (Chromium via Playwright) gives you true browser behavior: real TLS/HTTP2 stack, cookies, redirects, and JavaScript execution if needed. This post mirrors the functionality of the original browser_curl.sh wrapper but implemented with Playwright. It also includes an optional Selenium mini-variant at the end.
What this tool does
- Sends realistic browser headers (Chrome-like)
- Uses Chromium’s real network stack (HTTP/2, compression)
- Manages cookies (persist to a file)
- Follows redirects by default
- Supports JSON and form POSTs
- Async mode that returns immediately
--count Nto dispatch N async requests for quick load tests
Note: Advanced bot defenses (CAPTCHAs, JS/ML challenges, strict TLS/HTTP2 fingerprinting) may still require full page automation and real user-like behavior. Playwright can do that too by driving real pages.
Setup
Run these once to install Playwright and Chromium:
npm init -y && \
npm install playwright && \
npx playwright install chromiumThe complete Playwright CLI
Run this to create browser_playwright.mjs:
cat > browser_playwright.mjs << 'EOF'
#!/usr/bin/env node
import { chromium } from 'playwright';
import fs from 'fs';
import path from 'path';
import { spawn } from 'child_process';
const RED = '\u001b[31m';
const GRN = '\u001b[32m';
const YLW = '\u001b[33m';
const NC = '\u001b[0m';
function usage() {
const b = path.basename(process.argv[1]);
console.log(`Usage: ${b} [OPTIONS] URL
Advanced HTTP client using Playwright (Chromium) with browser-like behavior.
OPTIONS:
-X, --method METHOD HTTP method (GET, POST, PUT, DELETE) [default: GET]
-d, --data DATA Request body
-H, --header HEADER Add custom header (repeatable)
-o, --output FILE Write response body to file
-c, --cookie FILE Cookie storage file [default: /tmp/pw_cookies_<pid>.json]
-A, --user-agent UA Custom User-Agent
-t, --timeout SECONDS Request timeout [default: 30]
--async Run request(s) in background
--count N Number of async requests to fire [default: 1, requires --async]
--no-redirect Do not follow redirects (best-effort)
--show-headers Print response headers
--json Send data as JSON (sets Content-Type)
--form Send data as application/x-www-form-urlencoded
-v, --verbose Verbose output
-h, --help Show this help message
EXAMPLES:
${b} https://example.com
${b} --async https://example.com
${b} -X POST --json -d '{"a":1}' https://httpbin.org/post
${b} --async --count 10 https://httpbin.org/get
`);
}
function parseArgs(argv) {
const args = { method: 'GET', async: false, count: 1, followRedirects: true, showHeaders: false, timeout: 30, data: '', contentType: '', cookieFile: '', verbose: false, headers: [], url: '' };
for (let i = 0; i < argv.length; i++) {
const a = argv[i];
switch (a) {
case '-X': case '--method': args.method = String(argv[++i] || 'GET'); break;
case '-d': case '--data': args.data = String(argv[++i] || ''); break;
case '-H': case '--header': args.headers.push(String(argv[++i] || '')); break;
case '-o': case '--output': args.output = String(argv[++i] || ''); break;
case '-c': case '--cookie': args.cookieFile = String(argv[++i] || ''); break;
case '-A': case '--user-agent': args.userAgent = String(argv[++i] || ''); break;
case '-t': case '--timeout': args.timeout = Number(argv[++i] || '30'); break;
case '--async': args.async = true; break;
case '--count': args.count = Number(argv[++i] || '1'); break;
case '--no-redirect': args.followRedirects = false; break;
case '--show-headers': args.showHeaders = true; break;
case '--json': args.contentType = 'application/json'; break;
case '--form': args.contentType = 'application/x-www-form-urlencoded'; break;
case '-v': case '--verbose': args.verbose = true; break;
case '-h': case '--help': usage(); process.exit(0);
default:
if (!args.url && !a.startsWith('-')) args.url = a; else {
console.error(`${RED}Error: Unknown argument: ${a}${NC}`);
process.exit(1);
}
}
}
return args;
}
function parseHeaderList(list) {
const out = {};
for (const h of list) {
const idx = h.indexOf(':');
if (idx === -1) continue;
const name = h.slice(0, idx).trim();
const value = h.slice(idx + 1).trim();
if (!name) continue;
out[name] = value;
}
return out;
}
function buildDefaultHeaders(userAgent) {
const ua = userAgent || 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
return {
'User-Agent': ua,
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
'Accept-Language': 'en-US,en;q=0.9',
'Accept-Encoding': 'gzip, deflate, br',
'Connection': 'keep-alive',
'Upgrade-Insecure-Requests': '1',
'Sec-Fetch-Dest': 'document',
'Sec-Fetch-Mode': 'navigate',
'Sec-Fetch-Site': 'none',
'Sec-Fetch-User': '?1',
'Cache-Control': 'max-age=0'
};
}
async function performRequest(opts) {
// Cookie file handling
const defaultCookie = `/tmp/pw_cookies_${process.pid}.json`;
const cookieFile = opts.cookieFile || defaultCookie;
// Launch Chromium
const browser = await chromium.launch({ headless: true });
const extraHeaders = { ...buildDefaultHeaders(opts.userAgent), ...parseHeaderList(opts.headers) };
if (opts.contentType) extraHeaders['Content-Type'] = opts.contentType;
const context = await browser.newContext({ userAgent: extraHeaders['User-Agent'], extraHTTPHeaders: extraHeaders });
// Load cookies if present
if (fs.existsSync(cookieFile)) {
try {
const ss = JSON.parse(fs.readFileSync(cookieFile, 'utf8'));
if (ss.cookies?.length) await context.addCookies(ss.cookies);
} catch {}
}
const request = context.request;
// Build request options
const reqOpts = { headers: extraHeaders, timeout: opts.timeout * 1000 };
if (opts.data) {
// Playwright will detect JSON strings vs form strings by headers
reqOpts.data = opts.data;
}
if (opts.followRedirects === false) {
// Best-effort: limit redirects to 0
reqOpts.maxRedirects = 0;
}
const method = opts.method.toUpperCase();
let resp;
try {
if (method === 'GET') resp = await request.get(opts.url, reqOpts);
else if (method === 'POST') resp = await request.post(opts.url, reqOpts);
else if (method === 'PUT') resp = await request.put(opts.url, reqOpts);
else if (method === 'DELETE') resp = await request.delete(opts.url, reqOpts);
else if (method === 'PATCH') resp = await request.patch(opts.url, reqOpts);
else {
console.error(`${RED}Unsupported method: ${method}${NC}`);
await browser.close();
process.exit(2);
}
} catch (e) {
console.error(`${RED}[ERROR] ${e?.message || e}${NC}`);
await browser.close();
process.exit(3);
}
// Persist cookies
try {
const state = await context.storageState();
fs.writeFileSync(cookieFile, JSON.stringify(state, null, 2));
} catch {}
// Output
const status = resp.status();
const statusText = resp.statusText();
const headers = await resp.headers();
const body = await resp.text();
if (opts.verbose) {
console.error(`${YLW}Request: ${method} ${opts.url}${NC}`);
console.error(`${YLW}Headers: ${JSON.stringify(extraHeaders)}${NC}`);
}
if (opts.showHeaders) {
// Print a simple status line and headers to stdout before body
console.log(`HTTP ${status} ${statusText}`);
for (const [k, v] of Object.entries(headers)) {
console.log(`${k}: ${v}`);
}
console.log('');
}
if (opts.output) {
fs.writeFileSync(opts.output, body);
} else {
process.stdout.write(body);
}
if (!resp.ok()) {
console.error(`${RED}[ERROR] HTTP ${status} ${statusText}${NC}`);
await browser.close();
process.exit(4);
}
await browser.close();
}
async function main() {
const argv = process.argv.slice(2);
const opts = parseArgs(argv);
if (!opts.url) { console.error(`${RED}Error: URL is required${NC}`); usage(); process.exit(1); }
if ((opts.count || 1) > 1 && !opts.async) {
console.error(`${RED}Error: --count requires --async${NC}`);
process.exit(1);
}
if (opts.count < 1 || !Number.isInteger(opts.count)) {
console.error(`${RED}Error: --count must be a positive integer${NC}`);
process.exit(1);
}
if (opts.async) {
// Fire-and-forget background processes
const baseArgs = process.argv.slice(2).filter(a => a !== '--async' && !a.startsWith('--count'));
const pids = [];
for (let i = 0; i < opts.count; i++) {
const child = spawn(process.execPath, [process.argv[1], ...baseArgs], { detached: true, stdio: 'ignore' });
pids.push(child.pid);
child.unref();
}
if (opts.verbose) {
console.error(`${YLW}[ASYNC] Spawned ${opts.count} request(s).${NC}`);
}
if (opts.count === 1) console.error(`${GRN}[ASYNC] Request started with PID: ${pids[0]}${NC}`);
else console.error(`${GRN}[ASYNC] ${opts.count} requests started with PIDs: ${pids.join(' ')}${NC}`);
process.exit(0);
}
await performRequest(opts);
}
main().catch(err => {
console.error(`${RED}[FATAL] ${err?.stack || err}${NC}`);
process.exit(1);
});
EOF
chmod +x browser_playwright.mjsOptionally, move it into your PATH:
sudo mv browser_playwright.mjs /usr/local/bin/browser_playwrightQuick start
- Simple GET:
node browser_playwright.mjs https://example.com- Async GET (returns immediately):
node browser_playwright.mjs --async https://example.com- Fire 100 async requests in one command:
node browser_playwright.mjs --async --count 100 https://httpbin.org/get- POST JSON:
node browser_playwright.mjs -X POST --json \
-d '{"username":"user","password":"pass"}' \
https://httpbin.org/post- POST form data:
node browser_playwright.mjs -X POST --form \
-d "username=user&password=pass" \
https://httpbin.org/post- Include response headers:
node browser_playwright.mjs --show-headers https://example.com- Save response to a file:
node browser_playwright.mjs -o response.json https://httpbin.org/json- Custom headers:
node browser_playwright.mjs \
-H "X-API-Key: your-key" \
-H "Authorization: Bearer token" \
https://httpbin.org/headers- Persistent cookies across requests:
COOKIE_FILE="playwright_session.json"
# Login and save cookies
node browser_playwright.mjs -c "$COOKIE_FILE" \
-X POST --form \
-d "user=test&pass=secret" \
https://httpbin.org/post > /dev/null
# Authenticated-like follow-up (cookie file reused)
node browser_playwright.mjs -c "$COOKIE_FILE" \
https://httpbin.org/cookiesLoad testing patterns
- Simple load test with
--count:
node browser_playwright.mjs --async --count 100 https://httpbin.org/get- Loop-based alternative:
for i in {1..100}; do
node browser_playwright.mjs --async https://httpbin.org/get
done- Timed load test:
cat > pw_load_for_duration.sh << 'EOF'
#!/usr/bin/env bash
URL="${1:-https://httpbin.org/get}"
DURATION="${2:-60}"
COUNT=0
END_TIME=$(($(date +%s) + DURATION))
while [ "$(date +%s)" -lt "$END_TIME" ]; do
node browser_playwright.mjs --async "$URL" >/dev/null 2>&1
((COUNT++))
done
echo "Sent $COUNT requests in $DURATION seconds"
echo "Rate: $((COUNT / DURATION)) requests/second"
EOF
chmod +x pw_load_for_duration.sh
./pw_load_for_duration.sh https://httpbin.org/get 30- Parameterized load test:
cat > pw_load_test.sh << 'EOF'
#!/usr/bin/env bash
URL="${1:-https://httpbin.org/get}"
REQUESTS="${2:-50}"
echo "Load testing: $URL"
echo "Requests: $REQUESTS"
echo ""
START=$(date +%s)
node browser_playwright.mjs --async --count "$REQUESTS" "$URL"
echo ""
echo "Dispatched in $(($(date +%s) - START)) seconds"
EOF
chmod +x pw_load_test.sh
./pw_load_test.sh https://httpbin.org/get 200Options reference
-X, --methodHTTP method (GET/POST/PUT/DELETE/PATCH)-d, --dataRequest body-H, --headerAdd extra headers (repeatable)-o, --outputWrite response body to file-c, --cookieCookie file to use (and persist)-A, --user-agentOverride User-Agent-t, --timeoutMax request time in seconds (default 30)--asyncRun request(s) in the background--count NFire N async requests (requires--async)--no-redirectBest-effort disable following redirects--show-headersInclude response headers before body--jsonSetsContent-Type: application/json--formSetsContent-Type: application/x-www-form-urlencoded-v, --verboseVerbose diagnostics
Validation rules:
--countrequires--async--countmust be a positive integer
Under the hood: why this works better than plain curl
- Real Chromium network stack (HTTP/2, TLS, compression)
- Browser-like headers and a true User-Agent
- Cookie jar via Playwright context storageState
- Redirect handling by the browser stack
This helps pass simplistic bot checks and more closely resembles real user traffic.
Real-world examples
- API-style auth flow (demo endpoints):
cat > pw_auth_flow.sh << 'EOF'
#!/usr/bin/env bash
COOKIE_FILE="pw_auth_session.json"
BASE="https://httpbin.org"
echo "Login (simulated form POST)..."
node browser_playwright.mjs -c "$COOKIE_FILE" \
-X POST --form \
-d "user=user&pass=pass" \
"$BASE/post" > /dev/null
echo "Fetch cookies..."
node browser_playwright.mjs -c "$COOKIE_FILE" \
"$BASE/cookies"
echo "Load test a protected-like endpoint..."
node browser_playwright.mjs -c "$COOKIE_FILE" \
--async --count 20 \
"$BASE/get"
echo "Done"
rm -f "$COOKIE_FILE"
EOF
chmod +x pw_auth_flow.sh
./pw_auth_flow.sh- Scraping with rate limiting:
cat > pw_scrape.sh << 'EOF'
#!/usr/bin/env bash
URLS=(
"https://example.com/"
"https://example.com/"
"https://example.com/"
)
for url in "${URLS[@]}"; do
echo "Fetching: $url"
node browser_playwright.mjs -o "$(echo "$url" | sed 's#[/:]#_#g').html" "$url"
sleep 2
done
EOF
chmod +x pw_scrape.sh
./pw_scrape.sh- Health check monitoring:
cat > pw_health.sh << 'EOF'
#!/usr/bin/env bash
ENDPOINT="${1:-https://httpbin.org/status/200}"
while true; do
if node browser_playwright.mjs "$ENDPOINT" >/dev/null 2>&1; then
echo "$(date): Service healthy"
else
echo "$(date): Service unhealthy"
fi
sleep 30
done
EOF
chmod +x pw_health.sh
./pw_health.sh- Hanging or quoting issues: ensure your shell quoting is balanced. Prefer simple commands without complex inline quoting.
- Verbose mode too noisy: omit
-vin production. - Cookie file format: the script writes Playwright storageState JSON. It’s safe to keep or delete.
- 403 errors: site uses stronger protections. Drive a real page (Playwright
page.goto) and interact, or solve CAPTCHAs where required.
Performance notes
Dispatch time depends on process spawn and Playwright startup. For higher throughput, consider reusing the same Node process to issue many requests (modify the script to loop internally) or use k6/Locust/Artillery for large-scale load testing.
Limitations
- This CLI uses Playwright’s HTTP client bound to a Chromium context. It is much closer to real browsers than curl, but some advanced fingerprinting still detects automation.
- WebSocket flows, MFA, or complex JS challenges generally require full page automation (which Playwright supports).
When to use what
- Use this Playwright CLI when you need realistic browser behavior, cookies, and straightforward HTTP requests with quick async dispatch.
- Use full Playwright page automation for dynamic content, complex logins, CAPTCHAs, and JS-heavy sites.
Advanced combos
- With
jqfor JSON processing:
node browser_playwright.mjs https://httpbin.org/json | jq '.slideshow.title'- With
parallelfor concurrency:
echo -e "https://httpbin.org/get\nhttps://httpbin.org/headers" | \
parallel -j 5 "node browser_playwright.mjs -o {#}.json {}"- With
watchfor monitoring:
watch -n 5 "node browser_playwright.mjs https://httpbin.org/status/200 >/dev/null && echo ok || echo fail"- With
xargsfor batch processing:
echo -e "1\n2\n3" | xargs -I {} node browser_playwright.mjs "https://httpbin.org/anything/{}"Future enhancements
- Built-in rate limiting and retry logic
- Output modes (JSON-only, headers-only)
- Proxy support
- Response assertions (status codes, content patterns)
- Metrics collection (timings, success rates)
Minimal Selenium variant (Python)
If you prefer Selenium, here’s a minimal GET/headers/redirect/cookie-capable script. Note: issuing cross-origin POST bodies is more ergonomic with Playwright’s request client; Selenium focuses on page automation.
Install Selenium:
python3 -m venv .venv && source .venv/bin/activate
pip install --upgrade pip seleniumCreate browser_selenium.py:
cat > browser_selenium.py << 'EOF'
#!/usr/bin/env python3
import argparse, json, os, sys, time
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
RED='\033[31m'; GRN='\033[32m'; YLW='\033[33m'; NC='\033[0m'
def parse_args():
p = argparse.ArgumentParser(description='Minimal Selenium GET client')
p.add_argument('url')
p.add_argument('-o','--output')
p.add_argument('-c','--cookie', default=f"/tmp/selenium_cookies_{os.getpid()}.json")
p.add_argument('--show-headers', action='store_true')
p.add_argument('-t','--timeout', type=int, default=30)
p.add_argument('-A','--user-agent')
p.add_argument('-v','--verbose', action='store_true')
return p.parse_args()
args = parse_args()
opts = Options()
opts.add_argument('--headless=new')
if args.user_agent:
opts.add_argument(f'--user-agent={args.user_agent}')
with webdriver.Chrome(options=opts) as driver:
driver.set_page_load_timeout(args.timeout)
# Load cookies if present (domain-specific; best-effort)
if os.path.exists(args.cookie):
try:
ck = json.load(open(args.cookie))
for c in ck.get('cookies', []):
try:
driver.get('https://' + c.get('domain').lstrip('.'))
driver.add_cookie({
'name': c['name'], 'value': c['value'], 'path': c.get('path','/'),
'domain': c.get('domain'), 'secure': c.get('secure', False)
})
except Exception:
pass
except Exception:
pass
driver.get(args.url)
# Persist cookies (best-effort)
try:
cookies = driver.get_cookies()
json.dump({'cookies': cookies}, open(args.cookie, 'w'), indent=2)
except Exception:
pass
if args.output:
open(args.output, 'w').write(driver.page_source)
else:
sys.stdout.write(driver.page_source)
EOF
chmod +x browser_selenium.pyUse it:
./browser_selenium.py https://example.com > out.htmlConclusion
You now have a Playwright-powered CLI that mirrors the original curl-wrapper’s ergonomics but uses a real browser engine, plus a minimal Selenium alternative. Use the CLI for realistic headers, cookies, redirects, JSON/form POSTs, and async dispatch with --count. For tougher sites, scale up to full page automation with Playwright.
Resources
- Playwright docs: https://playwright.dev
- Selenium docs: https://www.selenium.dev
- httpbin test endpoints: https://httpbin.org