Category: Networking

Modern sites often block plain curl. Using a real browser engine (Chromium via Playwright) gives you true browser behavior: real TLS/HTTP2 stack, cookies, redirects, and JavaScript execution if needed. This post mirrors the functionality of the original browser_curl.sh wrapper but implemented with Playwright. It also includes an optional Selenium mini-variant at the end.

What this tool does

• Sends realistic browser headers (Chrome-like)
• Uses Chromium’s real network stack (HTTP/2, compression)
• Manages cookies (persist to a file)
• Follows redirects by default
• Supports JSON and form POSTs
• Async mode that returns immediately
• --count N to dispatch N async requests for quick load tests

Note: Advanced bot defenses (CAPTCHAs, JS/ML challenges, strict TLS/HTTP2 fingerprinting) may still require full page automation and real user-like behavior. Playwright can do that too by driving real pages.

Setup

Run these once to install Playwright and Chromium:

npm init -y && \
npm install playwright && \
npx playwright install chromium

The complete Playwright CLI

Run this to create browser_playwright.mjs:

cat > browser_playwright.mjs << 'EOF'
#!/usr/bin/env node
import { chromium } from 'playwright';
import fs from 'fs';
import path from 'path';
import { spawn } from 'child_process';
const RED = '\u001b[31m';
const GRN = '\u001b[32m';
const YLW = '\u001b[33m';
const NC  = '\u001b[0m';
function usage() {
const b = path.basename(process.argv[1]);
console.log(`Usage: ${b} [OPTIONS] URL
Advanced HTTP client using Playwright (Chromium) with browser-like behavior.
OPTIONS:
-X, --method METHOD        HTTP method (GET, POST, PUT, DELETE) [default: GET]
-d, --data DATA            Request body
-H, --header HEADER        Add custom header (repeatable)
-o, --output FILE          Write response body to file
-c, --cookie FILE          Cookie storage file [default: /tmp/pw_cookies_<pid>.json]
-A, --user-agent UA        Custom User-Agent
-t, --timeout SECONDS      Request timeout [default: 30]
--async                Run request(s) in background
--count N              Number of async requests to fire [default: 1, requires --async]
--no-redirect          Do not follow redirects (best-effort)
--show-headers         Print response headers
--json                 Send data as JSON (sets Content-Type)
--form                 Send data as application/x-www-form-urlencoded
-v, --verbose              Verbose output
-h, --help                 Show this help message
EXAMPLES:
${b} https://example.com
${b} --async https://example.com
${b} -X POST --json -d '{"a":1}' https://httpbin.org/post
${b} --async --count 10 https://httpbin.org/get
`);
}
function parseArgs(argv) {
const args = { method: 'GET', async: false, count: 1, followRedirects: true, showHeaders: false, timeout: 30, data: '', contentType: '', cookieFile: '', verbose: false, headers: [], url: '' };
for (let i = 0; i < argv.length; i++) {
const a = argv[i];
switch (a) {
case '-X': case '--method': args.method = String(argv[++i] || 'GET'); break;
case '-d': case '--data': args.data = String(argv[++i] || ''); break;
case '-H': case '--header': args.headers.push(String(argv[++i] || '')); break;
case '-o': case '--output': args.output = String(argv[++i] || ''); break;
case '-c': case '--cookie': args.cookieFile = String(argv[++i] || ''); break;
case '-A': case '--user-agent': args.userAgent = String(argv[++i] || ''); break;
case '-t': case '--timeout': args.timeout = Number(argv[++i] || '30'); break;
case '--async': args.async = true; break;
case '--count': args.count = Number(argv[++i] || '1'); break;
case '--no-redirect': args.followRedirects = false; break;
case '--show-headers': args.showHeaders = true; break;
case '--json': args.contentType = 'application/json'; break;
case '--form': args.contentType = 'application/x-www-form-urlencoded'; break;
case '-v': case '--verbose': args.verbose = true; break;
case '-h': case '--help': usage(); process.exit(0);
default:
if (!args.url && !a.startsWith('-')) args.url = a; else {
console.error(`${RED}Error: Unknown argument: ${a}${NC}`);
process.exit(1);
}
}
}
return args;
}
function parseHeaderList(list) {
const out = {};
for (const h of list) {
const idx = h.indexOf(':');
if (idx === -1) continue;
const name = h.slice(0, idx).trim();
const value = h.slice(idx + 1).trim();
if (!name) continue;
out[name] = value;
}
return out;
}
function buildDefaultHeaders(userAgent) {
const ua = userAgent || 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
return {
'User-Agent': ua,
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
'Accept-Language': 'en-US,en;q=0.9',
'Accept-Encoding': 'gzip, deflate, br',
'Connection': 'keep-alive',
'Upgrade-Insecure-Requests': '1',
'Sec-Fetch-Dest': 'document',
'Sec-Fetch-Mode': 'navigate',
'Sec-Fetch-Site': 'none',
'Sec-Fetch-User': '?1',
'Cache-Control': 'max-age=0'
};
}
async function performRequest(opts) {
// Cookie file handling
const defaultCookie = `/tmp/pw_cookies_${process.pid}.json`;
const cookieFile = opts.cookieFile || defaultCookie;
// Launch Chromium
const browser = await chromium.launch({ headless: true });
const extraHeaders = { ...buildDefaultHeaders(opts.userAgent), ...parseHeaderList(opts.headers) };
if (opts.contentType) extraHeaders['Content-Type'] = opts.contentType;
const context = await browser.newContext({ userAgent: extraHeaders['User-Agent'], extraHTTPHeaders: extraHeaders });
// Load cookies if present
if (fs.existsSync(cookieFile)) {
try {
const ss = JSON.parse(fs.readFileSync(cookieFile, 'utf8'));
if (ss.cookies?.length) await context.addCookies(ss.cookies);
} catch {}
}
const request = context.request;
// Build request options
const reqOpts = { headers: extraHeaders, timeout: opts.timeout * 1000 };
if (opts.data) {
// Playwright will detect JSON strings vs form strings by headers
reqOpts.data = opts.data;
}
if (opts.followRedirects === false) {
// Best-effort: limit redirects to 0
reqOpts.maxRedirects = 0;
}
const method = opts.method.toUpperCase();
let resp;
try {
if (method === 'GET') resp = await request.get(opts.url, reqOpts);
else if (method === 'POST') resp = await request.post(opts.url, reqOpts);
else if (method === 'PUT') resp = await request.put(opts.url, reqOpts);
else if (method === 'DELETE') resp = await request.delete(opts.url, reqOpts);
else if (method === 'PATCH') resp = await request.patch(opts.url, reqOpts);
else {
console.error(`${RED}Unsupported method: ${method}${NC}`);
await browser.close();
process.exit(2);
}
} catch (e) {
console.error(`${RED}[ERROR] ${e?.message || e}${NC}`);
await browser.close();
process.exit(3);
}
// Persist cookies
try {
const state = await context.storageState();
fs.writeFileSync(cookieFile, JSON.stringify(state, null, 2));
} catch {}
// Output
const status = resp.status();
const statusText = resp.statusText();
const headers = await resp.headers();
const body = await resp.text();
if (opts.verbose) {
console.error(`${YLW}Request: ${method} ${opts.url}${NC}`);
console.error(`${YLW}Headers: ${JSON.stringify(extraHeaders)}${NC}`);
}
if (opts.showHeaders) {
// Print a simple status line and headers to stdout before body
console.log(`HTTP ${status} ${statusText}`);
for (const [k, v] of Object.entries(headers)) {
console.log(`${k}: ${v}`);
}
console.log('');
}
if (opts.output) {
fs.writeFileSync(opts.output, body);
} else {
process.stdout.write(body);
}
if (!resp.ok()) {
console.error(`${RED}[ERROR] HTTP ${status} ${statusText}${NC}`);
await browser.close();
process.exit(4);
}
await browser.close();
}
async function main() {
const argv = process.argv.slice(2);
const opts = parseArgs(argv);
if (!opts.url) { console.error(`${RED}Error: URL is required${NC}`); usage(); process.exit(1); }
if ((opts.count || 1) > 1 && !opts.async) {
console.error(`${RED}Error: --count requires --async${NC}`);
process.exit(1);
}
if (opts.count < 1 || !Number.isInteger(opts.count)) {
console.error(`${RED}Error: --count must be a positive integer${NC}`);
process.exit(1);
}
if (opts.async) {
// Fire-and-forget background processes
const baseArgs = process.argv.slice(2).filter(a => a !== '--async' && !a.startsWith('--count'));
const pids = [];
for (let i = 0; i < opts.count; i++) {
const child = spawn(process.execPath, [process.argv[1], ...baseArgs], { detached: true, stdio: 'ignore' });
pids.push(child.pid);
child.unref();
}
if (opts.verbose) {
console.error(`${YLW}[ASYNC] Spawned ${opts.count} request(s).${NC}`);
}
if (opts.count === 1) console.error(`${GRN}[ASYNC] Request started with PID: ${pids[0]}${NC}`);
else console.error(`${GRN}[ASYNC] ${opts.count} requests started with PIDs: ${pids.join(' ')}${NC}`);
process.exit(0);
}
await performRequest(opts);
}
main().catch(err => {
console.error(`${RED}[FATAL] ${err?.stack || err}${NC}`);
process.exit(1);
});
EOF
chmod +x browser_playwright.mjs

Optionally, move it into your PATH:

sudo mv browser_playwright.mjs /usr/local/bin/browser_playwright

Quick start

• Simple GET:

node browser_playwright.mjs https://example.com

• Async GET (returns immediately):

node browser_playwright.mjs --async https://example.com

• Fire 100 async requests in one command:

node browser_playwright.mjs --async --count 100 https://httpbin.org/get

• POST JSON:

node browser_playwright.mjs -X POST --json \
-d '{"username":"user","password":"pass"}' \
https://httpbin.org/post

• POST form data:

node browser_playwright.mjs -X POST --form \
-d "username=user&password=pass" \
https://httpbin.org/post

• Include response headers:

node browser_playwright.mjs --show-headers https://example.com

• Save response to a file:

node browser_playwright.mjs -o response.json https://httpbin.org/json

• Custom headers:

node browser_playwright.mjs \
-H "X-API-Key: your-key" \
-H "Authorization: Bearer token" \
https://httpbin.org/headers

• Persistent cookies across requests:

COOKIE_FILE="playwright_session.json"
# Login and save cookies
node browser_playwright.mjs -c "$COOKIE_FILE" \
-X POST --form \
-d "user=test&pass=secret" \
https://httpbin.org/post > /dev/null
# Authenticated-like follow-up (cookie file reused)
node browser_playwright.mjs -c "$COOKIE_FILE" \
https://httpbin.org/cookies

Load testing patterns

• Simple load test with --count:

node browser_playwright.mjs --async --count 100 https://httpbin.org/get

• Loop-based alternative:

for i in {1..100}; do
node browser_playwright.mjs --async https://httpbin.org/get
done

• Timed load test:

cat > pw_load_for_duration.sh << 'EOF'
#!/usr/bin/env bash
URL="${1:-https://httpbin.org/get}"
DURATION="${2:-60}"
COUNT=0
END_TIME=$(($(date +%s) + DURATION))
while [ "$(date +%s)" -lt "$END_TIME" ]; do
node browser_playwright.mjs --async "$URL" >/dev/null 2>&1
((COUNT++))
done
echo "Sent $COUNT requests in $DURATION seconds"
echo "Rate: $((COUNT / DURATION)) requests/second"
EOF
chmod +x pw_load_for_duration.sh
./pw_load_for_duration.sh https://httpbin.org/get 30

• Parameterized load test:

cat > pw_load_test.sh << 'EOF'
#!/usr/bin/env bash
URL="${1:-https://httpbin.org/get}"
REQUESTS="${2:-50}"
echo "Load testing: $URL"
echo "Requests: $REQUESTS"
echo ""
START=$(date +%s)
node browser_playwright.mjs --async --count "$REQUESTS" "$URL"
echo ""
echo "Dispatched in $(($(date +%s) - START)) seconds"
EOF
chmod +x pw_load_test.sh
./pw_load_test.sh https://httpbin.org/get 200

Options reference

• -X, --method HTTP method (GET/POST/PUT/DELETE/PATCH)
• -d, --data Request body
• -H, --header Add extra headers (repeatable)
• -o, --output Write response body to file
• -c, --cookie Cookie file to use (and persist)
• -A, --user-agent Override User-Agent
• -t, --timeout Max request time in seconds (default 30)
• --async Run request(s) in the background
• --count N Fire N async requests (requires --async)
• --no-redirect Best-effort disable following redirects
• --show-headers Include response headers before body
• --json Sets Content-Type: application/json
• --form Sets Content-Type: application/x-www-form-urlencoded
• -v, --verbose Verbose diagnostics

Validation rules:

• --count requires --async
• --count must be a positive integer

Under the hood: why this works better than plain curl

• Real Chromium network stack (HTTP/2, TLS, compression)
• Browser-like headers and a true User-Agent
• Cookie jar via Playwright context storageState
• Redirect handling by the browser stack

This helps pass simplistic bot checks and more closely resembles real user traffic.

Real-world examples

• API-style auth flow (demo endpoints):

cat > pw_auth_flow.sh << 'EOF'
#!/usr/bin/env bash
COOKIE_FILE="pw_auth_session.json"
BASE="https://httpbin.org"
echo "Login (simulated form POST)..."
node browser_playwright.mjs -c "$COOKIE_FILE" \
-X POST --form \
-d "user=user&pass=pass" \
"$BASE/post" > /dev/null
echo "Fetch cookies..."
node browser_playwright.mjs -c "$COOKIE_FILE" \
"$BASE/cookies"
echo "Load test a protected-like endpoint..."
node browser_playwright.mjs -c "$COOKIE_FILE" \
--async --count 20 \
"$BASE/get"
echo "Done"
rm -f "$COOKIE_FILE"
EOF
chmod +x pw_auth_flow.sh
./pw_auth_flow.sh

• Scraping with rate limiting:

cat > pw_scrape.sh << 'EOF'
#!/usr/bin/env bash
URLS=(
"https://example.com/"
"https://example.com/"
"https://example.com/"
)
for url in "${URLS[@]}"; do
echo "Fetching: $url"
node browser_playwright.mjs -o "$(echo "$url" | sed 's#[/:]#_#g').html" "$url"
sleep 2
done
EOF
chmod +x pw_scrape.sh
./pw_scrape.sh

• Health check monitoring:

cat > pw_health.sh << 'EOF'
#!/usr/bin/env bash
ENDPOINT="${1:-https://httpbin.org/status/200}"
while true; do
if node browser_playwright.mjs "$ENDPOINT" >/dev/null 2>&1; then
echo "$(date): Service healthy"
else
echo "$(date): Service unhealthy"
fi
sleep 30
done
EOF
chmod +x pw_health.sh
./pw_health.sh

• Hanging or quoting issues: ensure your shell quoting is balanced. Prefer simple commands without complex inline quoting.
• Verbose mode too noisy: omit -v in production.
• Cookie file format: the script writes Playwright storageState JSON. It’s safe to keep or delete.
• 403 errors: site uses stronger protections. Drive a real page (Playwright page.goto) and interact, or solve CAPTCHAs where required.

Performance notes

Dispatch time depends on process spawn and Playwright startup. For higher throughput, consider reusing the same Node process to issue many requests (modify the script to loop internally) or use k6/Locust/Artillery for large-scale load testing.

Limitations

• This CLI uses Playwright’s HTTP client bound to a Chromium context. It is much closer to real browsers than curl, but some advanced fingerprinting still detects automation.
• WebSocket flows, MFA, or complex JS challenges generally require full page automation (which Playwright supports).

When to use what

• Use this Playwright CLI when you need realistic browser behavior, cookies, and straightforward HTTP requests with quick async dispatch.
• Use full Playwright page automation for dynamic content, complex logins, CAPTCHAs, and JS-heavy sites.

Advanced combos

• With jq for JSON processing:

node browser_playwright.mjs https://httpbin.org/json | jq '.slideshow.title'

• With parallel for concurrency:

echo -e "https://httpbin.org/get\nhttps://httpbin.org/headers" | \
parallel -j 5 "node browser_playwright.mjs -o {#}.json {}"

• With watch for monitoring:

watch -n 5 "node browser_playwright.mjs https://httpbin.org/status/200 >/dev/null && echo ok || echo fail"

• With xargs for batch processing:

echo -e "1\n2\n3" | xargs -I {} node browser_playwright.mjs "https://httpbin.org/anything/{}"

Future enhancements

• Built-in rate limiting and retry logic
• Output modes (JSON-only, headers-only)
• Proxy support
• Response assertions (status codes, content patterns)
• Metrics collection (timings, success rates)

Minimal Selenium variant (Python)

If you prefer Selenium, here’s a minimal GET/headers/redirect/cookie-capable script. Note: issuing cross-origin POST bodies is more ergonomic with Playwright’s request client; Selenium focuses on page automation.

Install Selenium:

python3 -m venv .venv && source .venv/bin/activate
pip install --upgrade pip selenium

Create browser_selenium.py:

cat > browser_selenium.py << 'EOF'
#!/usr/bin/env python3
import argparse, json, os, sys, time
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
RED='\033[31m'; GRN='\033[32m'; YLW='\033[33m'; NC='\033[0m'
def parse_args():
p = argparse.ArgumentParser(description='Minimal Selenium GET client')
p.add_argument('url')
p.add_argument('-o','--output')
p.add_argument('-c','--cookie', default=f"/tmp/selenium_cookies_{os.getpid()}.json")
p.add_argument('--show-headers', action='store_true')
p.add_argument('-t','--timeout', type=int, default=30)
p.add_argument('-A','--user-agent')
p.add_argument('-v','--verbose', action='store_true')
return p.parse_args()
args = parse_args()
opts = Options()
opts.add_argument('--headless=new')
if args.user_agent:
opts.add_argument(f'--user-agent={args.user_agent}')
with webdriver.Chrome(options=opts) as driver:
driver.set_page_load_timeout(args.timeout)
# Load cookies if present (domain-specific; best-effort)
if os.path.exists(args.cookie):
try:
ck = json.load(open(args.cookie))
for c in ck.get('cookies', []):
try:
driver.get('https://' + c.get('domain').lstrip('.'))
driver.add_cookie({
'name': c['name'], 'value': c['value'], 'path': c.get('path','/'),
'domain': c.get('domain'), 'secure': c.get('secure', False)
})
except Exception:
pass
except Exception:
pass
driver.get(args.url)
# Persist cookies (best-effort)
try:
cookies = driver.get_cookies()
json.dump({'cookies': cookies}, open(args.cookie, 'w'), indent=2)
except Exception:
pass
if args.output:
open(args.output, 'w').write(driver.page_source)
else:
sys.stdout.write(driver.page_source)
EOF
chmod +x browser_selenium.py

Use it:

./browser_selenium.py https://example.com > out.html

Conclusion

You now have a Playwright-powered CLI that mirrors the original curl-wrapper’s ergonomics but uses a real browser engine, plus a minimal Selenium alternative. Use the CLI for realistic headers, cookies, redirects, JSON/form POSTs, and async dispatch with --count. For tougher sites, scale up to full page automation with Playwright.

Resources

• Playwright docs: https://playwright.dev
• Selenium docs: https://www.selenium.dev
• httpbin test endpoints: https://httpbin.org

Modern websites deploy bot defenses that can block plain curl or naive scripts. In many cases, adding the right browser-like headers, HTTP/2, cookie persistence, and compression gets you past basic filters without needing a full browser.

This post walks through a small shell utility, browser_curl.sh, that wraps curl with realistic browser behavior. It also supports “fire-and-forget” async requests and a --count flag to dispatch many requests at once for quick load tests.

What this script does

• Sends browser-like headers (Chrome on macOS)
• Uses HTTP/2 and compression
• Manages cookies automatically (cookie jar)
• Follows redirects by default
• Supports JSON and form POSTs
• Async mode that returns immediately
• --count N to dispatch N async requests in one command

Note: This approach won’t solve advanced bot defenses that require JavaScript execution (e.g., Cloudflare Turnstile/CAPTCHAs or TLS/HTTP2 fingerprinting); for that, use a real browser automation tool like Playwright or Selenium.

The complete script

Save this as browser_curl.sh and make it executable in one command:

cat > browser_curl.sh << 'EOF' && chmod +x browser_curl.sh
#!/bin/bash
# browser_curl.sh - Advanced curl wrapper that mimics browser behavior
# Designed to bypass Cloudflare and other bot protection
# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m' # No Color
# Default values
METHOD="GET"
ASYNC=false
COUNT=1
FOLLOW_REDIRECTS=true
SHOW_HEADERS=false
OUTPUT_FILE=""
TIMEOUT=30
DATA=""
CONTENT_TYPE=""
COOKIE_FILE="/tmp/browser_curl_cookies_$$.txt"
VERBOSE=false
# Browser fingerprint (Chrome on macOS)
USER_AGENT="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
usage() {
cat << EOH
Usage: $(basename "$0") [OPTIONS] URL
Advanced curl wrapper that mimics browser behavior to bypass bot protection.
OPTIONS:
-X, --method METHOD        HTTP method (GET, POST, PUT, DELETE, etc.) [default: GET]
-d, --data DATA           POST/PUT data
-H, --header HEADER       Add custom header (can be used multiple times)
-o, --output FILE         Write output to file
-c, --cookie FILE         Use custom cookie file [default: temp file]
-A, --user-agent UA       Custom user agent [default: Chrome on macOS]
-t, --timeout SECONDS     Request timeout [default: 30]
--async                   Run request asynchronously in background
--count N                 Number of async requests to fire [default: 1, requires --async]
--no-redirect             Don't follow redirects
--show-headers            Show response headers
--json                    Send data as JSON (sets Content-Type)
--form                    Send data as form-urlencoded
-v, --verbose             Verbose output
-h, --help                Show this help message
EXAMPLES:
# Simple GET request
$(basename "$0") https://example.com
# Async GET request
$(basename "$0") --async https://example.com
# POST with JSON data
$(basename "$0") -X POST --json -d '{"username":"test"}' https://api.example.com/login
# POST with form data
$(basename "$0") -X POST --form -d "username=test&password=secret" https://example.com/login
# Multiple async requests (using loop)
for i in {1..10}; do
$(basename "$0") --async https://example.com/api/endpoint
done
# Multiple async requests (using --count)
$(basename "$0") --async --count 10 https://example.com/api/endpoint
EOH
exit 0
}
# Parse arguments
EXTRA_HEADERS=()
URL=""
while [[ $# -gt 0 ]]; do
case $1 in
-X|--method)
METHOD="$2"
shift 2
;;
-d|--data)
DATA="$2"
shift 2
;;
-H|--header)
EXTRA_HEADERS+=("$2")
shift 2
;;
-o|--output)
OUTPUT_FILE="$2"
shift 2
;;
-c|--cookie)
COOKIE_FILE="$2"
shift 2
;;
-A|--user-agent)
USER_AGENT="$2"
shift 2
;;
-t|--timeout)
TIMEOUT="$2"
shift 2
;;
--async)
ASYNC=true
shift
;;
--count)
COUNT="$2"
shift 2
;;
--no-redirect)
FOLLOW_REDIRECTS=false
shift
;;
--show-headers)
SHOW_HEADERS=true
shift
;;
--json)
CONTENT_TYPE="application/json"
shift
;;
--form)
CONTENT_TYPE="application/x-www-form-urlencoded"
shift
;;
-v|--verbose)
VERBOSE=true
shift
;;
-h|--help)
usage
;;
*)
if [[ -z "$URL" ]]; then
URL="$1"
else
echo -e "${RED}Error: Unknown argument '$1'${NC}" >&2
exit 1
fi
shift
;;
esac
done
# Validate URL
if [[ -z "$URL" ]]; then
echo -e "${RED}Error: URL is required${NC}" >&2
usage
fi
# Validate count
if [[ "$COUNT" -gt 1 ]] && [[ "$ASYNC" == false ]]; then
echo -e "${RED}Error: --count requires --async${NC}" >&2
exit 1
fi
if ! [[ "$COUNT" =~ ^[0-9]+$ ]] || [[ "$COUNT" -lt 1 ]]; then
echo -e "${RED}Error: --count must be a positive integer${NC}" >&2
exit 1
fi
# Execute curl
execute_curl() {
# Build curl arguments as array instead of string
local -a curl_args=()
# Basic options
curl_args+=("--compressed")
curl_args+=("--max-time" "$TIMEOUT")
curl_args+=("--connect-timeout" "10")
curl_args+=("--http2")
# Cookies (ensure file exists to avoid curl warning)
: > "$COOKIE_FILE" 2>/dev/null || true
curl_args+=("--cookie" "$COOKIE_FILE")
curl_args+=("--cookie-jar" "$COOKIE_FILE")
# Follow redirects
if [[ "$FOLLOW_REDIRECTS" == true ]]; then
curl_args+=("--location")
fi
# Show headers
if [[ "$SHOW_HEADERS" == true ]]; then
curl_args+=("--include")
fi
# Output file
if [[ -n "$OUTPUT_FILE" ]]; then
curl_args+=("--output" "$OUTPUT_FILE")
fi
# Verbose
if [[ "$VERBOSE" == true ]]; then
curl_args+=("--verbose")
else
curl_args+=("--silent" "--show-error")
fi
# Method
curl_args+=("--request" "$METHOD")
# Browser-like headers
curl_args+=("--header" "User-Agent: $USER_AGENT")
curl_args+=("--header" "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8")
curl_args+=("--header" "Accept-Language: en-US,en;q=0.9")
curl_args+=("--header" "Accept-Encoding: gzip, deflate, br")
curl_args+=("--header" "Connection: keep-alive")
curl_args+=("--header" "Upgrade-Insecure-Requests: 1")
curl_args+=("--header" "Sec-Fetch-Dest: document")
curl_args+=("--header" "Sec-Fetch-Mode: navigate")
curl_args+=("--header" "Sec-Fetch-Site: none")
curl_args+=("--header" "Sec-Fetch-User: ?1")
curl_args+=("--header" "Cache-Control: max-age=0")
# Content-Type for POST/PUT
if [[ -n "$DATA" ]]; then
if [[ -n "$CONTENT_TYPE" ]]; then
curl_args+=("--header" "Content-Type: $CONTENT_TYPE")
fi
curl_args+=("--data" "$DATA")
fi
# Extra headers
for header in "${EXTRA_HEADERS[@]}"; do
curl_args+=("--header" "$header")
done
# URL
curl_args+=("$URL")
if [[ "$ASYNC" == true ]]; then
# Run asynchronously in background
if [[ "$VERBOSE" == true ]]; then
echo -e "${YELLOW}[ASYNC] Running $COUNT request(s) in background...${NC}" >&2
echo -e "${YELLOW}Command: curl ${curl_args[*]}${NC}" >&2
fi
# Fire multiple requests if count > 1
local pids=()
for ((i=1; i<=COUNT; i++)); do
# Run in background detached, suppress all output
nohup curl "${curl_args[@]}" >/dev/null 2>&1 &
local pid=$!
disown $pid
pids+=("$pid")
done
if [[ "$COUNT" -eq 1 ]]; then
echo -e "${GREEN}[ASYNC] Request started with PID: ${pids[0]}${NC}" >&2
else
echo -e "${GREEN}[ASYNC] $COUNT requests started with PIDs: ${pids[*]}${NC}" >&2
fi
else
# Run synchronously
if [[ "$VERBOSE" == true ]]; then
echo -e "${YELLOW}Command: curl ${curl_args[*]}${NC}" >&2
fi
curl "${curl_args[@]}"
local exit_code=$?
if [[ $exit_code -ne 0 ]]; then
echo -e "${RED}[ERROR] Request failed with exit code: $exit_code${NC}" >&2
return $exit_code
fi
fi
}
# Cleanup temp cookie file on exit (only if using default temp file)
cleanup() {
if [[ "$COOKIE_FILE" == "/tmp/browser_curl_cookies_$$"* ]] && [[ -f "$COOKIE_FILE" ]]; then
rm -f "$COOKIE_FILE"
fi
}
# Only set cleanup trap for synchronous requests
if [[ "$ASYNC" == false ]]; then
trap cleanup EXIT
fi
# Main execution
execute_curl
# For async requests, exit immediately without waiting
if [[ "$ASYNC" == true ]]; then
exit 0
fi
EOF

Optionally, move it to your PATH:

sudo mv browser_curl.sh /usr/local/bin/browser_curl

Quick start

Simple GET request

./browser_curl.sh https://example.com

Async GET (returns immediately)

./browser_curl.sh --async https://example.com

Fire 100 async requests in one command

./browser_curl.sh --async --count 100 https://example.com/api

Common examples

POST JSON

./browser_curl.sh -X POST --json \
-d '{"username":"user","password":"pass"}' \
https://api.example.com/login

POST form data

./browser_curl.sh -X POST --form \
-d "username=user&password=pass" \
https://example.com/login

Include response headers

./browser_curl.sh --show-headers https://example.com

Save response to a file

./browser_curl.sh -o response.json https://api.example.com/data

Custom headers

./browser_curl.sh \
-H "X-API-Key: your-key" \
-H "Authorization: Bearer token" \
https://api.example.com/data

Persistent cookies across requests

COOKIE_FILE="session_cookies.txt"
# Login and save cookies
./browser_curl.sh -c "$COOKIE_FILE" \
-X POST --form \
-d "user=test&pass=secret" \
https://example.com/login
# Authenticated request using saved cookies
./browser_curl.sh -c "$COOKIE_FILE" \
https://example.com/dashboard

Load testing patterns

Simple load test with –count

The easiest way to fire multiple requests:

./browser_curl.sh --async --count 100 https://example.com/api

Example output:

[ASYNC] 100 requests started with PIDs: 1234 1235 1236 ... 1333

Performance: 100 requests dispatched in approximately 0.09 seconds

Loop-based approach (alternative)

for i in {1..100}; do
./browser_curl.sh --async https://example.com/api
done

Timed load test

Run continuous requests for a specific duration:

#!/bin/bash
URL="https://example.com/api"
DURATION=60  # seconds
COUNT=0
END_TIME=$(($(date +%s) + DURATION))
while [ "$(date +%s)" -lt "$END_TIME" ]; do
./browser_curl.sh --async "$URL" > /dev/null 2>&1
((COUNT++))
done
echo "Sent $COUNT requests in $DURATION seconds"
echo "Rate: $((COUNT / DURATION)) requests/second"

Parameterized load test script

#!/bin/bash
URL="${1:-https://httpbin.org/get}"
REQUESTS="${2:-50}"
echo "Load testing: $URL"
echo "Requests: $REQUESTS"
echo ""
START=$(date +%s)
./browser_curl.sh --async --count "$REQUESTS" "$URL"
echo ""
echo "Dispatched in $(($(date +%s) - START)) seconds"

Usage:

./load_test.sh https://api.example.com/endpoint 200

Options reference

Validation rules:

• --count requires --async
• --count must be a positive integer

Under the hood: why this works better than plain curl

Browser-like headers

The script automatically adds these headers to mimic Chrome:

User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36...
Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif...
Accept-Language: en-US,en;q=0.9
Accept-Encoding: gzip, deflate, br
Connection: keep-alive
Sec-Fetch-Dest: document
Sec-Fetch-Mode: navigate
Sec-Fetch-Site: none
Sec-Fetch-User: ?1
Cache-Control: max-age=0
Upgrade-Insecure-Requests: 1

HTTP/2 + compression

• Uses --http2 flag for HTTP/2 protocol support
• Enables --compressed for automatic gzip/brotli decompression
• Closer to modern browser behavior

Cookie jar

• Maintains session cookies across redirects and calls
• Persists cookies to file for reuse
• Automatically created and cleaned up

Redirect handling

• Follows redirects by default with --location
• Critical for login flows, SSO, and OAuth redirects

These features help bypass basic bot detection that blocks obvious non-browser clients.

Real-world examples

Example 1: API authentication flow

cd ~/Desktop/warp
bash -c 'cat > test_auth.sh << '\''SCRIPT'\''
#!/bin/bash
COOKIE_FILE="auth_session.txt"
API_BASE="https://api.example.com"
echo "Logging in..."
./browser_curl.sh -c "$COOKIE_FILE" -X POST --json -d "{\"username\":\"user\",\"password\":\"pass\"}" "$API_BASE/auth/login" > /dev/null
echo "Fetching profile..."
./browser_curl.sh -c "$COOKIE_FILE" "$API_BASE/user/profile" | jq .
echo "Load testing..."
./browser_curl.sh -c "$COOKIE_FILE" --async --count 50 "$API_BASE/api/data"
echo "Done!"
rm -f "$COOKIE_FILE"
SCRIPT
chmod +x test_auth.sh
./test_auth.sh'

Example 2: Scraping with rate limiting

#!/bin/bash
URLS=(
"https://example.com/page1"
"https://example.com/page2"
"https://example.com/page3"
)
for url in "${URLS[@]}"; do
echo "Fetching: $url"
./browser_curl.sh -o "$(basename "$url").html" "$url"
sleep 2  # Rate limiting
done

Example 3: Health check monitoring

#!/bin/bash
ENDPOINT="https://api.example.com/health"
while true; do
if ./browser_curl.sh "$ENDPOINT" | grep -q "healthy"; then
echo "$(date): Service healthy"
else
echo "$(date): Service unhealthy"
fi
sleep 30
done

Installing browser_curl to your PATH

If you want browser_curl.sh to be available anywhere then install it on your path using:

mkdir -p ~/.local/bin
echo "Installing browser_curl to ~/.local/bin/browser_curl"
install -m 0755 ~/Desktop/warp/browser_curl.sh ~/.local/bin/browser_curl
echo "Ensuring ~/.local/bin is on PATH via ~/.zshrc"
grep -q 'export PATH="$HOME/.local/bin:$PATH"' ~/.zshrc || \
echo 'export PATH="$HOME/.local/bin:$PATH"' >> ~/.zshrc
echo "Reloading shell config (~/.zshrc)"
source ~/.zshrc
echo "Verifying browser_curl is on PATH"
command -v browser_curl && echo "browser_curl is installed and on PATH" || echo "browser_curl not found on PATH"

Troubleshooting

Issue: Hanging with dquote> prompt

Cause: Shell quoting issue (unbalanced quotes)

Solution: Use simple, direct commands

# Good
./browser_curl.sh --async https://example.com
# Bad (unbalanced quotes)
echo "test && ./browser_curl.sh --async https://example.com && echo "done"

For chaining commands:

echo Start; ./browser_curl.sh --async https://example.com; echo Done

Issue: Verbose mode produces too much output

Cause: -v flag prints all curl diagnostics to stderr

Solution: Remove -v for production use:

# Debug mode
./browser_curl.sh -v https://example.com
# Production mode
./browser_curl.sh https://example.com

Issue: Cookie file warnings

Cause: First-time cookie file creation

Solution: The script now pre-creates the cookie file automatically. You can ignore any residual warnings.

Issue: 403 Forbidden errors

Cause: Site has stronger protections (JavaScript challenges, TLS fingerprinting)

Solution: Consider using real browser automation:

• Playwright (Python/Node.js)
• Selenium
• Puppeteer

Or combine approaches:

1. Use Playwright to initialize session and get cookies
2. Export cookies to file
3. Use browser_curl.sh -c cookies.txt for subsequent requests

Performance benchmarks

Tests conducted on 2023 MacBook Pro M2, macOS Sonoma:

Note: Dispatch time only; actual HTTP completion depends on target server.

Limitations

What this script CANNOT do

• JavaScript execution – Can’t solve JS challenges (use Playwright)
• CAPTCHA solving – Requires human intervention or services
• Advanced TLS fingerprinting – Can’t mimic exact browser TLS stack
• HTTP/2 fingerprinting – Can’t perfectly match browser HTTP/2 frames
• WebSocket connections – HTTP only
• Browser API access – No Canvas, WebGL, Web Crypto fingerprints

What this script CAN do

• Basic header spoofing – Pass simple User-Agent checks
• Cookie management – Maintain sessions
• Load testing – Quick async request dispatch
• API testing – POST/PUT/DELETE with JSON/form data
• Simple scraping – Pages without JS requirements
• Health checks – Monitoring endpoints

When to use what

Use browser_curl.sh when:

• Target has basic bot detection (header checks)
• API testing with authentication
• Quick load testing (less than 10k requests)
• Monitoring/health checks
• No JavaScript required
• You want a lightweight tool

Use Playwright/Selenium when:

• Target requires JavaScript execution
• CAPTCHA challenges present
• Advanced fingerprinting detected
• Need to interact with dynamic content
• Heavy scraping with anti-bot measures
• Login flows with MFA/2FA

Hybrid approach:

1. Use Playwright to bootstrap session
2. Extract cookies
3. Use browser_curl.sh for follow-up requests (faster)

Advanced: Combining with other tools

With jq for JSON processing

./browser_curl.sh https://api.example.com/users | jq '.[] | .name'

With parallel for concurrency control

cat urls.txt | parallel -j 10 "./browser_curl.sh -o {#}.html {}"

With watch for monitoring

watch -n 5 "./browser_curl.sh https://api.example.com/health | jq .status"

With xargs for batch processing

cat ids.txt | xargs -I {} ./browser_curl.sh "https://api.example.com/item/{}"

Future enhancements

Potential features to add:

• Rate limiting – Built-in requests/second throttling
• Retry logic – Exponential backoff on failures
• Output formats – JSON-only, CSV, headers-only modes
• Proxy support – SOCKS5/HTTP proxy options
• Custom TLS – Certificate pinning, client certs
• Response validation – Assert status codes, content patterns
• Metrics collection – Timing stats, success rates
• Configuration file – Default settings per domain

Conclusion

browser_curl.sh provides a pragmatic middle ground between plain curl and full browser automation. For many APIs and websites with basic bot filters, browser-like headers, proper protocol use, and cookie handling are sufficient.

Key takeaways:

• Simple wrapper around curl with realistic browser behavior
• Async mode with --count for easy load testing
• Works for basic bot detection, not advanced challenges
• Combine with Playwright for tough targets
• Lightweight and fast for everyday API work

The script is particularly useful for:

• API development and testing
• Quick load testing during development
• Monitoring and health checks
• Simple scraping tasks
• Learning curl features

For production load testing at scale, consider tools like k6, Locust, or Artillery. For heavy web scraping with anti-bot measures, invest in proper browser automation infrastructure.

Resources

• curl documentation
• HTTP headers reference
• Playwright for browser automation
• httpbin for testing

Below is a useful script when you want to see which processes are using a specific port.

#!/bin/bash
# Port Monitor Script for macOS
# Usage: ./port_monitor.sh <port_number>
# Check if port number is provided
if [ $# -eq 0 ]; then
echo "Usage: $0 <port_number>"
echo "Example: $0 8080"
exit 1
fi
PORT=$1
# Validate port number
if ! [[ $PORT =~ ^[0-9]+$ ]] || [ $PORT -lt 1 ] || [ $PORT -gt 65535 ]; then
echo "Error: Please provide a valid port number (1-65535)"
exit 1
fi
# Function to display processes using the port
show_port_usage() {
local timestamp=$(date "+%Y-%m-%d %H:%M:%S")
# Clear screen for better readability
clear
echo "=================================="
echo "Port Monitor - Port $PORT"
echo "Last updated: $timestamp"
echo "Press Ctrl+C to exit"
echo "=================================="
echo
# Check for processes using the port with lsof - both TCP and UDP
if lsof -i :$PORT &>/dev/null || netstat -an | grep -E "[:.]$PORT[[:space:]]" &>/dev/null; then
echo "Processes using port $PORT:"
echo
lsof -i :$PORT -P -n | head -1
echo "--------------------------------------------------------------------------------"
lsof -i :$PORT -P -n | tail -n +2
echo
# Also show netstat information for additional context
echo "Network connections on port $PORT:"
echo
printf "%-6s %-30s %-30s %-12s\n" "PROTO" "LOCAL ADDRESS" "FOREIGN ADDRESS" "STATE"
echo "--------------------------------------------------------------------------------------------"
# Show all connections (LISTEN, ESTABLISHED, etc.)
# Use netstat -n to show numeric addresses
netstat -anp tcp | grep -E "\.$PORT[[:space:]]" | while read line; do
# Extract the relevant fields from netstat output
proto=$(echo "$line" | awk '{print $1}')
local_addr=$(echo "$line" | awk '{print $4}')
foreign_addr=$(echo "$line" | awk '{print $5}')
state=$(echo "$line" | awk '{print $6}')
# Only print if we have valid data
if [ -n "$proto" ] && [ -n "$local_addr" ]; then
printf "%-6s %-30s %-30s %-12s\n" "$proto" "$local_addr" "$foreign_addr" "$state"
fi
done
# Also check UDP connections
netstat -anp udp | grep -E "\.$PORT[[:space:]]" | while read line; do
proto=$(echo "$line" | awk '{print $1}')
local_addr=$(echo "$line" | awk '{print $4}')
foreign_addr=$(echo "$line" | awk '{print $5}')
printf "%-6s %-30s %-30s %-12s\n" "$proto" "$local_addr" "$foreign_addr" "-"
done
# Also check for any established connections using lsof
echo
echo "Active connections with processes:"
echo "--------------------------------------------------------------------------------------------"
lsof -i :$PORT -P -n 2>/dev/null | grep -v LISTEN | tail -n +2 | while read line; do
if [ -n "$line" ]; then
echo "$line"
fi
done
else
echo "No processes found using port $PORT"
echo
# Check if the port might be in use but not showing up in lsof
local netstat_result=$(netstat -anv | grep -E "\.$PORT ")
if [ -n "$netstat_result" ]; then
echo "However, netstat shows activity on port $PORT:"
echo "$netstat_result"
fi
fi
echo
echo "Refreshing in 20 seconds... (Press Ctrl+C to exit)"
}
# Trap Ctrl+C to exit gracefully
trap 'echo -e "\n\nExiting port monitor..."; exit 0' INT
# Main loop - refresh every 20 seconds
while true; do
show_port_usage
sleep 20
done