Why Playwright for Proxy-Based Scraping
Playwright is a modern browser automation framework from Microsoft that supports Chromium, Firefox, and WebKit. Unlike HTTP-only libraries, Playwright renders full pages — executing JavaScript, handling dynamic content, and passing anti-bot checks that reject raw HTTP requests.
When combined with residential proxies, Playwright becomes one of the most effective tools for collecting data from heavily protected websites. This guide covers proxy configuration at every level: browser-wide, per-context, and per-page — with working code you can copy directly into your projects.
This guide assumes you have a ProxyHat account. If you are new to proxies, start with What Is a Proxy Server? and then review our best proxies for web scraping overview.
Installation and Setup
Node.js (Primary)
# Install Playwright
npm init -y
npm install playwright
# Download browser binaries
npx playwright install chromium
Python
# Install Playwright for Python
pip install playwright
python -m playwright install chromium
Browser-Level Proxy Configuration
The simplest approach sets a proxy at browser launch. Every context and page inherits this proxy automatically.
Node.js — Browser-Wide Proxy
const { chromium } = require('playwright');
(async () => {
const browser = await chromium.launch({
proxy: {
server: 'http://gate.proxyhat.com:8080',
username: 'USERNAME',
password: 'PASSWORD',
},
});
const page = await browser.newPage();
await page.goto('https://httpbin.org/ip');
console.log(await page.textContent('body'));
await browser.close();
})();
Python — Browser-Wide Proxy
from playwright.sync_api import sync_playwright
with sync_playwright() as p:
browser = p.chromium.launch(
proxy={
"server": "http://gate.proxyhat.com:8080",
"username": "USERNAME",
"password": "PASSWORD",
}
)
page = browser.new_page()
page.goto("https://httpbin.org/ip")
print(page.text_content("body"))
browser.close()
Per-Context Proxy Rotation
The real power of Playwright lies in browser contexts. Each context is an isolated session — separate cookies, storage, and cache — and can have its own proxy. This is the recommended pattern for proxy rotation because it avoids the overhead of launching a new browser for each IP.
Node.js — Per-Context Rotation
const { chromium } = require('playwright');
const crypto = require('crypto');
async function createProxiedContext(browser) {
const sessionId = crypto.randomBytes(4).toString('hex');
const context = await browser.newContext({
proxy: {
server: 'http://gate.proxyhat.com:8080',
username: `USERNAME-session-${sessionId}`,
password: 'PASSWORD',
},
});
return context;
}
(async () => {
// Launch browser WITHOUT a proxy — set it per context
const browser = await chromium.launch();
const urls = [
'https://example.com/page/1',
'https://example.com/page/2',
'https://example.com/page/3',
];
for (const url of urls) {
const context = await createProxiedContext(browser);
const page = await context.newPage();
try {
await page.goto(url, { timeout: 30000 });
const content = await page.content();
console.log(`Fetched ${url} — ${content.length} chars`);
} catch (err) {
console.error(`Failed ${url}: ${err.message}`);
} finally {
await context.close(); // Releases the session
}
}
await browser.close();
})();
Python — Per-Context Rotation
import uuid
from playwright.sync_api import sync_playwright
def create_proxied_context(browser):
session_id = uuid.uuid4().hex[:8]
context = browser.new_context(
proxy={
"server": "http://gate.proxyhat.com:8080",
"username": f"USERNAME-session-{session_id}",
"password": "PASSWORD",
}
)
return context
with sync_playwright() as p:
browser = p.chromium.launch()
urls = [
"https://example.com/page/1",
"https://example.com/page/2",
"https://example.com/page/3",
]
for url in urls:
context = create_proxied_context(browser)
page = context.new_page()
try:
page.goto(url, timeout=30000)
print(f"Fetched {url} — {len(page.content())} chars")
except Exception as e:
print(f"Failed {url}: {e}")
finally:
context.close()
browser.close()
Geo-Targeted Contexts
When scraping localized content, you can combine ProxyHat's geo-targeting with Playwright's locale and timezone settings for maximum authenticity. See all available locations on our locations page.
const { chromium } = require('playwright');
const GEO_PROFILES = {
us: { locale: 'en-US', timezone: 'America/New_York', country: 'us' },
de: { locale: 'de-DE', timezone: 'Europe/Berlin', country: 'de' },
jp: { locale: 'ja-JP', timezone: 'Asia/Tokyo', country: 'jp' },
};
async function createGeoContext(browser, region) {
const profile = GEO_PROFILES[region];
return browser.newContext({
proxy: {
server: 'http://gate.proxyhat.com:8080',
username: `USERNAME-country-${profile.country}`,
password: 'PASSWORD',
},
locale: profile.locale,
timezoneId: profile.timezone,
geolocation: null,
});
}
(async () => {
const browser = await chromium.launch();
for (const region of ['us', 'de', 'jp']) {
const context = await createGeoContext(browser, region);
const page = await context.newPage();
await page.goto('https://example.com/pricing');
console.log(`${region.toUpperCase()}: ${await page.title()}`);
await context.close();
}
await browser.close();
})();
Concurrent Scraping with Worker Pool
Playwright contexts are lightweight. You can run multiple contexts in parallel, each with a different proxy session, to dramatically increase throughput.
const { chromium } = require('playwright');
const crypto = require('crypto');
const MAX_CONCURRENCY = 5;
async function scrapeUrl(browser, url) {
const sessionId = crypto.randomBytes(4).toString('hex');
const context = await browser.newContext({
proxy: {
server: 'http://gate.proxyhat.com:8080',
username: `USERNAME-session-${sessionId}`,
password: 'PASSWORD',
},
});
const page = await context.newPage();
try {
await page.goto(url, { timeout: 30000, waitUntil: 'domcontentloaded' });
const title = await page.title();
return { url, title, success: true };
} catch (err) {
return { url, error: err.message, success: false };
} finally {
await context.close();
}
}
async function scrapeAll(urls) {
const browser = await chromium.launch();
const results = [];
// Process in batches of MAX_CONCURRENCY
for (let i = 0; i < urls.length; i += MAX_CONCURRENCY) {
const batch = urls.slice(i, i + MAX_CONCURRENCY);
const batchResults = await Promise.all(
batch.map(url => scrapeUrl(browser, url))
);
results.push(...batchResults);
console.log(`Completed batch ${Math.floor(i / MAX_CONCURRENCY) + 1}`);
}
await browser.close();
return results;
}
// Usage
const urls = Array.from({ length: 20 }, (_, i) =>
`https://example.com/product/${i + 1}`
);
scrapeAll(urls).then(results => {
const success = results.filter(r => r.success).length;
console.log(`Success: ${success}/${results.length}`);
});
For more advanced concurrency patterns, see our guide on scaling proxy requests with concurrency control.
Stealth Configuration
Default Playwright browsers have detectable automation markers. These settings reduce your fingerprint and help bypass anti-bot systems.
Essential Stealth Settings
const { chromium } = require('playwright');
(async () => {
const browser = await chromium.launch({
args: [
'--disable-blink-features=AutomationControlled',
'--disable-features=IsolateOrigins,site-per-process',
],
});
const context = await browser.newContext({
proxy: {
server: 'http://gate.proxyhat.com:8080',
username: 'USERNAME',
password: 'PASSWORD',
},
userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
viewport: { width: 1920, height: 1080 },
locale: 'en-US',
timezoneId: 'America/New_York',
deviceScaleFactor: 1,
hasTouch: false,
isMobile: false,
javaScriptEnabled: true,
});
// Remove automation markers
await context.addInitScript(() => {
// Override navigator.webdriver
Object.defineProperty(navigator, 'webdriver', {
get: () => undefined,
});
// Override navigator.plugins to look real
Object.defineProperty(navigator, 'plugins', {
get: () => [1, 2, 3, 4, 5],
});
// Override navigator.languages
Object.defineProperty(navigator, 'languages', {
get: () => ['en-US', 'en'],
});
// Override chrome.runtime to avoid detection
window.chrome = { runtime: {} };
});
const page = await context.newPage();
await page.goto('https://bot.sannysoft.com/');
await page.screenshot({ path: 'stealth-test.png' });
await browser.close();
})();
Python Stealth Configuration
from playwright.sync_api import sync_playwright
with sync_playwright() as p:
browser = p.chromium.launch(
args=[
"--disable-blink-features=AutomationControlled",
]
)
context = browser.new_context(
proxy={
"server": "http://gate.proxyhat.com:8080",
"username": "USERNAME",
"password": "PASSWORD",
},
user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/131.0.0.0 Safari/537.36",
viewport={"width": 1920, "height": 1080},
locale="en-US",
timezone_id="America/New_York",
)
context.add_init_script("""
Object.defineProperty(navigator, 'webdriver', {
get: () => undefined,
});
window.chrome = { runtime: {} };
""")
page = context.new_page()
page.goto("https://httpbin.org/headers")
print(page.text_content("body"))
browser.close()
Retry Logic with Proxy Rotation
Combining retry logic with automatic proxy rotation ensures failed requests are retried with a fresh IP and context.
const { chromium } = require('playwright');
const crypto = require('crypto');
async function fetchWithRetry(browser, url, maxRetries = 3) {
for (let attempt = 1; attempt <= maxRetries; attempt++) {
const sessionId = crypto.randomBytes(4).toString('hex');
const context = await browser.newContext({
proxy: {
server: 'http://gate.proxyhat.com:8080',
username: `USERNAME-session-${sessionId}`,
password: 'PASSWORD',
},
});
const page = await context.newPage();
try {
const response = await page.goto(url, {
timeout: 30000,
waitUntil: 'domcontentloaded',
});
if (response && response.status() >= 400) {
console.log(`Attempt ${attempt}: HTTP ${response.status()}, retrying...`);
await context.close();
continue;
}
const html = await page.content();
await context.close();
return html;
} catch (err) {
console.log(`Attempt ${attempt} failed: ${err.message}`);
await context.close();
if (attempt === maxRetries) {
throw new Error(`All ${maxRetries} attempts failed for ${url}`);
}
// Exponential backoff
await new Promise(r => setTimeout(r, 1000 * Math.pow(2, attempt - 1)));
}
}
}
(async () => {
const browser = await chromium.launch();
try {
const html = await fetchWithRetry(browser, 'https://example.com/data');
console.log(`Fetched ${html.length} chars`);
} catch (err) {
console.error(err.message);
}
await browser.close();
})();
SOCKS5 Proxy with Playwright
ProxyHat also supports SOCKS5 on port 1080. This is useful when you need protocol-agnostic proxying or want to avoid HTTP CONNECT overhead.
const { chromium } = require('playwright');
(async () => {
const browser = await chromium.launch({
proxy: {
server: 'socks5://gate.proxyhat.com:1080',
username: 'USERNAME',
password: 'PASSWORD',
},
});
const page = await browser.newPage();
await page.goto('https://httpbin.org/ip');
console.log(await page.textContent('body'));
await browser.close();
})();
Production Scraping Pattern
Here is a complete production-ready scraper that combines all the patterns above — per-context proxy rotation, stealth settings, retry logic, concurrency, and structured data extraction.
const { chromium } = require('playwright');
const crypto = require('crypto');
const fs = require('fs');
class PlaywrightScraper {
constructor({ concurrency = 3, maxRetries = 3 }) {
this.concurrency = concurrency;
this.maxRetries = maxRetries;
this.browser = null;
this.results = [];
this.stats = { success: 0, failed: 0 };
}
async init() {
this.browser = await chromium.launch({
args: ['--disable-blink-features=AutomationControlled'],
});
}
_createContext() {
const sessionId = crypto.randomBytes(4).toString('hex');
return this.browser.newContext({
proxy: {
server: 'http://gate.proxyhat.com:8080',
username: `USERNAME-session-${sessionId}`,
password: 'PASSWORD',
},
userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
viewport: { width: 1920, height: 1080 },
locale: 'en-US',
});
}
async scrapePage(url) {
for (let attempt = 1; attempt <= this.maxRetries; attempt++) {
const context = await this._createContext();
const page = await context.newPage();
try {
const response = await page.goto(url, {
timeout: 30000,
waitUntil: 'networkidle',
});
if (!response || response.status() >= 400) {
await context.close();
continue;
}
// Extract data — customize this for your target
const data = await page.evaluate(() => ({
title: document.title,
text: document.body.innerText.substring(0, 500),
}));
await context.close();
this.stats.success++;
return { url, ...data, success: true };
} catch (err) {
await context.close();
if (attempt === this.maxRetries) {
this.stats.failed++;
return { url, error: err.message, success: false };
}
await new Promise(r => setTimeout(r, 1000 * attempt));
}
}
}
async scrapeAll(urls) {
await this.init();
for (let i = 0; i < urls.length; i += this.concurrency) {
const batch = urls.slice(i, i + this.concurrency);
const batchResults = await Promise.all(
batch.map(url => this.scrapePage(url))
);
this.results.push(...batchResults);
}
await this.browser.close();
console.log(`Done: ${this.stats.success} OK, ${this.stats.failed} failed`);
return this.results;
}
}
// Usage
const scraper = new PlaywrightScraper({ concurrency: 5, maxRetries: 3 });
const urls = Array.from({ length: 50 }, (_, i) =>
`https://example.com/item/${i + 1}`
);
scraper.scrapeAll(urls).then(results => {
fs.writeFileSync('results.json', JSON.stringify(results, null, 2));
});
For building a reusable proxy abstraction layer, see Building a Proxy Middleware Layer. Explore the Node SDK and Python SDK for simplified proxy management, and check ProxyHat pricing to get started.
