Ensure you have the required libraries installed before proceeding.
Basic Web Scraping
Select Your Tech Stack:- Node.js
- Python
- C#
const playwright = require('playwright');
const AUTH = process.env.AUTH || 'USER:PASS';
const TARGET_URL = process.env.TARGET_URL || 'https://example.com';
async function scrape(url = TARGET_URL) {
if (AUTH === 'USER:PASS') {
throw new Error('🔑 Missing credentials! Update the AUTH environment variable.');
}
console.log(`🚀 Connecting to browser.ai...`);
const endpointURL = `wss://${AUTH}@brd.superproxy.io:9222`;
const browser = await playwright.chromium.connectOverCDP(endpointURL);
try {
console.log(`🌎 Navigating to ${url}...`);
const page = await browser.newPage();
await page.goto(url, { timeout: 120000 });
console.log(`📄 Extracting page content...`);
const data = await page.content();
console.log(`✅ Data scraped:`, data);
} finally {
await browser.close();
}
}
scrape().catch(err => console.error(`❌ Error:`, err));
const puppeteer = require('puppeteer-core');
const AUTH = process.env.AUTH || 'USER:PASS';
const TARGET_URL = process.env.TARGET_URL || 'https://example.com';
async function scrape(url = TARGET_URL) {
if (AUTH === 'USER:PASS') {
throw new Error('🔑 Missing credentials! Update the AUTH environment variable.');
}
console.log(`🚀 Connecting to browser.ai...`);
const browserWSEndpoint = `wss://${AUTH}@brd.superproxy.io:9222`;
const browser = await puppeteer.connect({ browserWSEndpoint });
try {
console.log(`🌎 Navigating to ${url}...`);
const page = await browser.newPage();
await page.goto(url, { timeout: 120000 });
console.log(`📄 Extracting page content...`);
const data = await page.content();
console.log(`✅ Data scraped:`, data);
} finally {
await browser.close();
}
}
scrape().catch(err => console.error(`❌ Error:`, err));
from playwright.sync_api import sync_playwright
import os
AUTH = os.getenv("AUTH", "USER:PASS")
TARGET_URL = os.getenv("TARGET_URL", "https://example.com")
def scrape(url=TARGET_URL):
if AUTH == "USER:PASS":
raise ValueError("🔑 Missing credentials! Update the AUTH environment variable.")
print("🚀 Connecting to browser.ai...")
endpoint_url = f"wss://{AUTH}@brd.superproxy.io:9222"
with sync_playwright() as p:
browser = p.chromium.connect_over_cdp(endpoint_url)
try:
print(f"🌎 Navigating to {url}...")
page = browser.new_page()
page.goto(url, timeout=120000)
print("📄 Extracting page content...")
data = page.content()
print(f"✅ Data scraped: {data}")
finally:
browser.close()
scrape()
using PuppeteerSharp;
using System;
class Program {
private static string AUTH = Environment.GetEnvironmentVariable("AUTH") ?? "USER:PASS";
private static string TARGET_URL = Environment.GetEnvironmentVariable("TARGET_URL") ?? "https://example.com";
static async Task Main() {
if (AUTH == "USER:PASS") {
throw new Exception("🔑 Missing credentials! Update the AUTH environment variable.");
}
Console.WriteLine("🚀 Connecting to browser.ai...");
var browser = await Puppeteer.ConnectAsync(new ConnectOptions {
BrowserWSEndpoint = $"wss://{AUTH}@brd.superproxy.io:9222"
});
try {
Console.WriteLine($"🌎 Navigating to {TARGET_URL}...");
var page = await browser.NewPageAsync();
await page.GoToAsync(TARGET_URL, new NavigationOptions { Timeout = 120000 });
Console.WriteLine("📄 Extracting page content...");
var data = await page.GetContentAsync();
Console.WriteLine($"✅ Data scraped: {data}");
} finally {
await browser.CloseAsync();
}
}
}
Debugging & Inspecting Your Session
Want to see what’s happening inside your browser.ai session? Open DevTools automatically:- Node.js
- Python
const { exec } = require('child_process');
const AUTH = process.env.AUTH || 'USER:PASS';
const browserWSEndpoint = `wss://${AUTH}@brd.superproxy.io:9222`;
(async () => {
console.log("🚀 Connecting to browser.ai...");
const browser = await puppeteer.connect({ browserWSEndpoint });
const page = await browser.newPage();
const client = await page.target().createCDPSession();
const { url: inspectUrl } = await client.send('Page.inspect');
console.log(`🛠 Open DevTools: ${inspectUrl}`);
exec(`google-chrome "${inspectUrl}"`);
})();
from playwright.sync_api import sync_playwright
import os, subprocess
AUTH = os.getenv("AUTH", "USER:PASS")
TARGET_URL = os.getenv("TARGET_URL", "https://example.com")
def open_devtools():
if AUTH == "USER:PASS":
raise ValueError("🔑 Missing credentials! Update the AUTH environment variable.")
print("🚀 Connecting to browser.ai...")
endpoint_url = f"wss://{AUTH}@brd.superproxy.io:9222"
with sync_playwright() as p:
browser = p.chromium.connect_over_cdp(endpoint_url)
page = browser.new_page()
client = page.context.new_cdp_session(page)
inspect_url = client.send("Page.inspect")["url"]
print(f"🛠 Open DevTools: {inspect_url}")
subprocess.run(["google-chrome", inspect_url])
open_devtools()
Handling CAPTCHAs
browser.ai can automatically detect and solve CAPTCHAs in real time:- Node.js
const puppeteer = require('puppeteer-core');
const AUTH = process.env.AUTH || 'USER:PASS';
const TARGET_URL = process.env.TARGET_URL || 'https://example.com';
(async () => {
const browser = await puppeteer.connect({ browserWSEndpoint: `wss://${AUTH}@brd.superproxy.io:9222` });
const page = await browser.newPage();
await page.goto(TARGET_URL, { timeout: 120000 });
console.log("🔎 Checking for CAPTCHA...");
const client = await page.target().createCDPSession();
const { status } = await client.send("Captcha.solve", { detectTimeout: 30000 });
console.log(`🛠 CAPTCHA status: ${status}`);
})();