SERP Scraping with Residential Proxies: A Practical Setup for SEO Teams

{
  "snapshot_id": "serp_us_desktop_2026-07-01T09-14-00Z_proxy-cost",
  "keyword": "residential proxy pricing",
  "market": {
    "requested_country": "US",
    "observed_country": "US",
    "language": "en-US",
    "device": "desktop"
  },
  "request": {
    "search_url": "https://www.google.com/search?q=residential+proxy+pricing&hl=en&gl=us&num=20",
    "http_status": 200,
    "duration_ms": 842,
    "bytes": 214392,
    "attempt": 1
  },
  "results": [
    {
      "rank": 1,
      "type": "organic",
      "title": "Residential Proxy Pricing",
      "url": "https://example.com/pricing",
      "domain": "example.com"
    }
  ],
  "serp_features": ["ads_top", "people_also_ask"],
  "raw_html_object_key": "s3://serp-archive/2026/07/01/..."
}

import asyncio
import time
from dataclasses import dataclass
from urllib.parse import urlencode

import httpx


PROXY_HOST = "p1.bytesflows.com:8001"
BASE_USER = "your-sub-user"
PASSWORD = "your-password"


@dataclass(frozen=True)
class Market:
    country: str
    hl: str
    gl: str
    accept_language: str


MARKETS = {
    "us": Market("us", "en", "us", "en-US,en;q=0.9"),
    "gb": Market("gb", "en", "gb", "en-GB,en;q=0.9"),
    "de": Market("de", "de", "de", "de-DE,de;q=0.9,en;q=0.6"),
    "jp": Market("jp", "ja", "jp", "ja-JP,ja;q=0.9,en;q=0.5"),
}


def proxy_url(country: str) -> str:
    username = f"{BASE_USER}-loc-{country}"
    return f"http://{username}:{PASSWORD}@{PROXY_HOST}"


def search_url(keyword: str, market: Market) -> str:
    params = urlencode({
        "q": keyword,
        "hl": market.hl,
        "gl": market.gl,
        "num": "20",
    })
    return f"https://www.google.com/search?{params}"


async def fetch_serp(keyword: str, country: str) -> dict:
    market = MARKETS[country]
    target = search_url(keyword, market)
    started = time.perf_counter()

    headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
                      "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",
        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
        "Accept-Language": market.accept_language,
        "Accept-Encoding": "gzip, deflate, br",
    }

    timeout = httpx.Timeout(connect=5.0, read=12.0, write=5.0, pool=5.0)

    async with httpx.AsyncClient(proxy=proxy_url(country), timeout=timeout, follow_redirects=False) as client:
        try:
            response = await client.get(target, headers=headers)
            elapsed_ms = round((time.perf_counter() - started) * 1000)
            return {
                "keyword": keyword,
                "country": country,
                "status": response.status_code,
                "duration_ms": elapsed_ms,
               

 "bytes": len(response.content),
                "url": str(response.url),
                "html_sample": response.text[:500],
            }
        except Exception as exc:
            elapsed_ms = round((time.perf_counter() - started) * 1000)
            return {
                "keyword": keyword,
                "country": country,
                "status": "error",
                "duration_ms": elapsed_ms,
                "error": type(exc).__name__,
            }


async def main():
    keywords = [
        "residential proxy pricing",
        "rank tracking software",
        "seo monitoring API",
    ]
    jobs = [fetch_serp(keyword, "us") for keyword in keywords]
    for result in await asyncio.gather(*jobs):
        print(result)


if __name__ == "__main__":
    asyncio.run(main())

import { chromium } from "playwright";

const market = {
  country: "us",
  hl: "en",
  gl: "us",
  acceptLanguage: "en-US,en;q=0.9",
};

const username = `your-sub-user-loc-${market.country}-session-serp_us_001-time-10`;

const browser = await chromium.launch({ headless: true });
const context = await browser.newContext({
  locale: "en-US",
  timezoneId: "America/New_York",
  proxy: {
    server: "http://p1.bytesflows.com:8001",
    username,
    password: process.env.BF_PROXY_PASS ?? "your-password",
  },
  extraHTTPHeaders: {
    "Accept-Language": market.acceptLanguage,
  },
});

const page = await context.newPage();

await page.route("**/*", (route) => {
  const type = route.request().resourceType();
  if (["image", "media", "font"].includes(type)) return route.abort();
  return route.continue();
});

const query = new URLSearchParams({
  q: "residential proxy pricing",
  hl: market.hl,
  gl: market.gl,
  num: "20",
});

await page.goto(`https://www.google.com/search?${query}`, {
  waitUntil: "domcontentloaded",
  timeout: 30_000,
});

await page.screenshot({ path: "serp-us-residential-proxy-pricing.png", fullPage: true });
console.log({ title: await page.title(), url: page.url() });

await context.close();
await browser.close();

240 KB x 100,000 x 1.10 x 30 / 1,048,576 = 755 GB/month