E-commerce Price Monitoring with Residential Proxies: From Test Run to Production

{
  "snapshot_id": "sku_B08N5_2026-07-01T10-15-00Z_us",
  "sku": "B08N5",
  "url": "https://example-market.test/product/B08N5",
  "market": {
    "requested_country": "US",
    "observed_country": "US",
    "currency": "USD",
    "delivery_region": "New York"
  },
  "price": {
    "amount": 199.99,
    "currency": "USD",
    "raw_text": "$199.99"
  },
  "availability": {
    "in_stock": true,
    "delivery_text": "Delivery by Friday"
  },
  "request": {
    "http_status": 200,
    "attempt": 1,
    "duration_ms": 731,
    "bytes": 184221,
    "proxy_mode": "rotating_country"
  },
  "evidence": {
    "html_object_key": "s3://price-monitor/2026/07/01/B08N5.html",
    "screenshot_object_key": null
  }
}

import asyncio
import json
import time
from dataclasses import dataclass

import aiohttp


PROXY_HOST = "http://p1.bytesflows.com:8001"
BASE_USER = "your-sub-user"
PASSWORD = "your-password"


@dataclass(frozen=True)
class SkuJob:
    sku: str
    url: str
    country: str
    needs_sticky: bool = False


def build_proxy(job: SkuJob) -> str:
    if job.needs_sticky:
        username = f"{BASE_USER}-loc-{job.country}-session-{job.sku.lower()}-time-15"
    else:
        username = f"{BASE_USER}-loc-{job.country}"
    return f"http://{username}:{PASSWORD}@p1.bytesflows.com:8001"


def headers_for_country(country: str) -> dict:
    languages = {
        "us": "en-US,en;q=0.9",
        "gb": "en-GB,en;q=0.9",
        "de": "de-DE,de;q=0.9,en;q=0.6",
        "jp": "ja-JP,ja;q=0.9,en;q=0.5",
    }
    return {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
                      "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",
        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
        "Accept-Language": languages.get(country, "en-US,en;q=0.9"),
        "Accept-Encoding": "gzip, deflate, br",
    }


async def fetch_sku(session: aiohttp.ClientSession, job: SkuJob, semaphore: asyncio.Semaphore) -> dict:
    async with semaphore:
        started = time.perf_counter()
        proxy = build_proxy(job)

        for attempt in range(1, 3):
            try:
                async with session.get(
                    job.url,
                    proxy=proxy,
                    headers=headers_for_country(job.country),
                    timeout=aiohttp.ClientTimeout(total=20, connect=6, sock_read=12),
                ) as response:
                    body = await response.text(errors="replace")
                    elapsed_ms = round((time.perf_counter() - started) * 1000)

                    return {
                        "sku": job.sku,
                        "country": job.country,
               

         "status": response.status,
                        "attempt": attempt,
                        "duration_ms": elapsed_ms,
                        "bytes": len(body.encode("utf-8")),
                        "html_sample": body[:300],
                    }
            except (aiohttp.ClientError, asyncio.TimeoutError) as exc:
                if attempt == 2:
                    elapsed_ms = round((time.perf_counter() - started) * 1000)
                    return {
                        "sku": job.sku,
                        "country": job.country,
                        "status": "error",
                        "attempt": attempt,
                        "duration_ms": elapsed_ms,
                        "error": type(exc).__name__,
                    }
                await asyncio.sleep(1.5 * attempt)


async def main():
    jobs = [
        SkuJob("SKU-101", "https://example.com/product/sku-101", "us"),
        SkuJob("SKU-102", "https://example.com/product/sku-102", "de"),
        SkuJob("SKU-103", "https://example.com/cart/shipping/sku-103", "gb", needs_sticky=True),
    ]

    semaphore = asyncio.Semaphore(10)
    connector = aiohttp.TCPConnector(limit=20, ttl_dns_cache=300)

    async with aiohttp.ClientSession(connector=connector) as session:
        results = await asyncio.gather(*(fetch_sku(session, job, semaphore) for job in jobs))
        print(json.dumps(results, indent=2))


if __name__ == "__main__":
    asyncio.run(main())

import { chromium } from "playwright";

const browser = await chromium.launch({ headless: true });
const context = await browser.newContext({
  locale: "en-US",
  timezoneId: "America/New_York",
  proxy: {
    server: "http://p1.bytesflows.com:8001",
    username: "your-sub-user-loc-us-session-price_evidence_101-time-15",
    password: process.env.BF_PROXY_PASS ?? "your-password",
  },
});

const page = await context.newPage();

await page.route("**/*", (route) => {
  const type = route.request().resourceType();
  if (["image", "media", "font"].includes(type)) return route.abort();
  return route.continue();
});

await page.goto("https://example.com/product/sku-101", {
  waitUntil: "domcontentloaded",
  timeout: 30_000,
});

await page.screenshot({ path: "sku-101-us-price.png", fullPage: true });
console.log({ title: await page.title(), url: page.url() });

await context.close();
await browser.close();

180 KB x 100,000 SKUs x 1.10 retry multiplier x 30 days / 1,048,576 = 566 GB/month

900 KB x 100,000 SKUs x 1.12 retry multiplier x 30 days / 1,048,576 = 2,884 GB/month