Connecting Crawl4AI & Scrapling to BytesFlows
Pass proxy configurations cleanly to ensure AI markdown extractors bypass Turnstile and Cloudflare protection shields.
example.py
import asyncio
from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig
async def main():
browser_cfg = BrowserConfig(
headless=True,
proxy="http://user-youraccount-rotate:password@proxy.bytesflows.com:31112"
)
run_cfg = CrawlerRunConfig(word_count_threshold=50)
async with AsyncWebCrawler(config=browser_cfg) as crawler:
result = await crawler.arun(
url="https://example.com/dynamic-article",
config=run_cfg
)
print("Extracted Markdown:
", result.markdown[:300])
if __name__ == "__main__":
asyncio.run(main())