diff --git a/async_scraper.py b/async_scraper.py index 5042d15..1caaa43 100644 --- a/async_scraper.py +++ b/async_scraper.py @@ -8,6 +8,8 @@ from datetime import datetime from tqdm.asyncio import tqdm from playwright.async_api import async_playwright # <- Playwright fallback +import rotate_vpn +import subprocess # CONFIG INPUT_PATH = "data.csv" @@ -16,9 +18,15 @@ CHUNK_SIZE = 100 RETRIES = 3 HEADERS = { - "User-Agent": "Mozilla/5.0 (X11; Linux x86_64)" + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64)", + "Accept-Language": "en-US,en;q=0.9", + "Accept-Encoding": "gzip, deflate, br", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", + "Connection": "keep-alive", } +ROTATE_EVERY = 10 + def parse_view_count(text): try: text = text.upper().replace(",", "").strip() @@ -114,7 +122,9 @@ async def run_scraper(urls, output_path): async with aiohttp.ClientSession(headers=HEADERS, connector=connector) as session: for i in range(0, len(urls), CHUNK_SIZE): batch = urls[i:i + CHUNK_SIZE] - print(f"\nā–¶ Processing batch {i // CHUNK_SIZE + 1} of {len(urls) // CHUNK_SIZE + 1}") + batch_num = i // CHUNK_SIZE + 1 + + print(f"\nProcessing batch {batch_num} of {len(urls) // CHUNK_SIZE + 1}") tasks = [get_data(session, url) for url in batch] results = [] @@ -130,9 +140,13 @@ async def run_scraper(urls, output_path): write_header = not os.path.exists(output_path) batch_df.to_csv(output_path, mode="a", header=write_header, index=False) - print(f"āœ” Appended {len(batch_df)} rows to {output_path}") + print(f"Appended {len(batch_df)} rows to {output_path}") + + if batch_num % ROTATE_EVERY == 0: + print("šŸ” Rotating VPN...") + subprocess.run(["./rotate_vpn.sh"]) + await asyncio.sleep(random.uniform(3.0, 5.0)) # give time for new IP to settle - await asyncio.sleep(random.uniform(0.8, 2.0)) def get_unprocessed_urls(): df = pd.read_csv(INPUT_PATH, delimiter='‽', encoding='utf-8', engine='python') diff --git a/rotate_vpn.py b/rotate_vpn.py new file mode 100644 index 0000000..7525236 --- /dev/null +++ b/rotate_vpn.py @@ -0,0 +1,4 @@ +import subprocess + +def rotate_vpn(): + subprocess.run(["bash", "rotate_vpn.sh"]) diff --git a/rotate_vpn.sh b/rotate_vpn.sh new file mode 100755 index 0000000..eee8f9e --- /dev/null +++ b/rotate_vpn.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +# Define a list of actual supported country names +COUNTRIES=( + "United_States" + "United_Kingdom" + "Germany" + "Netherlands" + "Sweden" + "Switzerland" + "Canada" + "France" + "Australia" + "Singapore" +) + +# Disconnect current VPN session +echo "Disconnecting VPN..." +nordvpn disconnect +sleep 3 + +# Pick a random country +RANDOM_COUNTRY=${COUNTRIES[$RANDOM % ${#COUNTRIES[@]}]} +echo "Connecting to $RANDOM_COUNTRY..." + +# Connect +nordvpn connect "$RANDOM_COUNTRY" +sleep 5