import requests
import random

user_agents = [
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/120.0",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) Safari/605.1.15",
    "Mozilla/5.0 (X11; Linux x86_64) Firefox/121.0"
]

headers = {
    "User-Agent": random.choice(user_agents),
    "Accept": "text/html,application/xhtml+xml",
    "Accept-Language": "en-US,en;q=0.9",
    "Accept-Encoding": "gzip, deflate, br",
    "Connection": "keep-alive"
}

response = requests.get(url, headers=headers)

import time
import random

def polite_request(url, min_delay=1, max_delay=3):
    """Wait between requests to avoid overloading the server"""
    time.sleep(random.uniform(min_delay, max_delay))
    return requests.get(url, headers=headers)

# In Scrapy
DOWNLOAD_DELAY = 1          # Wait 1 second between requests
RANDOMIZE_DOWNLOAD_DELAY = True  # Randomize by 0.5x to 1.5x

# Using a proxy rotation service
proxies = {
    "http": "http://proxy-server:8080",
    "https": "http://proxy-server:8080"
}
response = requests.get(url, proxies=proxies)

# Use undetected-chromedriver to bypass Cloudflare
import undetected_chromedriver as uc

driver = uc.Chrome(headless=True)
driver.get("https://cloudflare-protected-site.com")

# Always check robots.txt first
from urllib.robotparser import RobotFileParser

rp = RobotFileParser()
rp.set_url("https://example.com/robots.txt")
rp.read()

can_scrape = rp.can_fetch("*", "https://example.com/products")
print(f"Can scrape: {can_scrape}")

import re
from datetime import datetime

class DataCleaner:

    @staticmethod
    def clean_price(price_str: str) -> float:
        """Convert '₹75,999.00' to 75999.0"""
        cleaned = re.sub(r"[₹,\s]", "", price_str)
        return float(cleaned)

    @staticmethod
    def clean_text(text: str) -> str:
        """Remove extra whitespace and special characters"""
        return " ".join(text.split()).strip()

    @staticmethod
    def parse_date(date_str: str) -> datetime:
        """Parse various date formats"""
        formats = ["%B %d, %Y", "%d/%m/%Y", "%Y-%m-%d", "%d %b %Y"]
        for fmt in formats:
            try:
                return datetime.strptime(date_str.strip(), fmt)
            except ValueError:
                continue
        return None

    @staticmethod
    def clean_rating(rating_str: str) -> float:
        """Extract numeric rating from '4.5 out of 5' """
        match = re.search(r"(\d+\.?\d*)", rating_str)
        return float(match.group(1)) if match else None

# Usage
raw_data = {
    "title": "  Laptop Pro 15  \n",
    "price": "₹75,999.00",
    "rating": "4.5 out of 5 stars",
    "date": "January 15, 2026"
}

cleaner = DataCleaner()
clean_data = {
    "title": cleaner.clean_text(raw_data["title"]),      # "Laptop Pro 15"
    "price": cleaner.clean_price(raw_data["price"]),     # 75999.0
    "rating": cleaner.clean_rating(raw_data["rating"]), # 4.5
    "date": cleaner.parse_date(raw_data["date"])         # datetime object
}

import json
import csv
import sqlite3
import pandas as pd

# 1. JSON file
with open("products.json", "w", encoding="utf-8") as f:
    json.dump(all_products, f, ensure_ascii=False, indent=2, default=str)

# 2. CSV file
df = pd.DataFrame(all_products)
df.to_csv("products.csv", index=False, encoding="utf-8")

# 3. SQLite database
conn = sqlite3.connect("products.db")
df.to_sql("products", conn, if_exists="replace", index=False)

# 4. PostgreSQL
import psycopg2
from psycopg2.extras import execute_batch

conn = psycopg2.connect("postgresql://user:password@localhost/scraping")
cursor = conn.cursor()

execute_batch(cursor, """
    INSERT INTO products (title, price, rating, url, scraped_at)
    VALUES (%(title)s, %(price)s, %(rating)s, %(url)s, NOW())
    ON CONFLICT (url) DO UPDATE SET
        price = EXCLUDED.price,
        scraped_at = NOW()
""", all_products)

conn.commit()

# Example robots.txt at https://example.com/robots.txt
User-agent: *
Disallow: /admin/
Disallow: /user/profile/
Allow: /products/
Crawl-delay: 1           # Respect crawl rate

User-agent: Googlebot
Allow: /                 # Allow Google to crawl everything

# 1. Identify yourself in User-Agent
headers = {
    "User-Agent": "MyCompanyBot/1.0 (research purposes; contact@mycompany.com)"
}

# 2. Respect robots.txt
from urllib.robotparser import RobotFileParser
rp = RobotFileParser()
rp.set_url(f"{base_url}/robots.txt")
rp.read()
if not rp.can_fetch("*", url):
    print(f"Skipping {url} — blocked by robots.txt")
    continue

# 3. Add delays between requests
import time
time.sleep(2)  # Do not hammer the server

# 4. Cache responses to avoid re-requesting
import requests_cache
requests_cache.install_cache("scraping_cache", expire_after=3600)

# 5. Scrape during off-peak hours
# Run scrapers at night when traffic is lower

# 6. Use official APIs when available
# Always prefer official APIs over scraping

import requests
from bs4 import BeautifulSoup
import sqlite3
from datetime import datetime
import smtplib

def scrape_product_price(url: str) -> dict:
    headers = {"User-Agent": "Mozilla/5.0 ..."}
    response = requests.get(url, headers=headers)
    soup = BeautifulSoup(response.text, "html.parser")

    return {
        "url": url,
        "title": soup.select_one("h1.product-title").text.strip(),
        "price": float(soup.select_one(".price").text.replace("₹", "").replace(",", "")),
        "in_stock": bool(soup.select_one(".add-to-cart")),
        "scraped_at": datetime.now().isoformat()
    }

def check_and_alert(product_url: str, target_price: float):
    data = scrape_product_price(product_url)

    # Store in database
    conn = sqlite3.connect("prices.db")
    conn.execute("""
        INSERT INTO price_history (url, title, price, in_stock, scraped_at)
        VALUES (?, ?, ?, ?, ?)
    """, (data["url"], data["title"], data["price"], data["in_stock"], data["scraped_at"]))
    conn.commit()

    # Alert if price dropped below target
    if data["price"] <= target_price:
        send_email_alert(
            subject=f"Price Alert: {data['title']}",
            body=f"Price dropped to ₹{data['price']}! Buy now: {product_url}"
        )

# Schedule to run every hour
import schedule
import time

schedule.every(1).hours.do(
    check_and_alert,
    product_url="https://example.com/laptop",
    target_price=65000
)

while True:
    schedule.run_pending()
    time.sleep(60)

import anthropic
import requests
from bs4 import BeautifulSoup

def ai_scrape(url: str, extraction_goal: str) -> dict:
    """Use Claude to extract structured data from any webpage"""
    response = requests.get(url)
    soup = BeautifulSoup(response.text, "html.parser")
    # Clean HTML — remove scripts and styles
    for tag in soup(["script", "style", "nav", "footer"]):
        tag.decompose()
    clean_text = soup.get_text(separator="\n", strip=True)[:4000]

    client = anthropic.Anthropic()
    message = client.messages.create(
        model="claude-sonnet-4-6",
        max_tokens=1000,
        messages=[{
            "role": "user",
            "content": f"Extract the following from this webpage:\n{extraction_goal}\n\nWebpage content:\n{clean_text}\n\nReturn as JSON."
        }]
    )
    return message.content[0].text