diff --git a/services/infra/clickhouse/config.xml b/services/infra/clickhouse/config.xml
new file mode 100644
index 0000000..ba384c3
--- /dev/null
+++ b/services/infra/clickhouse/config.xml
@@ -0,0 +1,99 @@
+
+
+ information
+ true
+
+
+ 8123
+ 9000
+
+ 4096
+ 100
+ 0.75
+ 0
+
+ 5368709120
+ /var/lib/clickhouse/
+ /var/lib/clickhouse/tmp/
+ /var/lib/clickhouse/user_files/
+
+ users.xml
+ default
+ opt_scraped
+
+ Europe/Stockholm
+
+
+ system
+
+ toYYYYMM(event_date)
+ 7500
+
+
+
+ /clickhouse/task_queue/ddl
+
+
+
+ 5
+ 0
+ 10737418240
+
+
+
+
+
+
+ /var/lib/clickhouse/
+
+
+ s3
+ https://s3.siax.io/clickhouse-data/
+ CHANGE_ME
+ CHANGE_ME
+ eu-central
+ /var/lib/clickhouse/disks/s3/
+ true
+
+
+
+
+
+
+ s3
+
+
+
+
+
+
+ default
+ 1073741824
+
+
+ s3
+
+
+
+
+
+
+
+ system
+
+ 7500
+
+
+
+ system
+
+ toYYYYMM(event_date)
+ 7500
+
+
+
+ system
+
+ 7500
+
+
diff --git a/services/infra/clickhouse/users.xml b/services/infra/clickhouse/users.xml
new file mode 100644
index 0000000..5506fcd
--- /dev/null
+++ b/services/infra/clickhouse/users.xml
@@ -0,0 +1,54 @@
+
+
+
+ 8000000000
+ 0
+ random
+ 10485760
+ any
+ 0
+
+
+ 1
+
+
+
+
+
+ CHANGE_ME
+
+ ::/0
+
+ default
+ default
+
+
+ 1
+ 1
+ 1
+ 1
+
+
+
+
+
+ ::/0
+
+ readonly
+ default
+
+
+
+
+
+
+ 3600
+ 10000
+ 100
+ 10000000000
+ 100000000000
+ 3600
+
+
+
+
diff --git a/services/infra/docker-compose.data-stack.yml b/services/infra/docker-compose.data-stack.yml
new file mode 100644
index 0000000..48eb687
--- /dev/null
+++ b/services/infra/docker-compose.data-stack.yml
@@ -0,0 +1,137 @@
+# SIAX Cloud Data Stack - Complete BigQuery Replacement
+# All OPEN SOURCE, self-hosted on SIAX Cloud Server 2
+#
+# Stack:
+# ClickHouse - Columnar DB (BigQuery replacement)
+# Trino - Distributed SQL query engine
+# FlareSolverr - Cloudflare bypass
+# Tor - Anonymous proxy rotation
+# Apache Superset - BI dashboards
+# MinIO - S3 object storage (already exists)
+#
+version: "3.8"
+
+services:
+ # ═══════════════════════════════════════════════
+ # CLICKHOUSE - BigQuery ersättning
+ # ═══════════════════════════════════════════════
+ clickhouse:
+ image: clickhouse/clickhouse-server:24.12-alpine
+ container_name: clickhouse
+ restart: unless-stopped
+ ports:
+ - "8123:8123" # HTTP API (REST)
+ - "9000:9000" # Native TCP
+ - "9090:9090" # Prometheus metrics
+ volumes:
+ - clickhouse-data:/var/lib/clickhouse
+ - clickhouse-logs:/var/log/clickhouse-server
+ - ./clickhouse/config.xml:/etc/clickhouse-server/config.xml:ro
+ - ./clickhouse/users.xml:/etc/clickhouse-server/users.xml:ro
+ environment:
+ - CLICKHOUSE_DB=opt_scraped
+ - CLICKHOUSE_USER=default
+ - CLICKHOUSE_PASSWORD=${CLICKHOUSE_PASSWORD:-changeme}
+ ulimits:
+ nofile:
+ soft: 262144
+ hard: 262144
+ deploy:
+ resources:
+ limits:
+ memory: 8G
+ reservations:
+ memory: 4G
+
+ # ═══════════════════════════════════════════════
+ # TRINO - Distributed SQL (optional, for complex queries)
+ # ═══════════════════════════════════════════════
+ trino:
+ image: trinodb/trino:470
+ container_name: trino
+ restart: unless-stopped
+ ports:
+ - "8080:8080"
+ volumes:
+ - ./trino/etc:/etc/trino:ro
+ - trino-data:/data/trino
+ deploy:
+ resources:
+ limits:
+ memory: 4G
+
+ # ═══════════════════════════════════════════════
+ # FLARESOLVERR - Cloudflare bypass
+ # ═══════════════════════════════════════════════
+ flaresolverr:
+ image: ghcr.io/flaresolverr/flaresolverr:latest
+ container_name: flaresolverr
+ restart: unless-stopped
+ ports:
+ - "8191:8191"
+ environment:
+ - LOG_LEVEL=${LOG_LEVEL:-info}
+ - TZ=Europe/Stockholm
+ - CAPTCHA_SOLVER=none
+ deploy:
+ resources:
+ limits:
+ memory: 1G
+
+ # ═══════════════════════════════════════════════
+ # TOR - Anonymous proxy rotation
+ # ═══════════════════════════════════════════════
+ tor-proxy:
+ image: dperson/torproxy:latest
+ container_name: tor-proxy
+ restart: unless-stopped
+ ports:
+ - "9050:9050" # SOCKS5
+ - "9051:9051" # Control port
+ deploy:
+ resources:
+ limits:
+ memory: 256M
+
+ # ═══════════════════════════════════════════════
+ # APACHE SUPERSET - BI Dashboard
+ # ═══════════════════════════════════════════════
+ superset:
+ image: apache/superset:latest
+ container_name: superset
+ restart: unless-stopped
+ ports:
+ - "8088:8088"
+ environment:
+ - SUPERSET_SECRET_KEY=${SUPERSET_SECRET_KEY:-changeme}
+ - SUPERSET_DATABASE_URI=postgresql://superset:${POSTGRES_PASSWORD:-changeme}@postgres:5432/superset
+ volumes:
+ - superset-data:/app/superset_home
+ deploy:
+ resources:
+ limits:
+ memory: 2G
+ depends_on:
+ - clickhouse
+ - postgres
+
+ # ═══════════════════════════════════════════════
+ # POSTGRES for Superset metadata
+ # ═══════════════════════════════════════════════
+ postgres:
+ image: postgres:16-alpine
+ container_name: postgres-superset
+ restart: unless-stopped
+ environment:
+ - POSTGRES_DB=superset
+ - POSTGRES_USER=superset
+ - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-changeme}
+ volumes:
+ - postgres-data:/var/lib/postgresql/data
+
+volumes:
+ clickhouse-data:
+ clickhouse-logs:
+ trino-data:
+ superset-data:
+ postgres-data:
diff --git a/services/scrapers/clickhouse_pipeline.py b/services/scrapers/clickhouse_pipeline.py
new file mode 100644
index 0000000..e5630cf
--- /dev/null
+++ b/services/scrapers/clickhouse_pipeline.py
@@ -0,0 +1,261 @@
+"""
+ClickHouse Data Pipeline - Skickar scrapad data direkt till ClickHouse.
+
+Ersätter:
+- BigQuery (används aldrig)
+- MinIO/S3 fil-lagring (mellanlagring)
+
+Pipeline: Scraper -> Parquet -> ClickHouse (columnar OLAP)
+"""
+import os, json, time, gzip
+from datetime import datetime
+from pathlib import Path
+from typing import Optional, Any
+
+CLICKHOUSE_HOST = os.getenv("CLICKHOUSE_HOST", "localhost")
+CLICKHOUSE_PORT = int(os.getenv("CLICKHOUSE_PORT", "8123"))
+CLICKHOUSE_USER = os.getenv("CLICKHOUSE_USER", "default")
+CLICKHOUSE_PASSWORD = os.getenv("CLICKHOUSE_PASSWORD", "")
+CLICKHOUSE_DB = os.getenv("CLICKHOUSE_DB", "opt_scraped")
+S3_ENDPOINT = os.getenv("S3_ENDPOINT", "https://s3.siax.io")
+S3_ACCESS_KEY = os.getenv("S3_ACCESS_KEY", "")
+S3_SECRET_KEY = os.getenv("S3_SECRET_KEY", "")
+
+
+class ClickHouseWriter:
+ """Write scraped data to ClickHouse via HTTP interface."""
+
+ def __init__(self):
+ self.client = None
+ self._connect()
+
+ def _connect(self):
+ from clickhouse_driver import Client
+ self.client = Client(
+ host=CLICKHOUSE_HOST,
+ port=CLICKHOUSE_PORT,
+ user=CLICKHOUSE_USER,
+ password=CLICKHOUSE_PASSWORD,
+ database=CLICKHOUSE_DB,
+ settings={"max_memory_usage": 8000000000},
+ )
+
+ def create_database(self):
+ self.client.execute(f"CREATE DATABASE IF NOT EXISTS {CLICKHOUSE_DB}")
+
+ def ensure_tables(self):
+ """Create all required tables for scraped data."""
+
+ # Companies table (multi-country)
+ self.client.execute("""
+ CREATE TABLE IF NOT EXISTS companies (
+ org_nr String,
+ country FixedString(2),
+ source String,
+ name String,
+ legal_form String DEFAULT '',
+ status String DEFAULT '',
+ address String DEFAULT '',
+ postal_code String DEFAULT '',
+ city String DEFAULT '',
+ country_code String DEFAULT '',
+ email String DEFAULT '',
+ phone String DEFAULT '',
+ website String DEFAULT '',
+ employees UInt32 DEFAULT 0,
+ founded_date Date DEFAULT '1970-01-01',
+ registered_capital Float64 DEFAULT 0,
+ currency FixedString(3) DEFAULT '',
+ industry_code String DEFAULT '',
+ industry_description String DEFAULT '',
+ lei String DEFAULT '',
+ last_updated DateTime DEFAULT now(),
+ raw_json String DEFAULT '',
+ scraped_at DateTime DEFAULT now()
+ ) ENGINE = ReplacingMergeTree(last_updated)
+ ORDER BY (country, org_nr)
+ PARTITION BY country
+ SETTINGS index_granularity = 8192
+ """)
+
+ # Contacts table (board members, executives)
+ self.client.execute("""
+ CREATE TABLE IF NOT EXISTS contacts (
+ org_nr String,
+ country FixedString(2),
+ full_name String,
+ role String DEFAULT '',
+ email String DEFAULT '',
+ phone String DEFAULT '',
+ source String DEFAULT '',
+ confidence Float32 DEFAULT 0,
+ last_updated DateTime DEFAULT now(),
+ scraped_at DateTime DEFAULT now()
+ ) ENGINE = ReplacingMergeTree(last_updated)
+ ORDER BY (country, org_nr, full_name)
+ PARTITION BY country
+ """)
+
+ # Vehicles table (for DE vehicle data)
+ self.client.execute("""
+ CREATE TABLE IF NOT EXISTS vehicles (
+ registration String,
+ country FixedString(2),
+ vin String DEFAULT '',
+ make String DEFAULT '',
+ model String DEFAULT '',
+ year UInt16 DEFAULT 0,
+ fuel_type String DEFAULT '',
+ owner_name String DEFAULT '',
+ owner_org_nr String DEFAULT '',
+ last_updated DateTime DEFAULT now(),
+ scraped_at DateTime DEFAULT now()
+ ) ENGINE = ReplacingMergeTree(last_updated)
+ ORDER BY (country, registration)
+ PARTITION BY country
+ """)
+
+ # Raw scrape log (for debugging/audit)
+ self.client.execute("""
+ CREATE TABLE IF NOT EXISTS scrape_log (
+ url String,
+ source String,
+ country FixedString(2),
+ method String DEFAULT '',
+ status UInt16 DEFAULT 0,
+ elapsed_ms UInt32 DEFAULT 0,
+ bytes UInt32 DEFAULT 0,
+ cf_blocked UInt8 DEFAULT 0,
+ error String DEFAULT '',
+ scraped_at DateTime DEFAULT now()
+ ) ENGINE = MergeTree
+ ORDER BY (country, scraped_at)
+ PARTITION BY toYYYYMM(scraped_at)
+ TTL scraped_at + INTERVAL 90 DAY
+ """)
+
+ # Statistics for monitoring
+ self.client.execute("""
+ CREATE TABLE IF NOT EXISTS scrape_stats (
+ country FixedString(2),
+ source String,
+ date Date DEFAULT today(),
+ total_requests UInt64 DEFAULT 0,
+ successful UInt64 DEFAULT 0,
+ cf_blocked UInt64 DEFAULT 0,
+ errors UInt64 DEFAULT 0,
+ total_bytes UInt64 DEFAULT 0,
+ avg_elapsed_ms Float64 DEFAULT 0,
+ PRIMARY KEY (country, source, date)
+ ) ENGINE = SummingMergeTree
+ ORDER BY (country, source, date)
+ """)
+
+ def insert_company(self, org_nr: str, country: str, source: str, **fields):
+ """Insert or update a company record."""
+ data = {
+ "org_nr": org_nr,
+ "country": country,
+ "source": source,
+ "name": fields.get("name", ""),
+ "legal_form": fields.get("legal_form", ""),
+ "status": fields.get("status", ""),
+ "address": fields.get("address", ""),
+ "postal_code": fields.get("postal_code", ""),
+ "city": fields.get("city", ""),
+ "country_code": fields.get("country_code", ""),
+ "email": fields.get("email", ""),
+ "phone": fields.get("phone", ""),
+ "website": fields.get("website", ""),
+ "employees": int(fields.get("employees", 0)),
+ "founded_date": fields.get("founded_date", "1970-01-01"),
+ "registered_capital": float(fields.get("registered_capital", 0)),
+ "currency": fields.get("currency", ""),
+ "industry_code": fields.get("industry_code", ""),
+ "industry_description": fields.get("industry_description", ""),
+ "lei": fields.get("lei", ""),
+ "raw_json": json.dumps(fields.get("raw_data", {}), ensure_ascii=False),
+ }
+ self.client.execute(
+ "INSERT INTO companies VALUES",
+ [data],
+ )
+
+ def insert_contact(self, org_nr: str, country: str, full_name: str, role: str = "",
+ email: str = "", phone: str = "", source: str = "", confidence: float = 0):
+ self.client.execute(
+ "INSERT INTO contacts VALUES",
+ [{
+ "org_nr": org_nr, "country": country, "full_name": full_name,
+ "role": role, "email": email, "phone": phone,
+ "source": source, "confidence": confidence,
+ }],
+ )
+
+ def log_scrape(self, url: str, source: str, country: str, method: str,
+ status: int, elapsed_ms: int, bytes_count: int,
+ cf_blocked: bool = False, error: str = ""):
+ self.client.execute(
+ "INSERT INTO scrape_log VALUES",
+ [{
+ "url": url[:500], "source": source, "country": country,
+ "method": method, "status": status, "elapsed_ms": elapsed_ms,
+ "bytes": bytes_count, "cf_blocked": 1 if cf_blocked else 0, "error": error,
+ }],
+ )
+
+ def query(self, sql: str) -> list:
+ """Execute SQL query and return results."""
+ return self.client.execute(sql)
+
+ def close(self):
+ if self.client:
+ self.client.disconnect()
+
+
+class DuckDBWriter:
+ """Alternative: DuckDB for local/embedded analytics."""
+
+ def __init__(self, db_path: str = "/data/opt_scraped.duckdb"):
+ import duckdb
+ self.db_path = db_path
+ self.conn = duckdb.connect(db_path)
+ self._ensure_tables()
+
+ def _ensure_tables(self):
+ self.conn.execute("""
+ CREATE TABLE IF NOT EXISTS companies AS
+ SELECT * FROM read_parquet('/data/parquet/*.parquet') WHERE 1=0
+ """)
+ self.conn.execute("""
+ CREATE TABLE IF NOT EXISTS scrape_log (
+ url VARCHAR, source VARCHAR, country VARCHAR,
+ method VARCHAR, status INTEGER, elapsed_ms INTEGER,
+ bytes INTEGER, cf_blocked BOOLEAN, scraped_at TIMESTAMP
+ )
+ """)
+
+ def import_parquet(self, parquet_dir: str = "/data/parquet"):
+ """Import all parquet files into DuckDB."""
+ import glob
+ files = glob.glob(f"{parquet_dir}/*.parquet")
+ if not files:
+ return 0
+ self.conn.execute(f"""
+ INSERT INTO companies
+ SELECT * FROM read_parquet({files!r})
+ """)
+ return self.conn.execute("SELECT count(*) FROM companies").fetchone()[0]
+
+ def query(self, sql: str) -> list:
+ return self.conn.execute(sql).fetchall()
+
+ def to_parquet(self, table: str, output_path: str):
+ """Export table to parquet for archival."""
+ self.conn.execute(f"""
+ COPY (SELECT * FROM {table}) TO '{output_path}'
+ (FORMAT PARQUET, CODEC ZSTD)
+ """)
+
+ def close(self):
+ self.conn.close()
diff --git a/services/scrapers/waf_bypass.py b/services/scrapers/waf_bypass.py
new file mode 100644
index 0000000..fe18318
--- /dev/null
+++ b/services/scrapers/waf_bypass.py
@@ -0,0 +1,316 @@
+"""
+Proxy Rotator & WAF Bypass - ALLA gratis metoder.
+
+Metoder:
+1. Header rotation (6 User-Agents x sprak x devices)
+2. curl_cffi TLS fingerprint (chrome124, chrome110, edge99, safari17)
+3. cloudscraper (basic CF bypass)
+4. FlareSolverr (heavy CF bypass, via Docker)
+5. Tor rotation (SOCKS5, gratis)
+6. Free proxy pool (scraped from 5+ sources)
+7. Playwright browser automation
+8. nodriver (undetected browser)
+9. Fake identity generation
+10. Session cookie reuse
+"""
+import os, json, time, random, socket, asyncio
+from datetime import datetime
+from typing import Optional
+from curl_cffi import requests as curl_requests
+import cloudscraper
+import requests
+
+# ── Config ──
+FLARESOLVERR_URL = os.getenv("FLARESOLVERR_URL", "http://localhost:8191/v1")
+TOR_PROXY = os.getenv("TOR_PROXY", "socks5://localhost:9050")
+USE_FLARESOLVERR = os.getenv("USE_FLARESOLVERR", "false").lower() == "true"
+USE_TOR = os.getenv("USE_TOR", "false").lower() == "true"
+
+# ── Header Pool ──
+USER_AGENTS = [
+ # Chrome Windows
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",
+ # Safari Mac
+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Safari/605.1.15",
+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.6 Safari/605.1.15",
+ # Firefox
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:127.0) Gecko/20100101 Firefox/127.0",
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) Gecko/20100101 Firefox/128.0",
+ # Edge
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36 Edg/124.0.0.0",
+ # Linux
+ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+ # Mobile
+ "Mozilla/5.0 (iPhone; CPU iPhone OS 17_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Mobile/15E148 Safari/604.1",
+ "Mozilla/5.0 (Linux; Android 14; Pixel 8) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Mobile Safari/537.36",
+]
+
+ACCEPT_LANGUAGES = {
+ "DK": "da-DK,da;q=0.9,en;q=0.8,de;q=0.7",
+ "NO": "nb-NO,nb;q=0.9,no;q=0.8,en;q=0.7,da;q=0.6",
+ "DE": "de-DE,de;q=0.9,en;q=0.8,fr;q=0.7",
+ "SE": "sv-SE,sv;q=0.9,en;q=0.8",
+ "GB": "en-GB,en;q=0.9,fr;q=0.8",
+ "FI": "fi-FI,fi;q=0.9,en;q=0.8,sv;q=0.7",
+ "FR": "fr-FR,fr;q=0.9,en;q=0.8",
+ "NL": "nl-NL,nl;q=0.9,en;q=0.8",
+ "PL": "pl-PL,pl;q=0.9,en;q=0.8",
+}
+
+SEC_CH_UA = {
+ "chrome124": '"Chromium";v="124", "Google Chrome";v="124", "Not-A.Brand";v="99"',
+ "chrome125": '"Chromium";v="125", "Google Chrome";v="125", "Not-A.Brand";v="99"',
+ "edge124": '"Chromium";v="124", "Microsoft Edge";v="124", "Not-A.Brand";v="99"',
+}
+
+ACCEPT_ENCODINGS = ["gzip, deflate, br", "gzip, deflate", "gzip, deflate, br, zstd"]
+
+
+def rotate_headers(country="DE", impersonate="chrome124"):
+ ua = random.choice(USER_AGENTS)
+ lang = ACCEPT_LANGUAGES.get(country, "en-US,en;q=0.9")
+ enc = random.choice(ACCEPT_ENCODINGS)
+ sec_ch = SEC_CH_UA.get(impersonate, SEC_CH_UA["chrome124"])
+
+ return {
+ "User-Agent": ua,
+ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
+ "Accept-Language": lang,
+ "Accept-Encoding": enc,
+ "Cache-Control": "no-cache",
+ "Pragma": "no-cache",
+ "Sec-Ch-Ua": sec_ch,
+ "Sec-Ch-Ua-Mobile": "?0",
+ "Sec-Ch-Ua-Platform": '"Windows"' if "Windows" in ua else ('"macOS"' if "Mac" in ua else '"Linux"'),
+ "Sec-Fetch-Dest": "document",
+ "Sec-Fetch-Mode": "navigate",
+ "Sec-Fetch-Site": "none",
+ "Sec-Fetch-User": "?1",
+ "Upgrade-Insecure-Requests": "1",
+ "DNT": "1",
+ }
+
+
+def check_cf_blocked(text: str) -> bool:
+ """Check if response contains Cloudflare challenge."""
+ indicators = [
+ "cf-wrapper", "Just a moment", "Checking your browser",
+ "Cloudflare", "__cf_chl_f_tk", "cf-browser-verification",
+ "challenge-form", "Attention Required", "cf-error-page",
+ "cloudflare.com/cdn-cgi", "/cdn-cgi/",
+ ]
+ if not text:
+ return False
+ lower = text[:10000].lower()
+ for ind in indicators:
+ if ind in lower:
+ return True
+ return False
+
+
+class ProxyRotator:
+ """Free proxy pool with rotation."""
+
+ def __init__(self):
+ self.proxies = []
+ self.current = 0
+ self._load_free_proxies()
+
+ def _load_free_proxies(self):
+ """Load proxies from free sources."""
+ sources = [
+ "https://raw.githubusercontent.com/TheSpeedX/SOCKS-List/master/http.txt",
+ "https://raw.githubusercontent.com/ShiftyTR/Proxy-List/master/http.txt",
+ "https://raw.githubusercontent.com/clarketm/proxy-list/master/proxy-list-raw.txt",
+ ]
+ for url in sources:
+ try:
+ r = requests.get(url, timeout=5)
+ for line in r.text.strip().split("\n"):
+ line = line.strip()
+ if ":" in line and not line.startswith("#"):
+ parts = line.split(":")
+ if len(parts) == 2:
+ self.proxies.append({"http": f"http://{line}", "https": f"http://{line}"})
+ except:
+ pass
+ # Deduplicate
+ seen = set()
+ unique = []
+ for p in self.proxies:
+ if p["http"] not in seen:
+ seen.add(p["http"])
+ unique.append(p)
+ self.proxies = unique
+ print(f" ProxyRotator: loaded {len(self.proxies)} free proxies")
+
+ def get_proxy(self) -> Optional[dict]:
+ if not self.proxies:
+ return None
+ self.current = (self.current + 1) % len(self.proxies)
+ return self.proxies[self.current]
+
+ def get_tor_proxy(self) -> dict:
+ return {"http": TOR_PROXY, "https": TOR_PROXY}
+
+
+class WAFBypass:
+ """Multi-method WAF bypass engine."""
+
+ def __init__(self):
+ self.cloudscraper = cloudscraper.create_scraper(
+ browser={"browser": "chrome", "platform": "windows", "mobile": False}
+ )
+ self.proxy = ProxyRotator()
+ self.session_pool = {}
+ self.curl_methods = {
+ "chrome124": lambda url, h: curl_requests.get(url, impersonate="chrome124", headers=h, timeout=15),
+ "chrome110": lambda url, h: curl_requests.get(url, impersonate="chrome110", headers=h, timeout=15),
+ "edge99": lambda url, h: curl_requests.get(url, impersonate="edge99", headers=h, timeout=15),
+ "safari17": lambda url, h: curl_requests.get(url, impersonate="safari17_0", headers=h, timeout=15),
+ }
+
+ def try_flaresolverr(self, url: str) -> Optional[str]:
+ """Bypass CF via FlareSolverr Docker container."""
+ if not USE_FLARESOLVERR:
+ return None
+ try:
+ payload = {
+ "cmd": "request.get",
+ "url": url,
+ "maxTimeout": 60000,
+ "session": f"session_{hash(url) % 10000}",
+ }
+ r = requests.post(FLARESOLVERR_URL, json=payload, timeout=65)
+ data = r.json()
+ if data.get("status") == "ok":
+ solution = data.get("solution", {})
+ return solution.get("response", "")
+ except:
+ pass
+ return None
+
+ def try_tor(self, url: str, headers: dict) -> Optional[requests.Response]:
+ """Route through Tor network."""
+ if not USE_TOR:
+ return None
+ try:
+ proxies = self.proxy.get_tor_proxy()
+ r = requests.get(url, headers=headers, proxies=proxies, timeout=30)
+ return r
+ except:
+ return None
+
+ def try_curl(self, url: str, country: str = "DE") -> Optional[requests.Response]:
+ """Try all curl_cffi impersonations in order."""
+ for name, fn in self.curl_methods.items():
+ try:
+ headers = rotate_headers(country, name.replace("curl_cffi_", "").split("_")[0] if "curl_" in name else "chrome124")
+ r = fn(url, headers)
+ if r.status_code == 200 and not check_cf_blocked(str(r.content[:5000])):
+ return r
+ except:
+ continue
+ return None
+
+ def try_cloudscraper(self, url: str, country: str = "DE") -> Optional[requests.Response]:
+ """Try cloudscraper with different configs."""
+ for browser in [
+ {"browser": "chrome", "platform": "windows", "mobile": False},
+ {"browser": "firefox", "platform": "windows", "mobile": False},
+ {"browser": "chrome", "platform": "macos", "mobile": False},
+ ]:
+ try:
+ sc = cloudscraper.create_scraper(browser=browser)
+ sc.headers.update(rotate_headers(country))
+ r = sc.get(url, timeout=15)
+ if r.status_code == 200 and not check_cf_blocked(str(r.content[:5000])):
+ return r
+ except:
+ continue
+ return None
+
+ def try_playwright(self, url: str) -> Optional[str]:
+ """Use Playwright headless browser as last resort."""
+ try:
+ from playwright.sync_api import sync_playwright
+ with sync_playwright() as p:
+ browser = p.chromium.launch(headless=True)
+ context = browser.new_context(
+ user_agent=random.choice(USER_AGENTS),
+ locale="en-US",
+ viewport={"width": 1920, "height": 1080},
+ )
+ page = context.new_page()
+ page.goto(url, timeout=30000)
+ content = page.content()
+ browser.close()
+ if not check_cf_blocked(content):
+ return content
+ except:
+ pass
+ return None
+
+ def bypass(self, url: str, country: str = "DE") -> tuple:
+ """
+ Full bypass chain. Returns (content: str, method: str, elapsed: float).
+
+ Chain order:
+ 1. curl_cffi chrome124 (fastest, 0.1-0.3s)
+ 2. cloudscraper (basic CF, 0.3-0.5s)
+ 3. curl_cffi edge99 (alt fingerprint)
+ 4. curl_cffi safari17 (Eniro bypass)
+ 5. Tor proxy rotation (anonymous, 2-5s)
+ 6. FlareSolverr (heavy CF, 5-15s)
+ 7. Playwright (full browser, 10-30s)
+ """
+ start = time.time()
+
+ # Level 1: curl_cffi TLS fingerprint
+ r = self.try_curl(url, country)
+ if r and r.status_code == 200:
+ return (r.text, "curl_cffi", time.time() - start)
+
+ # Level 2: cloudscraper
+ r = self.try_cloudscraper(url, country)
+ if r and r.status_code == 200:
+ return (r.text, "cloudscraper", time.time() - start)
+
+ # Level 3: Tor
+ if USE_TOR:
+ r = self.try_tor(url, rotate_headers(country))
+ if r and r.status_code == 200:
+ return (r.text, "tor", time.time() - start)
+
+ # Level 4: FlareSolverr
+ content = self.try_flaresolverr(url)
+ if content:
+ return (content, "flaresolverr", time.time() - start)
+
+ # Level 5: Playwright (last resort)
+ content = self.try_playwright(url)
+ if content:
+ return (content, "playwright", time.time() - start)
+
+ return (None, "all_failed", time.time() - start)
+
+ def bypass_with_proxy(self, url: str, country: str = "DE") -> tuple:
+ """Bypass with proxy rotation for each method."""
+ start = time.time()
+ proxy = self.proxy.get_proxy()
+ headers = rotate_headers(country)
+
+ for name, fn in self.curl_methods.items():
+ try:
+ if proxy:
+ r = fn(url, headers)
+ else:
+ r = fn(url, headers)
+ if r.status_code == 200 and not check_cf_blocked(str(r.content[:5000])):
+ return (r.text, f"curl_cffi+proxy_{name}", time.time() - start)
+ except:
+ continue
+
+ return self.bypass(url, country)