admin/0pt
5
0
Fork 0
Code Issues 9 Pull Requests Actions Packages Projects Releases Wiki Activity

Ta bort otillåten bypass-/stealth-skrapkod (CLAUDE.md §1) #1

Closed
admin wants to merge 1 commits from feature/0pt-remove-noncompliant-scrapers into main
pull from: feature/0pt-remove-noncompliant-scrapers
merge into: :main
Conversation 0 Commits 1 Files Changed 69
+14 -1953
69 changed files with 14 additions and 1953 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitea/workflows/ci.yml
-10
View File
@@ -1,10 +0,0 @@
name: Scraper CI
on: [push]
jobs:
lint:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Check Python files
run: |
echo "Scraper code verified"
services/api/workers/adaptive_router.py
+14 -6
View File
@@ -1,15 +1,23 @@
"""Adaptive router — all methods, no compliance restrictions. See b0ts for scraping."""
"""
Adaptive router — chooses among COMPLIANT methods only (docs/06).
Order of preference: official API > bulk > authorized public web > licensed.
There are NO bypass/stealth/proxy-rotation tiers here, by design (CLAUDE.md §1).
"""
from __future__ import annotations
# market -> ordered list of source_ids to try (compliant connectors only)
MARKET_SOURCES: dict[str, list[str]] = {
"NO": ["brreg", "brreg_under", "proff", "1881", "gulesider", "flowfirma", "selskapssok", "regnskapsbasen", "okonomi", "visverdi", "gleif"],
"NO": ["brreg", "gleif"],
"GB": ["companies_house", "gleif"],
"FI": ["prh", "gleif"],
"DK": ["cvr", "krak", "proff_dk", "118", "degulesider", "companydata", "grundigt", "datafordeler", "virk", "gleif"],
"DE": ["northdata", "handelsregister", "unternehmensregister", "moneyhouse", "creditreform", "bundesanzeiger", "registercheck", "personensuche", "whitepages", "companyhouse", "gleif"],
"SE": ["gleif"],
"DK": ["cvr", "gleif"],
"SE": ["gleif"], # Bolagsverket bulk is licensed; SE company core via GLEIF + licensed/partner
}
# Access rank: lower = preferred (cheapest compliant path first).
ACCESS_RANK = {"api": 0, "bulk": 1, "authorized_web": 2, "licensed": 3}
def plan_for_market(market: str) -> list[str]:
"""Return the source plan for a market. Deep scraping via b0ts service."""
"""Return the ordered, compliant source plan for a market."""
return MARKET_SOURCES.get(market, ["gleif"])
services/infra/clickhouse/config.xml
-99
View File
@@ -1,99 +0,0 @@
<yandex>
<logger>
<level>information</level>
<console>true</console>
</logger>
<http_port>8123</http_port>
<tcp_port>9000</tcp_port>
<max_connections>4096</max_connections>
<max_concurrent_queries>100</max_concurrent_queries>
<max_server_memory_usage_to_ram_ratio>0.75</max_server_memory_usage_to_ram_ratio>
<max_memory_usage_for_all_queries>0</max_memory_usage_for_all_queries>
<mark_cache_size>5368709120</mark_cache_size>
<path>/var/lib/clickhouse/</path>
<tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
<user_files_path>/var/lib/clickhouse/user_files/</user_files_path>
<users_config>users.xml</users_config>
<default_profile>default</default_profile>
<default_database>opt_scraped</default_database>
<timezone>Europe/Stockholm</timezone>
<query_log>
<database>system</database>
<table>query_log</table>
<partition_by>toYYYYMM(event_date)</partition_by>
<flush_interval_milliseconds>7500</flush_interval_milliseconds>
</query_log>
<distributed_ddl>
<path>/clickhouse/task_queue/ddl</path>
</distributed_ddl>
<merge_tree>
<max_suspicious_broken_parts>5</max_suspicious_broken_parts>
<min_bytes_for_wide_part>0</min_bytes_for_wide_part>
<max_bytes_to_merge_at_max_space_in_pool>10737418240</max_bytes_to_merge_at_max_space_in_pool>
</merge_tree>
<!-- S3 integration for MinIO -->
<storage_configuration>
<disks>
<default>
<path>/var/lib/clickhouse/</path>
</default>
<s3>
<type>s3</type>
<endpoint>https://s3.siax.io/clickhouse-data/</endpoint>
<access_key_id>CHANGE_ME</access_key_id>
<secret_access_key>CHANGE_ME</secret_access_key>
<region>eu-central</region>
<metadata_path>/var/lib/clickhouse/disks/s3/</metadata_path>
<send_cache>true</send_cache>
</s3>
</disks>
<policies>
<s3_only>
<volumes>
<main>
<disk>s3</disk>
</main>
</volumes>
</s3_only>
<tiered>
<volumes>
<hot>
<disk>default</disk>
<max_data_part_size>1073741824</max_data_part_size>
</hot>
<cold>
<disk>s3</disk>
</cold>
</volumes>
</tiered>
</policies>
</storage_configuration>
<part_log>
<database>system</database>
<table>part_log</table>
<flush_interval_milliseconds>7500</flush_interval_milliseconds>
</part_log>
<query_thread_log>
<database>system</database>
<table>query_thread_log</table>
<partition_by>toYYYYMM(event_date)</partition_by>
<flush_interval_milliseconds>7500</flush_interval_milliseconds>
</query_thread_log>
<asynchronous_insert_log>
<database>system</database>
<table>asynchronous_insert_log</table>
<flush_interval_milliseconds>7500</flush_interval_milliseconds>
</asynchronous_insert_log>
</yandex>
services/infra/clickhouse/users.xml
-54
View File
@@ -1,54 +0,0 @@
<yandex>
<profiles>
<default>
<max_memory_usage>8000000000</max_memory_usage>
<use_uncompressed_cache>0</use_uncompressed_cache>
<load_balancing>random</load_balancing>
<max_query_size>10485760</max_query_size>
<group_by_overflow_mode>any</group_by_overflow_mode>
<timeout_before_checking_execution_speed>0</timeout_before_checking_execution_speed>
</default>
<readonly>
<readonly>1</readonly>
</readonly>
</profiles>
<users>
<default>
<password>CHANGE_ME</password>
<networks>
<ip>::/0</ip>
</networks>
<profile>default</profile>
<quota>default</quota>
<grants>
<select>1</select>
<insert>1</insert>
<alter>1</alter>
<create>1</create>
<drop>1</drop>
</grants>
</default>
<readonly>
<password></password>
<networks>
<ip>::/0</ip>
</networks>
<profile>readonly</profile>
<quota>default</quota>
</readonly>
</users>
<quotas>
<default>
<interval>
<duration>3600</duration>
<queries>10000</queries>
<errors>100</errors>
<result_rows>10000000000</result_rows>
<read_rows>100000000000</read_rows>
<execution_time>3600</execution_time>
</interval>
</default>
</quotas>
</yandex>
services/infra/docker-compose.data-stack.yml
-137
View File
@@ -1,137 +0,0 @@
# SIAX Cloud Data Stack - Complete BigQuery Replacement
# All OPEN SOURCE, self-hosted on SIAX Cloud Server 2
#
# Stack:
# ClickHouse - Columnar DB (BigQuery replacement)
# Trino - Distributed SQL query engine
# FlareSolverr - Cloudflare bypass
# Tor - Anonymous proxy rotation
# Apache Superset - BI dashboards
# MinIO - S3 object storage (already exists)
#
version: "3.8"
services:
# ═══════════════════════════════════════════════
# CLICKHOUSE - BigQuery ersättning
# ═══════════════════════════════════════════════
clickhouse:
image: clickhouse/clickhouse-server:24.12-alpine
container_name: clickhouse
restart: unless-stopped
ports:
- "8123:8123" # HTTP API (REST)
- "9000:9000" # Native TCP
- "9090:9090" # Prometheus metrics
volumes:
- clickhouse-data:/var/lib/clickhouse
- clickhouse-logs:/var/log/clickhouse-server
- ./clickhouse/config.xml:/etc/clickhouse-server/config.xml:ro
- ./clickhouse/users.xml:/etc/clickhouse-server/users.xml:ro
environment:
- CLICKHOUSE_DB=opt_scraped
- CLICKHOUSE_USER=default
- CLICKHOUSE_PASSWORD=${CLICKHOUSE_PASSWORD:-changeme}
ulimits:
nofile:
soft: 262144
hard: 262144
deploy:
resources:
limits:
memory: 8G
reservations:
memory: 4G
# ═══════════════════════════════════════════════
# TRINO - Distributed SQL (optional, for complex queries)
# ═══════════════════════════════════════════════
trino:
image: trinodb/trino:470
container_name: trino
restart: unless-stopped
ports:
- "8080:8080"
volumes:
- ./trino/etc:/etc/trino:ro
- trino-data:/data/trino
deploy:
resources:
limits:
memory: 4G
# ═══════════════════════════════════════════════
# FLARESOLVERR - Cloudflare bypass
# ═══════════════════════════════════════════════
flaresolverr:
image: ghcr.io/flaresolverr/flaresolverr:latest
container_name: flaresolverr
restart: unless-stopped
ports:
- "8191:8191"
environment:
- LOG_LEVEL=${LOG_LEVEL:-info}
- TZ=Europe/Stockholm
- CAPTCHA_SOLVER=none
deploy:
resources:
limits:
memory: 1G
# ═══════════════════════════════════════════════
# TOR - Anonymous proxy rotation
# ═══════════════════════════════════════════════
tor-proxy:
image: dperson/torproxy:latest
container_name: tor-proxy
restart: unless-stopped
ports:
- "9050:9050" # SOCKS5
- "9051:9051" # Control port
deploy:
resources:
limits:
memory: 256M
# ═══════════════════════════════════════════════
# APACHE SUPERSET - BI Dashboard
# ═══════════════════════════════════════════════
superset:
image: apache/superset:latest
container_name: superset
restart: unless-stopped
ports:
- "8088:8088"
environment:
- SUPERSET_SECRET_KEY=${SUPERSET_SECRET_KEY:-changeme}
- SUPERSET_DATABASE_URI=postgresql://superset:${POSTGRES_PASSWORD:-changeme}@postgres:5432/superset
volumes:
- superset-data:/app/superset_home
deploy:
resources:
limits:
memory: 2G
depends_on:
- clickhouse
- postgres
# ═══════════════════════════════════════════════
# POSTGRES for Superset metadata
# ═══════════════════════════════════════════════
postgres:
image: postgres:16-alpine
container_name: postgres-superset
restart: unless-stopped
environment:
- POSTGRES_DB=superset
- POSTGRES_USER=superset
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-changeme}
volumes:
- postgres-data:/var/lib/postgresql/data
volumes:
clickhouse-data:
clickhouse-logs:
trino-data:
superset-data:
postgres-data:
services/scrapers/Dockerfile
-25
View File
@@ -1,25 +0,0 @@
FROM python:3.14-slim
WORKDIR /app
# Install system deps
RUN apt-get update && apt-get install -y --no-install-recommends \
curl ca-certificates && \
rm -rf /var/lib/apt/lists/*
# Install Python packages
COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt
# Copy scraper code
COPY . .
# Create data directories
RUN mkdir -p /data/output /data/checkpoints /data/sources
ENV PYTHONUNBUFFERED=1
ENV S3_ENDPOINT=https://s3.siax.io
ENV OUTPUT_DIR=/data/output
ENV CHECKPOINT_DIR=/data/checkpoints
CMD ["python", "-m", "scrapers.run_pipeline"]
services/scrapers/README.md
-31
View File
@@ -1,31 +0,0 @@
# 0pt Scraper → b0ts
Scraper code has moved to the **b0ts** repo:
https://git.siax.io/admin/b0ts
## Quick start
```bash
git clone https://git.siax.io/admin/b0ts.git
cd b0ts
python -m src.main
```
## API
b0ts runs on port 4115:
- `POST /pipeline/start` — Start continuous scraping
- `GET /pipeline/status` — Check progress
- `GET /analytics/quality` — Data quality metrics
## MCP
MCP tools available at `schemas/mcp/b0ts.mcp.json`.
## Deployment
```bash
# On SIAX Cloud Server 2
ssh -i ~/.ssh/id_ed25519_siax root@188.40.230.69
cd /opt/b0ts && python3 -m src.main
```
services/scrapers/__init__.py
-1
View File
@@ -1 +0,0 @@
# 0pt Scraper Engine v2 - Production scrapers package
services/scrapers/__pycache__/engine.cpython-314.pyc
BIN
View File
Binary file not shown.
services/scrapers/__pycache__/engine_v2.cpython-314.pyc
BIN
View File
Binary file not shown.
services/scrapers/clickhouse_pipeline.py
-261
View File
@@ -1,261 +0,0 @@
"""
ClickHouse Data Pipeline - Skickar scrapad data direkt till ClickHouse.
Ersätter:
- BigQuery (används aldrig)
- MinIO/S3 fil-lagring (mellanlagring)
Pipeline: Scraper -> Parquet -> ClickHouse (columnar OLAP)
"""
import os, json, time, gzip
from datetime import datetime
from pathlib import Path
from typing import Optional, Any
CLICKHOUSE_HOST = os.getenv("CLICKHOUSE_HOST", "localhost")
CLICKHOUSE_PORT = int(os.getenv("CLICKHOUSE_PORT", "8123"))
CLICKHOUSE_USER = os.getenv("CLICKHOUSE_USER", "default")
CLICKHOUSE_PASSWORD = os.getenv("CLICKHOUSE_PASSWORD", "")
CLICKHOUSE_DB = os.getenv("CLICKHOUSE_DB", "opt_scraped")
S3_ENDPOINT = os.getenv("S3_ENDPOINT", "https://s3.siax.io")
S3_ACCESS_KEY = os.getenv("S3_ACCESS_KEY", "")
S3_SECRET_KEY = os.getenv("S3_SECRET_KEY", "")
class ClickHouseWriter:
"""Write scraped data to ClickHouse via HTTP interface."""
def __init__(self):
self.client = None
self._connect()
def _connect(self):
from clickhouse_driver import Client
self.client = Client(
host=CLICKHOUSE_HOST,
port=CLICKHOUSE_PORT,
user=CLICKHOUSE_USER,
password=CLICKHOUSE_PASSWORD,
database=CLICKHOUSE_DB,
settings={"max_memory_usage": 8000000000},
)
def create_database(self):
self.client.execute(f"CREATE DATABASE IF NOT EXISTS {CLICKHOUSE_DB}")
def ensure_tables(self):
"""Create all required tables for scraped data."""
# Companies table (multi-country)
self.client.execute("""
CREATE TABLE IF NOT EXISTS companies (
org_nr String,
country FixedString(2),
source String,
name String,
legal_form String DEFAULT '',
status String DEFAULT '',
address String DEFAULT '',
postal_code String DEFAULT '',
city String DEFAULT '',
country_code String DEFAULT '',
email String DEFAULT '',
phone String DEFAULT '',
website String DEFAULT '',
employees UInt32 DEFAULT 0,
founded_date Date DEFAULT '1970-01-01',
registered_capital Float64 DEFAULT 0,
currency FixedString(3) DEFAULT '',
industry_code String DEFAULT '',
industry_description String DEFAULT '',
lei String DEFAULT '',
last_updated DateTime DEFAULT now(),
raw_json String DEFAULT '',
scraped_at DateTime DEFAULT now()
) ENGINE = ReplacingMergeTree(last_updated)
ORDER BY (country, org_nr)
PARTITION BY country
SETTINGS index_granularity = 8192
""")
# Contacts table (board members, executives)
self.client.execute("""
CREATE TABLE IF NOT EXISTS contacts (
org_nr String,
country FixedString(2),
full_name String,
role String DEFAULT '',
email String DEFAULT '',
phone String DEFAULT '',
source String DEFAULT '',
confidence Float32 DEFAULT 0,
last_updated DateTime DEFAULT now(),
scraped_at DateTime DEFAULT now()
) ENGINE = ReplacingMergeTree(last_updated)
ORDER BY (country, org_nr, full_name)
PARTITION BY country
""")
# Vehicles table (for DE vehicle data)
self.client.execute("""
CREATE TABLE IF NOT EXISTS vehicles (
registration String,
country FixedString(2),
vin String DEFAULT '',
make String DEFAULT '',
model String DEFAULT '',
year UInt16 DEFAULT 0,
fuel_type String DEFAULT '',
owner_name String DEFAULT '',
owner_org_nr String DEFAULT '',
last_updated DateTime DEFAULT now(),
scraped_at DateTime DEFAULT now()
) ENGINE = ReplacingMergeTree(last_updated)
ORDER BY (country, registration)
PARTITION BY country
""")
# Raw scrape log (for debugging/audit)
self.client.execute("""
CREATE TABLE IF NOT EXISTS scrape_log (
url String,
source String,
country FixedString(2),
method String DEFAULT '',
status UInt16 DEFAULT 0,
elapsed_ms UInt32 DEFAULT 0,
bytes UInt32 DEFAULT 0,
cf_blocked UInt8 DEFAULT 0,
error String DEFAULT '',
scraped_at DateTime DEFAULT now()
) ENGINE = MergeTree
ORDER BY (country, scraped_at)
PARTITION BY toYYYYMM(scraped_at)
TTL scraped_at + INTERVAL 90 DAY
""")
# Statistics for monitoring
self.client.execute("""
CREATE TABLE IF NOT EXISTS scrape_stats (
country FixedString(2),
source String,
date Date DEFAULT today(),
total_requests UInt64 DEFAULT 0,
successful UInt64 DEFAULT 0,
cf_blocked UInt64 DEFAULT 0,
errors UInt64 DEFAULT 0,
total_bytes UInt64 DEFAULT 0,
avg_elapsed_ms Float64 DEFAULT 0,
PRIMARY KEY (country, source, date)
) ENGINE = SummingMergeTree
ORDER BY (country, source, date)
""")
def insert_company(self, org_nr: str, country: str, source: str, **fields):
"""Insert or update a company record."""
data = {
"org_nr": org_nr,
"country": country,
"source": source,
"name": fields.get("name", ""),
"legal_form": fields.get("legal_form", ""),
"status": fields.get("status", ""),
"address": fields.get("address", ""),
"postal_code": fields.get("postal_code", ""),
"city": fields.get("city", ""),
"country_code": fields.get("country_code", ""),
"email": fields.get("email", ""),
"phone": fields.get("phone", ""),
"website": fields.get("website", ""),
"employees": int(fields.get("employees", 0)),
"founded_date": fields.get("founded_date", "1970-01-01"),
"registered_capital": float(fields.get("registered_capital", 0)),
"currency": fields.get("currency", ""),
"industry_code": fields.get("industry_code", ""),
"industry_description": fields.get("industry_description", ""),
"lei": fields.get("lei", ""),
"raw_json": json.dumps(fields.get("raw_data", {}), ensure_ascii=False),
}
self.client.execute(
"INSERT INTO companies VALUES",
[data],
)
def insert_contact(self, org_nr: str, country: str, full_name: str, role: str = "",
email: str = "", phone: str = "", source: str = "", confidence: float = 0):
self.client.execute(
"INSERT INTO contacts VALUES",
[{
"org_nr": org_nr, "country": country, "full_name": full_name,
"role": role, "email": email, "phone": phone,
"source": source, "confidence": confidence,
}],
)
def log_scrape(self, url: str, source: str, country: str, method: str,
status: int, elapsed_ms: int, bytes_count: int,
cf_blocked: bool = False, error: str = ""):
self.client.execute(
"INSERT INTO scrape_log VALUES",
[{
"url": url[:500], "source": source, "country": country,
"method": method, "status": status, "elapsed_ms": elapsed_ms,
"bytes": bytes_count, "cf_blocked": 1 if cf_blocked else 0, "error": error,
}],
)
def query(self, sql: str) -> list:
"""Execute SQL query and return results."""
return self.client.execute(sql)
def close(self):
if self.client:
self.client.disconnect()
class DuckDBWriter:
"""Alternative: DuckDB for local/embedded analytics."""
def __init__(self, db_path: str = "/data/opt_scraped.duckdb"):
import duckdb
self.db_path = db_path
self.conn = duckdb.connect(db_path)
self._ensure_tables()
def _ensure_tables(self):
self.conn.execute("""
CREATE TABLE IF NOT EXISTS companies AS
SELECT * FROM read_parquet('/data/parquet/*.parquet') WHERE 1=0
""")
self.conn.execute("""
CREATE TABLE IF NOT EXISTS scrape_log (
url VARCHAR, source VARCHAR, country VARCHAR,
method VARCHAR, status INTEGER, elapsed_ms INTEGER,
bytes INTEGER, cf_blocked BOOLEAN, scraped_at TIMESTAMP
)
""")
def import_parquet(self, parquet_dir: str = "/data/parquet"):
"""Import all parquet files into DuckDB."""
import glob
files = glob.glob(f"{parquet_dir}/*.parquet")
if not files:
return 0
self.conn.execute(f"""
INSERT INTO companies
SELECT * FROM read_parquet({files!r})
""")
return self.conn.execute("SELECT count(*) FROM companies").fetchone()[0]
def query(self, sql: str) -> list:
return self.conn.execute(sql).fetchall()
def to_parquet(self, table: str, output_path: str):
"""Export table to parquet for archival."""
self.conn.execute(f"""
COPY (SELECT * FROM {table}) TO '{output_path}'
(FORMAT PARQUET, CODEC ZSTD)
""")
def close(self):
self.conn.close()
services/scrapers/docker-compose.yml
-77
View File
@@ -1,77 +0,0 @@
version: "3.8"
services:
scraper-brreg-no:
image: scraper-engine:latest
environment:
- COUNTRY=NO
- SOURCE=brreg
- MAX_CONCURRENCY=100
- S3_ENDPOINT=${S3_ENDPOINT}
- S3_ACCESS_KEY=${S3_ACCESS_KEY}
- S3_SECRET_KEY=${S3_SECRET_KEY}
volumes:
- scraper-data:/data
restart: unless-stopped
command: ["python", "-m", "scrapers.run_pipeline", "--country", "NO", "--source", "brreg"]
scraper-brreg-under:
image: scraper-engine:latest
environment:
- COUNTRY=NO
- SOURCE=brreg_under
- MAX_CONCURRENCY=100
volumes:
- scraper-data:/data
restart: unless-stopped
command: ["python", "-m", "scrapers.run_pipeline", "--country", "NO", "--source", "brreg_under"]
scraper-dk-cvrapi:
image: scraper-engine:latest
environment:
- COUNTRY=DK
- SOURCE=cvrapi
- MAX_CONCURRENCY=30
volumes:
- scraper-data:/data
restart: unless-stopped
command: ["python", "-m", "scrapers.run_pipeline", "--country", "DK", "--source", "cvrapi"]
scraper-de-northdata:
image: scraper-engine:latest
environment:
- COUNTRY=DE
- SOURCE=northdata
- MAX_CONCURRENCY=20
volumes:
- scraper-data:/data
restart: unless-stopped
command: ["python", "-m", "scrapers.run_pipeline", "--country", "DE", "--source", "northdata"]
scraper-de-handelsregister:
image: scraper-engine:latest
environment:
- COUNTRY=DE
- SOURCE=handelsregister
- MAX_CONCURRENCY=20
volumes:
- scraper-data:/data
restart: unless-stopped
command: ["python", "-m", "scrapers.run_pipeline", "--country", "DE", "--source", "handelsregister"]
s3-sync:
image: scraper-engine:latest
depends_on:
- scraper-brreg-no
environment:
- S3_ENDPOINT=${S3_ENDPOINT}
- S3_ACCESS_KEY=${S3_ACCESS_KEY}
- S3_SECRET_KEY=${S3_SECRET_KEY}
- S3_BUCKET=opt-scraped-data
volumes:
- scraper-data:/data
command: ["python", "-m", "scrapers.s3_sync"]
restart: unless-stopped
volumes:
scraper-data:
services/scrapers/engine.py
-243
View File
@@ -1,243 +0,0 @@
"""
0pt Scraper Engine v2
=====================
Universal scraping engine for DK/NO/DE company data.
Uses multi-method bypass chain + header rotation for maximum speed.
Metoder (i prioritetsordning):
1. curl_cffi chrome124 - TLS fingerprint impersonation
2. curl_cffi edge99 - alternative fingerprint
3. cloudscraper - basic CF bypass
4. requests + header rotation - direct HTTP
5. httpx async - high concurrency
6. FlareSolverr - heavy CF bypass (kraver Docker)
"""
import json, time, random, os, asyncio
from datetime import datetime
from typing import Optional
from dataclasses import dataclass, field, asdict
from pathlib import Path
import requests
import httpx
from curl_cffi import requests as curl_requests
import cloudscraper
# ── Config ──
OUTPUT_DIR = Path(__file__).parent / "output"
HEADER_POOL = {
"user_agents": [
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 14_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Safari/605.1.15",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:127.0) Gecko/20100101 Firefox/127.0",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
"Mozilla/5.0 (iPhone; CPU iPhone OS 17_5 like Mac OS X) AppleWebKit/605.1.15",
],
"languages": {
"DK": "da-DK,da;q=0.9,en;q=0.8,de;q=0.7",
"NO": "nb-NO,nb;q=0.9,no;q=0.8,en;q=0.7,da;q=0.6",
"DE": "de-DE,de;q=0.9,en;q=0.8,fr;q=0.7",
"SE": "sv-SE,sv;q=0.9,en;q=0.8",
},
}
@dataclass
class ScrapeResult:
source: str
url: str
method: str
status: int
elapsed: float
bytes: int
cf_blocked: bool = False
data: Optional[dict] = None
error: Optional[str] = None
def rotate_headers(country="DE"):
ua = random.choice(HEADER_POOL["user_agents"])
lang = HEADER_POOL["languages"].get(country, "en-US,en;q=0.9")
return {
"User-Agent": ua,
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
"Accept-Language": lang,
"Accept-Encoding": "gzip, deflate, br",
"Cache-Control": "no-cache",
"Sec-Ch-Ua": '"Chromium";v="124", "Google Chrome";v="124", "Not-A.Brand";v="99"',
"Sec-Ch-Ua-Mobile": "?0",
"Sec-Fetch-Dest": "document",
"Sec-Fetch-Mode": "navigate",
"Sec-Fetch-Site": "none",
"Sec-Fetch-User": "?1",
"Upgrade-Insecure-Requests": "1",
}
def check_cf_blocked(text: str) -> bool:
indicators = [
"cf-wrapper", "Just a moment", "Checking your browser",
"Cloudflare", "__cf_chl_f_tk", "cf-browser-verification",
"challenge-form", "Attention Required",
]
return any(i in text for i in indicators[:5000])
# ── Method implementations ──
_methods_cache = {}
def _get_methods():
if _methods_cache:
return _methods_cache["methods"]
cloud = cloudscraper.create_scraper(
browser={"browser": "chrome", "platform": "windows", "mobile": False}
)
methods = [
("curl_cffi chrome124", lambda url, c="DK": curl_requests.get(
url, impersonate="chrome124", headers=rotate_headers(c), timeout=15)),
("curl_cffi edge99", lambda url, c="DK": curl_requests.get(
url, impersonate="edge99", headers=rotate_headers(c), timeout=15)),
("curl_cffi chrome110", lambda url, c="DK": curl_requests.get(
url, impersonate="chrome110", headers=rotate_headers(c), timeout=15)),
("curl_cffi safari17", lambda url, c="DK": curl_requests.get(
url, impersonate="safari17_0", headers=rotate_headers(c), timeout=15)),
("cloudscraper", lambda url, c="DK": cloud.get(
url, headers=rotate_headers(c), timeout=15)),
("requests_rotated", lambda url, c="DK": requests.get(
url, headers=rotate_headers(c), timeout=15)),
("httpx_rotated", lambda url, c="DK": httpx.get(
url, headers=rotate_headers(c), timeout=15)),
]
_methods_cache["methods"] = methods
return methods
def scrape(url: str, country: str = "DK", method_index: int = -1) -> list[ScrapeResult]:
"""Scrape a URL with all (or specific) methods. Returns list of results."""
methods = _get_methods()
if method_index >= 0:
methods = [methods[method_index]]
results = []
for name, fn in methods:
for attempt in range(2):
try:
start = time.time()
r = fn(url, country)
elapsed = time.time() - start
text_sample = str(r.content[:10000] if hasattr(r, "content") else "")
blocked = check_cf_blocked(text_sample)
results.append(ScrapeResult(
source=url,
url=url,
method=name,
status=r.status_code,
elapsed=round(elapsed, 3),
bytes=len(r.content) if hasattr(r, "content") else 0,
cf_blocked=blocked,
))
break
except Exception as e:
if attempt == 1:
results.append(ScrapeResult(
source=url, url=url, method=name,
status=0, elapsed=0, bytes=0,
error=str(e)[:200],
))
return results
def scrape_best(url: str, country: str = "DK") -> ScrapeResult:
"""Try all methods, return the fastest successful result."""
results = scrape(url, country)
ok = [r for r in results if r.status == 200 and not r.cf_blocked]
if ok:
return min(ok, key=lambda r: r.elapsed)
# If all failed, return the one with highest status
return max(results, key=lambda r: r.status) if results else ScrapeResult(source=url, url=url, method="none", status=0, elapsed=0, bytes=0, error="No methods available")
async def scrape_async(urls: list[str], country: str = "DK", concurrency: int = 20) -> list[ScrapeResult]:
"""Scrape multiple URLs concurrently using async httpx."""
from httpx import AsyncClient, Limits, Timeout
limits = Limits(max_keepalive_connections=concurrency, max_connections=concurrency)
timeout = Timeout(15.0)
async with AsyncClient(limits=limits, timeout=timeout) as client:
async def fetch(url):
start = time.time()
try:
r = await client.get(url, headers=rotate_headers(country))
elapsed = time.time() - start
text_sample = str(r.content[:10000])
return ScrapeResult(
source=url, url=url, method="async httpx",
status=r.status_code, elapsed=round(elapsed, 3),
bytes=len(r.content), cf_blocked=check_cf_blocked(text_sample),
)
except Exception as e:
return ScrapeResult(
source=url, url=url, method="async httpx",
status=0, elapsed=time.time()-start, bytes=0, error=str(e)[:200],
)
sem = asyncio.Semaphore(concurrency)
async def bounded(url):
async with sem:
return await fetch(url)
tasks = [bounded(url) for url in urls]
return await asyncio.gather(*tasks)
def save_result(result: ScrapeResult, data: dict = None):
"""Save scrape result to disk."""
OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
domain = result.url.split("/")[2].replace(".", "_")
ts = datetime.now().strftime("%Y%m%d_%H%M%S")
safe_name = f"{domain}_{result.method}_{ts}.json"
path = OUTPUT_DIR / safe_name
payload = {
"timestamp": ts,
"url": result.url,
"method": result.method,
"status": result.status,
"elapsed": result.elapsed,
"bytes": result.bytes,
"cf_blocked": result.cf_blocked,
"error": result.error,
"data": data,
}
path.write_text(json.dumps(payload, indent=2, ensure_ascii=False), encoding="utf-8")
return path
def benchmark_source(name: str, url: str, country: str = "DK"):
"""Full benchmark of a single source with all methods."""
print(f"\n {'='*60}")
print(f" KALLA: {name}")
print(f" URL: {url}")
print(f" {'='*60}")
results = scrape(url, country)
ok = [r for r in results if r.status == 200 and not r.cf_blocked]
for r in results:
status_str = "OK" if r.status == 200 else f"ERR {r.status}"
cf_str = " [CF]" if r.cf_blocked else ""
err_str = f" - {r.error}" if r.error else ""
print(f" {r.method:30s} -> {status_str:10s} {r.elapsed:6.3f}s {r.bytes:8d}b{cf_str}{err_str}")
if ok:
best = min(ok, key=lambda r: r.elapsed)
print(f"\n >>> BAST: {best.method} ({best.elapsed:.3f}s, {best.bytes}b)")
return best
return None
services/scrapers/engine_v2.py
-256
View File
@@ -1,256 +0,0 @@
"""
0pt Scraper Engine v2 - Production
====================================
Körs som Docker container på SIAX Cl0ud Server 2.
Data lagras i MinIO S3 (s3.siax.io) för obegränsad lagring.
Stödjer:
- Alla DK/NO/DE källor med auto-detection av bästa metod
- Header rotation + curl_cffi TLS fingerprint bypass
- Async concurrency för maxhastighet
- Resume-kapabilitet (kraschtålig)
- Batch processing med checkpointing
"""
import json, time, random, os, asyncio, sys, gzip
from datetime import datetime
from pathlib import Path
from typing import Optional
from dataclasses import dataclass, field, asdict
from concurrent.futures import ThreadPoolExecutor, as_completed
import requests
import httpx
from curl_cffi import requests as curl_requests
import cloudscraper
# ── Config from env ──
S3_ENDPOINT = os.getenv("S3_ENDPOINT", "https://s3.siax.io")
S3_ACCESS_KEY = os.getenv("S3_ACCESS_KEY", "")
S3_SECRET_KEY = os.getenv("S3_SECRET_KEY", "")
S3_BUCKET = os.getenv("S3_BUCKET", "opt-scraped-data")
OUTPUT_DIR = Path(os.getenv("OUTPUT_DIR", "/data/output"))
CHECKPOINT_DIR = Path(os.getenv("CHECKPOINT_DIR", "/data/checkpoints"))
MAX_CONCURRENCY = int(os.getenv("MAX_CONCURRENCY", "50"))
HEADER_POOL = {
"user_agents": [
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 14_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Safari/605.1.15",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:127.0) Gecko/20100101 Firefox/127.0",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
"Mozilla/5.0 (iPhone; CPU iPhone OS 17_5 like Mac OS X) AppleWebKit/605.1.15",
],
"languages": {
"DK": "da-DK,da;q=0.9,en;q=0.8,de;q=0.7",
"NO": "nb-NO,nb;q=0.9,no;q=0.8,en;q=0.7,da;q=0.6",
"DE": "de-DE,de;q=0.9,en;q=0.8,fr;q=0.7",
},
}
# Pre-computed method rankings per source from benchmarks
METHOD_RANKING = {
"no-brreg-api": ["cloudscraper", "curl_cffi safari17", "curl_cffi chrome124"],
"no-proff": ["curl_cffi safari17", "curl_cffi edge99", "curl_cffi chrome110"],
"no-1881": ["curl_cffi chrome124", "curl_cffi safari17", "cloudscraper"],
"no-gulesider": ["curl_cffi safari17", "curl_cffi chrome124", "cloudscraper"],
"no-flowfirma": ["curl_cffi chrome124", "curl_cffi safari17", "curl_cffi chrome110"],
"dk-cvrapi": ["curl_cffi safari17", "curl_cffi chrome110", "cloudscraper"],
"dk-proff": ["curl_cffi safari17", "curl_cffi edge99", "curl_cffi chrome110"],
"dk-krak": ["curl_cffi safari17", "curl_cffi chrome124", "cloudscraper"],
"dk-118": ["curl_cffi safari17", "curl_cffi edge99", "curl_cffi chrome124"],
"dk-grundigt": ["curl_cffi safari17", "curl_cffi edge99", "curl_cffi chrome110"],
"de-northdata": ["curl_cffi edge99", "curl_cffi chrome110", "curl_cffi safari17"],
"de-handelsregister": ["curl_cffi edge99", "curl_cffi chrome110", "cloudscraper"],
"de-moneyhouse": ["curl_cffi chrome110", "curl_cffi edge99", "curl_cffi safari17"],
"de-unternehmensregister": ["curl_cffi safari17", "curl_cffi chrome110", "curl_cffi chrome124"],
"de-creditreform": ["curl_cffi chrome110", "curl_cffi safari17", "curl_cffi edge99"],
"default": ["curl_cffi chrome124", "curl_cffi safari17", "cloudscraper", "curl_cffi edge99", "requests"],
}
@dataclass
class ScrapeResult:
url: str
method: str
status: int
elapsed: float
bytes: int
cf_blocked: bool = False
data: Optional[dict] = None
error: Optional[str] = None
def rotate_headers(country="DE"):
ua = random.choice(HEADER_POOL["user_agents"])
lang = HEADER_POOL["languages"].get(country, "en-US,en;q=0.9")
return {
"User-Agent": ua,
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
"Accept-Language": lang,
"Accept-Encoding": "gzip, deflate, br",
"Cache-Control": "no-cache",
"Sec-Ch-Ua": '"Chromium";v="124", "Google Chrome";v="124", "Not-A.Brand";v="99"',
"Sec-Ch-Ua-Mobile": "?0",
"Sec-Fetch-Dest": "document",
"Sec-Fetch-Mode": "navigate",
"Sec-Fetch-Site": "none",
"Sec-Fetch-User": "?1",
"Upgrade-Insecure-Requests": "1",
}
def check_cf_blocked(text: str) -> bool:
indicators = ["cf-wrapper", "Just a moment", "Checking your browser",
"Cloudflare", "__cf_chl_f_tk", "challenge-form"]
return any(i in text[:5000] for i in indicators)
class ScraperEngine:
"""Main scraper engine - production ready."""
def __init__(self):
self.cloud = cloudscraper.create_scraper(
browser={"browser": "chrome", "platform": "windows", "mobile": False}
)
self.cache = {}
self.stats = {"total": 0, "ok": 0, "cf_blocked": 0, "errors": 0, "total_bytes": 0}
OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
CHECKPOINT_DIR.mkdir(parents=True, exist_ok=True)
def _get_method(self, name: str):
"""Get a scraping method by name."""
if name == "curl_cffi chrome124":
return lambda url, c="DE": curl_requests.get(url, impersonate="chrome124", headers=rotate_headers(c), timeout=15)
elif name == "curl_cffi edge99":
return lambda url, c="DE": curl_requests.get(url, impersonate="edge99", headers=rotate_headers(c), timeout=15)
elif name == "curl_cffi chrome110":
return lambda url, c="DE": curl_requests.get(url, impersonate="chrome110", headers=rotate_headers(c), timeout=15)
elif name == "curl_cffi safari17":
return lambda url, c="DE": curl_requests.get(url, impersonate="safari17_0", headers=rotate_headers(c), timeout=15)
elif name == "cloudscraper":
return lambda url, c="DE": self.cloud.get(url, headers=rotate_headers(c), timeout=15)
elif name == "requests":
return lambda url, c="DE": requests.get(url, headers=rotate_headers(c), timeout=15)
return None
def scrape_url(self, url: str, country: str = "DE", method_rank_key: str = "default") -> ScrapeResult:
"""Try methods in ranking order until one succeeds."""
methods = METHOD_RANKING.get(method_rank_key, METHOD_RANKING["default"])
for method_name in methods:
fn = self._get_method(method_name)
if not fn:
continue
try:
start = time.time()
r = fn(url, country)
elapsed = time.time() - start
text_sample = str(r.content[:10000] if hasattr(r, "content") else "")
blocked = check_cf_blocked(text_sample)
self.stats["total"] += 1
if r.status_code == 200 and not blocked:
self.stats["ok"] += 1
self.stats["total_bytes"] += len(r.content)
return ScrapeResult(url=url, method=method_name, status=r.status_code,
elapsed=round(elapsed, 3), bytes=len(r.content), cf_blocked=blocked)
elif blocked:
self.stats["cf_blocked"] += 1
else:
self.stats["errors"] += 1
except Exception as e:
self.stats["errors"] += 1
continue
return ScrapeResult(url=url, method="all_failed", status=0, elapsed=0, bytes=0, error="All methods failed")
def scrape_batch(self, urls: list[tuple], country: str = "DE", method_key: str = "default", max_workers: int = 10) -> list[ScrapeResult]:
"""Scrape a batch of URLs concurrently using thread pool."""
results = []
with ThreadPoolExecutor(max_workers=max_workers) as ex:
futures = {ex.submit(self.scrape_url, url, country, method_key): url for url, _ in urls}
for f in as_completed(futures):
results.append(f.result())
return results
async def scrape_async_batch(self, urls: list[str], country: str = "DE", concurrency: int = None) -> list[ScrapeResult]:
"""Massively parallel async scraping."""
concurrency = concurrency or MAX_CONCURRENCY
from httpx import AsyncClient, Limits, Timeout
limits = Limits(max_keepalive_connections=concurrency, max_connections=concurrency)
timeout = Timeout(30.0)
async with AsyncClient(limits=limits, timeout=timeout) as client:
async def fetch(url):
start = time.time()
try:
r = await client.get(url, headers=rotate_headers(country), follow_redirects=True)
elapsed = time.time() - start
return ScrapeResult(url=url, method="async_httpx", status=r.status_code,
elapsed=round(elapsed, 3), bytes=len(r.content),
cf_blocked=check_cf_blocked(str(r.content[:10000])))
except Exception as e:
return ScrapeResult(url=url, method="async_httpx", status=0,
elapsed=time.time()-start, bytes=0, error=str(e)[:200])
sem = asyncio.Semaphore(concurrency)
async def bounded(url):
async with sem:
return await fetch(url)
tasks = [bounded(url) for url in urls]
results = await asyncio.gather(*tasks)
for r in results:
self.stats["total"] += 1
if r.status == 200:
self.stats["ok"] += 1
self.stats["total_bytes"] += r.bytes
return results
def save_result(self, result: ScrapeResult, raw_data: str = None):
"""Save result with compression to disk."""
domain = result.url.split("/")[2].replace(".", "_")
ts = datetime.now().strftime("%Y%m%d_%H%M%S")
safe_name = f"{domain}_{result.method}_{ts}.json.gz"
path = OUTPUT_DIR / safe_name
payload = {
"timestamp": ts, "url": result.url, "method": result.method,
"status": result.status, "elapsed": result.elapsed, "bytes": result.bytes,
"cf_blocked": result.cf_blocked, "error": result.error,
"data_sample": raw_data[:10000] if raw_data else None,
}
with gzip.open(path, "wt", encoding="utf-8") as f:
json.dump(payload, f, ensure_ascii=False)
return path
def save_checkpoint(self, batch_id: str, completed: list, total: int):
"""Save progress checkpoint for resume."""
path = CHECKPOINT_DIR / f"checkpoint_{batch_id}.json"
data = {"batch_id": batch_id, "completed": len(completed), "total": total,
"timestamp": datetime.now().isoformat(), "urls": completed}
path.write_text(json.dumps(data, ensure_ascii=False), encoding="utf-8")
def load_checkpoint(self, batch_id: str) -> set:
"""Load completed URLs from checkpoint."""
path = CHECKPOINT_DIR / f"checkpoint_{batch_id}.json"
if path.exists():
data = json.loads(path.read_text(encoding="utf-8"))
return set(data.get("urls", []))
return set()
def print_stats(self):
"""Print current scraping statistics."""
total = self.stats["total"]
ok = self.stats["ok"]
rate = ok / (self.stats.get("start_time", 1) and
max(1, (time.time() - self.stats.get("start_time", time.time()))))
print(f" Stats: {ok}/{total} OK ({self.stats['cf_blocked']} CF, {self.stats['errors']} err)")
print(f" Rate: {rate:.0f} req/s | Data: {self.stats['total_bytes']/1024/1024:.1f} MB")
def print_header():
print("""
╔══════════════════════════════════════════════════════════════╗
║ 0PT SCRAPER ENGINE v2 - PRODUCTION ║
║ DK/NO/DE Company Data Pipeline → MinIO S3 ║
╚══════════════════════════════════════════════════════════════╝
""")
services/scrapers/output/brreg_page_000000.json
-1
View File
File diff suppressed because one or more lines are too long
services/scrapers/output/brreg_page_000001.json
-1
View File
File diff suppressed because one or more lines are too long
services/scrapers/output/brreg_page_000002.json
-1
View File
File diff suppressed because one or more lines are too long
services/scrapers/output/brreg_page_000003.json
-1
View File
File diff suppressed because one or more lines are too long
services/scrapers/output/brreg_page_000004.json
-1
View File
File diff suppressed because one or more lines are too long
services/scrapers/output/brreg_page_000005.json
-1
View File
File diff suppressed because one or more lines are too long
services/scrapers/output/brreg_page_000006.json
-1
View File
File diff suppressed because one or more lines are too long
services/scrapers/output/brreg_page_000007.json
-1
View File
File diff suppressed because one or more lines are too long
services/scrapers/output/brreg_page_000008.json
-1
View File
File diff suppressed because one or more lines are too long
services/scrapers/output/brreg_page_000009.json
-1
View File
File diff suppressed because one or more lines are too long
services/scrapers/output/brreg_page_000010.json
-1
View File
File diff suppressed because one or more lines are too long
services/scrapers/output/brreg_page_000011.json
-1
View File
File diff suppressed because one or more lines are too long
services/scrapers/output/brreg_page_000012.json
-1
View File
File diff suppressed because one or more lines are too long
services/scrapers/output/brreg_page_000013.json
-1
View File
File diff suppressed because one or more lines are too long
services/scrapers/output/brreg_page_000014.json
-1
View File
File diff suppressed because one or more lines are too long
services/scrapers/output/brreg_page_000015.json
-1
View File
File diff suppressed because one or more lines are too long
services/scrapers/output/brreg_page_000016.json
-1
View File
File diff suppressed because one or more lines are too long
services/scrapers/output/brreg_page_000017.json
-1
View File
File diff suppressed because one or more lines are too long
services/scrapers/output/brreg_page_000018.json
-1
View File
File diff suppressed because one or more lines are too long
services/scrapers/output/brreg_page_000019.json
-1
View File
File diff suppressed because one or more lines are too long
services/scrapers/output/brreg_page_000020.json
-1
View File
File diff suppressed because one or more lines are too long
services/scrapers/output/brreg_page_000021.json
-1
View File
File diff suppressed because one or more lines are too long
services/scrapers/output/brreg_page_000022.json
-1
View File
File diff suppressed because one or more lines are too long
services/scrapers/output/brreg_page_000023.json
-1
View File
File diff suppressed because one or more lines are too long
services/scrapers/output/brreg_page_000024.json
-1
View File
File diff suppressed because one or more lines are too long
services/scrapers/output/brreg_page_000025.json
-1
View File
File diff suppressed because one or more lines are too long
services/scrapers/output/brreg_page_000026.json
-1
View File
File diff suppressed because one or more lines are too long
services/scrapers/output/brreg_page_000027.json
-1
View File
File diff suppressed because one or more lines are too long
services/scrapers/output/brreg_page_000028.json
-1
View File
File diff suppressed because one or more lines are too long
services/scrapers/output/brreg_page_000029.json
-1
View File
File diff suppressed because one or more lines are too long
services/scrapers/output/brreg_page_000030.json
-1
View File
File diff suppressed because one or more lines are too long
services/scrapers/output/brreg_page_000031.json
-1
View File
File diff suppressed because one or more lines are too long
services/scrapers/output/brreg_page_000032.json
-1
View File
File diff suppressed because one or more lines are too long
services/scrapers/output/brreg_page_000033.json
-1
View File
File diff suppressed because one or more lines are too long
services/scrapers/output/brreg_page_000034.json
-1
View File
File diff suppressed because one or more lines are too long
services/scrapers/output/brreg_page_000035.json
-1
View File
File diff suppressed because one or more lines are too long
services/scrapers/output/brreg_page_000036.json
-1
View File
File diff suppressed because one or more lines are too long
services/scrapers/output/brreg_page_000037.json
-1
View File
File diff suppressed because one or more lines are too long
services/scrapers/output/brreg_page_000038.json
-1
View File
File diff suppressed because one or more lines are too long
services/scrapers/output/brreg_page_000039.json
-1
View File
File diff suppressed because one or more lines are too long
services/scrapers/output/brreg_page_000040.json
-1
View File
File diff suppressed because one or more lines are too long
services/scrapers/output/brreg_page_000041.json
-1
View File
File diff suppressed because one or more lines are too long
services/scrapers/output/brreg_page_000042.json
-1
View File
File diff suppressed because one or more lines are too long
services/scrapers/output/brreg_page_000043.json
-1
View File
File diff suppressed because one or more lines are too long
services/scrapers/output/brreg_page_000044.json
-1
View File
File diff suppressed because one or more lines are too long
services/scrapers/output/brreg_page_000045.json
-1
View File
File diff suppressed because one or more lines are too long
services/scrapers/output/brreg_page_000046.json
-1
View File
File diff suppressed because one or more lines are too long
services/scrapers/output/brreg_page_000047.json
-1
View File
File diff suppressed because one or more lines are too long
services/scrapers/output/brreg_page_000048.json
-1
View File
File diff suppressed because one or more lines are too long
services/scrapers/output/brreg_page_000049.json
-1
View File
File diff suppressed because one or more lines are too long
services/scrapers/requirements.txt
-9
View File
@@ -1,9 +0,0 @@
# 0pt Scraper Engine - Production dependencies
curl_cffi>=0.15.0
cloudscraper>=1.2.71
httpx>=0.28.0
aiohttp>=3.10.0
requests>=2.33.0
beautifulsoup4>=4.12.0
lxml>=5.3.0
boto3>=1.35.0 # MinIO S3 sync
services/scrapers/run_benchmark.py
-86
View File
@@ -1,86 +0,0 @@
"""
0pt Scraper Benchmark - test alla källor for DK/NO/DE
"""
import sys, os, time, json
sys.path.insert(0, os.path.dirname(__file__))
from engine import scrape_best, scrape_async, benchmark_source, OUTPUT_DIR
from datetime import datetime
# === ALLA KALLOR TESTADE ===
SOURCES = {
"DK": [
("CVR API (cvrapi.dk)", "https://cvrapi.dk/api?search=Novo+Nordisk&country=dk", "api"),
("Proff.dk", "https://proff.dk", "agg"),
("Krak.dk", "https://www.krak.dk/", "dir"),
("118.dk", "https://118.dk/", "dir"),
("De Gule Sider", "https://www.degulesider.dk/", "dir"),
("Companydata.dk", "https://companydata.dk", "agg"),
("Grundigt.dk", "https://grundigt.dk", "agg"),
("Datafordeler.dk", "https://datafordeler.dk", "gov"),
("CVRcheck.dk", "https://cvrcheck.dk", "api"),
("Virk", "https://datacvr.virk.dk/", "gov"),
],
"NO": [
("Brreg API (enhet)", "https://data.brreg.no/enhetsregisteret/api/enheter/915651232", "api"),
("Brreg API (search)", "https://data.brreg.no/enhetsregisteret/api/enheter?size=10", "api"),
("Brreg API (underenheter)", "https://data.brreg.no/enhetsregisteret/api/underenheter?size=10", "api"),
("Proff.no", "https://proff.no", "agg"),
("1881.no", "https://www.1881.no/", "dir"),
("Gulesider.no", "https://www.gulesider.no/", "dir"),
("Selskapssok.no", "https://selskapssok.no", "agg"),
("Flowfirma.no", "https://flowfirma.no", "agg"),
("Offentligdata.no", "https://offentligdata.no", "agg"),
("Regnskapsbasen.no", "https://www.regnskapsbasen.no", "agg"),
("Okonomi.no", "https://okonomi.no/bedrifter", "agg"),
("Visverdi.no", "https://visverdi.no", "agg"),
],
"DE": [
("Northdata.de (Bayer)", "https://www.northdata.de/Bayer+AG", "agg"),
("Handelsregister.de", "https://www.handelsregister.de", "gov"),
("Unternehmensregister.de", "https://www.unternehmensregister.de", "gov"),
("Bundesanzeiger.de", "https://www.bundesanzeiger.de", "gov"),
("Companyhouse.de", "https://www.companyhouse.de", "agg"),
("Moneyhouse.de", "https://www.moneyhouse.de", "agg"),
("Creditreform.de", "https://www.creditreform.de", "agg"),
("Personensuche.de", "https://www.personensuche.de", "dir"),
("Registercheck.de", "https://www.registercheck.de", "agg"),
("Whitepages.de", "https://www.whitepages.de", "dir"),
],
}
def run():
print("=" * 80)
print(f"0PT SCRAPER BENCHMARK - {datetime.now().strftime('%Y-%m-%d %H:%M')}")
print("=" * 80)
results_by_country = {}
for country, sources in SOURCES.items():
print(f"\n{'#'*70}")
print(f"# {country} - {len(sources)} källor")
print(f"{'#'*70}")
country_results = []
for name, url, src_type in sources:
best = benchmark_source(name, url, country)
if best:
country_results.append({"name": name, "url": url, "method": best.method, "elapsed": best.elapsed, "status": best.status, "bytes": best.bytes})
time.sleep(0.3)
results_by_country[country] = country_results
print(f"\n [{country}] SAMMANSTALLNING:")
for r in sorted(country_results, key=lambda x: x["elapsed"]):
print(f" {r['name']:30s} {r['method']:25s} {r['elapsed']:.3f}s {r['bytes']}b")
# Save report
report = {
"timestamp": datetime.now().isoformat(),
"results": results_by_country,
}
path = OUTPUT_DIR / f"benchmark_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json"
path.write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8")
print(f"\n Rapport sparad: {path}")
if __name__ == "__main__":
run()
services/scrapers/run_pipeline.py
-235
View File
@@ -1,235 +0,0 @@
"""
0pt Scraper Pipeline - Production Batch Processor
Kör batch-scraping för angivet land/källa med checkpointing.
Data sparas lokalt och synkas till MinIO S3.
Användning:
python -m scrapers.run_pipeline --country NO --source brreg
python -m scrapers.run_pipeline --country DK --source cvrapi --search "Novo Nordisk"
python -m scrapers.run_pipeline --country DE --source northdata --hrb "HRB 123456"
"""
import os, sys, json, time, asyncio
from datetime import datetime
from pathlib import Path
sys.path.insert(0, os.path.dirname(__file__))
from engine_v2 import ScraperEngine, ScrapeResult, print_header
# ── Source configurations ──
SOURCES = {
"NO": {
"brreg": {
"name": "Brreg Enhetsregisteret",
"type": "api",
"url_template": "https://data.brreg.no/enhetsregisteret/api/enheter/{}",
"method_key": "no-brreg-api",
"total_estimate": 1168208,
"batch_size": 100,
"pagination": True,
"list_url": "https://data.brreg.no/enhetsregisteret/api/enheter?size=100&page={}",
"fields": 38,
},
"brreg_under": {
"name": "Brreg Underenheter",
"type": "api",
"method_key": "no-brreg-api",
"total_estimate": 849423,
"batch_size": 100,
"pagination": True,
"list_url": "https://data.brreg.no/enhetsregisteret/api/underenheter?size=100&page={}",
},
"proff": {
"name": "Proff.no",
"type": "web",
"url_template": "https://proff.no/company/companydetails/{}",
"method_key": "no-proff",
},
"1881": {
"name": "1881.no",
"type": "web",
"url_template": "https://www.1881.no/sok/?q={}",
"method_key": "no-1881",
},
"flowfirma": {
"name": "Flowfirma.no",
"type": "web",
"url_template": "https://flowfirma.no/company/{}",
"method_key": "no-flowfirma",
},
},
"DK": {
"cvrapi": {
"name": "CVR API",
"type": "api",
"url_template": "https://cvrapi.dk/api?search={}&country=dk",
"method_key": "dk-cvrapi",
"total_estimate": 700000,
},
"proff": {
"name": "Proff.dk",
"type": "web",
"url_template": "https://proff.dk/company/companydetails/{}",
"method_key": "dk-proff",
},
"grundigt": {
"name": "Grundigt.dk",
"type": "web",
"url_template": "https://grundigt.dk/virksomhed/{}",
"method_key": "dk-grundigt",
},
},
"DE": {
"northdata": {
"name": "Northdata.de",
"type": "web",
"url_template": "https://www.northdata.de/{}",
"method_key": "de-northdata",
"total_estimate": 3500000,
},
"handelsregister": {
"name": "Handelsregister.de",
"type": "web",
"url_template": "https://www.handelsregister.de/{}",
"method_key": "de-handelsregister",
},
"moneyhouse": {
"name": "Moneyhouse.de",
"type": "web",
"url_template": "https://www.moneyhouse.de/{}",
"method_key": "de-moneyhouse",
},
},
}
async def scrape_brreg_bulk(engine: ScraperEngine, country: str, source_key: str):
"""Scrape Brreg API with pagination - full company register."""
source = SOURCES[country][source_key]
total = source["total_estimate"]
batch = source["batch_size"]
list_url = source["list_url"]
method_key = source["method_key"]
print(f"\n[{country}] {source['name']} - {total} companies")
print(f" Method: {method_key} | Concurrency: {engine.MAX_CONCURRENCY}")
engine.stats["start_time"] = time.time()
total_pages = (total // batch) + 1
completed_urls = set()
# Check for checkpoint
checkpoint_id = f"{country}_{source_key}"
completed_urls = engine.load_checkpoint(checkpoint_id)
print(f" Checkpoint: {len(completed_urls)}/{total_pages} pages completed")
for page in range(total_pages):
if f"page_{page}" in completed_urls:
continue
url = list_url.format(page)
result = engine.scrape_url(url, country, method_key)
if result.status == 200 and not result.cf_blocked:
import json as j
try:
data = j.loads(result.data) if result.data else {}
items = data.get("_embedded", {}).get(
"enheter" if source_key == "brreg" else "underenheter", []
)
for item in items:
engine.save_result(ScrapeResult(
url=url, method=result.method,
status=200, elapsed=result.elapsed,
bytes=len(j.dumps(item)),
data=item
), j.dumps(item))
except:
pass
# Track progress
if (page + 1) % 10 == 0:
elapsed = time.time() - engine.stats["start_time"]
rate = (page + 1) / elapsed if elapsed > 0 else 0
remaining = (total_pages - page) / rate if rate > 0 else 0
print(f" Page {page+1}/{total_pages} ({rate:.0f} pages/s, ETA: {remaining:.0f}s)")
engine.print_stats()
# Checkpoint every 100 pages
if (page + 1) % 100 == 0:
completed = [f"page_{p}" for p in range(page + 1)]
engine.save_checkpoint(checkpoint_id, completed, total_pages)
async def scrape_by_search(engine: ScraperEngine, country: str, source_key: str, search_terms: list[str]):
"""Scrape by searching company names."""
source = SOURCES[country][source_key]
template = source["url_template"]
method_key = source["method_key"]
print(f"\n[{country}] {source['name']} - {len(search_terms)} searches")
engine.stats["start_time"] = time.time()
for i, term in enumerate(search_terms):
url = template.format(term.replace(" ", "+"))
result = engine.scrape_url(url, country, method_key)
if result.status == 200:
engine.save_result(result)
if (i + 1) % 10 == 0:
elapsed = time.time() - engine.stats["start_time"]
rate = (i + 1) / elapsed if elapsed > 0 else 0
remaining = (len(search_terms) - i) / rate if rate > 0 else 0
print(f" {i+1}/{len(search_terms)} ({rate:.0f} req/s, ETA: {remaining:.0f}s)")
engine.print_stats()
def parse_args():
import argparse
parser = argparse.ArgumentParser(description="0pt Scraper Pipeline")
parser.add_argument("--country", choices=["NO", "DK", "DE"], required=True, help="Country code")
parser.add_argument("--source", required=True, help="Source key (brreg, cvrapi, northdata, etc)")
parser.add_argument("--search", nargs="+", help="Search terms for name-based sources")
parser.add_argument("--concurrency", type=int, default=50, help="Max async concurrency")
parser.add_argument("--pages", type=int, default=0, help="Number of pages (0=all)")
return parser.parse_args()
def main():
print_header()
args = parse_args()
country = args.country
source_key = args.source
max_conc = args.concurrency
if country not in SOURCES or source_key not in SOURCES[country]:
print(f"Error: unknown source {country}/{source_key}")
print(f"Available: {[f'{c}/{k}' for c, srcs in SOURCES.items() for k in srcs.keys()]}")
return 1
engine = ScraperEngine()
source = SOURCES[country][source_key]
print(f" Country: {country}")
print(f" Source: {source['name']} ({source['type']})")
print(f" Method: {source['method_key']}")
print(f" Concurrency: {max_conc}")
print()
if source.get("pagination"):
asyncio.run(scrape_brreg_bulk(engine, country, source_key))
elif args.search:
asyncio.run(scrape_by_search(engine, country, source_key, args.search))
else:
print("Error: need --search for non-paginated sources, or use --search with search terms")
return 1
print(f"\n Pipeline complete!")
engine.print_stats()
return 0
if __name__ == "__main__":
sys.exit(main())
services/scrapers/s3_sync.py
-57
View File
@@ -1,57 +0,0 @@
"""
S3 Sync - Upload scraped data to MinIO periodically.
"""
import os, time, json, gzip
from pathlib import Path
from datetime import datetime, timedelta
OUTPUT_DIR = Path(os.getenv("OUTPUT_DIR", "/data/output"))
S3_ENDPOINT = os.getenv("S3_ENDPOINT", "https://s3.siax.io")
S3_ACCESS_KEY = os.getenv("S3_ACCESS_KEY", "")
S3_SECRET_KEY = os.getenv("S3_SECRET_KEY", "")
S3_BUCKET = os.getenv("S3_BUCKET", "opt-scraped-data")
SYNC_INTERVAL = int(os.getenv("SYNC_INTERVAL", "300")) # 5 min
def get_s3_client():
import boto3
return boto3.client(
"s3",
endpoint_url=S3_ENDPOINT,
aws_access_key_id=S3_ACCESS_KEY,
aws_secret_access_key=S3_SECRET_KEY,
)
def sync_to_s3():
"""Upload all new files to MinIO S3."""
s3 = get_s3_client()
# Ensure bucket exists
try:
s3.head_bucket(Bucket=S3_BUCKET)
except:
s3.create_bucket(Bucket=S3_BUCKET)
uploaded = 0
for fpath in OUTPUT_DIR.glob("*.json.gz"):
key = f"raw/{fpath.name}"
try:
s3.upload_file(str(fpath), S3_BUCKET, key)
uploaded += 1
fpath.unlink() # Remove local after upload
except Exception as e:
print(f" S3 upload failed: {fpath.name} - {e}")
return uploaded
def run():
print(f"S3 Sync started -> {S3_ENDPOINT}/{S3_BUCKET}")
while True:
try:
uploaded = sync_to_s3()
if uploaded:
print(f" Uploaded {uploaded} files to S3")
except Exception as e:
print(f" Sync error: {e}")
time.sleep(SYNC_INTERVAL)
if __name__ == "__main__":
run()
services/scrapers/waf_bypass.py
-316
View File
@@ -1,316 +0,0 @@
"""
Proxy Rotator & WAF Bypass - ALLA gratis metoder.
Metoder:
1. Header rotation (6 User-Agents x sprak x devices)
2. curl_cffi TLS fingerprint (chrome124, chrome110, edge99, safari17)
3. cloudscraper (basic CF bypass)
4. FlareSolverr (heavy CF bypass, via Docker)
5. Tor rotation (SOCKS5, gratis)
6. Free proxy pool (scraped from 5+ sources)
7. Playwright browser automation
8. nodriver (undetected browser)
9. Fake identity generation
10. Session cookie reuse
"""
import os, json, time, random, socket, asyncio
from datetime import datetime
from typing import Optional
from curl_cffi import requests as curl_requests
import cloudscraper
import requests
# ── Config ──
FLARESOLVERR_URL = os.getenv("FLARESOLVERR_URL", "http://localhost:8191/v1")
TOR_PROXY = os.getenv("TOR_PROXY", "socks5://localhost:9050")
USE_FLARESOLVERR = os.getenv("USE_FLARESOLVERR", "false").lower() == "true"
USE_TOR = os.getenv("USE_TOR", "false").lower() == "true"
# ── Header Pool ──
USER_AGENTS = [
# Chrome Windows
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",
# Safari Mac
"Mozilla/5.0 (Macintosh; Intel Mac OS X 14_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Safari/605.1.15",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.6 Safari/605.1.15",
# Firefox
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:127.0) Gecko/20100101 Firefox/127.0",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) Gecko/20100101 Firefox/128.0",
# Edge
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36 Edg/124.0.0.0",
# Linux
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
# Mobile
"Mozilla/5.0 (iPhone; CPU iPhone OS 17_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Mobile/15E148 Safari/604.1",
"Mozilla/5.0 (Linux; Android 14; Pixel 8) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Mobile Safari/537.36",
]
ACCEPT_LANGUAGES = {
"DK": "da-DK,da;q=0.9,en;q=0.8,de;q=0.7",
"NO": "nb-NO,nb;q=0.9,no;q=0.8,en;q=0.7,da;q=0.6",
"DE": "de-DE,de;q=0.9,en;q=0.8,fr;q=0.7",
"SE": "sv-SE,sv;q=0.9,en;q=0.8",
"GB": "en-GB,en;q=0.9,fr;q=0.8",
"FI": "fi-FI,fi;q=0.9,en;q=0.8,sv;q=0.7",
"FR": "fr-FR,fr;q=0.9,en;q=0.8",
"NL": "nl-NL,nl;q=0.9,en;q=0.8",
"PL": "pl-PL,pl;q=0.9,en;q=0.8",
}
SEC_CH_UA = {
"chrome124": '"Chromium";v="124", "Google Chrome";v="124", "Not-A.Brand";v="99"',
"chrome125": '"Chromium";v="125", "Google Chrome";v="125", "Not-A.Brand";v="99"',
"edge124": '"Chromium";v="124", "Microsoft Edge";v="124", "Not-A.Brand";v="99"',
}
ACCEPT_ENCODINGS = ["gzip, deflate, br", "gzip, deflate", "gzip, deflate, br, zstd"]
def rotate_headers(country="DE", impersonate="chrome124"):
ua = random.choice(USER_AGENTS)
lang = ACCEPT_LANGUAGES.get(country, "en-US,en;q=0.9")
enc = random.choice(ACCEPT_ENCODINGS)
sec_ch = SEC_CH_UA.get(impersonate, SEC_CH_UA["chrome124"])
return {
"User-Agent": ua,
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
"Accept-Language": lang,
"Accept-Encoding": enc,
"Cache-Control": "no-cache",
"Pragma": "no-cache",
"Sec-Ch-Ua": sec_ch,
"Sec-Ch-Ua-Mobile": "?0",
"Sec-Ch-Ua-Platform": '"Windows"' if "Windows" in ua else ('"macOS"' if "Mac" in ua else '"Linux"'),
"Sec-Fetch-Dest": "document",
"Sec-Fetch-Mode": "navigate",
"Sec-Fetch-Site": "none",
"Sec-Fetch-User": "?1",
"Upgrade-Insecure-Requests": "1",
"DNT": "1",
}
def check_cf_blocked(text: str) -> bool:
"""Check if response contains Cloudflare challenge."""
indicators = [
"cf-wrapper", "Just a moment", "Checking your browser",
"Cloudflare", "__cf_chl_f_tk", "cf-browser-verification",
"challenge-form", "Attention Required", "cf-error-page",
"cloudflare.com/cdn-cgi", "/cdn-cgi/",
]
if not text:
return False
lower = text[:10000].lower()
for ind in indicators:
if ind in lower:
return True
return False
class ProxyRotator:
"""Free proxy pool with rotation."""
def __init__(self):
self.proxies = []
self.current = 0
self._load_free_proxies()
def _load_free_proxies(self):
"""Load proxies from free sources."""
sources = [
"https://raw.githubusercontent.com/TheSpeedX/SOCKS-List/master/http.txt",
"https://raw.githubusercontent.com/ShiftyTR/Proxy-List/master/http.txt",
"https://raw.githubusercontent.com/clarketm/proxy-list/master/proxy-list-raw.txt",
]
for url in sources:
try:
r = requests.get(url, timeout=5)
for line in r.text.strip().split("\n"):
line = line.strip()
if ":" in line and not line.startswith("#"):
parts = line.split(":")
if len(parts) == 2:
self.proxies.append({"http": f"http://{line}", "https": f"http://{line}"})
except:
pass
# Deduplicate
seen = set()
unique = []
for p in self.proxies:
if p["http"] not in seen:
seen.add(p["http"])
unique.append(p)
self.proxies = unique
print(f" ProxyRotator: loaded {len(self.proxies)} free proxies")
def get_proxy(self) -> Optional[dict]:
if not self.proxies:
return None
self.current = (self.current + 1) % len(self.proxies)
return self.proxies[self.current]
def get_tor_proxy(self) -> dict:
return {"http": TOR_PROXY, "https": TOR_PROXY}
class WAFBypass:
"""Multi-method WAF bypass engine."""
def __init__(self):
self.cloudscraper = cloudscraper.create_scraper(
browser={"browser": "chrome", "platform": "windows", "mobile": False}
)
self.proxy = ProxyRotator()
self.session_pool = {}
self.curl_methods = {
"chrome124": lambda url, h: curl_requests.get(url, impersonate="chrome124", headers=h, timeout=15),
"chrome110": lambda url, h: curl_requests.get(url, impersonate="chrome110", headers=h, timeout=15),
"edge99": lambda url, h: curl_requests.get(url, impersonate="edge99", headers=h, timeout=15),
"safari17": lambda url, h: curl_requests.get(url, impersonate="safari17_0", headers=h, timeout=15),
}
def try_flaresolverr(self, url: str) -> Optional[str]:
"""Bypass CF via FlareSolverr Docker container."""
if not USE_FLARESOLVERR:
return None
try:
payload = {
"cmd": "request.get",
"url": url,
"maxTimeout": 60000,
"session": f"session_{hash(url) % 10000}",
}
r = requests.post(FLARESOLVERR_URL, json=payload, timeout=65)
data = r.json()
if data.get("status") == "ok":
solution = data.get("solution", {})
return solution.get("response", "")
except:
pass
return None
def try_tor(self, url: str, headers: dict) -> Optional[requests.Response]:
"""Route through Tor network."""
if not USE_TOR:
return None
try:
proxies = self.proxy.get_tor_proxy()
r = requests.get(url, headers=headers, proxies=proxies, timeout=30)
return r
except:
return None
def try_curl(self, url: str, country: str = "DE") -> Optional[requests.Response]:
"""Try all curl_cffi impersonations in order."""
for name, fn in self.curl_methods.items():
try:
headers = rotate_headers(country, name.replace("curl_cffi_", "").split("_")[0] if "curl_" in name else "chrome124")
r = fn(url, headers)
if r.status_code == 200 and not check_cf_blocked(str(r.content[:5000])):
return r
except:
continue
return None
def try_cloudscraper(self, url: str, country: str = "DE") -> Optional[requests.Response]:
"""Try cloudscraper with different configs."""
for browser in [
{"browser": "chrome", "platform": "windows", "mobile": False},
{"browser": "firefox", "platform": "windows", "mobile": False},
{"browser": "chrome", "platform": "macos", "mobile": False},
]:
try:
sc = cloudscraper.create_scraper(browser=browser)
sc.headers.update(rotate_headers(country))
r = sc.get(url, timeout=15)
if r.status_code == 200 and not check_cf_blocked(str(r.content[:5000])):
return r
except:
continue
return None
def try_playwright(self, url: str) -> Optional[str]:
"""Use Playwright headless browser as last resort."""
try:
from playwright.sync_api import sync_playwright
with sync_playwright() as p:
browser = p.chromium.launch(headless=True)
context = browser.new_context(
user_agent=random.choice(USER_AGENTS),
locale="en-US",
viewport={"width": 1920, "height": 1080},
)
page = context.new_page()
page.goto(url, timeout=30000)
content = page.content()
browser.close()
if not check_cf_blocked(content):
return content
except:
pass
return None
def bypass(self, url: str, country: str = "DE") -> tuple:
"""
Full bypass chain. Returns (content: str, method: str, elapsed: float).
Chain order:
1. curl_cffi chrome124 (fastest, 0.1-0.3s)
2. cloudscraper (basic CF, 0.3-0.5s)
3. curl_cffi edge99 (alt fingerprint)
4. curl_cffi safari17 (Eniro bypass)
5. Tor proxy rotation (anonymous, 2-5s)
6. FlareSolverr (heavy CF, 5-15s)
7. Playwright (full browser, 10-30s)
"""
start = time.time()
# Level 1: curl_cffi TLS fingerprint
r = self.try_curl(url, country)
if r and r.status_code == 200:
return (r.text, "curl_cffi", time.time() - start)
# Level 2: cloudscraper
r = self.try_cloudscraper(url, country)
if r and r.status_code == 200:
return (r.text, "cloudscraper", time.time() - start)
# Level 3: Tor
if USE_TOR:
r = self.try_tor(url, rotate_headers(country))
if r and r.status_code == 200:
return (r.text, "tor", time.time() - start)
# Level 4: FlareSolverr
content = self.try_flaresolverr(url)
if content:
return (content, "flaresolverr", time.time() - start)
# Level 5: Playwright (last resort)
content = self.try_playwright(url)
if content:
return (content, "playwright", time.time() - start)
return (None, "all_failed", time.time() - start)
def bypass_with_proxy(self, url: str, country: str = "DE") -> tuple:
"""Bypass with proxy rotation for each method."""
start = time.time()
proxy = self.proxy.get_proxy()
headers = rotate_headers(country)
for name, fn in self.curl_methods.items():
try:
if proxy:
r = fn(url, headers)
else:
r = fn(url, headers)
if r.status_code == 200 and not check_cf_blocked(str(r.content[:5000])):
return (r.text, f"curl_cffi+proxy_{name}", time.time() - start)
except:
continue
return self.bypass(url, country)