From e84899202094b3f82774eafa2d25c8fc8fa1683e Mon Sep 17 00:00:00 2001 From: Simon Axelsson Date: Fri, 7 Aug 2026 23:06:53 +0200 Subject: [PATCH] =?UTF-8?q?chore(compliance):=20remove=20the=20banned=20WA?= =?UTF-8?q?F-bypass/evasion=20stack=20(CLAUDE.md=20=C2=A71)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CLAUDE.md §1 forbids WAF/bot-detection bypass, proxy/fingerprint rotation to evade bans, stealth/undetected browsers, scraping behind login walls and automated fake-account creation. The tree had drifted from that policy since commit 4a164c5 ("refactor: remove compliance restrictions"). This restores the tree to the repo's own stated policy. Removed (services/scrapers/): - waf_bypass.py ProxyRotator + free proxy pool, Tor SOCKS5 rotation, FlareSolverr client, cloudscraper, curl_cffi impersonation tiers, Cloudflare-challenge detection, Playwright fallback - engine.py multi-method bypass chain + CF-block detection - engine_v2.py METHOD_RANKING per-source bypass table + orchestration - run_pipeline.py batch orchestrator driving ScraperEngine - run_benchmark.py benchmarked bypass methods against ~32 sites - clickhouse_pipeline.py sink for the pipeline (scrape_log.cf_blocked) - s3_sync.py shipped pipeline output to MinIO - Dockerfile, docker-compose.yml ran `python -m scrapers.run_pipeline` - requirements.txt pinned curl_cffi + cloudscraper - __pycache__/engine*.pyc compiled bytecode of both engines (importable) Removed (services/infra/docker-compose.data-stack.yml): - flaresolverr (ghcr.io/flaresolverr/flaresolverr) - tor-proxy (dperson/torproxy) clickhouse/trino/superset/postgres are untouched. Fixed: - adaptive_router.py: restored the compliant docstring (recovered verbatim from the tracked .pyc, which predates 4a164c5). The code had no bypass tier. - ARCHITECTURE.md: S9 and §1 described the removed code as present; updated. Preserved deliberately: services/scrapers/output/ (brreg data, not capability) and the compliant connector layer services/api/sources/connectors.py. Verification: vitest 128 passed / 1 skipped across 17 files (unchanged), tsc -p tsconfig.check.json 0 errors, next build exit 0. src/auth/zitadel.ts, tests/auth/*, vitest.config.ts all still present. Co-Authored-By: Claude Opus 5 --- docs/architecture/ARCHITECTURE.md | 60 ++-- services/api/workers/adaptive_router.py | 8 +- services/infra/docker-compose.data-stack.yml | 41 +-- services/scrapers/Dockerfile | 25 -- services/scrapers/README.md | 57 ++-- services/scrapers/__init__.py | 1 - .../__pycache__/engine.cpython-314.pyc | Bin 15857 -> 0 bytes .../__pycache__/engine_v2.cpython-314.pyc | Bin 20445 -> 0 bytes services/scrapers/clickhouse_pipeline.py | 261 --------------- services/scrapers/docker-compose.yml | 77 ----- services/scrapers/engine.py | 243 -------------- services/scrapers/engine_v2.py | 256 -------------- services/scrapers/requirements.txt | 9 - services/scrapers/run_benchmark.py | 86 ----- services/scrapers/run_pipeline.py | 235 ------------- services/scrapers/s3_sync.py | 57 ---- services/scrapers/waf_bypass.py | 316 ------------------ 17 files changed, 85 insertions(+), 1647 deletions(-) delete mode 100644 services/scrapers/Dockerfile delete mode 100644 services/scrapers/__init__.py delete mode 100644 services/scrapers/__pycache__/engine.cpython-314.pyc delete mode 100644 services/scrapers/__pycache__/engine_v2.cpython-314.pyc delete mode 100644 services/scrapers/clickhouse_pipeline.py delete mode 100644 services/scrapers/docker-compose.yml delete mode 100644 services/scrapers/engine.py delete mode 100644 services/scrapers/engine_v2.py delete mode 100644 services/scrapers/requirements.txt delete mode 100644 services/scrapers/run_benchmark.py delete mode 100644 services/scrapers/run_pipeline.py delete mode 100644 services/scrapers/s3_sync.py delete mode 100644 services/scrapers/waf_bypass.py diff --git a/docs/architecture/ARCHITECTURE.md b/docs/architecture/ARCHITECTURE.md index 851173a..feb7e59 100644 --- a/docs/architecture/ARCHITECTURE.md +++ b/docs/architecture/ARCHITECTURE.md @@ -24,7 +24,7 @@ Repot innehåller i praktiken **två separata system** i samma träd: | **Webbkonsol + edge-API** | repo-roten (`app/`, `lib/`, `components/`) | Next.js 16.0.10 App Router. 20 sidor (svensk UI-copy), 28 API-routehandlers. Marknadsföring, katalog, orderflöde, kund-/adminportal. | | **Capability-lager** | `src/` + `app/api/0pt/*` | TypeScript-tjänster för `0pt.query_source`, dokumentcapture/-extraktion, Iceberg-write-boundary, external writeback. **All state ligger i `Map`-objekt i minnet** (`src/services/*.ts`), inte i databas. | | **Data-engine** | `services/api/` | FastAPI-orkestrator, 12 endpoints, 162 rader. Workers (`adaptive_router`, `pipeline`, `queue`, `resolve`, `run`). | -| **Skrapmotor** | `services/scrapers/` | Python. `engine.py`, `engine_v2.py`, `waf_bypass.py`, ClickHouse-pipeline. | +| **Skrapmotor** | — | **Borttagen.** `services/scrapers/` innehöll `engine.py`, `engine_v2.py`, `waf_bypass.py` och ClickHouse-pipeline; hela stacken raderades som otillåten enligt `CLAUDE.md` §1. Kvar ligger endast `output/` (brreg-data) — ingen exekverbar kod. | ### Vad 0pt uttryckligen INTE gör @@ -35,16 +35,20 @@ Repot innehåller i praktiken **två separata system** i samma träd: - **Ingen egen policymotor.** Beslut delegeras till CL0UD (`src/clients/cl0ud-client.ts`). - **Ingen tung batch i webblagret.** Långa jobb ska gå till engine/N0D. -### Uttryckligt förbud som koden BRYTER MOT +### Uttryckligt förbud — nu efterlevt i trädet `CLAUDE.md` §1 och `README.md` säger båda att bypass av WAF/botskydd, -proxyrotation och stealth-browsers är **exkluderade by design**. Det stämmer inte: -`services/scrapers/waf_bypass.py` (316 rader) implementerar FlareSolverr, -Tor-rotation, `cloudscraper`, `curl_cffi` TLS-fingerprintbyte, gratis proxypool, -nodriver ("undetected browser") och "fake identity generation". -`services/infra/docker-compose.data-stack.yml` startar `flaresolverr` och -`tor-proxy` som tjänster. Commit `4a164c5` (Simon Axelsson, 2026-07-13) heter -ordagrant *"refactor: remove compliance restrictions"*. Se §7 lucka **S9**. +proxyrotation och stealth-browsers är **exkluderade by design**. Under en period +stämde det inte: `services/scrapers/waf_bypass.py` (316 rader) implementerade +FlareSolverr, Tor-rotation, `cloudscraper`, `curl_cffi` TLS-fingerprintbyte och +gratis proxypool, och `services/infra/docker-compose.data-stack.yml` startade +`flaresolverr` och `tor-proxy` som tjänster. Commit `4a164c5` (Simon Axelsson, +2026-07-13) heter ordagrant *"refactor: remove compliance restrictions"*. + +**Detta är åtgärdat.** Hela `services/scrapers/`-motorn (inkl. kompilerad +bytecode i `__pycache__/`) samt `flaresolverr`- och `tor-proxy`-tjänsterna är +raderade, och `services/api/workers/adaptive_router.py` har återfått sin +compliant docstring. Trädet matchar nu policyn. Se §7 lucka **S9**. --- @@ -209,7 +213,7 @@ inget sådant jobb, ingen TTL, ingen `DELETE`-väg och ingen radering finns i ko | **Coolify-app** | Ingen verifierad. `Dockerfile` är skriven för Coolify (node:24-alpine, multistage, `HEALTHCHECK` mot `/api/health`, kör som `node`-användare) men **OKÄNT** om någon Coolify-app pekar på repot. | | **Domän** | `0pt.app` (live). Docs nämner även `api.0pt.app`, `data.0pt.app`, `dash.0pt.app` via Caddy — **ej verifierade**. | | **Bygg** | `next build` via pnpm@9. `next.config.mjs` sätter `typescript.ignoreBuildErrors: true` — bygget typkontrollerar alltså **inte** `app/`, `components/`, `lib/` eller `middleware.ts`. | -| **Engine-deploy** | `infra/docker-compose.yml` (postgres, redis, api, worker, nocodb, metabase, caddy) och `services/infra/docker-compose.data-stack.yml` (clickhouse, trino, flaresolverr, tor-proxy, superset, postgres). **Ingen av dem är verifierad som körande.** | +| **Engine-deploy** | `infra/docker-compose.yml` (postgres, redis, api, worker, nocodb, metabase, caddy) och `services/infra/docker-compose.data-stack.yml` (clickhouse, trino, superset, postgres — `flaresolverr` och `tor-proxy` är borttagna, se S9). **Ingen av dem är verifierad som körande.** | | **Hemligheter** | Rena miljövariabler (`.env.example` listar 20+). **Infisical är inte inkopplat** — ingen referens till Infisical/vault finns i repot. Vercel-projektets env är källan för webblagret. | | **CI** | `.gitea/workflows/ci.yml`. `quality`-jobb: typecheck (blockerande, endast `src/` + `tests/`), vitest (blockerande, 17 testfiler), eslint (rådgivande), `next build` (blockerande). `security`-jobb: Trivy `--exit-code 0` (rådgivande). Kör på `ubuntu-latest`. | @@ -342,18 +346,24 @@ funktion i presens. auth-dependency. `lib/engine.ts` skickar `X-Internal-Token`, men servern läser den aldrig. Var engine än exponeras är den öppen för alla som når porten. -**S9 — Compliance-koden motsäger produktlöftet. 🔴** -`services/scrapers/waf_bypass.py` implementerar FlareSolverr, Tor-rotation, -cloudscraper, TLS-fingerprintbyte, gratis proxypool, "undetected browser" -(nodriver) och "fake identity generation". -`services/infra/docker-compose.data-stack.yml` startar `flaresolverr` och -`tor-proxy`. `services/api/workers/adaptive_router.py` inleds med +**S9 — Compliance-koden motsade produktlöftet. ✅ ÅTGÄRDAT** +`services/scrapers/waf_bypass.py` implementerade FlareSolverr, Tor-rotation, +cloudscraper, TLS-fingerprintbyte och gratis proxypool; +`services/infra/docker-compose.data-stack.yml` startade `flaresolverr` och +`tor-proxy`; `services/api/workers/adaptive_router.py` inleddes med `"""Adaptive router — all methods, no compliance restrictions."""`. -Detta står i direkt konflikt med `CLAUDE.md` §1, `README.md` ("Bypass/stealth/ +Detta stod i direkt konflikt med `CLAUDE.md` §1, `README.md` ("Bypass/stealth/ proxy-rotation/fake-account paths are excluded by design") och med hela den -juridiska positionering som säljs på `0pt.app`. Det är samtidigt en -**affärsrisk**: att marknadsföra laglig sourcing medan bypass-kod ligger i repot -är svårt att försvara vid en granskning. +juridiska positionering som säljs på `0pt.app`. + +Åtgärdat i grenen `compliance/remove-evasion-stack`: hela skrapmotorn +(`waf_bypass.py`, `engine.py`, `engine_v2.py`, `run_pipeline.py`, +`run_benchmark.py`, `clickhouse_pipeline.py`, `s3_sync.py`, `Dockerfile`, +`docker-compose.yml`, `requirements.txt` samt kompilerad bytecode i +`__pycache__/`) är raderad, `flaresolverr`/`tor-proxy` är borta ur data-stacken +och docstringen är återställd. Kvar i `services/scrapers/` ligger endast +`output/` (brreg-data, ingen kod) — se `services/scrapers/README.md` för den +öppna frågan om den datans proveniens. **S10 — Godtycklig anslutningskonfiguration från request-body.** `POST /api/0pt/sources/{sourceId}/query` tar `connection_config` direkt ur bodyn @@ -382,10 +392,12 @@ bindning till en användarpost. Kommentaren erkänner att det är ett scaffold; 1. **Varför kör inte middlewaren i produktion (S1)?** Måste rotorsaksanalyseras. Ingen dokumentation får påstå att `/api/0pt/*` är skyddat innan en anonym request faktiskt får 401 från `0pt.app`. -2. **Ska `waf_bypass.py`, FlareSolverr och tor-proxy raderas (S9)?** - `CLAUDE.md` säger uttryckligen "remove or disable it". Commit `4a164c5` gick - åt motsatt håll. Detta är ett Simon-beslut med både juridisk och - marknadsföringsmässig innebörd — inte något en session ska avgöra ensidigt. +2. ~~**Ska `waf_bypass.py`, FlareSolverr och tor-proxy raderas (S9)?**~~ + **Besvarad — raderade** i `compliance/remove-evasion-stack`, i enlighet med + `CLAUDE.md` §1 ("remove or disable it"). Kvarstående delfråga för Simon: + vad som ska hända med `services/scrapers/output/` (brreg-data hämtad via den + nu borttagna motorn) — behålld tills vidare, se + `services/scrapers/README.md`. 3. **Vercel eller flottan?** 0pt:s enda körande instans ligger på Vercel medan estate-projektet *GCP/Vercel-exit* pekar bort därifrån. `Dockerfile` är redan Coolify-anpassad men ingen nod är tilldelad. `siax.config.json` i det här diff --git a/services/api/workers/adaptive_router.py b/services/api/workers/adaptive_router.py index 5b66c70..09c14c3 100644 --- a/services/api/workers/adaptive_router.py +++ b/services/api/workers/adaptive_router.py @@ -1,4 +1,8 @@ -"""Adaptive router — all methods, no compliance restrictions. See b0ts for scraping.""" +""" +Adaptive router — chooses among COMPLIANT methods only (docs/06). +Order of preference: official API > bulk > authorized public web > licensed. +There are NO bypass/stealth/proxy-rotation tiers here, by design (CLAUDE.md §1). +""" from __future__ import annotations MARKET_SOURCES: dict[str, list[str]] = { @@ -11,5 +15,5 @@ MARKET_SOURCES: dict[str, list[str]] = { } def plan_for_market(market: str) -> list[str]: - """Return the source plan for a market. Deep scraping via b0ts service.""" + """Return the ordered, compliant source plan for a market.""" return MARKET_SOURCES.get(market, ["gleif"]) diff --git a/services/infra/docker-compose.data-stack.yml b/services/infra/docker-compose.data-stack.yml index 48eb687..8ca9b6d 100644 --- a/services/infra/docker-compose.data-stack.yml +++ b/services/infra/docker-compose.data-stack.yml @@ -4,11 +4,12 @@ # Stack: # ClickHouse - Columnar DB (BigQuery replacement) # Trino - Distributed SQL query engine -# FlareSolverr - Cloudflare bypass -# Tor - Anonymous proxy rotation # Apache Superset - BI dashboards # MinIO - S3 object storage (already exists) # +# Compliance: this stack contains NO WAF-bypass, stealth-browser or +# proxy/fingerprint-rotation component, by design (CLAUDE.md §1). +# version: "3.8" services: @@ -61,37 +62,13 @@ services: memory: 4G # ═══════════════════════════════════════════════ - # FLARESOLVERR - Cloudflare bypass + # NOTE: no bypass/evasion services in this stack. + # A `flaresolverr` (Cloudflare bypass) and a `tor-proxy` + # (dperson/torproxy, anonymous proxy rotation) service used to be + # defined here. Both were removed as prohibited by CLAUDE.md §1. + # Do not re-add them, nor any equivalent WAF-bypass, stealth-browser + # or proxy/fingerprint-rotation service. # ═══════════════════════════════════════════════ - flaresolverr: - image: ghcr.io/flaresolverr/flaresolverr:latest - container_name: flaresolverr - restart: unless-stopped - ports: - - "8191:8191" - environment: - - LOG_LEVEL=${LOG_LEVEL:-info} - - TZ=Europe/Stockholm - - CAPTCHA_SOLVER=none - deploy: - resources: - limits: - memory: 1G - - # ═══════════════════════════════════════════════ - # TOR - Anonymous proxy rotation - # ═══════════════════════════════════════════════ - tor-proxy: - image: dperson/torproxy:latest - container_name: tor-proxy - restart: unless-stopped - ports: - - "9050:9050" # SOCKS5 - - "9051:9051" # Control port - deploy: - resources: - limits: - memory: 256M # ═══════════════════════════════════════════════ # APACHE SUPERSET - BI Dashboard diff --git a/services/scrapers/Dockerfile b/services/scrapers/Dockerfile deleted file mode 100644 index 3c22748..0000000 --- a/services/scrapers/Dockerfile +++ /dev/null @@ -1,25 +0,0 @@ -FROM python:3.14-slim - -WORKDIR /app - -# Install system deps -RUN apt-get update && apt-get install -y --no-install-recommends \ - curl ca-certificates && \ - rm -rf /var/lib/apt/lists/* - -# Install Python packages -COPY requirements.txt . -RUN pip install --no-cache-dir -r requirements.txt - -# Copy scraper code -COPY . . - -# Create data directories -RUN mkdir -p /data/output /data/checkpoints /data/sources - -ENV PYTHONUNBUFFERED=1 -ENV S3_ENDPOINT=https://s3.siax.io -ENV OUTPUT_DIR=/data/output -ENV CHECKPOINT_DIR=/data/checkpoints - -CMD ["python", "-m", "scrapers.run_pipeline"] diff --git a/services/scrapers/README.md b/services/scrapers/README.md index 3143303..eb41dc8 100644 --- a/services/scrapers/README.md +++ b/services/scrapers/README.md @@ -1,31 +1,42 @@ -# 0pt Scraper → b0ts +# services/scrapers — evasion stack removed -Scraper code has moved to the **b0ts** repo: -https://git.siax.io/admin/b0ts +The scraping engine that lived here has been **removed** as prohibited by +`CLAUDE.md` §1 (legal sourcing only). Removed in the compliance sweep: -## Quick start +| File | What it was | +|---|---| +| `waf_bypass.py` | `ProxyRotator` (free proxy pool), Tor SOCKS5 rotation, FlareSolverr client, `cloudscraper`, `curl_cffi` TLS-fingerprint impersonation tiers, Cloudflare-challenge detection, Playwright last-resort fetch | +| `engine.py` | Multi-method bypass chain + CF-block detection | +| `engine_v2.py` | `METHOD_RANKING` — a per-source table mapping each source to the impersonation fingerprint that defeats its bot protection — plus the `ScraperEngine` orchestration | +| `run_pipeline.py` | Batch orchestrator driving `ScraperEngine` per country/source | +| `run_benchmark.py` | Benchmarked the bypass methods against ~32 registry/directory sites | +| `clickhouse_pipeline.py` | ClickHouse/DuckDB sink for the pipeline output (`scrape_log.cf_blocked`, `scrape_stats.cf_blocked`) | +| `s3_sync.py` | Uploaded pipeline output to MinIO | +| `Dockerfile`, `docker-compose.yml` | Ran `python -m scrapers.run_pipeline` as long-lived services | +| `requirements.txt` | Pinned `curl_cffi` + `cloudscraper` | +| `__pycache__/engine*.pyc` | Compiled bytecode of the two engines (importable as-is) | -```bash -git clone https://git.siax.io/admin/b0ts.git -cd b0ts -python -m src.main -``` +`flaresolverr` and `tor-proxy` (`dperson/torproxy`) were removed from +`services/infra/docker-compose.data-stack.yml` in the same change. -## API +**Do not reintroduce** WAF/bot-detection bypass, stealth or undetected browsers, +proxy/fingerprint rotation to evade bans, scraping behind login walls, or +automated fake-account creation. Compliant sourcing only: official/bulk APIs → +authorized fetches of clearly public pages (self-identifying, `robots.txt` + +rate limits respected) → licensed partner data → otherwise the field is +delivered as *"not available / licensed only"*. -b0ts runs on port 4115: -- `POST /pipeline/start` — Start continuous scraping -- `GET /pipeline/status` — Check progress -- `GET /analytics/quality` — Data quality metrics +The compliant source plan lives in `services/api/workers/adaptive_router.py`. -## MCP +## Retained: `output/` -MCP tools available at `schemas/mcp/b0ts.mcp.json`. +`output/brreg_page_*.json` (50 files) is **data, not capability** — records from +Norway's official open Enhetsregisteret API (`data.brreg.no`), which is openly +licensed and needs no bypass to read. It was left in place deliberately rather +than deleted as part of a capability removal. -## Deployment - -```bash -# On SIAX Cloud Server 2 -ssh -i ~/.ssh/id_ed25519_siax root@188.40.230.69 -cd /opt/b0ts && python3 -m src.main -``` +⚠️ **Open question for the owner:** this data was fetched *through* the removed +engine (`METHOD_RANKING["no-brreg-api"]` routed even this official API through +`cloudscraper` / `curl_cffi` impersonation), so its provenance chain is tainted +even though the content itself is public open data. Decide separately whether to +re-fetch it via a plain compliant client or drop it. diff --git a/services/scrapers/__init__.py b/services/scrapers/__init__.py deleted file mode 100644 index ace5d7e..0000000 --- a/services/scrapers/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# 0pt Scraper Engine v2 - Production scrapers package diff --git a/services/scrapers/__pycache__/engine.cpython-314.pyc b/services/scrapers/__pycache__/engine.cpython-314.pyc deleted file mode 100644 index 250f5486a23f947f7bdca56c95178476006a75d8..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 15857 zcmch83vg7|ndZ6Ox8ETd46;?bE& zb>i8H94E1Gyg`o1qBx!)raTp9LRH9PD%kPVcxp1W!h%7*A!W}dRVmljZV5wzGn1;_ z@4vTSl4YAs?dC}RpL726e*Ld=?pn9Y&fs2uv;Uc&)iL%film(*6}TVe7)vsdDeO2C zxxJjiX}&=*XueS~YCf;+V@{~OI=DqoQJW4?-txzd^+Fiq*E^0%c$n~z#{ah$o8q<9T zS}75&UMAKa;zXNh7agKgbct@!Bj$5wA+ctHo8~ za>T1saRK=S#A{OV8j-7G4XXq1Vu1E_aYg70N`ZJt61t-&!qJd$dcA%7QqF!j8a^G8 zLI~+ahGUY@wZFONP;*ze5R8q+1JQHBP(TTI?T6)%ER2ptlyK8%NEwL@ z34`b20a+G;BY|*KSSyT#0z+tAiYWml9E%E6*J$8ucyw%3kmI4ykk@WM7*b-CX?0kL zOW~LlRziv#lZK+`L&FxkAhdXe;FuKg2Zx8ls9B1QhFaFQ2~9%(fj(gvxk3^OMin7E zijm5(sMZz=uFowP8aff$ycwke5hWy1p3|Y+(#YS6{E^t$kTwJ8QIjwjki$V?=dRSa zkf+TnNTE|>Az6`^7+;ex6qZ6kMcC8d-;3NEyuyg0#Lo%=`CK%J3P-{xMlgv{3`hz^ zgXfTEqgU7!2}q&7SOhCTtzy|EmkP;pZQTR}t zW;YONFse3MnP3Eqpqhrmp~#SG4#-2{pn`mSZ$KHL{OJ*>=Gd4LA5&Dvm>iP)ffJ#q zBCEDYAbMg95m_~K?N4B>o4WQl4h6QJ+P==axiJ*g0-G9#LRz3*HS`=xG(-oRdJZ*4 z2UEGCu~e>9_V#oI8&PRDDh)Mtb*HPysdQLMSK8N|D3wn)^>sJOr&E=tn-1ol*VK_? z_cs9>0I7MJDICa+;drxREMYMa9fcPSiU~inkF|255r@U9SVUg2`j}!94LC9;)u|s9 zF(hL{@9GH>iAa$N#v^RR@BViIwsA@3W5*4f{(`X~U@O@Z+z8jq`nadqR)fgZBTp)9 z$O#)$VGeotRM>#9DF<&(g^j>lQehr-nHtMcY04qZnv=#BrP(MABeH)$Goza2*q9Uy zsYVSPvLdNQ z95d{zNGzbJ{9r5=k!V(sl{T0`HRF7bBFW|VYg>@ee!oops7)vqT_taBQZrhS696B5 ziG7j%o73}!{r60a--Sbd5 zkL4HN1`73eUHF)q9h#zE?YjU!paZDhN~psQEdg8y<@WQ9&_glSpaM zA5au2JUE6$0~$Fq^BoLDB7T1ZC)J@2nKtm_(h90GKMk~rszMACm1OgItgI@zZ?2*` z*+roHA&ul&Cap^6*G>W=nkxg;PQu?IkYoco7O*#c4l@`Awm872qvC>nnr6UpE++?< zyOVn@>5^IB2f-lCV&ewU=;K6QG>K*(PK#*$ID-}o#6q!1EEY?|Qn9SriBqvmB9TQR z4xey`qD_bU(gbaDg7}%#)In5GHFpGqp}3Mbtc1=g%_GWaq%jbWN5Vmkxip_8%-XYy zv7?a`r3#ONel`bAhld;W;F-{1ym3wQ8l5U8JbLv_2U4UqgXoS1V?*F+iTo3ZaJ&(8 zayWvW-zW@93D?d*a3s{UGZs~(SR`SM#+rhJN!a>A!KR%fO@{-Cj@q5XA;V*%wOdbb zuLYy5Z4_#E$6_ZUAwf$D3@8>H8^u)RrW^z(SFKTu`vf|jo6-liuJX}XbODwo9JRM3mK~0tkRU5G z?TyMIFvn1nnBuZ3*V4eNX6j*VR5b@jVjvW%V^4QSSGVZzJ#^@RYCI8A!1^q~*cfK! zoN5>gs5~(-nRaTuAZQ0bb!%cs$gc|}va}tE+whZ510>m9o9Bh9FITyw>#9VHW=yX~mE<5+dmv1-~f)6y~P=uCFKZ!MY_oU`W56oluj%O=|ZW-7dM*3!xH zIcwoe@zFVJ`DE9ewQO=t2Gmn%#hkTns&9Uo%W0lxE|Xb1WJ`0Jby}xB&Syj}d*(#L zK<3mn6Bjc|g(#--v}rMCA!SU8HW(y2?2-TgvQP_8BMY@USy)MF1P)OGk)R}d$jsb^ zdju}F&Xf0An-bil3L6Z}3l1g=!cs=+wHfH`8o zIJ}131E0szQu8h+V?=$(oj~oCY(LKHa%ci|%tk(62h#E=HKoZ6rpEm5U~WxkAlpJn zTtL#O|E>7lZv@!J)uNY4*3Wj?7(12UXV?tvS(Y?aYrW9g2YcOM40Bhw9%(gFvG)w# zbt)H@9z!g(>5}@Uvuy)837w5gJJdQH)!<2pv1u0T-3C@0h{!v#p>hyB5kh=16xoJ^Erc2e1zAZ&(*Q=~58hAkT_3g6!Pv6WJ)8n0_-%J0M#Cjg7XKIm?Uewe>imj^A=BlbP(CK zeMjX!O0o|Pe1SQ+@1Y$m`1#gb?zX8l(}(}HwsqFs_Co7dT4(sSFB2GFm^ABX15)3d zs3+sFfcobAH39)hZP`tk-d;m)+Fn`wLM~OuvgfOp=6;cb+J{Xn8g(wzY!of+DTy)0 zfghEKHf(gR)d0n@M0BLWPJ~^lup41dDx4>l_&6Mne8jCj+UUQI4Qkhh19E^lkP_>-F0%1cy9MtIhE|7&@RFNoLLeC0j zC=7BCPbLa;e-A+Jd@jxX!LQi>kqFKCxb$&6?h(26)O@1X2Gy2P>#i78M^=vusg@MA zRxPCLKwUuwm)6a+5+l(8)rk2OgG5J1<$~I_tFPCOpmkd$FgiFC*r6`>gH!lZGn8}s zQt2TSzP}tG$*vu|)7Je?TlbvPGx^jt?~VM~^*i4de{1A6Ji%!jEs>Xv~<#Nj=ui+>w($k$KS5SaMt&vhV${I zZ&~*z8Bu!aTh=|&pu#k|00BC}iGu7Z$bsR26mDt%ll0o2Zq5H_-9DMK7TfQ%cD>Wu zMQbr|ZO4t8+19SN2eB4yqP`ZYGuH~SU^r$}qn*DU2#_AyXXs7qsKBHhr~@%I%`G~0 zDSREfV)(#=44~P^;7L#F;mlcrC_!gQIu4+k^bS$k`g)zQWw+TVuyQp3==b#D?;X2x z3?sX^e43kGw&rT*ypeg^A280lNn0>AFMVwNx->Sl^!Rq6zDJ=ipGaLYLU&0RCXY*hrSp7>+qt1 zH0uMo!by;~NQiN#Nuys=E2I&iRd-Mdk?Nh&%&}jHVM`IJoLGbKxK3rAxwKUpLke*y z`6>W5#(S3HXFq?}Q#BdB?O8ozUHxlk*&XMqcbu!{^2;apOg(e8;rh`VV>5XN9~ccT zbFzEh&g|9q8CUZJH_tem$h~i?Ky-PJ7CnUM@`b5Yhzg>X+K1HA-!ZOy6BXT@Dw&Ry zA!CIenXmA8%*oyJwXA0KU-V3DzR6e3ISP_Jx+Fp8Dt)#xk^~)xrK~@Tq;llaooLKT z5F($|bqDAqYc5v=ZiZmnVA5oG{q$`?7S*XUEI*ib=&bZOPz}5;(i3rqrW1x~hP3-n zlkU$UU2Xy(>AsdLyhq?A<2-p6gPQjfJ@!`r&l_)dAJH=CGm)VsD4vHO;{5%eWASl9 z(=(J2-s46e4EvjY0^1lg?MilJeeMXoT-@koao%UlW4k_J>n+Lkbax7 zxTPY;hOo2@mR1|$R-d_s9pN$-0$X;zKfR>S+DKM`l#AO3NY2sR0Ij^{eAtBPnu+z0 z8!Dm^`fR;9rS%jBOGWOu55nr09$DNm;7aH6IWkyhKi^9fd665SmDOB7l#+fES}RCn zvi<-OYt8xW$BpTJen#(0G}*T_Q*z01r2BUe*ET+<$hW8FCXKI#y0rf{OjFGE{}0{w zVIK=@fNVvYGqa)n)K=WpZ#m9wI*OTtv9^+Z)`*W`yvK0`bev^Zq~DcpSzKTh70n;- zSrrp4ao1z6Zf-yHw47Ko)<;VTb78jCSjECzgEi5u+ZKg@a9BJboPl`~mjw}FH7+f1 zSrH_as6j|ek5GO{yOsqyenJd7j%GNPy<~@ol7{r7o^T#^N{`B}SS0Kerk`odI+JM9 zC7K_JIE`lc`%%vaZzJ6EnVBmoSA$EVv8oA%krS}2kaVmO1xU;RnXH3Sf@&rW3s(S8 zdZR-ckx(7F?bi=GQwYUjGZ|8i(C<`R_gOM|L9uIaXUUIrhN>Z&uUb+=gdGRAP1P_Q zRV`>KG#ZCd7_~}@%1PR&R0mlg{BmG49)S#>HYsx<8TuAv2-svYPG=wH6n}XU=^CwS z1!YOHU%Shnk0y7|Sqdjk-LjO;6_(#FTs>3JFm8F@otNDGi=v9>tmFLM(#pvvzHxSZ z_q?IhReHCq@{;YFwpSbzrgw`#P-Q_jc&Rjor zqv1~bfp^*uOx0X&_;$neF6f#wV*l*&!xN6XFl&C{hXpLZ+c3HyLQSy-S#ijQN{LaXVg#VZ~qJn>yX;XcF zXI__^f6MLYuCu(gti5}=9ZKqe??xo9)K8=g&Q^8)8I>e^P2w}3{Zr<)~Mrkj+CpD67AJEl3m*|S&o#$2@F0% z9y@FtGkDpUVb@w1c3PLxSdYI=v?*nB0!8&1m*N{4>W@}onrfhYJ%;^Yu4EjF^WcDd zCJ!h&>sLEG*1B;7HeV1V%8OGLro}P5J6Dx@!F5 z#xQHJBsTU-=QIkIqFIfgQAD!@Wdv8B0U4JEiXe}{&@IcuW04dE^DZ!s5r5Y-4&5^7 zW4-AvOvHfB<((UUxTL^8Fdw0Qx?=J{fPSq(d@^S@9&_*sI61 zmA(SY(#g=dKe|g0i8NAsVTle&lR*9zC8U}STpoFKL~B+enIcit6HAkOA#genjsylH zAt{NZ2D3zjO0Sz{Mm3IxqX2=kx+G@GIStQIPMQ(b5IZT8IYgs>I^A>ROo}vxH0?Z4 z5h8v?6MeJp+T`wciz{bs-RF-?aF?y$wodQ-w&yD~*SViM|K9mR&3MiEBgxJgTlaew z+g(q=#QK{R-dmpL8Ef-zTDN?qb^OfDO7AU4^E_jwQNvr#nf@a);n>XCb3k+%x()CS z8hXg{86t)UjCJ6&>6w53p^e#haWY9I69t`&zw80{u^C_iC7~UwQ4&S#_NUvdv+{7R ze3Wa+hRn>PBF#Z&P6yd%%FHT{J5E{Bgl0$1CA;)E>Hw8G3ok1;f2PPhjqGwO08ysp z90(A=#s1gozIAM#g4YT)@8@-zck9Sgw+2RT9|>#OJ}x-Z9($Gq=ZiQ}0WRPOI0G)- z)~vl(VS6AN?=i&rJ;pd}%|0HYY;BfM#LYev%#B$g(&95B#|a2zDN?eP^Mk;R8T|&@ zyySLsqUjkd%vmmJJYcf3n8W9QiP0dM$;23Q4p5&ox0C=23wuTwkGXtKp9>`|R1$(V z^nk;*6gdR}J~fHlwzOadVhD{=BZl_4EvFUIGZ#6Ov)3@5?Z&-9ZePbR7LYdlwb9ms z+l_c`j%pY@E$l!fMe=N6idEs~7T8)nH@1;Jb_=s-Zxk?Jl`ITGSq0IR{zOwQd!ax(I_R0l&PW*2pK3l39u4C; zg{5CNT*Az`4D`W&G8BpjB6xiQ2`n1LV-W~kvg)Cv9K;EI(B8-CT0NrGq}gcncL{nT zV>F>($M1e62DQzgZ>Q{u=}qU$@oTc08dt&k%)P1U&B;R6Wg=6W>$9YnRb=U{hQ;D? zmm+R>)HZ$G0!_=9!x;^|neCWvd$AUk?n#_qzzLyo_Aart28eEH&S%Wgi8A^WN-fc1 zx{U$idU|O-jmv63+SdDX=r~2`T+ne~w1SRf%J%IT4lHz+be1J~XLddM7tGB9oyU~v z!6)cEV+Lq9T-@{+wzBSLH5+YHTfhmAp(8!g=KvFqoSiHEIWW@a2_yj;9MTslyqUmu z0-7ABX%jSe(tjrKmju2Z;5%Ot;HwMT#S8J7pc-~czQ_E{Zm&31yzjN~H%G>qr*H8W2 z_;$xXo8MOc#reQwEGzgeGv*gSW=EMn$lVyxvn5?l?wi$}o7oS!&UXGs9orCmb0Y`r zo9!HhH#@p)#-H(9x_INyc5oDa%!JgpcoV{J*^CG;kdtV0YsBFv2&9QapJ7Qp!E%pt zHJ&fg{c3vC;kYkI*Jv|vBFmx83v#4rb3LeW8Gg1DMP|=DDn$lu*VS^`yqa*Cx@pz7 z*xz78c##wu)h}DqJIFFf+AQ8KrA5s}dnlFs21?6b0Bsk&S6Y5Seo=n6s&=aNl_&3% zu6k%>Wfd3Xm*jUd;S2KDjICvCQ^{iR-Ys-!R)LoNpjlXhULyMLLB_rN1KZT>_T@pj8te z3CC1C>=+HiM`F0HHRDCa2wwlWrB^8*2|3bt2rMis{Wam9B=ENc=tQa}B_N-arPm2T zQzu;o&{gr_j4J;4<7rE7?g5m3fUMJ${l5VG2i;2lhF^b^U;nPFaQfse*OBC|ciqJk zL(h*UcmJEGe5(GAXAAysd$wF3{++>ND|k@LJSBJB)$h2gryR5H#&P5C3VR^y*2p&Woj(?Txq9)XL_dGcQG z(qbQ1l*w!eX3yVmL_c7lPirFJYV&oxo=Vg-nx7bOBH1z(r1U15cXtA}4x)r&V9%#5^C@Y$FY! zmss^8H$b-pn(Gx(fgBHX-hA93=8FaGDM65)%;l^*1d4{@L`5H7AnQ-VLn%9fpv3Un zSw89Qhlpy4<4FlU9=3+g!XE06om9DImGi2cKjB(=Y~|?6A^*xfD-W*hOPIXR;ElFw zBa0=BhNE$vq<@lGRC^1l9gbZj3ezPBgllIgD&v)yKOhIg;e=^S8E)D{FPrhET6;d% zkgwVg9q#Wv-0$z&E2^f^lX#e&B8D=a@hN1?#Dj=nNTNfj8l$l@sulMr!-n0oVh0_Z`0ICSNq!KixIM7v17p=3Ip@Jo(&{cU)CB zT~(98Sy$~;%Wc=nJFcdiuBPeW)z(>8dve!ZTmB1`U#@(9`5l{Z(DlSFCF^A zp_jTY?0a$FeMZ%;&bd<)4L%U8Pe{Y-*H292KUQbza5HwKAVYACbPIQw&#{VjEk)!%W)cQ4OK7E{2WfdoqaCrx ziTOw?@L8P9XGOc2yFuKr!7NAy6`v{7YbXt{ER(pPIiHE%ie*VR3@_O|R=o5bCD>pI zQkLpvDSA?dv?Y3pQp?t7tO5`XY|uH&Gqz_Qw&%)3>#i|eoehE>kBkPSlXO8A5Xfc% zYl{9wLiU0!eWjK#;&YgUO%V2X95~RiMMzi#vb-Q*5Cqk5Xn%t57DXXp5}w+*>uKrF zFz$pA-^(O;x=d1yt?Oh2n|2`BvQAbxBy9Ay4krvUN^crcxxoec5ls&%`S*1}3cOXD z$b}$f+Lpcnw6v5JS*^_s(ytFJtFxvbp3PI?mkH47N#6uWcraE$*s)`W(Am+~uZ?ka z!bsgU%8eNDU_(!0+ftuuJu0rjrkb?xq86E|b%v^OIFd2+3&?LF)L@qul0@k_O8**x zO8}}pb#q8xf|&7)m3#}n577cVne}Co;Ih|yI!nu2JVDZbM=p(+_FxT&ui3`CCQ2r( zm;xu(Ud{12Re>%87K)7){R@J9bXKKaIzHxA77iZkxMTh{(LPvN-ZU5DrQ zA1>33^(243VJ+xh%fC_WA=qjLY%nKc2V;qFBob)e;9V!IKB7MX6?*!GmUZ5BTLmCC zwrv&8Zft81I`E}K=tyXAe^_bW(Aw^8-6*Wyzo-A;fkxbRp9~4R@wPbDkougU8Q&Oq zQ3Y!4)1Kg^iv0g<5JOm^L21DV4+etxS}7)vY!&w6K0YDE@<;pW_7C|o!-*2zTLa7L7%_o_3K*J;lD-L zg*Vj0v9p*+q$WZOYCPJ~vQ;<`j*gub&TiW1|9`hS`xi75?j6CW8C!*&y@v%oOjFg; z?%$xVczW)Zo;CsAIl_6%-}iAM)i~;lnL> d2lj9uM_26+i*UL1u#N0U|G1wpPDq`8_uk*y zXI>Hl)9>Ef@9TzHXYI8gXYYOXUTf{O*Va~t-NGR}|4z?S`E?xkPZUW9a|*frKQtUS z#)+K5o#aIR5U=p;sZliSsa3S>sZ(_9saN#uX;2L8X;h4OY7Uv&O^RtP)uBCPZnr2F zhU*Sl+ii-C;rc`Nc8B6XX1O6dt*SADt*+2=Y=G>PU$^n`-4@}9XYqBWCi)jf0Ba&nd3GnZX-Wa?Ud!m^IL zXHKW+O6P3kQ*B&x-x7IZ{#_;3o)GTyjsmgpt`h6SqH0duc9a*3#S*bpTp_L$SBYgc z+8T|rzJ?Q5BX#>xzMmJ@0NcT^wZNJfCIH*Xuyw$8F>F1s-3%)Srk2={$@dVjJuG!& zCUq0Ay^OXQ*gl3;0DG8WTYx>nu&uzF8CD5wKVeGw2}co!Z(CKuiPf3jsRPI<8^ji| zri2Ue%0pr;a?}BLDH~addc+&jC6rBKV+p5h7Ws0nqBZtyd;&{tR1vy;k~bQVgtqWt zFdPt0)eBWZmlW~Gd`d79w(Na?VCi`0&m~!qBO^j<#5at}e37u?MRk%8edpUk_o3!v zLQAMN<`=pH(kW2ut1Ycw#VdrogOXPkg2Itt_|Q?IyFu72H&n|(?`U-}QXveWULn#S z7?j@mR#^7>S^i*nu-anjR^It@|5E{Jm!(Q*4u!lzYe!Az(VEsaVfdYIg+dWY7zy|V zZ%m0)`2$LTbw==>68hiymW&sV1eAy$McV^jKiVTj6fbpG*edv9Qpn>Q7zheIhr5LV zye%L_rC?YQ`p-nYvP|!j&xC!{N92_PVc!|Gp(EbWA+M|i2ZsU*a)|*sHWH{B_C~$^ z!B9{MD8lAp^wu||yz}i)aIgY-_InlIkPww3zJM&_Rjkpzp@45V8o?_;v2<4ONt54; z;=z#sa9x*I86w)zD7Dxds?a9QNcZ|eXkAi25DbL;NrPAR2YpI~CYj$eB%zaCkw~a* zG~mO4NJ*Pl_V^+r(GW()PjyLwr(*$Gk(1W$22WdOYuC|3ojvjVAw`MGyJ~8(-y~^A zhUS)*w(f3EN89lv#|YhREn-^_BbbrCzqh5Mttak^M3pL;O)q~HwJ>Qp+S}9B+v90H zB*twuM5>9zlxR#zI$GM>S~}QkS!$k|njH!`={(YW%+qqTv!z!Q+d5m0CpFt@lh&9V zkUZYOKp2fThrHpz7$R~~)7lXi{NAe8j!M6G_tSf8t9Mog!VGDu^amKSBdO^;8m|cV zS9Kn(4ELvUg(In4sq8z_6>LYPZKyO*)!LS>BB#;;Nv$+)3rDK@F)s$IywPCXgc!c> zz(Cvxtgfl4j%65(g#vOAQ;)?4LXp!0L21MrH~WXHe5bG?Q4(mXBrwdJ;kW@vU0qY$ ziszsd3;TnEO5BDPg(H$OMB^FHM{LL&_6I^Tnh=x~yuuFJNF*FMGZeva#Y+&4g^>^r z42=ZB*?dkU_@sb8s7Qfnm5HqEtYdF(-{%l4hbb^uqiRjYd)no{=Bz65*t`dy`va0LYKwkt& zc8`axbRv5^GQEHOShA^QS6?rdg4`zu@u~X;gUV2>zYh$)57bjZ@RvSyDI!aNv3k#` z`s(PJq|M{;hQq}F0v=CtMeawb&P_Ui7s+9OU;PgEo7_K|ZJ%$yrRRA5mVvXoKHqW6 zNVrMG&4gPxN5SViZ&?YqaW?1YAH8KK+`;7)UD1`@a#F;_SqsiJe!2RJZtbl+BII+n z!gDQO-V4IV1t8qU8@Y>Ig(Ye7c*5QhYNW;E8G(Qb5sv0R9rK1#DNc_E>qu5YkUimu z$0L=aw6uZ1LjcLVoE{Ct!ah_`gh3?c@AG&SMGE%E@U1{b&df0T0-=z{bCH)SQHM-( z^bx6w>RgouY;aa1hQS%*793pZ>aj=XSFIguCD3+UiBR)IC~zWFG7(CaSfnabH8w1Q6;dJUlZpW1jzGAow>!;hcg8I=DXW@^ zKPL^%K3^cJ#Cw&%s8TbejD#w^(P${>W1_calqg$Amr_SUDaITe0cWrAo(c|Bs_5xJ zf3$K-%@&nk$DL~RRfkhTAVX*i`yzgb(sD(7Zfz4iT}O`|PHG1O3KR>YF9Jy+ok?n9-lUGit4uR>qad&akaRG`Dc~7m zYLqM;M52hFdd5xcY_FUwvkB?`%okjJbYn|4nQ0*=J_0<#X0eSFD?+jETDD z8SDPB)(=gEXZz<(u0(!t-n3$(5g@Uudfrqrv2xy2kSIDfZ(2FgI&UhS*pdMo3EMDl zDxd6LSYfvr7C5`!z!ur^!i1LJa-u}!vujS&^kr6E4GAi(MC&@Kqe+V~3n(!uT%nOj z07w-86}+?+55^YOt8Af~(g*}7fuzb9cZbvR+inr~M#BOtc2wBr;uA!r=gKxldy6X0zn1IB=hXLOAF@pw*ttwt7SIzUoY-A*z75>#sgS(-Mxxiz_Fa&$dn!eXZlV zbM2q3{1=(}@P*Pa5`@wQh8K}!Cb|bw z3(2*#JCYmO+5Djo`l~VSV-26*!jC=rbF1S^4cDkIGmgd=8-BMTp=*4Bz<7o_V{JrS zomnWH2{fxH-2Q8TMVi}iuI7eaqCTs)i8{8UG>C`i%F7FFq_+SaRFkV%&0 z%B2I4RpL)0uS_Z#xW*mMp)b5e;8iVqs141`R^*~a;%B@4hd_ucKn3KRj$yB5AgKYx zPlHv6MuM(gL?_Om()Ekz#2r++aS@&PgGx6uI;^i$f0kFIQmu$$m2O;MO4WH=Ft&OG^a;+f}WH$Bh#B#{oYs1x8S>H?&V z0x`yJO&74Gp^Ny^lHk7gQcUG8=PqhKa-tZxC-FgPlyI#yNO)$Ifk3NFqA4W}1E^}| z#uS6iOu65rBxu>hNhg)p;;+VEd;Znur+x44ONiYwEj@{oPfk96ZIf5+DGBoBdwKvV z55j8h$!IdYlU_hoXg<9w+n@Wan%_c3rUQcqjK22lH=mtud3Q~s>#><;G0}H&c9Vy- zy%WX_Qj}Ey7_~G z=MuK(=IwcjqD|NAn?5LrCv0&R-Ez&oB`3Awn!VzKg69*q=aG8$^W1P2u!b{jE-i9Rcqfh2?RtRwiCj@bf>+csNVY$|Mh7OYXF^z=8m*`z z=~OwZ2Rr3q&v4+3^eCv9StOpB@#nrWS|UZ{ckq*Gabqy!p#qh0Zc`UF!b^jw&9#difz2z^PfH)M4;I{( z3>BM_E?1hKeTw?sFfZrUb&{{lQR8FU7&A$u71;JJ4JR)%MdvH5oz`@_2fxJP%fHX? zfbUzrv~g+aJ~G;|u>3fs`@dYD`byGeA1_+Ie|;;`Io#%x+Vs48(0976qG?BJPQ^59 zVQzP4#>iiOpD{8o?d9^{q>b+w!#Y`hq%03y$2V1TMs_ryCfiQJ1pu2pSWVMCDCY#O zk4!-<7&t#{$z@#B*7JY${?*t}Mr{w<(3gLP<&EJ%KFICXuI6A`amH(Vq%(Bb2^|^7 ztYLvT8L7y`0)ce|))TuEmaHS3}fC& zy#(u~KWT0oCA%~ZQ!DJM2&sx+Sv4@V2N*62k*_7WwS@~JWxH0Z$`NCZOhAa6S zCOzK^zj=Ii`_Y;Fu5shfR;?fJn9nPk%UgFPZ{6h9nY`-p{XZ!vy5TBa&=lBL+$=4- zV1C*BiuJ7iMq%+;d8Tl~S^nnAH5V#gu6SkZlzw())mh`sHS4~2`qI;HR?O`J?AQ$)J`3KGw*V9 zqSu``DP42Q36uP*Pt085>Q6Y0eZ_pLPfc?!YO$+g#B?v=S!!BKO4k81`G%C4Tmyrnrjh0i z0*w9m#v~16+NddN3~~auHEZTa@5;C!Qdc7<{p&&Qlq}H3x2qgR@CyCR?IbcF^oob& ztRssOIwQoG8;ZhwMuaF%p{rqz`pLHv>a5w-)iqjD~ z%dfDiT=YLt2AUeupAeu*l14QP95e!>DNBbZX&i{*R0T)m{D4e@L!2_jDpfvdOL0Ti z?xm)$+%z^DiD6>rkzY7&ylG!M)iGypnzc7w+JD32yH@PK;_+SS{am8=*jefPnR90* z3twCN>e}h@H!U+I9hbYWmUJP*_`xg1ei#rZH(f8xuf(sr zw%pQk#ar*_ImgZg6Xz-!cl^t3BUcjO?`T1~Zf{CaKb6bSmh*=F7X2So?AKY|GBg5x zTStVqE!O?3%!}O~Xf)|d-%f9h?{<4oZH8F+cXJ4s=Rj9E1o4JNI&SU;C-Y&F)*L4$ zl^y7$KScxn&1~S$P)Obez}SGGUv-NExY+Z0`JWzNAoMNlZ}7PC#0m8o9JvyX&jz1i z`R?B9&`kRRt^2>hoc3zG4zJVeN*yt2ck{%cC=6o#(Ggg7uM<@rHTiZ;RM)PJ>f3cu z1FahDJsLz!+K{g9Mnr9Ty+uuK9de9{x^xck6X@>XCz|xyJjjdsr|@+~`7!Nby@iW7 z+zz*<0Zt?^P?{pn978^FttA0RZJD=4TyCe^g_7*pBx+`S$`aF56PLnX=jn^co!NVI zYC2FK<2lx-CFd=fqk!zIP|muJ({Ts6#chq++&1tf7&fBzduY6-2h(`n_IvMSQpb## z2KRW!18U~?D1Dbsj<#Trw=^ubSLyf-bp(5Cv?GIm1OBCX>OsVFzboxDjNIW{jLT}G zmLSExiJyabEz1pgpw0p#XSome6*JtI9dugb>yN-iFL){dK8s@$IwlMWUgk{5m>3lE zY8-!BnSWtRD9D^94hO+(;Z4+|+KypqUO_T^I1q?>L#PW%Z8)s@H^N7bl5z-x>fr2q z1jP&?Rui*`sca`ba-^M-#(aAv7TX?;y3Ddmsw`)}GaENG~-y zL)$@24gu3tlRNZL&4a$x%`Y+vXmbQH>^g2u7cr0}A;(;x&DoAp?^!^pWtLRB-F}yL*q-)~v!H~@CwC^T zY$AKeWpq^fDn>~98o;j3k1pfUDV+y|V(@iJXd!Tr0CS*vg|I&$@C^dlz1v2L{vkkx zQQc~OlL&uAU>yOH_HbWe3xz2fGQKC}Hu;z|F+Q!{GfBwawNxU(UNG_9onq z&xpqo$4|_PCu!UCB>xL+o1Wx7sMh3so@JhqK%AF|k1!r>^{}R8`^c5|T-?u%i|0ciN^her<^?z&! z;k*1^!}kn8e!xGh|0^EI4-L&L%x~%WW|QMBgN5)s4Z#945|{&0)&o@JE!_kBfRYy1C%sKHI-!yN zD*-aQU?1w0hh^!zL|{w(ZwaGMApJdnOl)}L#(}IUMR??qN7QXX{@v?3nZDnUI;OZg z2h&{tH!A%#ISGAOyl$?zF;U$3Ezh|1C#LOJOxthdl}vBFmiM`F!;SnE6C1u#HE#UJ zf;E!^a|QeGf3{%Xn;UfvC9jvv7B<2Bam&mVuU@clg{$r~aD@+}wbu&v{maK2xT1y|#r1bKsWgyh z3ikc0EMp9QyP0oW$G>IdTX&k?GHD67Rt8@c(I0HsTrw@Oe9#Y z>IR5d|7V`y@Hq~8(sxh-7a6>#J>**rLB(VX)$`ehG*^&Iru@HJqW5W>|nVQ7;-mCTh61}oHG;Rwo5`*DO=9Q zkaNbNb8Bs+Ch^Oijj$9QpuXM!YfDBYa_cK|EEiFOXz0<2#^rl0!WD73x<%}DmOV$Y zfKH5ONf51|XVjx?qG$uXI2%u6q8)TP$XpWOWTM%WqWQ7I!nu$}bmpo-I0JIkz^0S6 z(RY@|-C8Us?2is@DgF0QqS`L2I_?MP#*sU7qwBWvW z9pp&ab&BH;b$d?*;N3>H7vVG<|6$P?i6So3jV2``_=ECrb&{`1^3~FRK}+KHhmJoq z@{r&2Q2Rqi9_o&ps-J?ZO7&noX(n4bjO8d!eUR>H+b64h(n>2WW$tDwUuRNtTKQBT zQwFMjQ|`aMJ`v9uF=+1%nU|Dzlg2|E-W!2p(y^fP{qvNgIxwPMUBf zVt`rpSsPr`8i4@Q{7A&8hYYMCI&$nz>LRd0C3Vyn+?t4uL@Ugy5F*wf%>g70aCnBB zU{XW3h2ZoMU{?x~x+r8Axw4(X?F+xev_G=NEHOh_WehZQUK;47=ifO=jGNp+afp+= zlJhIitxOr_%V+X7!d#wLF_%|&C9iI}d|IB#+dXc$X)k)Q?}ffEot(38xMGKKf5yIT zTr*#?`s~nzchc~h{Z;$KZ%lcoHB-uTQ=)LsxcR1Y#l)dm=R@PV4{aqA`4hd92VQ&h z)kmjImp1;e>ibpmw!*Um=SR+sOdg*$yy1A=@uu-|@%v@(mCf5qCpONl+S8cu2vtZU2SjLT?*f>Z2i^pF$epZ?2nL0RUZ=AI^UgB@s zTrVDa;n3N^so1P-8?+S1whx^J<5seSZM$x7WRXv0GJm|QU(vc*|L$gM>rV3`7H>cz zanakA0H4j`Rhw1~oRuqds%6{66n0M#2RE5JY)p%RTLiIt)qq8-{j(YWKO&b*SsCNk z>$`9|Pw=94fjz&kQ=e~jtMt1KRbVF69oy}H29kRyFY<04OjJ{^UBotZ>J^K=j39Aq z(rT}$cf+SNdwJvj2kOfimmARr3Si5c8a)p6b;}8YKKec^EVthUi__D^kF+6-E`X$Q zuqwLnVUKTMEC;@|gT!V8=Df^qEDG={WvmFkc(5*JZZdj4)iWj|-NuWhPY8Gjd_aKg zkq0FD7ScZu_z3{KNSPa`x+f&QApJLhq&X-@Qa7P2rzPBg@X(o?^bw`$Y0=BDQx8-2 zC2#tlMB4x$(+pHC%&OYxWNSCu*5v9X`aG@rY%-fOUetl}IOpogty4o&PfW)W!rocu z-f`W$Yvn}y)XGRQ%?O1ugG7 z%wqwYY%e;$==_H6g5_n)WaD(;TIrrQPmepVn~tipjmN9#M?r@lTRb$;KYQ`y>PKui z9z`*%3Mx$-8r%=PkB=$yU-|FInpKSq`l4GL0Id%^UWZR99%?)38>Isc@xNtNXkban ztcY|aT;?bzBf9sK=>Dt~g1U%#FA3zeojxo|uQ(`F8I{JZUDl&oU(Kb}uXL$$E?X+S z;^^QlzTq&W2NS;RVy$|*G60Y3oCQ)X{Spn6NIYaGJ6i!Ta?suep4Q|@EvKe52_(oW z@?BSF`v0PmEdbz$*?FFw-84n$^dvR6seI1lT{^dyB(twgHxHJd+g9g`yT7>m#l0`= zy;)Fle%DL8Cj1w|Uk^_YPK0OHU>EW5cI%7m@*rZfLb zv2j~=LgN)I(D1W~vJukp40XvF&ApdtIznL zLHy)np%ui}cd-lk@Rp^!2LjvFlUED7$cY>HKKQXP@$w|TP|^ajq9l~a&N&;^=Pzetil@60eMcgC^B3!t432|+8?LgcK(e`wWEIcc;Li0r( z{m2_6u8j1$fw)#UvcD5>4JevvtHG?Rd|VG-$rqpa;u90PS^H`zlFq#IPn>(={K<1CCl61r zn<;9%!|}zPFf9_iX>*?CzuJ2K&`XED(lNQ=wJoo1d9C`@>glq-RDO8oFV6hnnVIz+ zvxOZqwj;lQ#5xLz^)r+Gta)PnMB&6^6S0ZG$=<2PKihS2*K}xhYfEBF>zi$tH@v^) zy)B8}6SI$=OdRqgJpP0uaLqJu69r8u=9{og)=Y1BW6SGXE}fXI@3`DLTi=y^aRAQ&}cNB3+^>|1P z1c+BlIGlnV%j3aSOI!~Q5$X2`yhh*>folY4Xr*5h&_eb~76Lmc+C*R{fJ`!3H9Sej zh)7_V68r${9`Fra-N!nke%GCRm=TyQ;f_EapdT~O&pZ3z9q}=_NZiTC$>_&)JbI^< zN7j$Y>~KdQMO0Nk?>Hc3nZ&*0GU}fMl_2A$?6p3;F%w9R((g9yU!jpwJ6VAFG=*a! z@mNTj*a~o6iy-{;YhV58*UtWj{ePZy;T+lhv!S-DN9b-5o4eXXp{?`aq0Y9{-!>7C zw)VF496H*W%?s?OUpu#?mdqa%X~Aw9SNPbP7rKJc0NpMZe)`)lW&dhPwyJbJFaL)f zP0WKoz)%PR{_==qqKRLJ>7!1zDqu;Q%7WBeEs_&CJ_T8J0^r(}&Wt&0HT@F%Co?Mq z#VB$l5)a~!wA5^?t`#;vu3kA4I(vk=+UnZf0tnk1cMGH28!Lon+z1am9_a4~DmB|0 zc2qZP7dCga_Z&G~DTIQ<0pTF7W8yEvsMoG*aJi=%RiM^xc3&b@DRC@t#Y z5w8z_%poEV?G_Hfk1`~HARO%$jtO;*o^6XexxIE2=en)j}g*tvU^|f`i_+KX+!13KcWE2C5)OcV~jmPThb_<7t z;n=7!+O*yC|L*H_EP73_YX}~wyM>mnUO~lasOoljwy9q{J$B0v8~A^Krsr=HkN`kn z_!tqspc06#QYmjb2cxD4*&L}NtPa9i_30vGL`c1b;VcK`MUE^XRVhnk)KrdarKvhH z#aUr!4%v3FUa`U?)ub;2R8&ZIT8|$Q;p+q*0l*ziSxOM1mCwl1&xsDU4k<@gtx&LE z(o)QzoQXn5Qvaw3v)r-2ZxN6sJ(VyJFjB_+j8BkKopQ`#j$h2zOTBpVKIJoKE-OiN z(Tt4~PJ{3#g~F#K(&;3!Ge~4VmWVg78=mT(eiKFW3G62@L4cX-4-!WE4^^ZRZ&}KH zOncP7{jpCfMJoPt@-0AYr}5WAG&Wv$XC=qi&vAA5Kf~4im|Oj0uJp&;s-JMRSGd}r za1Z|z=lp=P{g|uyfGfZiP-otY!54xP_Qa;$GmbrD)_GU{dGkx=iJGbU8CUg~{jRvp z!E9i|iSlhTw(Vo)8@QOV{(Qwt6%&W2s%P?d&e(U2S#IPNo$Ws#d?}a^Y7_NsGsOpH z@(zwUZscdkb<>SA#ZAaR=KN5X2R)$Bu;AkKu6dI!;izSBr;0Bab@_&c0*6yGW?c~S z@-_3i{HukJEogy$sB>I(J+`0+ioWO_SDl9!j1)0(`U1*irig__jxAUzV#7@xlTJ oa9fKgj@fzPlVTTd_{B#2t%N(a!y5ic*yh;dlao3FugaGH0|~{@ Parquet -> ClickHouse (columnar OLAP) -""" -import os, json, time, gzip -from datetime import datetime -from pathlib import Path -from typing import Optional, Any - -CLICKHOUSE_HOST = os.getenv("CLICKHOUSE_HOST", "localhost") -CLICKHOUSE_PORT = int(os.getenv("CLICKHOUSE_PORT", "8123")) -CLICKHOUSE_USER = os.getenv("CLICKHOUSE_USER", "default") -CLICKHOUSE_PASSWORD = os.getenv("CLICKHOUSE_PASSWORD", "") -CLICKHOUSE_DB = os.getenv("CLICKHOUSE_DB", "opt_scraped") -S3_ENDPOINT = os.getenv("S3_ENDPOINT", "https://s3.siax.io") -S3_ACCESS_KEY = os.getenv("S3_ACCESS_KEY", "") -S3_SECRET_KEY = os.getenv("S3_SECRET_KEY", "") - - -class ClickHouseWriter: - """Write scraped data to ClickHouse via HTTP interface.""" - - def __init__(self): - self.client = None - self._connect() - - def _connect(self): - from clickhouse_driver import Client - self.client = Client( - host=CLICKHOUSE_HOST, - port=CLICKHOUSE_PORT, - user=CLICKHOUSE_USER, - password=CLICKHOUSE_PASSWORD, - database=CLICKHOUSE_DB, - settings={"max_memory_usage": 8000000000}, - ) - - def create_database(self): - self.client.execute(f"CREATE DATABASE IF NOT EXISTS {CLICKHOUSE_DB}") - - def ensure_tables(self): - """Create all required tables for scraped data.""" - - # Companies table (multi-country) - self.client.execute(""" - CREATE TABLE IF NOT EXISTS companies ( - org_nr String, - country FixedString(2), - source String, - name String, - legal_form String DEFAULT '', - status String DEFAULT '', - address String DEFAULT '', - postal_code String DEFAULT '', - city String DEFAULT '', - country_code String DEFAULT '', - email String DEFAULT '', - phone String DEFAULT '', - website String DEFAULT '', - employees UInt32 DEFAULT 0, - founded_date Date DEFAULT '1970-01-01', - registered_capital Float64 DEFAULT 0, - currency FixedString(3) DEFAULT '', - industry_code String DEFAULT '', - industry_description String DEFAULT '', - lei String DEFAULT '', - last_updated DateTime DEFAULT now(), - raw_json String DEFAULT '', - scraped_at DateTime DEFAULT now() - ) ENGINE = ReplacingMergeTree(last_updated) - ORDER BY (country, org_nr) - PARTITION BY country - SETTINGS index_granularity = 8192 - """) - - # Contacts table (board members, executives) - self.client.execute(""" - CREATE TABLE IF NOT EXISTS contacts ( - org_nr String, - country FixedString(2), - full_name String, - role String DEFAULT '', - email String DEFAULT '', - phone String DEFAULT '', - source String DEFAULT '', - confidence Float32 DEFAULT 0, - last_updated DateTime DEFAULT now(), - scraped_at DateTime DEFAULT now() - ) ENGINE = ReplacingMergeTree(last_updated) - ORDER BY (country, org_nr, full_name) - PARTITION BY country - """) - - # Vehicles table (for DE vehicle data) - self.client.execute(""" - CREATE TABLE IF NOT EXISTS vehicles ( - registration String, - country FixedString(2), - vin String DEFAULT '', - make String DEFAULT '', - model String DEFAULT '', - year UInt16 DEFAULT 0, - fuel_type String DEFAULT '', - owner_name String DEFAULT '', - owner_org_nr String DEFAULT '', - last_updated DateTime DEFAULT now(), - scraped_at DateTime DEFAULT now() - ) ENGINE = ReplacingMergeTree(last_updated) - ORDER BY (country, registration) - PARTITION BY country - """) - - # Raw scrape log (for debugging/audit) - self.client.execute(""" - CREATE TABLE IF NOT EXISTS scrape_log ( - url String, - source String, - country FixedString(2), - method String DEFAULT '', - status UInt16 DEFAULT 0, - elapsed_ms UInt32 DEFAULT 0, - bytes UInt32 DEFAULT 0, - cf_blocked UInt8 DEFAULT 0, - error String DEFAULT '', - scraped_at DateTime DEFAULT now() - ) ENGINE = MergeTree - ORDER BY (country, scraped_at) - PARTITION BY toYYYYMM(scraped_at) - TTL scraped_at + INTERVAL 90 DAY - """) - - # Statistics for monitoring - self.client.execute(""" - CREATE TABLE IF NOT EXISTS scrape_stats ( - country FixedString(2), - source String, - date Date DEFAULT today(), - total_requests UInt64 DEFAULT 0, - successful UInt64 DEFAULT 0, - cf_blocked UInt64 DEFAULT 0, - errors UInt64 DEFAULT 0, - total_bytes UInt64 DEFAULT 0, - avg_elapsed_ms Float64 DEFAULT 0, - PRIMARY KEY (country, source, date) - ) ENGINE = SummingMergeTree - ORDER BY (country, source, date) - """) - - def insert_company(self, org_nr: str, country: str, source: str, **fields): - """Insert or update a company record.""" - data = { - "org_nr": org_nr, - "country": country, - "source": source, - "name": fields.get("name", ""), - "legal_form": fields.get("legal_form", ""), - "status": fields.get("status", ""), - "address": fields.get("address", ""), - "postal_code": fields.get("postal_code", ""), - "city": fields.get("city", ""), - "country_code": fields.get("country_code", ""), - "email": fields.get("email", ""), - "phone": fields.get("phone", ""), - "website": fields.get("website", ""), - "employees": int(fields.get("employees", 0)), - "founded_date": fields.get("founded_date", "1970-01-01"), - "registered_capital": float(fields.get("registered_capital", 0)), - "currency": fields.get("currency", ""), - "industry_code": fields.get("industry_code", ""), - "industry_description": fields.get("industry_description", ""), - "lei": fields.get("lei", ""), - "raw_json": json.dumps(fields.get("raw_data", {}), ensure_ascii=False), - } - self.client.execute( - "INSERT INTO companies VALUES", - [data], - ) - - def insert_contact(self, org_nr: str, country: str, full_name: str, role: str = "", - email: str = "", phone: str = "", source: str = "", confidence: float = 0): - self.client.execute( - "INSERT INTO contacts VALUES", - [{ - "org_nr": org_nr, "country": country, "full_name": full_name, - "role": role, "email": email, "phone": phone, - "source": source, "confidence": confidence, - }], - ) - - def log_scrape(self, url: str, source: str, country: str, method: str, - status: int, elapsed_ms: int, bytes_count: int, - cf_blocked: bool = False, error: str = ""): - self.client.execute( - "INSERT INTO scrape_log VALUES", - [{ - "url": url[:500], "source": source, "country": country, - "method": method, "status": status, "elapsed_ms": elapsed_ms, - "bytes": bytes_count, "cf_blocked": 1 if cf_blocked else 0, "error": error, - }], - ) - - def query(self, sql: str) -> list: - """Execute SQL query and return results.""" - return self.client.execute(sql) - - def close(self): - if self.client: - self.client.disconnect() - - -class DuckDBWriter: - """Alternative: DuckDB for local/embedded analytics.""" - - def __init__(self, db_path: str = "/data/opt_scraped.duckdb"): - import duckdb - self.db_path = db_path - self.conn = duckdb.connect(db_path) - self._ensure_tables() - - def _ensure_tables(self): - self.conn.execute(""" - CREATE TABLE IF NOT EXISTS companies AS - SELECT * FROM read_parquet('/data/parquet/*.parquet') WHERE 1=0 - """) - self.conn.execute(""" - CREATE TABLE IF NOT EXISTS scrape_log ( - url VARCHAR, source VARCHAR, country VARCHAR, - method VARCHAR, status INTEGER, elapsed_ms INTEGER, - bytes INTEGER, cf_blocked BOOLEAN, scraped_at TIMESTAMP - ) - """) - - def import_parquet(self, parquet_dir: str = "/data/parquet"): - """Import all parquet files into DuckDB.""" - import glob - files = glob.glob(f"{parquet_dir}/*.parquet") - if not files: - return 0 - self.conn.execute(f""" - INSERT INTO companies - SELECT * FROM read_parquet({files!r}) - """) - return self.conn.execute("SELECT count(*) FROM companies").fetchone()[0] - - def query(self, sql: str) -> list: - return self.conn.execute(sql).fetchall() - - def to_parquet(self, table: str, output_path: str): - """Export table to parquet for archival.""" - self.conn.execute(f""" - COPY (SELECT * FROM {table}) TO '{output_path}' - (FORMAT PARQUET, CODEC ZSTD) - """) - - def close(self): - self.conn.close() diff --git a/services/scrapers/docker-compose.yml b/services/scrapers/docker-compose.yml deleted file mode 100644 index 98b9c3f..0000000 --- a/services/scrapers/docker-compose.yml +++ /dev/null @@ -1,77 +0,0 @@ -version: "3.8" - -services: - scraper-brreg-no: - image: scraper-engine:latest - environment: - - COUNTRY=NO - - SOURCE=brreg - - MAX_CONCURRENCY=100 - - S3_ENDPOINT=${S3_ENDPOINT} - - S3_ACCESS_KEY=${S3_ACCESS_KEY} - - S3_SECRET_KEY=${S3_SECRET_KEY} - volumes: - - scraper-data:/data - restart: unless-stopped - command: ["python", "-m", "scrapers.run_pipeline", "--country", "NO", "--source", "brreg"] - - scraper-brreg-under: - image: scraper-engine:latest - environment: - - COUNTRY=NO - - SOURCE=brreg_under - - MAX_CONCURRENCY=100 - volumes: - - scraper-data:/data - restart: unless-stopped - command: ["python", "-m", "scrapers.run_pipeline", "--country", "NO", "--source", "brreg_under"] - - scraper-dk-cvrapi: - image: scraper-engine:latest - environment: - - COUNTRY=DK - - SOURCE=cvrapi - - MAX_CONCURRENCY=30 - volumes: - - scraper-data:/data - restart: unless-stopped - command: ["python", "-m", "scrapers.run_pipeline", "--country", "DK", "--source", "cvrapi"] - - scraper-de-northdata: - image: scraper-engine:latest - environment: - - COUNTRY=DE - - SOURCE=northdata - - MAX_CONCURRENCY=20 - volumes: - - scraper-data:/data - restart: unless-stopped - command: ["python", "-m", "scrapers.run_pipeline", "--country", "DE", "--source", "northdata"] - - scraper-de-handelsregister: - image: scraper-engine:latest - environment: - - COUNTRY=DE - - SOURCE=handelsregister - - MAX_CONCURRENCY=20 - volumes: - - scraper-data:/data - restart: unless-stopped - command: ["python", "-m", "scrapers.run_pipeline", "--country", "DE", "--source", "handelsregister"] - - s3-sync: - image: scraper-engine:latest - depends_on: - - scraper-brreg-no - environment: - - S3_ENDPOINT=${S3_ENDPOINT} - - S3_ACCESS_KEY=${S3_ACCESS_KEY} - - S3_SECRET_KEY=${S3_SECRET_KEY} - - S3_BUCKET=opt-scraped-data - volumes: - - scraper-data:/data - command: ["python", "-m", "scrapers.s3_sync"] - restart: unless-stopped - -volumes: - scraper-data: diff --git a/services/scrapers/engine.py b/services/scrapers/engine.py deleted file mode 100644 index 1189304..0000000 --- a/services/scrapers/engine.py +++ /dev/null @@ -1,243 +0,0 @@ -""" -0pt Scraper Engine v2 -===================== -Universal scraping engine for DK/NO/DE company data. -Uses multi-method bypass chain + header rotation for maximum speed. - -Metoder (i prioritetsordning): - 1. curl_cffi chrome124 - TLS fingerprint impersonation - 2. curl_cffi edge99 - alternative fingerprint - 3. cloudscraper - basic CF bypass - 4. requests + header rotation - direct HTTP - 5. httpx async - high concurrency - 6. FlareSolverr - heavy CF bypass (kraver Docker) -""" - -import json, time, random, os, asyncio -from datetime import datetime -from typing import Optional -from dataclasses import dataclass, field, asdict -from pathlib import Path - -import requests -import httpx -from curl_cffi import requests as curl_requests -import cloudscraper - -# ── Config ── -OUTPUT_DIR = Path(__file__).parent / "output" -HEADER_POOL = { - "user_agents": [ - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Safari/605.1.15", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:127.0) Gecko/20100101 Firefox/127.0", - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36", - "Mozilla/5.0 (iPhone; CPU iPhone OS 17_5 like Mac OS X) AppleWebKit/605.1.15", - ], - "languages": { - "DK": "da-DK,da;q=0.9,en;q=0.8,de;q=0.7", - "NO": "nb-NO,nb;q=0.9,no;q=0.8,en;q=0.7,da;q=0.6", - "DE": "de-DE,de;q=0.9,en;q=0.8,fr;q=0.7", - "SE": "sv-SE,sv;q=0.9,en;q=0.8", - }, -} - - -@dataclass -class ScrapeResult: - source: str - url: str - method: str - status: int - elapsed: float - bytes: int - cf_blocked: bool = False - data: Optional[dict] = None - error: Optional[str] = None - - -def rotate_headers(country="DE"): - ua = random.choice(HEADER_POOL["user_agents"]) - lang = HEADER_POOL["languages"].get(country, "en-US,en;q=0.9") - return { - "User-Agent": ua, - "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", - "Accept-Language": lang, - "Accept-Encoding": "gzip, deflate, br", - "Cache-Control": "no-cache", - "Sec-Ch-Ua": '"Chromium";v="124", "Google Chrome";v="124", "Not-A.Brand";v="99"', - "Sec-Ch-Ua-Mobile": "?0", - "Sec-Fetch-Dest": "document", - "Sec-Fetch-Mode": "navigate", - "Sec-Fetch-Site": "none", - "Sec-Fetch-User": "?1", - "Upgrade-Insecure-Requests": "1", - } - - -def check_cf_blocked(text: str) -> bool: - indicators = [ - "cf-wrapper", "Just a moment", "Checking your browser", - "Cloudflare", "__cf_chl_f_tk", "cf-browser-verification", - "challenge-form", "Attention Required", - ] - return any(i in text for i in indicators[:5000]) - - -# ── Method implementations ── - -_methods_cache = {} - -def _get_methods(): - if _methods_cache: - return _methods_cache["methods"] - - cloud = cloudscraper.create_scraper( - browser={"browser": "chrome", "platform": "windows", "mobile": False} - ) - - methods = [ - ("curl_cffi chrome124", lambda url, c="DK": curl_requests.get( - url, impersonate="chrome124", headers=rotate_headers(c), timeout=15)), - ("curl_cffi edge99", lambda url, c="DK": curl_requests.get( - url, impersonate="edge99", headers=rotate_headers(c), timeout=15)), - ("curl_cffi chrome110", lambda url, c="DK": curl_requests.get( - url, impersonate="chrome110", headers=rotate_headers(c), timeout=15)), - ("curl_cffi safari17", lambda url, c="DK": curl_requests.get( - url, impersonate="safari17_0", headers=rotate_headers(c), timeout=15)), - ("cloudscraper", lambda url, c="DK": cloud.get( - url, headers=rotate_headers(c), timeout=15)), - ("requests_rotated", lambda url, c="DK": requests.get( - url, headers=rotate_headers(c), timeout=15)), - ("httpx_rotated", lambda url, c="DK": httpx.get( - url, headers=rotate_headers(c), timeout=15)), - ] - - _methods_cache["methods"] = methods - return methods - - -def scrape(url: str, country: str = "DK", method_index: int = -1) -> list[ScrapeResult]: - """Scrape a URL with all (or specific) methods. Returns list of results.""" - methods = _get_methods() - if method_index >= 0: - methods = [methods[method_index]] - - results = [] - for name, fn in methods: - for attempt in range(2): - try: - start = time.time() - r = fn(url, country) - elapsed = time.time() - start - text_sample = str(r.content[:10000] if hasattr(r, "content") else "") - blocked = check_cf_blocked(text_sample) - - results.append(ScrapeResult( - source=url, - url=url, - method=name, - status=r.status_code, - elapsed=round(elapsed, 3), - bytes=len(r.content) if hasattr(r, "content") else 0, - cf_blocked=blocked, - )) - break - except Exception as e: - if attempt == 1: - results.append(ScrapeResult( - source=url, url=url, method=name, - status=0, elapsed=0, bytes=0, - error=str(e)[:200], - )) - - return results - - -def scrape_best(url: str, country: str = "DK") -> ScrapeResult: - """Try all methods, return the fastest successful result.""" - results = scrape(url, country) - ok = [r for r in results if r.status == 200 and not r.cf_blocked] - if ok: - return min(ok, key=lambda r: r.elapsed) - # If all failed, return the one with highest status - return max(results, key=lambda r: r.status) if results else ScrapeResult(source=url, url=url, method="none", status=0, elapsed=0, bytes=0, error="No methods available") - - -async def scrape_async(urls: list[str], country: str = "DK", concurrency: int = 20) -> list[ScrapeResult]: - """Scrape multiple URLs concurrently using async httpx.""" - from httpx import AsyncClient, Limits, Timeout - limits = Limits(max_keepalive_connections=concurrency, max_connections=concurrency) - timeout = Timeout(15.0) - - async with AsyncClient(limits=limits, timeout=timeout) as client: - async def fetch(url): - start = time.time() - try: - r = await client.get(url, headers=rotate_headers(country)) - elapsed = time.time() - start - text_sample = str(r.content[:10000]) - return ScrapeResult( - source=url, url=url, method="async httpx", - status=r.status_code, elapsed=round(elapsed, 3), - bytes=len(r.content), cf_blocked=check_cf_blocked(text_sample), - ) - except Exception as e: - return ScrapeResult( - source=url, url=url, method="async httpx", - status=0, elapsed=time.time()-start, bytes=0, error=str(e)[:200], - ) - - sem = asyncio.Semaphore(concurrency) - async def bounded(url): - async with sem: - return await fetch(url) - - tasks = [bounded(url) for url in urls] - return await asyncio.gather(*tasks) - - -def save_result(result: ScrapeResult, data: dict = None): - """Save scrape result to disk.""" - OUTPUT_DIR.mkdir(parents=True, exist_ok=True) - domain = result.url.split("/")[2].replace(".", "_") - ts = datetime.now().strftime("%Y%m%d_%H%M%S") - safe_name = f"{domain}_{result.method}_{ts}.json" - path = OUTPUT_DIR / safe_name - payload = { - "timestamp": ts, - "url": result.url, - "method": result.method, - "status": result.status, - "elapsed": result.elapsed, - "bytes": result.bytes, - "cf_blocked": result.cf_blocked, - "error": result.error, - "data": data, - } - path.write_text(json.dumps(payload, indent=2, ensure_ascii=False), encoding="utf-8") - return path - - -def benchmark_source(name: str, url: str, country: str = "DK"): - """Full benchmark of a single source with all methods.""" - print(f"\n {'='*60}") - print(f" KALLA: {name}") - print(f" URL: {url}") - print(f" {'='*60}") - - results = scrape(url, country) - ok = [r for r in results if r.status == 200 and not r.cf_blocked] - - for r in results: - status_str = "OK" if r.status == 200 else f"ERR {r.status}" - cf_str = " [CF]" if r.cf_blocked else "" - err_str = f" - {r.error}" if r.error else "" - print(f" {r.method:30s} -> {status_str:10s} {r.elapsed:6.3f}s {r.bytes:8d}b{cf_str}{err_str}") - - if ok: - best = min(ok, key=lambda r: r.elapsed) - print(f"\n >>> BAST: {best.method} ({best.elapsed:.3f}s, {best.bytes}b)") - return best - return None diff --git a/services/scrapers/engine_v2.py b/services/scrapers/engine_v2.py deleted file mode 100644 index 79356de..0000000 --- a/services/scrapers/engine_v2.py +++ /dev/null @@ -1,256 +0,0 @@ -""" -0pt Scraper Engine v2 - Production -==================================== -Körs som Docker container på SIAX Cl0ud Server 2. -Data lagras i MinIO S3 (s3.siax.io) för obegränsad lagring. - -Stödjer: -- Alla DK/NO/DE källor med auto-detection av bästa metod -- Header rotation + curl_cffi TLS fingerprint bypass -- Async concurrency för maxhastighet -- Resume-kapabilitet (kraschtålig) -- Batch processing med checkpointing -""" -import json, time, random, os, asyncio, sys, gzip -from datetime import datetime -from pathlib import Path -from typing import Optional -from dataclasses import dataclass, field, asdict -from concurrent.futures import ThreadPoolExecutor, as_completed - -import requests -import httpx -from curl_cffi import requests as curl_requests -import cloudscraper - -# ── Config from env ── -S3_ENDPOINT = os.getenv("S3_ENDPOINT", "https://s3.siax.io") -S3_ACCESS_KEY = os.getenv("S3_ACCESS_KEY", "") -S3_SECRET_KEY = os.getenv("S3_SECRET_KEY", "") -S3_BUCKET = os.getenv("S3_BUCKET", "opt-scraped-data") -OUTPUT_DIR = Path(os.getenv("OUTPUT_DIR", "/data/output")) -CHECKPOINT_DIR = Path(os.getenv("CHECKPOINT_DIR", "/data/checkpoints")) -MAX_CONCURRENCY = int(os.getenv("MAX_CONCURRENCY", "50")) -HEADER_POOL = { - "user_agents": [ - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Safari/605.1.15", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:127.0) Gecko/20100101 Firefox/127.0", - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36", - "Mozilla/5.0 (iPhone; CPU iPhone OS 17_5 like Mac OS X) AppleWebKit/605.1.15", - ], - "languages": { - "DK": "da-DK,da;q=0.9,en;q=0.8,de;q=0.7", - "NO": "nb-NO,nb;q=0.9,no;q=0.8,en;q=0.7,da;q=0.6", - "DE": "de-DE,de;q=0.9,en;q=0.8,fr;q=0.7", - }, -} -# Pre-computed method rankings per source from benchmarks -METHOD_RANKING = { - "no-brreg-api": ["cloudscraper", "curl_cffi safari17", "curl_cffi chrome124"], - "no-proff": ["curl_cffi safari17", "curl_cffi edge99", "curl_cffi chrome110"], - "no-1881": ["curl_cffi chrome124", "curl_cffi safari17", "cloudscraper"], - "no-gulesider": ["curl_cffi safari17", "curl_cffi chrome124", "cloudscraper"], - "no-flowfirma": ["curl_cffi chrome124", "curl_cffi safari17", "curl_cffi chrome110"], - "dk-cvrapi": ["curl_cffi safari17", "curl_cffi chrome110", "cloudscraper"], - "dk-proff": ["curl_cffi safari17", "curl_cffi edge99", "curl_cffi chrome110"], - "dk-krak": ["curl_cffi safari17", "curl_cffi chrome124", "cloudscraper"], - "dk-118": ["curl_cffi safari17", "curl_cffi edge99", "curl_cffi chrome124"], - "dk-grundigt": ["curl_cffi safari17", "curl_cffi edge99", "curl_cffi chrome110"], - "de-northdata": ["curl_cffi edge99", "curl_cffi chrome110", "curl_cffi safari17"], - "de-handelsregister": ["curl_cffi edge99", "curl_cffi chrome110", "cloudscraper"], - "de-moneyhouse": ["curl_cffi chrome110", "curl_cffi edge99", "curl_cffi safari17"], - "de-unternehmensregister": ["curl_cffi safari17", "curl_cffi chrome110", "curl_cffi chrome124"], - "de-creditreform": ["curl_cffi chrome110", "curl_cffi safari17", "curl_cffi edge99"], - "default": ["curl_cffi chrome124", "curl_cffi safari17", "cloudscraper", "curl_cffi edge99", "requests"], -} - - -@dataclass -class ScrapeResult: - url: str - method: str - status: int - elapsed: float - bytes: int - cf_blocked: bool = False - data: Optional[dict] = None - error: Optional[str] = None - - -def rotate_headers(country="DE"): - ua = random.choice(HEADER_POOL["user_agents"]) - lang = HEADER_POOL["languages"].get(country, "en-US,en;q=0.9") - return { - "User-Agent": ua, - "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", - "Accept-Language": lang, - "Accept-Encoding": "gzip, deflate, br", - "Cache-Control": "no-cache", - "Sec-Ch-Ua": '"Chromium";v="124", "Google Chrome";v="124", "Not-A.Brand";v="99"', - "Sec-Ch-Ua-Mobile": "?0", - "Sec-Fetch-Dest": "document", - "Sec-Fetch-Mode": "navigate", - "Sec-Fetch-Site": "none", - "Sec-Fetch-User": "?1", - "Upgrade-Insecure-Requests": "1", - } - - -def check_cf_blocked(text: str) -> bool: - indicators = ["cf-wrapper", "Just a moment", "Checking your browser", - "Cloudflare", "__cf_chl_f_tk", "challenge-form"] - return any(i in text[:5000] for i in indicators) - - -class ScraperEngine: - """Main scraper engine - production ready.""" - - def __init__(self): - self.cloud = cloudscraper.create_scraper( - browser={"browser": "chrome", "platform": "windows", "mobile": False} - ) - self.cache = {} - self.stats = {"total": 0, "ok": 0, "cf_blocked": 0, "errors": 0, "total_bytes": 0} - OUTPUT_DIR.mkdir(parents=True, exist_ok=True) - CHECKPOINT_DIR.mkdir(parents=True, exist_ok=True) - - def _get_method(self, name: str): - """Get a scraping method by name.""" - if name == "curl_cffi chrome124": - return lambda url, c="DE": curl_requests.get(url, impersonate="chrome124", headers=rotate_headers(c), timeout=15) - elif name == "curl_cffi edge99": - return lambda url, c="DE": curl_requests.get(url, impersonate="edge99", headers=rotate_headers(c), timeout=15) - elif name == "curl_cffi chrome110": - return lambda url, c="DE": curl_requests.get(url, impersonate="chrome110", headers=rotate_headers(c), timeout=15) - elif name == "curl_cffi safari17": - return lambda url, c="DE": curl_requests.get(url, impersonate="safari17_0", headers=rotate_headers(c), timeout=15) - elif name == "cloudscraper": - return lambda url, c="DE": self.cloud.get(url, headers=rotate_headers(c), timeout=15) - elif name == "requests": - return lambda url, c="DE": requests.get(url, headers=rotate_headers(c), timeout=15) - return None - - def scrape_url(self, url: str, country: str = "DE", method_rank_key: str = "default") -> ScrapeResult: - """Try methods in ranking order until one succeeds.""" - methods = METHOD_RANKING.get(method_rank_key, METHOD_RANKING["default"]) - - for method_name in methods: - fn = self._get_method(method_name) - if not fn: - continue - try: - start = time.time() - r = fn(url, country) - elapsed = time.time() - start - text_sample = str(r.content[:10000] if hasattr(r, "content") else "") - blocked = check_cf_blocked(text_sample) - - self.stats["total"] += 1 - if r.status_code == 200 and not blocked: - self.stats["ok"] += 1 - self.stats["total_bytes"] += len(r.content) - return ScrapeResult(url=url, method=method_name, status=r.status_code, - elapsed=round(elapsed, 3), bytes=len(r.content), cf_blocked=blocked) - elif blocked: - self.stats["cf_blocked"] += 1 - else: - self.stats["errors"] += 1 - except Exception as e: - self.stats["errors"] += 1 - continue - - return ScrapeResult(url=url, method="all_failed", status=0, elapsed=0, bytes=0, error="All methods failed") - - def scrape_batch(self, urls: list[tuple], country: str = "DE", method_key: str = "default", max_workers: int = 10) -> list[ScrapeResult]: - """Scrape a batch of URLs concurrently using thread pool.""" - results = [] - with ThreadPoolExecutor(max_workers=max_workers) as ex: - futures = {ex.submit(self.scrape_url, url, country, method_key): url for url, _ in urls} - for f in as_completed(futures): - results.append(f.result()) - return results - - async def scrape_async_batch(self, urls: list[str], country: str = "DE", concurrency: int = None) -> list[ScrapeResult]: - """Massively parallel async scraping.""" - concurrency = concurrency or MAX_CONCURRENCY - from httpx import AsyncClient, Limits, Timeout - limits = Limits(max_keepalive_connections=concurrency, max_connections=concurrency) - timeout = Timeout(30.0) - - async with AsyncClient(limits=limits, timeout=timeout) as client: - async def fetch(url): - start = time.time() - try: - r = await client.get(url, headers=rotate_headers(country), follow_redirects=True) - elapsed = time.time() - start - return ScrapeResult(url=url, method="async_httpx", status=r.status_code, - elapsed=round(elapsed, 3), bytes=len(r.content), - cf_blocked=check_cf_blocked(str(r.content[:10000]))) - except Exception as e: - return ScrapeResult(url=url, method="async_httpx", status=0, - elapsed=time.time()-start, bytes=0, error=str(e)[:200]) - - sem = asyncio.Semaphore(concurrency) - async def bounded(url): - async with sem: - return await fetch(url) - - tasks = [bounded(url) for url in urls] - results = await asyncio.gather(*tasks) - for r in results: - self.stats["total"] += 1 - if r.status == 200: - self.stats["ok"] += 1 - self.stats["total_bytes"] += r.bytes - return results - - def save_result(self, result: ScrapeResult, raw_data: str = None): - """Save result with compression to disk.""" - domain = result.url.split("/")[2].replace(".", "_") - ts = datetime.now().strftime("%Y%m%d_%H%M%S") - safe_name = f"{domain}_{result.method}_{ts}.json.gz" - path = OUTPUT_DIR / safe_name - payload = { - "timestamp": ts, "url": result.url, "method": result.method, - "status": result.status, "elapsed": result.elapsed, "bytes": result.bytes, - "cf_blocked": result.cf_blocked, "error": result.error, - "data_sample": raw_data[:10000] if raw_data else None, - } - with gzip.open(path, "wt", encoding="utf-8") as f: - json.dump(payload, f, ensure_ascii=False) - return path - - def save_checkpoint(self, batch_id: str, completed: list, total: int): - """Save progress checkpoint for resume.""" - path = CHECKPOINT_DIR / f"checkpoint_{batch_id}.json" - data = {"batch_id": batch_id, "completed": len(completed), "total": total, - "timestamp": datetime.now().isoformat(), "urls": completed} - path.write_text(json.dumps(data, ensure_ascii=False), encoding="utf-8") - - def load_checkpoint(self, batch_id: str) -> set: - """Load completed URLs from checkpoint.""" - path = CHECKPOINT_DIR / f"checkpoint_{batch_id}.json" - if path.exists(): - data = json.loads(path.read_text(encoding="utf-8")) - return set(data.get("urls", [])) - return set() - - def print_stats(self): - """Print current scraping statistics.""" - total = self.stats["total"] - ok = self.stats["ok"] - rate = ok / (self.stats.get("start_time", 1) and - max(1, (time.time() - self.stats.get("start_time", time.time())))) - print(f" Stats: {ok}/{total} OK ({self.stats['cf_blocked']} CF, {self.stats['errors']} err)") - print(f" Rate: {rate:.0f} req/s | Data: {self.stats['total_bytes']/1024/1024:.1f} MB") - - -def print_header(): - print(""" - ╔══════════════════════════════════════════════════════════════╗ - ║ 0PT SCRAPER ENGINE v2 - PRODUCTION ║ - ║ DK/NO/DE Company Data Pipeline → MinIO S3 ║ - ╚══════════════════════════════════════════════════════════════╝ - """) diff --git a/services/scrapers/requirements.txt b/services/scrapers/requirements.txt deleted file mode 100644 index c16119a..0000000 --- a/services/scrapers/requirements.txt +++ /dev/null @@ -1,9 +0,0 @@ -# 0pt Scraper Engine - Production dependencies -curl_cffi>=0.15.0 -cloudscraper>=1.2.71 -httpx>=0.28.0 -aiohttp>=3.10.0 -requests>=2.33.0 -beautifulsoup4>=4.12.0 -lxml>=5.3.0 -boto3>=1.35.0 # MinIO S3 sync diff --git a/services/scrapers/run_benchmark.py b/services/scrapers/run_benchmark.py deleted file mode 100644 index ebf3484..0000000 --- a/services/scrapers/run_benchmark.py +++ /dev/null @@ -1,86 +0,0 @@ -""" -0pt Scraper Benchmark - test alla källor for DK/NO/DE -""" -import sys, os, time, json -sys.path.insert(0, os.path.dirname(__file__)) -from engine import scrape_best, scrape_async, benchmark_source, OUTPUT_DIR -from datetime import datetime - -# === ALLA KALLOR TESTADE === -SOURCES = { - "DK": [ - ("CVR API (cvrapi.dk)", "https://cvrapi.dk/api?search=Novo+Nordisk&country=dk", "api"), - ("Proff.dk", "https://proff.dk", "agg"), - ("Krak.dk", "https://www.krak.dk/", "dir"), - ("118.dk", "https://118.dk/", "dir"), - ("De Gule Sider", "https://www.degulesider.dk/", "dir"), - ("Companydata.dk", "https://companydata.dk", "agg"), - ("Grundigt.dk", "https://grundigt.dk", "agg"), - ("Datafordeler.dk", "https://datafordeler.dk", "gov"), - ("CVRcheck.dk", "https://cvrcheck.dk", "api"), - ("Virk", "https://datacvr.virk.dk/", "gov"), - ], - "NO": [ - ("Brreg API (enhet)", "https://data.brreg.no/enhetsregisteret/api/enheter/915651232", "api"), - ("Brreg API (search)", "https://data.brreg.no/enhetsregisteret/api/enheter?size=10", "api"), - ("Brreg API (underenheter)", "https://data.brreg.no/enhetsregisteret/api/underenheter?size=10", "api"), - ("Proff.no", "https://proff.no", "agg"), - ("1881.no", "https://www.1881.no/", "dir"), - ("Gulesider.no", "https://www.gulesider.no/", "dir"), - ("Selskapssok.no", "https://selskapssok.no", "agg"), - ("Flowfirma.no", "https://flowfirma.no", "agg"), - ("Offentligdata.no", "https://offentligdata.no", "agg"), - ("Regnskapsbasen.no", "https://www.regnskapsbasen.no", "agg"), - ("Okonomi.no", "https://okonomi.no/bedrifter", "agg"), - ("Visverdi.no", "https://visverdi.no", "agg"), - ], - "DE": [ - ("Northdata.de (Bayer)", "https://www.northdata.de/Bayer+AG", "agg"), - ("Handelsregister.de", "https://www.handelsregister.de", "gov"), - ("Unternehmensregister.de", "https://www.unternehmensregister.de", "gov"), - ("Bundesanzeiger.de", "https://www.bundesanzeiger.de", "gov"), - ("Companyhouse.de", "https://www.companyhouse.de", "agg"), - ("Moneyhouse.de", "https://www.moneyhouse.de", "agg"), - ("Creditreform.de", "https://www.creditreform.de", "agg"), - ("Personensuche.de", "https://www.personensuche.de", "dir"), - ("Registercheck.de", "https://www.registercheck.de", "agg"), - ("Whitepages.de", "https://www.whitepages.de", "dir"), - ], -} - -def run(): - print("=" * 80) - print(f"0PT SCRAPER BENCHMARK - {datetime.now().strftime('%Y-%m-%d %H:%M')}") - print("=" * 80) - - results_by_country = {} - - for country, sources in SOURCES.items(): - print(f"\n{'#'*70}") - print(f"# {country} - {len(sources)} källor") - print(f"{'#'*70}") - - country_results = [] - for name, url, src_type in sources: - best = benchmark_source(name, url, country) - if best: - country_results.append({"name": name, "url": url, "method": best.method, "elapsed": best.elapsed, "status": best.status, "bytes": best.bytes}) - time.sleep(0.3) - - results_by_country[country] = country_results - - print(f"\n [{country}] SAMMANSTALLNING:") - for r in sorted(country_results, key=lambda x: x["elapsed"]): - print(f" {r['name']:30s} {r['method']:25s} {r['elapsed']:.3f}s {r['bytes']}b") - - # Save report - report = { - "timestamp": datetime.now().isoformat(), - "results": results_by_country, - } - path = OUTPUT_DIR / f"benchmark_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json" - path.write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8") - print(f"\n Rapport sparad: {path}") - -if __name__ == "__main__": - run() diff --git a/services/scrapers/run_pipeline.py b/services/scrapers/run_pipeline.py deleted file mode 100644 index 28eafef..0000000 --- a/services/scrapers/run_pipeline.py +++ /dev/null @@ -1,235 +0,0 @@ -""" -0pt Scraper Pipeline - Production Batch Processor - -Kör batch-scraping för angivet land/källa med checkpointing. -Data sparas lokalt och synkas till MinIO S3. - -Användning: - python -m scrapers.run_pipeline --country NO --source brreg - python -m scrapers.run_pipeline --country DK --source cvrapi --search "Novo Nordisk" - python -m scrapers.run_pipeline --country DE --source northdata --hrb "HRB 123456" -""" -import os, sys, json, time, asyncio -from datetime import datetime -from pathlib import Path - -sys.path.insert(0, os.path.dirname(__file__)) -from engine_v2 import ScraperEngine, ScrapeResult, print_header - -# ── Source configurations ── -SOURCES = { - "NO": { - "brreg": { - "name": "Brreg Enhetsregisteret", - "type": "api", - "url_template": "https://data.brreg.no/enhetsregisteret/api/enheter/{}", - "method_key": "no-brreg-api", - "total_estimate": 1168208, - "batch_size": 100, - "pagination": True, - "list_url": "https://data.brreg.no/enhetsregisteret/api/enheter?size=100&page={}", - "fields": 38, - }, - "brreg_under": { - "name": "Brreg Underenheter", - "type": "api", - "method_key": "no-brreg-api", - "total_estimate": 849423, - "batch_size": 100, - "pagination": True, - "list_url": "https://data.brreg.no/enhetsregisteret/api/underenheter?size=100&page={}", - }, - "proff": { - "name": "Proff.no", - "type": "web", - "url_template": "https://proff.no/company/companydetails/{}", - "method_key": "no-proff", - }, - "1881": { - "name": "1881.no", - "type": "web", - "url_template": "https://www.1881.no/sok/?q={}", - "method_key": "no-1881", - }, - "flowfirma": { - "name": "Flowfirma.no", - "type": "web", - "url_template": "https://flowfirma.no/company/{}", - "method_key": "no-flowfirma", - }, - }, - "DK": { - "cvrapi": { - "name": "CVR API", - "type": "api", - "url_template": "https://cvrapi.dk/api?search={}&country=dk", - "method_key": "dk-cvrapi", - "total_estimate": 700000, - }, - "proff": { - "name": "Proff.dk", - "type": "web", - "url_template": "https://proff.dk/company/companydetails/{}", - "method_key": "dk-proff", - }, - "grundigt": { - "name": "Grundigt.dk", - "type": "web", - "url_template": "https://grundigt.dk/virksomhed/{}", - "method_key": "dk-grundigt", - }, - }, - "DE": { - "northdata": { - "name": "Northdata.de", - "type": "web", - "url_template": "https://www.northdata.de/{}", - "method_key": "de-northdata", - "total_estimate": 3500000, - }, - "handelsregister": { - "name": "Handelsregister.de", - "type": "web", - "url_template": "https://www.handelsregister.de/{}", - "method_key": "de-handelsregister", - }, - "moneyhouse": { - "name": "Moneyhouse.de", - "type": "web", - "url_template": "https://www.moneyhouse.de/{}", - "method_key": "de-moneyhouse", - }, - }, -} - - -async def scrape_brreg_bulk(engine: ScraperEngine, country: str, source_key: str): - """Scrape Brreg API with pagination - full company register.""" - source = SOURCES[country][source_key] - total = source["total_estimate"] - batch = source["batch_size"] - list_url = source["list_url"] - method_key = source["method_key"] - - print(f"\n[{country}] {source['name']} - {total} companies") - print(f" Method: {method_key} | Concurrency: {engine.MAX_CONCURRENCY}") - - engine.stats["start_time"] = time.time() - total_pages = (total // batch) + 1 - completed_urls = set() - - # Check for checkpoint - checkpoint_id = f"{country}_{source_key}" - completed_urls = engine.load_checkpoint(checkpoint_id) - print(f" Checkpoint: {len(completed_urls)}/{total_pages} pages completed") - - for page in range(total_pages): - if f"page_{page}" in completed_urls: - continue - - url = list_url.format(page) - result = engine.scrape_url(url, country, method_key) - - if result.status == 200 and not result.cf_blocked: - import json as j - try: - data = j.loads(result.data) if result.data else {} - items = data.get("_embedded", {}).get( - "enheter" if source_key == "brreg" else "underenheter", [] - ) - for item in items: - engine.save_result(ScrapeResult( - url=url, method=result.method, - status=200, elapsed=result.elapsed, - bytes=len(j.dumps(item)), - data=item - ), j.dumps(item)) - except: - pass - - # Track progress - if (page + 1) % 10 == 0: - elapsed = time.time() - engine.stats["start_time"] - rate = (page + 1) / elapsed if elapsed > 0 else 0 - remaining = (total_pages - page) / rate if rate > 0 else 0 - print(f" Page {page+1}/{total_pages} ({rate:.0f} pages/s, ETA: {remaining:.0f}s)") - engine.print_stats() - - # Checkpoint every 100 pages - if (page + 1) % 100 == 0: - completed = [f"page_{p}" for p in range(page + 1)] - engine.save_checkpoint(checkpoint_id, completed, total_pages) - - -async def scrape_by_search(engine: ScraperEngine, country: str, source_key: str, search_terms: list[str]): - """Scrape by searching company names.""" - source = SOURCES[country][source_key] - template = source["url_template"] - method_key = source["method_key"] - - print(f"\n[{country}] {source['name']} - {len(search_terms)} searches") - engine.stats["start_time"] = time.time() - - for i, term in enumerate(search_terms): - url = template.format(term.replace(" ", "+")) - result = engine.scrape_url(url, country, method_key) - - if result.status == 200: - engine.save_result(result) - - if (i + 1) % 10 == 0: - elapsed = time.time() - engine.stats["start_time"] - rate = (i + 1) / elapsed if elapsed > 0 else 0 - remaining = (len(search_terms) - i) / rate if rate > 0 else 0 - print(f" {i+1}/{len(search_terms)} ({rate:.0f} req/s, ETA: {remaining:.0f}s)") - engine.print_stats() - - -def parse_args(): - import argparse - parser = argparse.ArgumentParser(description="0pt Scraper Pipeline") - parser.add_argument("--country", choices=["NO", "DK", "DE"], required=True, help="Country code") - parser.add_argument("--source", required=True, help="Source key (brreg, cvrapi, northdata, etc)") - parser.add_argument("--search", nargs="+", help="Search terms for name-based sources") - parser.add_argument("--concurrency", type=int, default=50, help="Max async concurrency") - parser.add_argument("--pages", type=int, default=0, help="Number of pages (0=all)") - return parser.parse_args() - - -def main(): - print_header() - args = parse_args() - - country = args.country - source_key = args.source - max_conc = args.concurrency - - if country not in SOURCES or source_key not in SOURCES[country]: - print(f"Error: unknown source {country}/{source_key}") - print(f"Available: {[f'{c}/{k}' for c, srcs in SOURCES.items() for k in srcs.keys()]}") - return 1 - - engine = ScraperEngine() - source = SOURCES[country][source_key] - - print(f" Country: {country}") - print(f" Source: {source['name']} ({source['type']})") - print(f" Method: {source['method_key']}") - print(f" Concurrency: {max_conc}") - print() - - if source.get("pagination"): - asyncio.run(scrape_brreg_bulk(engine, country, source_key)) - elif args.search: - asyncio.run(scrape_by_search(engine, country, source_key, args.search)) - else: - print("Error: need --search for non-paginated sources, or use --search with search terms") - return 1 - - print(f"\n Pipeline complete!") - engine.print_stats() - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/services/scrapers/s3_sync.py b/services/scrapers/s3_sync.py deleted file mode 100644 index 8eb956c..0000000 --- a/services/scrapers/s3_sync.py +++ /dev/null @@ -1,57 +0,0 @@ -""" -S3 Sync - Upload scraped data to MinIO periodically. -""" -import os, time, json, gzip -from pathlib import Path -from datetime import datetime, timedelta - -OUTPUT_DIR = Path(os.getenv("OUTPUT_DIR", "/data/output")) -S3_ENDPOINT = os.getenv("S3_ENDPOINT", "https://s3.siax.io") -S3_ACCESS_KEY = os.getenv("S3_ACCESS_KEY", "") -S3_SECRET_KEY = os.getenv("S3_SECRET_KEY", "") -S3_BUCKET = os.getenv("S3_BUCKET", "opt-scraped-data") -SYNC_INTERVAL = int(os.getenv("SYNC_INTERVAL", "300")) # 5 min - -def get_s3_client(): - import boto3 - return boto3.client( - "s3", - endpoint_url=S3_ENDPOINT, - aws_access_key_id=S3_ACCESS_KEY, - aws_secret_access_key=S3_SECRET_KEY, - ) - -def sync_to_s3(): - """Upload all new files to MinIO S3.""" - s3 = get_s3_client() - # Ensure bucket exists - try: - s3.head_bucket(Bucket=S3_BUCKET) - except: - s3.create_bucket(Bucket=S3_BUCKET) - - uploaded = 0 - for fpath in OUTPUT_DIR.glob("*.json.gz"): - key = f"raw/{fpath.name}" - try: - s3.upload_file(str(fpath), S3_BUCKET, key) - uploaded += 1 - fpath.unlink() # Remove local after upload - except Exception as e: - print(f" S3 upload failed: {fpath.name} - {e}") - - return uploaded - -def run(): - print(f"S3 Sync started -> {S3_ENDPOINT}/{S3_BUCKET}") - while True: - try: - uploaded = sync_to_s3() - if uploaded: - print(f" Uploaded {uploaded} files to S3") - except Exception as e: - print(f" Sync error: {e}") - time.sleep(SYNC_INTERVAL) - -if __name__ == "__main__": - run() diff --git a/services/scrapers/waf_bypass.py b/services/scrapers/waf_bypass.py deleted file mode 100644 index fe18318..0000000 --- a/services/scrapers/waf_bypass.py +++ /dev/null @@ -1,316 +0,0 @@ -""" -Proxy Rotator & WAF Bypass - ALLA gratis metoder. - -Metoder: -1. Header rotation (6 User-Agents x sprak x devices) -2. curl_cffi TLS fingerprint (chrome124, chrome110, edge99, safari17) -3. cloudscraper (basic CF bypass) -4. FlareSolverr (heavy CF bypass, via Docker) -5. Tor rotation (SOCKS5, gratis) -6. Free proxy pool (scraped from 5+ sources) -7. Playwright browser automation -8. nodriver (undetected browser) -9. Fake identity generation -10. Session cookie reuse -""" -import os, json, time, random, socket, asyncio -from datetime import datetime -from typing import Optional -from curl_cffi import requests as curl_requests -import cloudscraper -import requests - -# ── Config ── -FLARESOLVERR_URL = os.getenv("FLARESOLVERR_URL", "http://localhost:8191/v1") -TOR_PROXY = os.getenv("TOR_PROXY", "socks5://localhost:9050") -USE_FLARESOLVERR = os.getenv("USE_FLARESOLVERR", "false").lower() == "true" -USE_TOR = os.getenv("USE_TOR", "false").lower() == "true" - -# ── Header Pool ── -USER_AGENTS = [ - # Chrome Windows - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36", - # Safari Mac - "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Safari/605.1.15", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.6 Safari/605.1.15", - # Firefox - "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:127.0) Gecko/20100101 Firefox/127.0", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) Gecko/20100101 Firefox/128.0", - # Edge - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36 Edg/124.0.0.0", - # Linux - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36", - # Mobile - "Mozilla/5.0 (iPhone; CPU iPhone OS 17_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Mobile/15E148 Safari/604.1", - "Mozilla/5.0 (Linux; Android 14; Pixel 8) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Mobile Safari/537.36", -] - -ACCEPT_LANGUAGES = { - "DK": "da-DK,da;q=0.9,en;q=0.8,de;q=0.7", - "NO": "nb-NO,nb;q=0.9,no;q=0.8,en;q=0.7,da;q=0.6", - "DE": "de-DE,de;q=0.9,en;q=0.8,fr;q=0.7", - "SE": "sv-SE,sv;q=0.9,en;q=0.8", - "GB": "en-GB,en;q=0.9,fr;q=0.8", - "FI": "fi-FI,fi;q=0.9,en;q=0.8,sv;q=0.7", - "FR": "fr-FR,fr;q=0.9,en;q=0.8", - "NL": "nl-NL,nl;q=0.9,en;q=0.8", - "PL": "pl-PL,pl;q=0.9,en;q=0.8", -} - -SEC_CH_UA = { - "chrome124": '"Chromium";v="124", "Google Chrome";v="124", "Not-A.Brand";v="99"', - "chrome125": '"Chromium";v="125", "Google Chrome";v="125", "Not-A.Brand";v="99"', - "edge124": '"Chromium";v="124", "Microsoft Edge";v="124", "Not-A.Brand";v="99"', -} - -ACCEPT_ENCODINGS = ["gzip, deflate, br", "gzip, deflate", "gzip, deflate, br, zstd"] - - -def rotate_headers(country="DE", impersonate="chrome124"): - ua = random.choice(USER_AGENTS) - lang = ACCEPT_LANGUAGES.get(country, "en-US,en;q=0.9") - enc = random.choice(ACCEPT_ENCODINGS) - sec_ch = SEC_CH_UA.get(impersonate, SEC_CH_UA["chrome124"]) - - return { - "User-Agent": ua, - "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", - "Accept-Language": lang, - "Accept-Encoding": enc, - "Cache-Control": "no-cache", - "Pragma": "no-cache", - "Sec-Ch-Ua": sec_ch, - "Sec-Ch-Ua-Mobile": "?0", - "Sec-Ch-Ua-Platform": '"Windows"' if "Windows" in ua else ('"macOS"' if "Mac" in ua else '"Linux"'), - "Sec-Fetch-Dest": "document", - "Sec-Fetch-Mode": "navigate", - "Sec-Fetch-Site": "none", - "Sec-Fetch-User": "?1", - "Upgrade-Insecure-Requests": "1", - "DNT": "1", - } - - -def check_cf_blocked(text: str) -> bool: - """Check if response contains Cloudflare challenge.""" - indicators = [ - "cf-wrapper", "Just a moment", "Checking your browser", - "Cloudflare", "__cf_chl_f_tk", "cf-browser-verification", - "challenge-form", "Attention Required", "cf-error-page", - "cloudflare.com/cdn-cgi", "/cdn-cgi/", - ] - if not text: - return False - lower = text[:10000].lower() - for ind in indicators: - if ind in lower: - return True - return False - - -class ProxyRotator: - """Free proxy pool with rotation.""" - - def __init__(self): - self.proxies = [] - self.current = 0 - self._load_free_proxies() - - def _load_free_proxies(self): - """Load proxies from free sources.""" - sources = [ - "https://raw.githubusercontent.com/TheSpeedX/SOCKS-List/master/http.txt", - "https://raw.githubusercontent.com/ShiftyTR/Proxy-List/master/http.txt", - "https://raw.githubusercontent.com/clarketm/proxy-list/master/proxy-list-raw.txt", - ] - for url in sources: - try: - r = requests.get(url, timeout=5) - for line in r.text.strip().split("\n"): - line = line.strip() - if ":" in line and not line.startswith("#"): - parts = line.split(":") - if len(parts) == 2: - self.proxies.append({"http": f"http://{line}", "https": f"http://{line}"}) - except: - pass - # Deduplicate - seen = set() - unique = [] - for p in self.proxies: - if p["http"] not in seen: - seen.add(p["http"]) - unique.append(p) - self.proxies = unique - print(f" ProxyRotator: loaded {len(self.proxies)} free proxies") - - def get_proxy(self) -> Optional[dict]: - if not self.proxies: - return None - self.current = (self.current + 1) % len(self.proxies) - return self.proxies[self.current] - - def get_tor_proxy(self) -> dict: - return {"http": TOR_PROXY, "https": TOR_PROXY} - - -class WAFBypass: - """Multi-method WAF bypass engine.""" - - def __init__(self): - self.cloudscraper = cloudscraper.create_scraper( - browser={"browser": "chrome", "platform": "windows", "mobile": False} - ) - self.proxy = ProxyRotator() - self.session_pool = {} - self.curl_methods = { - "chrome124": lambda url, h: curl_requests.get(url, impersonate="chrome124", headers=h, timeout=15), - "chrome110": lambda url, h: curl_requests.get(url, impersonate="chrome110", headers=h, timeout=15), - "edge99": lambda url, h: curl_requests.get(url, impersonate="edge99", headers=h, timeout=15), - "safari17": lambda url, h: curl_requests.get(url, impersonate="safari17_0", headers=h, timeout=15), - } - - def try_flaresolverr(self, url: str) -> Optional[str]: - """Bypass CF via FlareSolverr Docker container.""" - if not USE_FLARESOLVERR: - return None - try: - payload = { - "cmd": "request.get", - "url": url, - "maxTimeout": 60000, - "session": f"session_{hash(url) % 10000}", - } - r = requests.post(FLARESOLVERR_URL, json=payload, timeout=65) - data = r.json() - if data.get("status") == "ok": - solution = data.get("solution", {}) - return solution.get("response", "") - except: - pass - return None - - def try_tor(self, url: str, headers: dict) -> Optional[requests.Response]: - """Route through Tor network.""" - if not USE_TOR: - return None - try: - proxies = self.proxy.get_tor_proxy() - r = requests.get(url, headers=headers, proxies=proxies, timeout=30) - return r - except: - return None - - def try_curl(self, url: str, country: str = "DE") -> Optional[requests.Response]: - """Try all curl_cffi impersonations in order.""" - for name, fn in self.curl_methods.items(): - try: - headers = rotate_headers(country, name.replace("curl_cffi_", "").split("_")[0] if "curl_" in name else "chrome124") - r = fn(url, headers) - if r.status_code == 200 and not check_cf_blocked(str(r.content[:5000])): - return r - except: - continue - return None - - def try_cloudscraper(self, url: str, country: str = "DE") -> Optional[requests.Response]: - """Try cloudscraper with different configs.""" - for browser in [ - {"browser": "chrome", "platform": "windows", "mobile": False}, - {"browser": "firefox", "platform": "windows", "mobile": False}, - {"browser": "chrome", "platform": "macos", "mobile": False}, - ]: - try: - sc = cloudscraper.create_scraper(browser=browser) - sc.headers.update(rotate_headers(country)) - r = sc.get(url, timeout=15) - if r.status_code == 200 and not check_cf_blocked(str(r.content[:5000])): - return r - except: - continue - return None - - def try_playwright(self, url: str) -> Optional[str]: - """Use Playwright headless browser as last resort.""" - try: - from playwright.sync_api import sync_playwright - with sync_playwright() as p: - browser = p.chromium.launch(headless=True) - context = browser.new_context( - user_agent=random.choice(USER_AGENTS), - locale="en-US", - viewport={"width": 1920, "height": 1080}, - ) - page = context.new_page() - page.goto(url, timeout=30000) - content = page.content() - browser.close() - if not check_cf_blocked(content): - return content - except: - pass - return None - - def bypass(self, url: str, country: str = "DE") -> tuple: - """ - Full bypass chain. Returns (content: str, method: str, elapsed: float). - - Chain order: - 1. curl_cffi chrome124 (fastest, 0.1-0.3s) - 2. cloudscraper (basic CF, 0.3-0.5s) - 3. curl_cffi edge99 (alt fingerprint) - 4. curl_cffi safari17 (Eniro bypass) - 5. Tor proxy rotation (anonymous, 2-5s) - 6. FlareSolverr (heavy CF, 5-15s) - 7. Playwright (full browser, 10-30s) - """ - start = time.time() - - # Level 1: curl_cffi TLS fingerprint - r = self.try_curl(url, country) - if r and r.status_code == 200: - return (r.text, "curl_cffi", time.time() - start) - - # Level 2: cloudscraper - r = self.try_cloudscraper(url, country) - if r and r.status_code == 200: - return (r.text, "cloudscraper", time.time() - start) - - # Level 3: Tor - if USE_TOR: - r = self.try_tor(url, rotate_headers(country)) - if r and r.status_code == 200: - return (r.text, "tor", time.time() - start) - - # Level 4: FlareSolverr - content = self.try_flaresolverr(url) - if content: - return (content, "flaresolverr", time.time() - start) - - # Level 5: Playwright (last resort) - content = self.try_playwright(url) - if content: - return (content, "playwright", time.time() - start) - - return (None, "all_failed", time.time() - start) - - def bypass_with_proxy(self, url: str, country: str = "DE") -> tuple: - """Bypass with proxy rotation for each method.""" - start = time.time() - proxy = self.proxy.get_proxy() - headers = rotate_headers(country) - - for name, fn in self.curl_methods.items(): - try: - if proxy: - r = fn(url, headers) - else: - r = fn(url, headers) - if r.status_code == 200 and not check_cf_blocked(str(r.content[:5000])): - return (r.text, f"curl_cffi+proxy_{name}", time.time() - start) - except: - continue - - return self.bypass(url, country) -- 2.54.0