#!/usr/bin/env python3 """Scraper-detaljsidor: /scraper/ - spec + live-data per scraper.""" SCRAPERS = { "sf": { "label": "SF multi-scrape", "unit": "sf-multi-scrape.service", "desc": "Multi-source vehicle scraper: biluppgifter/fordonskollen via pool, fordonssok/car.info direkt", "cmd": "Documents/SF_Extractor/.venv/bin/python scrape_sources.py", "sources": ["biluppgifter", "fordonskollen", "fordonssok", "carinfo"], "spec": [ ("Källor", "biluppgifter.se, fordonskollen.se, fordonssok.se, car.info (sv-se)"), ("Indata", "scrape_urls/*.txt (14,5M regnr, sitemap-enumererade)"), ("Utdata", "spool/*.jsonl.gz (rå HTML) + full_data/*/records-*.jsonl.gz (extraherat)"), ("Takt", "~4 parallella per domän; biluppgifter via pool (CF IP-block)"), ("Drift", "systemd always-restart (startar om efter stop-timeout, återupptar)"), ("Logg", "Documents/SF_Extractor/logs/multi_scrape.log — [källa] n= ok= fail= @pos/total"), ], }, "carinfo": { "label": "car.info specs", "unit": "carinfo-crawl.service", "desc": "car.info spec crawler, resumable (vehiq VEH-HIST-030)", "cmd": "/usr/bin/python3 ~/.local/share/carinfo/crawl.py", "sources": ["car.info"], "spec": [ ("Källa", "car.info modell/generation/motorvarianter (en-se)"), ("Utdata", "~/.local/share/carinfo/trims.jsonl {url, make, model, specs}"), ("Checkpoint", "ckpt.json (makes_done, models_done, gens_done) — återupptar"), ("Drift", "systemd; cooldown-loop vid tunna modeller"), ("Logg", "~/.local/share/carinfo/crawl.log"), ], }, "enrich": { "label": "SIAX enrich", "unit": "siax-enrich.service", "desc": "Job Description Enricher (LinkedIn jobbsidor)", "cmd": "job-automation/deploy/scanner/enrich-descriptions-laptop.py", "sources": ["linkedin"], "spec": [ ("Källa", "LinkedIn jobbsidor"), ("Utdata", "ingen lokal resultatfil (fjeder/nätverk) — endast status synlig"), ("Drift", "systemd"), ], }, "scanner": { "label": "SIAX laptop-scanner", "unit": "siax-laptop-scanner.service", "desc": "Laptop Job Scanner (residential IP, pushar till server6)", "cmd": "job-automation/deploy/scanner/laptop-scanner.py", "sources": ["jobbsajter"], "spec": [ ("Källa", "jobbsajter via residential IP"), ("Utdata", "pushas till server6 — ingen lokal resultatfil"), ("Drift", "systemd"), ], }, } DETAIL = """ Scrape Console · __LABEL__

🕷️ Scrape Console ·

Specifikation

Dataset

Logg



Körningar (runs-historik, 3 dygn)




Schema

"""