#!/usr/bin/env python3 """Data-vyer för Scrape Console: dataset-register, tabell, postdetalj, export. Endast stdlib. Anropas från server.py (routes) och serverar /data-sidan.""" import csv, gzip, json, os, time HOME = os.path.expanduser("~") SF = os.path.join(HOME, "Documents/SF_Extractor") CI = os.path.join(HOME, ".local/share/carinfo") COUNT_CACHE = {"ts": 0, "data": {}} def _files_jsonl_gz(d): try: return sorted( [ os.path.join(d, f) for f in os.listdir(d) if f.endswith(".jsonl.gz") and os.path.getsize(os.path.join(d, f)) > 0 ] ) except Exception: return [] def _iter_jsonl_gz(path): try: with gzip.open(path, "rt", errors="replace") as f: for line in f: line = line.strip() if line: try: yield json.loads(line) except Exception: continue except Exception: return def _iter_jsonl(path): try: with open(path, errors="replace") as f: for line in f: line = line.strip() if line: try: yield json.loads(line) except Exception: continue except Exception: return def _iter_csv(path): try: with open(path, newline="", errors="replace") as f: for row in csv.DictReader(f): yield row except Exception: return def _count(it, cap=2000000): n = 0 for _ in it: n += 1 if n >= cap: break return n SCRAPER_OF = {"biluppgifter_records": "sf", "sf_spool": "sf", "carinfo_trims": "carinfo", "vehicles_clean": "sf", "allabolag": "sf", "hitta": "sf", "ratsit": "sf", "bolagsfakta": "sf", "hitta_companies": "sf", "reco_companies": "sf", "regnr": "sf", "queue_biluppgifter": "sf", "queue_biluppgifter_crawled": "sf", "queue_multi_fordonssok": "sf", "queue_multi_fordonskollen": "sf", "queue_multi_carinfo": "sf"} DATASETS = [ { "id": "biluppgifter_records", "label": "SF · biluppgifter records (extraherat)", "kind": "multi-jsonl-gz", "dir": os.path.join(SF, "full_data/biluppgifter"), "prefix": "records-", "desc": "titel, tabeller, json_ld, text per fordonssida", }, { "id": "sf_spool", "label": "SF · spool (råcaptures, live)", "kind": "multi-jsonl-gz", "dir": os.path.join(SF, "spool"), "prefix": "multi-", "desc": "{url, html} per hämtad sida, nyast först", }, { "id": "carinfo_trims", "label": "car.info · trims/specs", "kind": "jsonl", "path": os.path.join(CI, "trims.jsonl"), "desc": "{url, make, model, specs{...}}", }, { "id": "vehicles_clean", "label": "SF · vehicles_clean.csv", "kind": "csv", "path": os.path.join(SF, "cleaned_allabolag/vehicles_clean.csv"), "desc": "40+ fordonskolumner (regnr, vin, make, model, skatt, besiktning ...)", }, { "id": "allabolag", "label": "SF · extracted_allabolag.csv", "kind": "csv", "path": os.path.join(SF, "cleaned_allabolag/extracted_allabolag.csv"), "desc": "", }, { "id": "hitta", "label": "SF · extracted_hitta.csv", "kind": "csv", "path": os.path.join(SF, "cleaned_allabolag/extracted_hitta.csv"), "desc": "", }, { "id": "ratsit", "label": "SF · extracted_ratsit.csv", "kind": "csv", "path": os.path.join(SF, "cleaned_allabolag/extracted_ratsit.csv"), "desc": "", }, { "id": "bolagsfakta", "label": "SF · extracted_bolagsfakta.csv", "kind": "csv", "path": os.path.join(SF, "cleaned_allabolag/extracted_bolagsfakta.csv"), "desc": "", }, { "id": "hitta_companies", "label": "SF · hitta companies.csv", "kind": "csv", "path": os.path.join(SF, "multisite_data/hitta/companies.csv"), "desc": "", }, { "id": "regnr", "label": "Register · regnr x domän (14,5M, indexerat)", "kind": "sqlite", "desc": "prefixsök på regnr — ms-svar via index", }, { "id": "queue_biluppgifter", "label": "SF · kö: biluppgifter frontier (1,8M)", "kind": "lines", "path": os.path.join(SF, "full_data/biluppgifter/frontier.txt"), "desc": "", }, { "id": "queue_biluppgifter_crawled", "label": "SF · kö: biluppgifter crawled", "kind": "lines", "path": os.path.join(SF, "full_data/biluppgifter/crawled.txt"), "desc": "", }, { "id": "queue_multi_fordonssok", "label": "SF · kö: multi/fordonssok crawled", "kind": "lines", "path": os.path.join(SF, "full_data/multi/fordonssok/crawled.txt"), "desc": "", }, { "id": "queue_multi_fordonskollen", "label": "SF · kö: multi/fordonskollen crawled", "kind": "lines", "path": os.path.join(SF, "full_data/multi/fordonskollen/crawled.txt"), "desc": "", }, { "id": "queue_multi_carinfo", "label": "SF · kö: multi/carinfo crawled", "kind": "lines", "path": os.path.join(SF, "full_data/multi/carinfo/crawled.txt"), "desc": "", }, { "id": "reco_companies", "label": "SF · reco companies.csv", "kind": "csv", "path": os.path.join(SF, "multisite_data/reco/companies.csv"), "desc": "", }, { "id": "hitta_person_reg", "label": "Register · hitta personer (6,08M)", "kind": "lines", "path": os.path.join(SF, "hitta_person_urls.txt"), "desc": "person-URL:er från hitta.se sitemaps", }, { "id": "hitta_company_reg", "label": "Register · hitta företag (1,59M rena)", "kind": "lines", "path": os.path.join(SF, "hitta_company_clean.txt"), "desc": "malformade sök-URL:er borttagna", }, { "id": "ratsit_person_reg", "label": "Register · ratsit personer (6,2M)", "kind": "lines", "path": os.path.join(SF, "ratsit_person_urls.txt"), "desc": "namn+ort+födelsedatum i URL", }, { "id": "ratsit_company_reg", "label": "Register · ratsit företag (1,31M)", "kind": "lines", "path": os.path.join(SF, "ratsit_company_urls.txt"), "desc": "", }, { "id": "bolagsfakta_reg", "label": "Register · bolagsfakta företag (1,54M)", "kind": "lines", "path": os.path.join(SF, "bolagsfakta_company_urls.txt"), "desc": "", }, { "id": "allabolag_reg", "label": "Register · allabolag företag (1,29M)", "kind": "lines", "path": os.path.join(SF, "allabolag_urls_for_sf.txt"), "desc": "", }, { "id": "relations", "label": "VEHIQ · relationsgraf fordon→ägare→person→företag→adress", "kind": "multi-jsonl-gz", "dir": os.path.join(HOME, "sf-s10/full_data/relations"), "prefix": "relations-", "desc": "kanter: src_kind/src_id/relation/dst_kind/dst_id/weight", }, {"id":"queue_hitta_person","label":"SF · kö: multi/hitta_person crawled","kind":"lines","path":os.path.join(SF,"full_data/multi/hitta_person/crawled.txt"),"desc":""}, {"id":"queue_hitta_company","label":"SF · kö: multi/hitta_company crawled","kind":"lines","path":os.path.join(SF,"full_data/multi/hitta_company/crawled.txt"),"desc":""}, {"id":"queue_ratsit_person","label":"SF · kö: multi/ratsit_person crawled","kind":"lines","path":os.path.join(SF,"full_data/multi/ratsit_person/crawled.txt"),"desc":""}, {"id":"queue_bolagsfakta_company","label":"SF · kö: multi/bolagsfakta_company crawled","kind":"lines","path":os.path.join(SF,"full_data/multi/bolagsfakta_company/crawled.txt"),"desc":""}, { "id": "identified", "label": "SF · identifierade records (_id + regnr/orgnr/personnummer)", "kind": "multi-jsonl-gz", "dir": os.path.join(HOME, "sf-s10/full_data/identified"), "prefix": "identified", "desc": "_id (vehicle:/company:/person:), id_type, regnr, orgnr, personnummer", }, { "id": "relations_v2", "label": "VEHIQ · relations v2 (företag→adress, företag→person, samma adress)", "kind": "multi-jsonl-gz", "dir": os.path.join(HOME, "sf-s10/full_data/relations"), "prefix": "relations_v2", "desc": "located_at / related_person / related_company / same_address / brand / model", }, ] def _wc(path): try: with open(path, "rb") as f: return sum(1 for _ in f) except Exception: return -1 def _ds(dsid): for d in DATASETS: if d["id"] == dsid: return d return None def _iter_ds(d, reverse_spool=True): if d["kind"] == "multi-jsonl-gz": files = [ f for f in _files_jsonl_gz(d["dir"]) if os.path.basename(f).startswith(d.get("prefix", "")) ] if reverse_spool: files = sorted(files, key=os.path.getmtime, reverse=True) for fp in files: yield from _iter_jsonl_gz(fp) elif d["kind"] == "jsonl": yield from _iter_jsonl(d["path"]) elif d["kind"] == "csv": yield from _iter_csv(d["path"]) elif d["kind"] == "lines": yield from _iter_lines(d["path"]) def counts(): if time.time() - COUNT_CACHE["ts"] < 120: return COUNT_CACHE["data"] cfile = os.path.join(HOME, ".local/share/scrape-console/counts.json") try: saved = json.load(open(cfile)) except Exception: saved = {} def fkey(d): try: if d["kind"] in ("csv", "jsonl", "lines"): st = os.stat(d["path"]) return "%s:%s" % (st.st_mtime, st.st_size) fs = sorted(_files_jsonl_gz(d["dir"])) return "%s:%s" % (len(fs), sum(os.path.getsize(f) for f in fs)) except Exception: return "?" out, dirty = {}, False for d in DATASETS: if d["kind"] == "sqlite": out[d["id"]] = {"rows": sum(DOMAIN_COUNTS.values()), "bytes": 4100000000} continue key = fkey(d) if saved.get(d["id"], {}).get("key") == key: out[d["id"]] = saved[d["id"]] continue try: if d["kind"] == "csv": n = _count(_iter_csv(d["path"])) elif d["kind"] == "jsonl": n = _count(_iter_jsonl(d["path"])) elif d["kind"] == "lines": n = _count(_iter_lines(d["path"])) else: files = [ f for f in _files_jsonl_gz(d["dir"]) if os.path.basename(f).startswith(d.get("prefix", "")) ] n = 0 for fp in files: n += _count(_iter_jsonl_gz(fp)) if n >= 2000000: break except Exception as e: out[d["id"]] = {"rows": -1, "bytes": -1, "err": str(e)[:80]} continue out[d["id"]] = {"rows": n, "bytes": -1, "key": key} saved[d["id"]] = out[d["id"]] dirty = True if dirty: try: json.dump(saved, open(cfile, "w")) except Exception: pass COUNT_CACHE.update(ts=time.time(), data=out) return out REGDB = os.path.expanduser("~/.local/share/regnr-register/regnr.db") DOMAIN_COUNTS = {"fordonssok": 11524863, "fordonskollen": 13366899, "carinfo": 11075425, "minbilkoll": 98658} # snapshot 2026-10-05 def _sqlite_sightings(q="", page=1, per=50): import sqlite3 if not os.path.exists(REGDB): raise FileNotFoundError("registret återbyggs — klart inom ~1h") con = sqlite3.connect("file:" + REGDB + "?mode=ro", uri=True) cur = con.cursor() q = (q or "").strip().upper() if q: cur.execute("SELECT regnr, domain, url FROM sightings WHERE regnr LIKE ? " "ORDER BY regnr, domain LIMIT 500", (q + "%",)) rows = [{"regnr": r[0], "domain": r[1], "url": r[2]} for r in cur.fetchall()] else: cur.execute("SELECT regnr, domain, url FROM sightings ORDER BY regnr LIMIT 50") rows = [{"regnr": r[0], "domain": r[1], "url": r[2]} for r in cur.fetchall()] con.close() return rows def _iter_lines(path): try: with open(path, errors="replace") as f: for line in f: line = line.strip() if line: yield {"rad": line[:300]} except Exception: return def _spool_title(html): import re m = re.search(r"(.*?)", html or "", re.S | re.I) return (m.group(1).strip()[:140] if m else "-") def _flat(rec): """Platta till en nivå för tabellvisning (kapslade -> JSON-sträng, trunkerad).""" if not isinstance(rec, dict): return {"value": str(rec)[:300]} if set(rec.keys()) == {"u", "h"}: return {"url": rec.get("u", "")[:200], "titel": _spool_title(rec.get("h", "")), "html_kb": round(len(rec.get("h", "")) / 1024, 1)} out = {} for k, v in rec.items(): if isinstance(v, (dict, list)): s = json.dumps(v, ensure_ascii=False) out[k] = (s[:220] + "…") if len(s) > 220 else s else: s = str(v) out[k] = (s[:220] + "…") if len(s) > 220 else s return out def query(dsid, q="", page=1, per=50, scan_cap=400000, sort=None, sdir="asc"): d = _ds(dsid) if not d: return {"error": "okänt dataset"} if d["kind"] == "sqlite": try: rows = _sqlite_sightings(q, page, per) except Exception as e: return {"error": str(e)[:120], "total": 0, "page": page, "per": per, "pages": 1, "cols": [], "rows": [], "scanned": 0, "truncated": False} cols = ["regnr", "domain", "url"] if sort == "domain": rows.sort(key=lambda r: r["domain"], reverse=(sdir == "desc")) start = (page - 1) * per total = len(rows) if q else sum(DOMAIN_COUNTS.values()) return {"total": total, "page": page, "per": per, "pages": max(1, (min(total, 500) + per - 1) // per), "cols": cols, "rows": rows[start:start + per], "scanned": len(rows), "truncated": len(rows) >= 500, "sort": sort, "dir": sdir} if d["kind"] == "lines" and not q and not sort: total = _wc(d["path"]) start = (page - 1) * per rows, i = [], 0 for rec in _iter_lines(d["path"]): if i >= start and len(rows) < per: rows.append(_flat(rec)) i += 1 if i >= start + per: break return {"total": total if total >= 0 else 0, "page": page, "per": per, "pages": max(1, (max(total, 0) + per - 1) // per), "cols": ["rad"], "rows": rows, "scanned": i, "truncated": False, "sort": sort, "dir": sdir} q = (q or "").lower() matched, scanned = [], 0 for rec in _iter_ds(d): scanned += 1 if scanned > scan_cap: break if q and q not in json.dumps(rec, ensure_ascii=False).lower(): continue matched.append(rec) cols = list(_flat(matched[0]).keys())[:12] if matched else [] if sort and sort in cols: try: matched.sort(key=lambda r: str(_flat(r).get(sort, "")), reverse=(sdir == "desc")) except Exception: pass total = len(matched) start = (page - 1) * per return { "total": total, "page": page, "per": per, "pages": max(1, (total + per - 1) // per), "cols": cols, "rows": [_flat(r) for r in matched[start:start + per]], "scanned": scanned, "truncated": scanned >= scan_cap, "sort": sort, "dir": sdir, } def record_at(dsid, n, q=""): d = _ds(dsid) if not d: return {"error": "okänt dataset"} q = (q or "").lower() i = -1 for rec in _iter_ds(d): if q and q not in json.dumps(rec, ensure_ascii=False).lower(): continue i += 1 if i == n: return {"n": n, "record": rec} if i > n + 5: break return {"error": "finns ej"} def export(dsid, q="", fmt="csv", cap=100000): d = _ds(dsid) q = (q or "").lower() rows, cols, n = [], [], 0 for rec in _iter_ds(d): if q and q not in json.dumps(rec, ensure_ascii=False).lower(): continue f = _flat(rec) if not cols: cols = list(f.keys()) rows.append(f) n += 1 if n >= cap: break if fmt == "json": body = json.dumps(rows, ensure_ascii=False, indent=1).encode() return body, "application/json", f"{dsid}.json" if fmt == "html": h = ["" + "".join(f"" for c in cols) + ""] for r in rows: h.append("" + "".join(f"" for c in cols) + "") h.append("
{c}
{str(r.get(c, ''))}
") return ("".join(h)).encode(), "text/html", f"{dsid}.html" if fmt == "xml": import xml.etree.ElementTree as ET root = ET.Element("rows") for r in rows: it = ET.SubElement(root, "row") for c in cols: e = ET.SubElement(it, "f", name=c) e.text = str(r.get(c, "")) return ET.tostring(root, encoding="unicode").encode(), "application/xml", f"{dsid}.xml" import io buf = io.StringIO() w = csv.DictWriter(buf, fieldnames=cols, extrasaction="ignore") w.writeheader() w.writerows(rows) return buf.getvalue().encode(), "text/csv", f"{dsid}.csv" PAGE = """ Scrape Console · Alla data

🕷️ Scrape Console · Min dashboard · Alla data

"""