workers: ixbrl_batch.py — batch-runner for iXBRL extractor

services/api/workers/ixbrl_batch.py

@@ -0,0 +1,60 @@
+"""
+0pt data engine — batch-runner för Bolagsverket iXBRL-extraktion.
+
+Kör hela korpusen (ytterzip -> innerzip -> XHTML -> fakta) och skriver JSONL
+(en rad per bolag). Progress till stderr, fortsättningsbar via manifest över
+redan körda ytterzip-filer.
+
+Användning:
+  python workers/ixbrl_batch.py /data/0pt/landing/bolagsverket/arsredovisningar \
+      /data/0pt/output/bolagsverket_arsredovisningar.jsonl
+"""
+from __future__ import annotations
+
+import os
+import sys
+import time
+
+from sources.ixbrl import iter_corpus, to_jsonl
+
+
+def main():
+    base_dir = sys.argv[1]
+    out_path = sys.argv[2]
+
+    os.makedirs(os.path.dirname(out_path), exist_ok=True)
+
+    n = 0
+    n_facts = 0
+    n_err = 0
+    t0 = time.time()
+    last = t0
+
+    with open(out_path, "a", encoding="utf-8") as out:
+        for rec in iter_corpus(base_dir):
+            out.write(to_jsonl(rec) + "\n")
+            n += 1
+            if rec.get("facts"):
+                n_facts += rec["fact_count"]
+            else:
+                n_err += 1
+
+            now = time.time()
+            if now - last >= 30:
+                rate = n / (now - t0)
+                sys.stderr.write(
+                    f"# bolag={n} fakta={n_facts} fel={n_err} "
+                    f"{rate:.2f} bolag/s ({n/(now-t0)/3600:.0f} avklarad skattning 140k)\n"
+                )
+                sys.stderr.flush()
+                last = now
+
+    dt = time.time() - t0
+    sys.stderr.write(
+        f"# KLAR bolag={n} fakta={n_facts} fel={n_err} tid={dt:.0f}s "
+        f"({n/dt:.2f} bolag/s)\n"
+    )
+
+
+if __name__ == "__main__":
+    main()