From be6909bce2896972db9c56dce6c757dcfb36f173 Mon Sep 17 00:00:00 2001 From: Simon Axelsson Date: Thu, 25 Jun 2026 01:45:38 +0200 Subject: [PATCH] feat(engine+edge): live vehicles + screening endpoints, entity-resolution + job-queue + financials modules Edge /api/vehicles (RDW NL, live now) + /api/screen (OpenSanctions, live now), both defer to engine when ENGINE_URL set. Engine: /vehicles/{plate} + /screen endpoints; workers/resolve (blocking+fuzzy entity resolution, now used by pipeline); workers/queue (in-process job queue); sources/financials (Companies House accounts + XBRL path). py_compile + tsc + next build green. Co-Authored-By: Claude Opus 4.8 --- app/api/screen/route.ts | 27 ++++++++++++++ app/api/vehicles/route.ts | 34 ++++++++++++++++++ .../__pycache__/main.cpython-314.pyc | Bin 11706 -> 12665 bytes services/api/orchestrator/main.py | 12 +++++++ .../__pycache__/financials.cpython-314.pyc | Bin 0 -> 2404 bytes services/api/sources/financials.py | 34 ++++++++++++++++++ .../__pycache__/pipeline.cpython-314.pyc | Bin 5377 -> 4771 bytes .../workers/__pycache__/queue.cpython-314.pyc | Bin 0 -> 2462 bytes .../__pycache__/resolve.cpython-314.pyc | Bin 0 -> 2408 bytes services/api/workers/pipeline.py | 12 ++----- services/api/workers/queue.py | 31 ++++++++++++++++ services/api/workers/resolve.py | 33 +++++++++++++++++ 12 files changed, 174 insertions(+), 9 deletions(-) create mode 100644 app/api/screen/route.ts create mode 100644 app/api/vehicles/route.ts create mode 100644 services/api/sources/__pycache__/financials.cpython-314.pyc create mode 100644 services/api/sources/financials.py create mode 100644 services/api/workers/__pycache__/queue.cpython-314.pyc create mode 100644 services/api/workers/__pycache__/resolve.cpython-314.pyc create mode 100644 services/api/workers/queue.py create mode 100644 services/api/workers/resolve.py diff --git a/app/api/screen/route.ts b/app/api/screen/route.ts new file mode 100644 index 0000000..3dac028 --- /dev/null +++ b/app/api/screen/route.ts @@ -0,0 +1,27 @@ +import { NextRequest, NextResponse } from "next/server"; +import { engineFetch } from "@/lib/engine"; + +// GET /api/screen?q=Name — OpenSanctions screening (sanctions/PEP). VERIFY +// commercial terms before production volume. Defers to engine when running. +export async function GET(req: NextRequest) { + const q = req.nextUrl.searchParams.get("q") ?? ""; + if (!q) return NextResponse.json({ matches: [] }); + + const eng = await engineFetch(`/screen?q=${encodeURIComponent(q)}`); + if (eng && eng.ok) return NextResponse.json(await eng.json()); + + try { + const r = await fetch(`https://api.opensanctions.org/search/default?q=${encodeURIComponent(q)}&limit=5`, { + headers: { "User-Agent": "0pt/0.1 (+https://0pt.app; SIAX)" }, + cache: "no-store", + }); + if (!r.ok) return NextResponse.json({ matches: [], source: "opensanctions" }); + const data = (await r.json()) as { results?: { caption?: string; score?: number; schema?: string }[] }; + return NextResponse.json({ + source: "opensanctions", + matches: (data.results ?? []).map((x) => ({ name: x.caption ?? null, score: x.score ?? null, schema: x.schema ?? null })), + }); + } catch { + return NextResponse.json({ matches: [], source: "opensanctions" }); + } +} diff --git a/app/api/vehicles/route.ts b/app/api/vehicles/route.ts new file mode 100644 index 0000000..ee0dc1a --- /dev/null +++ b/app/api/vehicles/route.ts @@ -0,0 +1,34 @@ +import { NextRequest, NextResponse } from "next/server"; +import { engineFetch } from "@/lib/engine"; + +// GET /api/vehicles?plate=XX-99-YY — RDW (NL) open vehicle data (technical/fleet +// only, no owner PII). Defers to the engine when running; else calls RDW directly. +export async function GET(req: NextRequest) { + const plate = (req.nextUrl.searchParams.get("plate") ?? "").toUpperCase().replace(/[-\s]/g, ""); + if (!plate) return NextResponse.json({ error: { code: "bad_request", message: "plate krävs" } }, { status: 400 }); + + const eng = await engineFetch(`/vehicles/${plate}`); + if (eng && eng.ok) return NextResponse.json(await eng.json()); + + try { + const r = await fetch(`https://opendata.rdw.nl/resource/m9d7-ebf2.json?kenteken=${plate}`, { + headers: { "User-Agent": "0pt/0.1 (+https://0pt.app; SIAX)" }, + cache: "no-store", + }); + if (!r.ok) return NextResponse.json({ vehicle: null }, { status: 200 }); + const arr = (await r.json()) as Record[]; + if (!arr.length) return NextResponse.json({ vehicle: null }); + const d = arr[0]; + return NextResponse.json({ + source: "rdw", + vehicle: { + reg_plate: plate, market: "NL", + make: d.merk ?? null, model: d.handelsbenaming ?? null, + year: (d.datum_eerste_toelating ?? "").slice(0, 4) || null, + }, + provenance: { level: 3, source: "rdw" }, + }); + } catch { + return NextResponse.json({ vehicle: null, error: "fetch_failed" }, { status: 200 }); + } +} diff --git a/services/api/orchestrator/__pycache__/main.cpython-314.pyc b/services/api/orchestrator/__pycache__/main.cpython-314.pyc index 307123e45278e772c89ccc1cb1cd832fb23d07de..ef0a4a7b4a8cc14ca5d837025e8f1a1c00597ddc 100644 GIT binary patch delta 3968 zcma)8du)@}75}bZN$ijpPMjAcP8?$=5EBSEFJ4Omq)SY=gfdElz;GI&rUp^C}}HM=TzYL-r2r0|8&onRFj|1Z_2hFbQ|sl<)lK3hAIzM z8C4C+c(GTZPU@n?VG9Bj#qf-KMmy&3RMxF)Cv@ds61JJg0CA3Pn_sGN)pindYc$O7 zG}i7P?J}47^m%)_v$t9zKuBUEVNvo)b zR)?%cBdvjTEv=*VbOYT8`wg^_dTCS0VR&gX_0bmUr<-Vi24UVx10lQ7RH=adZ61Yg z9xv~-ni7m`Gu;AP?H(mrX81yi(LzJxe!A5U=dju&R~@*L@NK}mJr5oL2OfE%?8=OG z=8W0lQ975mjn-$mvv@{#=Gen|>}}7oOYW{5cQ>fhgR195Y+eJ?o5S=0W>*e_@{KKP zfOh9VdjPaI2SQ9cV4|qb>L1;g!+Z@e{i`rrpT&Fw;dJ2SAL-!AtEBq@cwiN5Sg`E46=^M6@G+3hlD!*Jh7UeG$>{OoGAr4~cE+aiECv`T}W&#EplqdClBz|d8g zaL*cQAIX8n0CaQ}sEb;fmEfxjuiL&ud=}x>(rQYNrQ$|HOX*QP&QvueTx@bSI-{OW z#FOg6L;FQSd^}z-Pu%=J1-FJMqjmABq1Q8G8)y9 zHy(`{21`t23{{Ut*;f8};fT|RY)E{(yNEi%a>?h54uV)wRKH~_Yy^pLOlfR7X6TsD zO$pq^|H z$n$UO=0oM*&TfJ8V41K^$Ic0BQcW@GPOGjO(tE*?sY$ptk&3@*xN(<|fvKUUdbMtz#URe5)y0#1TO$c-t;UkZX?6j)h8M0@IS6@+b5DEVJW4X-Er>42 zSs9^Sp;ApPD4c_+oVfxNE8UdLN=d*Q7QP zOR`I_fY%{!dK=MsFHE>}EyLoF0`R2S6*zsB|F-5(d3k6#>GaT>=_G`Nzvd0opp~Ig z!Y;R7POaBb;(8!+%6}CLX?ld@#(FuhFuz$_Lq_?n#`XMetuuQ-p~@NPSm(%`b=dAy z(qxSE5=?0Pow6%Rz0#miLX{9{Rtl8iiXGMj30fkeU|C=A5Pl?pz3;#v$V|Sq>?-U@ zasnoLE08ZiufLbn+-SL1(s;kOuyJnR16T2Vmv_M3eh94jzPyM2t0_;6Fn&L%u%uEgBG;y`^SOuWu+1LK?A=S`2u zLi7_@zTBg}2%Nc?+eqM%0t7Q228j7X@7*3$!nE0eGyF!mZDA-v5G_ArMG+L-Ws5$B z<2n9CQ-)mOEjubVud zK-=HbE9;$9%(Pv;If;DHn|qfZ_d)yT-}L=>>}M$I=SWc9piKrL&wjaUHN{8?zXUcj zxXWJx!U=6zez;_G_9>iRV@CY6 z40nh0AVagj@;|^JE#GUF_`<$vh*nbF%^tuB7Ek?SzPqi>_Jy+dZwqJI(qxC{qGz$F w{vO%zl?D3mzpaX8;}Xu6oQlPM8BVX%+#~D1vOxdAL-!KS_+K`EZo`cEA3QA}`~Uy| delta 3163 zcma)8S!`5Q7@jjT9olIr(~Y*!(q-CCZ#zrr!qx=}ZP};Ip@>+^IPKiRz;tds_YMW4 zPI(Y2Mq@c>L?dwtni^kZ5?o&Ng+vn`+=6)GgAeM1Q3Ed;dGi0y?Q|hjXwv^||N8yQ zoqtC@8_I3WUFo#IU)S1>& zw);G~Xx_#-^=UI8ux5eNSbOBQQAYD=0WG9ObTusw*o<;o0_Re?hOVV$v>fi&(F$5g zs{#&V9j&HrT0?7T9re(9*elc%$S{%>MHaxIq1Zwjhx1x(rU|2xdf~=bY>Byys({6) zrv70!4b;IioNDB$2~SdbJ@7U!lCA{|_b)h`-qDu%U|NeUCPqns+_MaO+dTV*G&^wD zFT=fYo_iCxvl(4ar@>G_=3v^5g_Uk;P0qng+d-&fiBRJ*LY+sxqFw2Fp<6+2+Y&kN zG7fgmJJ`JtPSWr#!@hl?Nyii+?hBbSB%fc%e0!f8JP@lwp>RB| z8DS%;#r05#)xs^q(qSGX4M@C5d_Y7|D5N9rXeewLEIN`fR21^_WY&IXHL@Xb^P5@J zft|yux!<{K4_*ofQ;h_R-_4CgM~xxfVBX5^_~~Khhew47$C$>(!-kF-uVx!iB4^$? z6^wMkm5+bnbd$E!FV0~*SRQtB5f#yb-;d6J<1A{vE&d&?nwIDA#-JIshWxptU0py?Z)W|mF06$Ri2ciq#n)4 zd4k;{oU-5|YJ5~xR+tt2{{tvfV-b~wq7hk#!hzE$p^Jpap?dPSeu*ae>+)$BloD9bjs(x?HW+R6wud~a*sPSiNCd?wd1jmOf!UQj`#iYNX zl=1_$sV!*H4A6nQD3DnzLw$m1OA0XyI&04T_u<*#+v*bJX@0HF9g^jL5#_NAf{Nq5 zGeEZg{W^aZC6QnU=qrFg=jW_Cx7Y0SS@;@kZI|UO9Ww)+P76J-&d>0x9vJ`prswkD zJ1F%o5_DIwOT-t&FK=z88EN4?U^6p0_cahs7|rr~meARU@H)l6tbe)gE!aKKlk@yp z#RUJh!}NsDG!*QX$l+WJ;^*#URGm~Z?sY5+^<#F}6(CCzV`GUp`v`8I=YKX_e>hKG z;kO#iSyamNyr+CpGuAn@KPG>1`#Xt@AlT+ z*-C5FMtx8dBg6;#Sff#VBk@I}iGVfn#pIwUb)%7lC*D9}8WSJ<=KcU7kT}9T!_>>JyuDa8-1NY1 zB_+S~oO&>AvQai&nNAUTQg>-WCZ}|R%4OXm@^-6AZ>CkZu}oO!;hGijDht0aM(R-IXSyF>JHLY^kG$&yY< zQIU14C_{{1g)o=QwjLolMI8vstXi^5iJg)Z>_>D9;?o(^9fvuev1aEI12#<-b*d+c zRWdD-9!NbQZ||R?)OLr`X^^IL+a8t=k7Zs`3=H{ZowWdT9;SC>Am&jXo#6Joi%=1G zkD(Xw6jrgysk|CUvlI(A*`fKOqj-MnE|3{P81x+p}QMlm`AIq9ZR>_aIWtb`icE{Yn{;FVf+3ZcGQbD zgK?oQU_-81ns*sMY=HtDY-Y{IdAzMDEQML7RGf!$O+l$bruaUj3V9TT^4!7OTp`f% zlM`Fiy)XC;7DBCg8lQ-%p(mT`4`Yq@IFHx3!KT7KtcFj**_^`j{6RrNHlOEnk(Twn z@3Aeaw3AaKEl>N<19pLYAjgvTKs)i%p4Z^?H{8FcG(_NWYMA{7VQtp-!TkO>HlscS z9%BvZ{bMWghefGIQ)sZcS6fi=5u~=M?JAcS)L4R@XtTx9QPG3!dffAS5VA!zo);hL zFWx#Kwc`;Q9d|W4AED9N(n#~;+<_s4!e|q$Y>R3B|K%Zw4*m@NE9f730A$S8L@857Z<3so_2}%=aJ@@8V6Ytns_EhhE5K#>YkvYx@r! zJv^#qa%0+o(U<+8QAes-KiI$;*ADGmGs<}bYszE6X2dT0M$x>bbDI}I47}tpUjj1-xSUd zP7%EbIN%4VSs_)Aj{6KP0=h@1e8Hro;`(Ca`unlQGJwI_6vN=5@tf>P8TSK#cf=3S zS)2HgB5@7Jv>7TYA)l*)eTh(F&G-V_)hR!cac3TU*;bX_?W=w3lLHIr{*PFxcpWppJoLciBoPLnj zA!;X}$4$VQ-=*kB$_GlVE4eJC=CglB+E?Nm7IrT_b#CJPeae~w#JPH$cA zOk9g5-oq=A=tq0s-@A~#8ri(k)w3|OID5Xgwr1CA5XE1_V0|T)yA?tmooB@};>V%0 z(KFG7V~Z2ZapgVXdUWmSnU!c)ZQZt&=-Qe$S9|@ki`#Y6TfsnFT8%UV zxEjm-7ERpd+4P^k;>`DIM^gszEyr^Ix~Cz5Z)dtkJMhH~cw}4X;zpj)J?lpXg^RoJ zh!VQEdn3@7;%&gc)PqO*#Y>w6M)wT>|A#ftj3hgkI(VR$x|q(=7Cf>;T|zY)$heO`G)C{~YihF)(vB zv3d}pFyYa67+^Xw8+zHms=OTGh17J}J literal 0 HcmV?d00001 diff --git a/services/api/sources/financials.py b/services/api/sources/financials.py new file mode 100644 index 0000000..188ee3f --- /dev/null +++ b/services/api/sources/financials.py @@ -0,0 +1,34 @@ +""" +Structured financials (docs/06). XBRL/iXBRL is the fast, exact lane (not OCR). +Companies House publishes accounts as iXBRL; this fetches the accounts filing +history (real, key-gated) and documents the XBRL extraction step. +""" +from __future__ import annotations + +import os +from typing import Any + +import httpx + +USER_AGENT = "0pt-data-engine/0.1 (+https://0pt.app; SIAX)" + + +async def companies_house_filings(number: str) -> dict[str, Any]: + key = os.getenv("COMPANIES_HOUSE_API_KEY") + if not key: + return {"company": number, "filings": [], "note": "level0: kräver COMPANIES_HOUSE_API_KEY"} + url = f"https://api.company-information.service.gov.uk/company/{number}/filing-history?category=accounts" + async with httpx.AsyncClient(headers={"User-Agent": USER_AGENT}, timeout=15) as c: + r = await c.get(url, auth=(key, "")) + if r.status_code != 200: + return {"company": number, "filings": []} + items = r.json().get("items", []) + return { + "company": number, + "source": "companies_house", + "filings": [{"date": i.get("date"), "type": i.get("type"), "description": i.get("description")} for i in items[:10]], + } + +# XBRL extraction (extension): with `arelle` installed, download each accounts +# filing's iXBRL document and pull tagged facts (turnover, profit, employees) +# into fact_financials with provenance. Structured, exact — never OCR for XBRL. diff --git a/services/api/workers/__pycache__/pipeline.cpython-314.pyc b/services/api/workers/__pycache__/pipeline.cpython-314.pyc index 77e79eacd9660fd0e84381e80c0ea1de50e28131..30cbb051b698a56a2a794ec0e29fb54d26a933c2 100644 GIT binary patch delta 783 zcmZWnO-vI(6n?YaZvW`+Z2xJYKa{@$7D7ObA>sznH5g*nxE^f8m^OsA&^k@k8|c*w z$y~VW!K>-TgFg`osV76cn8qt-fz$*N4$javA@L>iy>Gsm_hsJXy&QQnAdkv!C$Kmw z%;rCJy_Y{DyN%>3rQ)+(Z6Ar&NE%5;z+ij+kf(Ri^YmefiS$tFbvzaU`XC0IG-C_Z zCT!n@kxYy`@-VGasKv^PsbIK|t3n#8!c9Lw8Jn1XamF_AlCT&Aa6lBwY8ht|tqNAH zbgD0CYDXJ8e|Ov6W2!^#vg{>*9>^p+E}4N2GjI@06~k0pXgSFM7wu`mmDIe3YIqFq zqpaaG{6@eCUcAwHfDtl1Mz<*$VbeP<8WB?+wzbtv9TsR9(+DES2h7%11REm*&|S z(dY`kERZYoE$*B0F=cQt{$G%8Zet_W_2qo&2{GAjl&y+O0H~eeIV3m7KnZNE{FI}c zw~p-ewP$-{93nF(zk~56AT{)5R)9Q9rcm*R-?gj-M3kKWr@d0rxg9I2$+gGm+N(;K|A6e16Jb9~0Gw>r1au9dRY pV(uDQW+FeREHOA?4I$JNL?r%o0!ki0-~f_z*EelvLro&=%0JI!p!Wa( delta 1474 zcmZt`T})e5_&fLZ2MWEt_qP0u0xd9#Ep;#p3jvgM{LH4L!2`Q7vA~cr>;i4MUAKjo zV(SSI#rU1lTVUBq?svZb|9ie@ zecvav3swu@k*E4E|75)@d{2yqOPS^6%}2yw9PA>deL%22_>^56jfHC~Qedi1w43Fa zb@80Qdli8E`=0?|MrsBZtZ0+w+RfAE9WHwcQZ~gX!pK_yNh#{mtrRIhs8|CFWr|IN zt|FTnyNa$;q9V2xxiL3DJw%&yTPi6c<@|Lz4M1i8ApyD%)XH)m6ok(LD7G=%5uGAB zE_IOu)CJ*jfO2R=O%X0g8SWHhxREM=l%z<}mArHWrIRNSh&iRuEN_8L&+!;a~mUh2I9w8xg(tb;me z6|J6%(HdGyPtdxTDO0Cnr}c`7HYkn`18r2~4r2?LVVI0=xglCg)C4gzOU?zdcjR5=E~uip26nP2*C4$GxJm-$G@3t#%_A#V3j-nzElwDIOw?OW1T&v(v;zM-4JUmd^I{Zf~kxR6t#dlQqn>5I9U`Mv4) zc0=*pLgHa4@k#LG;HTG~r$zL$S{-zd&)N{&vse($-#P!+vg6J_1M|0UvGeu7Y;m(u z8Gxk~(7vgR)T_OSrdiiK?iaFDv}WrK;?Op&?=?lY5?E|9lZfT{ zbY{_8rFjImX#{J4nUvbAeIPWS!CwYE9{2Hc5y?Wdyjjg8rY|SoRoM(# zJJS(N=GkUm7$9R${m|OBa{jT^u|}Q<@*~!#U3)UOl-U)ouJM0V2v6KCpAT)cY%J{g z`nG2Ge8FA!xwXL?mAjP*^6+yS_rvJ+b2n=-m=|lV7S-F_yHTjAI-s9yzaXkp};Hm1Lqo%cp!

W8Vi9vs`6Ina;1Pymw)=+2pj-O|o4#MzGE8-DvRo73L`2G*H0~Q4U diff --git a/services/api/workers/__pycache__/queue.cpython-314.pyc b/services/api/workers/__pycache__/queue.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..55557b13974e2437516708cb722e8f4df76cbc47 GIT binary patch literal 2462 zcmb7FU2GIp6ux)v%26d(6$m)7|NI+U@Le z?_ElH+MrR0iJ*z0#0M07fH$KLKDR#jDp4#>NYwDeTS+Bo@WFFuc1vkc;!Wn5iV%@ilK<7KOCY0u}1S=O^_cdA?K%UQ?ECbzVyBGs7f z_G@h3EQR^BrA?aISYF!hZcY{~H#h*wg6qyc!~mgc zsCaE3NGm9VI0j&)V-Zet3}8r2s1W0_PJov^UhzTlcvS~@^&r3!a)XD9lSf4KSq!Gc zCx);OiNtHoz*NvUIwrmP9zr>gHKQYV8tYh>dT<-MM(zk*SEUv&Q^%D@ijH*+L1Zc* zH)nDtOt!iQ2KJ@jW)O9n*>goFod;0Llj*YEwX4Lt%u*>0izjSINZKsfX&-8q_VUtQ zT5z`*MinW`Fx+^}>F$lt8jN^cU%&fwchd~F8f)zJt7`9_`2%=#_ z5WdI@KxkFat=A8IA6brk7d<0?6rhbDdn7+92)z8Xd)klBk4|)1S9G<9hzmf)6?LU4 z$0cyEE1NXWDA+)>PLM5vM1Z6)_5ASi=J`xxVyyv)_*fE11qp;Z1h5^SlT|TY9q^{F z-tdIJ%j+<~78$+|5DKlm157JQfQ@l93jF3VDxBIhCK~rhd#lBH2ztm2=$ynAGOUEb zl1M@T4XZB^8`Gy?Yf7R`Fu9~u<|#tA15ZUI)8~LNVZWner=O32<#0yz;x;Fwq-xY`-2IPFipZy=0*E%TRjW^;w4?)pJ zu!NInTuMS>H)|yR?G#aIn?~)F1bP#S0kZfae61Qvi+b^figKE0J0+495qU$G(gHAV zjFDPp)LwuDY5|{1*;qqkkFUD_^sTmAR9K3Pjhw2RfmdV7v4^|0? zAFQg6Y`qa{`LJWZ=GS0ohU<1H05z*N@3k zYrL$aJ{`=^jBwmhtDs;5_YVC^odxU1H&J7rl#;ot&R2g19aF2?MPUApal`oXwmG@h# zNGnwY5skz<%T>I7v&k1X*z1RSfW8cxqSdhjKp+tq-<1fK?*$QVyN&99MV;4C=WlBL k8FqeXG5hJr57E{eYFmZeSECp@f*|cE)bL^WwBq^w55;u%%K!iX literal 0 HcmV?d00001 diff --git a/services/api/workers/__pycache__/resolve.cpython-314.pyc b/services/api/workers/__pycache__/resolve.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..19968640267080a69736a04ee5da45d68200f6fd GIT binary patch literal 2408 zcmai0OKcNI7@oD)cH%5~HE93=S(3)Ziu2$hgb3p0=HbwWbVUs*j;+Nzc9yJn&Fnbf z2&q2w5Y$6+=plU^f_flaC{>Y4r5@5#IXbmS(xr#emIEgeMTkpP^`G^Fhe-QNJ2U@3 z|HJdo_y6P0+AUrLbZ7F&#YPuGe=?>_&f2o8v{u85kdPl7TC6jPMr#4t_KG%_TP*(}lt>^0>S7UM=L zE$dk^L5!3*_-=n_Kr?o z6RLvsIPMmSoK}@YW_lV-v}~WKV2Vjf)m4+K@jg*EM0^D#x~z$DS<}pJub;DOWL-C? z%noM)>pY=nVd9yaI0qSJA0r+ww(DBahkX zM*I%TnZ(p`WzuO({2cLs)V?zxsez?cZ5Kxj;*w_I?VKNhMGpNg)O|X5ZE$v|AnaTb z+VevDeB*rlM)G=cA@Q}b;_J)%`tCN~mF{;OU+EZF>KG_=4CQM=c|Js#mzOZ<27!&q zw`|v&V^VTSjvgoA(oL9+H`MSHOGlZgyAolPVeb{%Bp=n~J1Nht=+qi~;?5VEtyO4>U@8Xz{Qhaq`u@ZogRttZE<7g~R^{L7MG ze9q{HV^*!4Bp55P((_~0q%cVW&z>C1U6lGgc5xDh2dkbMlUN6RLSyi>s0Mk1iJkC% zSY8uT1t;W;MlCmVU92lmNiDEXw%STYhFTo4T+q4W6BcKftjc^o5vbS?py1#Mox+4& zzFjF#V39+sVp}fwU`x{sw_4vc<9g80G;`v&x~+2!pIrSayU_6E)#bX*JN!y-|1Z7$ zi}KI-Cwy-peCUi4&TU!hk0z>!q-w%uw+D^~(&jyx+)>U8U4fVR3?>)Y_yWo3g z@zQkQyXU_9XjvF!O>Owv zaK2^l+-?~1JG&MxEw$`j7WS=gL$%Fo+mTS08@8){#Lqp3l0g1y%M**izlm7P^2TC` z47`c#iDJcOxoI{HeVaf>fL>a8*X^4s-WkiI-I8pl#Uxi9Nf}B;!$(OM?6NLlcEGUa g3ilaH!9oBzprH; dict[str, Any]: break # first compliant source that answers wins per_market[market] = got - # 3) structuring — dedup by fuzzy name within market - deduped: list[dict[str, Any]] = [] - for r in records: - dup = next((d for d in deduped if d["market"] == r["market"] and fuzz.ratio(d["name"], r["name"]) > 92), None) - if dup: - dup["confidence"] = min(0.99, dup["confidence"] + 0.1) # corroboration bump - else: - deduped.append(r) + # 3) structuring — entity resolution (blocking + fuzzy match + corroboration) + deduped = resolve(records) # 4) enrichment/fusion already folded into confidence above. # 5) QA — drop rows below the order's confidence threshold. diff --git a/services/api/workers/queue.py b/services/api/workers/queue.py new file mode 100644 index 0000000..8aa79fe --- /dev/null +++ b/services/api/workers/queue.py @@ -0,0 +1,31 @@ +""" +Lightweight in-process job queue (docs/02). Adequate for single-worker dev; swap +the backend for Redis/Celery at scale without changing callers. +""" +from __future__ import annotations + +import asyncio +from typing import Any, Awaitable, Callable + +Job = Callable[..., Awaitable[Any]] + + +class JobQueue: + def __init__(self) -> None: + self._q: "asyncio.Queue[tuple[Job, tuple[Any, ...]]]" = asyncio.Queue() + + async def enqueue(self, fn: Job, *args: Any) -> None: + await self._q.put((fn, args)) + + async def worker(self) -> None: + while True: + fn, args = await self._q.get() + try: + await fn(*args) + except Exception as e: # keep the worker alive on a failed job + print("job failed:", repr(e)) + finally: + self._q.task_done() + + +queue = JobQueue() diff --git a/services/api/workers/resolve.py b/services/api/workers/resolve.py new file mode 100644 index 0000000..92330fd --- /dev/null +++ b/services/api/workers/resolve.py @@ -0,0 +1,33 @@ +""" +Entity resolution (docs/06): block, score, and link records across sources so the +same company from GLEIF + Brreg + CVR collapses to one record with corroborated +confidence. rapidfuzz only; deterministic; no external calls. +""" +from __future__ import annotations + +from typing import Any + +from rapidfuzz import fuzz + + +def block_key(rec: dict[str, Any]) -> tuple[str, str]: + """Cheap blocking key — same market + first initial — to avoid O(n^2).""" + return (rec.get("market") or "", (rec.get("name") or "")[:1].upper()) + + +def resolve(records: list[dict[str, Any]], threshold: int = 92) -> list[dict[str, Any]]: + out: list[dict[str, Any]] = [] + for r in records: + match = None + for o in out: + if block_key(o) == block_key(r) and fuzz.token_sort_ratio(o.get("name", ""), r.get("name", "")) >= threshold: + match = o + break + if match: + # corroboration: independent source agreeing bumps confidence + level + match["confidence"] = min(0.99, match.get("confidence", 0.0) + 0.1) + match["agreed_sources"] = match.get("agreed_sources", 1) + 1 + match.setdefault("merged_from", []).append(r.get("source")) + else: + out.append({**r, "agreed_sources": 1}) + return out