| NB. ============================================================================= |
| NB. Synthetic David — hybrid evidence retrieve (PURE J) |
| NB. Product path against sql/001 evidence indexes; Python evidence/ stays harness. |
| NB. Locale: david. Verbs prefixed ev_ to avoid clobbering scorer/execution names. |
| NB. Smoke embeddings dim 8; production SQL uses vector(1536). |
| NB. ============================================================================= |
|
|
| cocurrent 'david' |
|
|
| NB. |
| ev_field =: 4 : '> (x { y)' |
|
|
| NB. Open one level if boxed |
| ev_unbox1 =: 3 : 'if. 32 = 3!:0 y do. > y else. y end.' |
|
|
| NB. |
| EV_AZ =: 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' |
| EV_az =: 'abcdefghijklmnopqrstuvwxyz' |
| EV_alnum =: EV_az , '0123456789' |
|
|
| ev_lc1 =: 3 : 0 |
| i =. EV_AZ i. y |
| if. i < #EV_AZ do. i { EV_az else. y end. |
| ) |
|
|
| ev_lc =: 3 : 'ev_lc1"0 y' |
|
|
| ev_tokens =: 3 : 0 |
| if. 0 = #y do. 0 $ a: return. end. |
| s =. ev_lc , y |
| ok =. s e. EV_alnum |
| s =. ok } (' ' #~ #s) ,: s |
| parts =. <;._1 ' ' , s |
| parts =. parts #~ 1 < #&> parts |
| ~. parts |
| ) |
|
|
| NB. lexical overlap = |q ∩ c| / |q| (0 if empty query tokens) |
| ev_lexical =: 4 : 0 |
| q =. ev_tokens x |
| if. 0 = #q do. 0.0 return. end. |
| c =. ev_tokens y |
| if. 0 = #c do. 0.0 return. end. |
| (# q -. q -. c) % #q |
| ) |
|
|
| NB. cosine similarity of two numeric lists |
| ev_cosine =: 4 : 0 |
| a =. , x |
| b =. , y |
| if. (#a) ~: #b do. 0.0 return. end. |
| if. (0 = #a) +. (0 = #b) do. 0.0 return. end. |
| na =. %: +/ *: a |
| nb =. %: +/ *: b |
| if. (0 = na) +. (0 = nb) do. 0.0 return. end. |
| (+/ a * b) % (na * nb) |
| ) |
|
|
| NB. Axes for dim-8 smoke query embed (prod = vector(1536) via encoder) |
| NB. [cobol, balance, payment, schema, pricing, vol, greek, junk] |
| EV_AXES =: 'cobol';'balance';'payment';'schema';'pricing';'vol';'greek';'junk' |
|
|
| ev_qembed =: 3 : 0 |
| toks =. ev_tokens y |
| NB. synonym: volatility → vol |
| if. (<'volatility') e. toks do. toks =. toks , <'vol' end. |
| v =. 0 #~ #EV_AXES |
| i =. 0 |
| while. i < #EV_AXES do. |
| if. (i { EV_AXES) e. toks do. v =. 1 i} v end. |
| i =. i + 1 |
| end. |
| NB. fallback canned COBOL qvec when axis hits empty but query non-blank |
| if. (0 = +/ v) *. (0 < #y) do. |
| v =. 0.93 0.91 0.88 0.25 0.05 0 0 0 |
| end. |
| v |
| ) |
|
|
| NB. |
| NB. row: id ; source_id ; source_type ; document_id ; location ; |
| NB. content ; authority ; parent_boxed ; embedding (dim 8 smoke) |
|
|
| ev_mk =: 3 : 0 |
| NB. y already a 9-field boxed row |
| y |
| ) |
|
|
| EV_CORPUS =: 0 $ a: |
|
|
| NB. ev_copybook_01 |
| EV_CORPUS =: EV_CORPUS , < ('ev_copybook_01' ; 'src_copybook_acct' ; 'copybook' ; 'ACCTREC.cpy' ; '01-ACCOUNT-BALANCE' ; '01 ACCOUNT-RECORD. 05 ACCOUNT-BALANCE PIC S9(9)V99. 05 PAYMENT-AMOUNT PIC S9(7)V99. 05 LAST-PAYMENT-DATE PIC X(8).' ; 0.95 ; (<'ev_provenance_root') ; 0.95 0.90 0.85 0.20 0.05 0 0 0) |
|
|
| NB. ev_cobol_src_01 |
| EV_CORPUS =: EV_CORPUS , < ('ev_cobol_src_01' ; 'src_cobol_pay' ; 'cobol_source' ; 'PAYROLL.cbl' ; 'PROCEDURE DIVISION / CALC-BALANCE' ; 'COMPUTE ACCOUNT-BALANCE = ACCOUNT-BALANCE - PAYMENT-AMOUNT. IF ACCOUNT-BALANCE < ZERO THEN PERFORM OVERDRAFT-CHECK. MOVE PAYMENT-AMOUNT TO WS-LAST-PAYMENT.' ; 0.90 ; (<'ev_provenance_root') ; 0.92 0.88 0.90 0.15 0.05 0 0 0) |
|
|
| NB. ev_db2_schema_01 |
| EV_CORPUS =: EV_CORPUS , < ('ev_db2_schema_01' ; 'src_db2_acct' ; 'db2_schema' ; 'ACCT.DDL' ; 'TABLE ACCOUNT_BALANCES' ; 'CREATE TABLE ACCOUNT_BALANCES (ACCT_ID CHAR(12), BALANCE DECIMAL(11,2), LAST_PAYMENT DECIMAL(9,2), UPDATED_TS TIMESTAMP).' ; 0.85 ; (<'ev_provenance_root') ; 0.70 0.80 0.55 0.95 0.05 0 0 0) |
|
|
| NB. ev_quant_pricing_01 |
| EV_CORPUS =: EV_CORPUS , < ('ev_quant_pricing_01' ; 'src_quant_report' ; 'quant_report' ; 'PRICING_Q3.pdf' ; 'section:monte_carlo' ; 'Monte Carlo pricing of exotic options. Implied volatility surface and greek sensitivities. No COBOL or account balance fields.' ; 0.80 ; (<'ev_provenance_root') ; 0.05 0.05 0.05 0.05 0.95 0.90 0.85 0.10) |
|
|
| NB. ev_lexical_lift_01 — weak dense, strong lexical (no dense floor) |
| EV_CORPUS =: EV_CORPUS , < ('ev_lexical_lift_01' ; 'src_glossary' ; 'glossary' ; 'LEGACY-GLOSSARY.txt' ; 'entry:PAYMENT-BALANCE' ; 'cobol payment balance glossary entry: PAYMENT and BALANCE fields on ACCOUNT records in legacy batch.' ; 0.70 ; (<'ev_provenance_root') ; 0.10 0.15 0.10 0.05 0.40 0.35 0.30 0.20) |
|
|
| NB. ev_junk_low_auth — authority ~0.1, no parents |
| EV_CORPUS =: EV_CORPUS , < ('ev_junk_low_auth' ; 'src_web_scrape' ; 'web_scrape' ; 'random-blog.html' ; 'body' ; 'Someone said COBOL payment balance might work like a spreadsheet. Unverified forum post.' ; 0.10 ; (0 $ a:) ; 0.60 0.55 0.50 0.10 0.10 0 0 0.9) |
|
|
| NB. |
| NB. score = dense + lexical + authority - parent_penalty |
| NB. (no dense floor — lexical can lift weak dense) |
| NB. x = query ; qvec ; require_parent y = corpus row |
| ev_score_row =: 4 : 0 |
| q =. 0 ev_field x |
| qv =. 1 ev_field x |
| reqp =. 2 ev_field x |
| row =. y |
| emb =. 8 ev_field row |
| content =. 5 ev_field row |
| auth =. 6 ev_field row |
| parents =. boxopen 7 ev_field row |
| dense =. qv ev_cosine emb |
| lex =. q ev_lexical content |
| pen =. 0.0 |
| if. reqp *. (0 = # parents) do. pen =. 1.0 end. |
| dense + lex + auth - pen |
| ) |
|
|
| NB. |
| NB. y = query ; min_authority ; top_k [; require_parent] |
| NB. returns boxed list of (evidence_id ; score) sorted score DESC, id ASC |
| ev_retrieve =: 3 : 0 |
| args =. y |
| q =. 0 ev_field args |
| min_a =. 1 ev_field args |
| top_k =. 2 ev_field args |
| reqp =. 0 |
| if. 3 < # args do. reqp =. 3 ev_field args end. |
| NB. empty / blank query → empty result |
| if. 0 = # q do. 0 $ a: return. end. |
| if. 0 = # ev_tokens q do. 0 $ a: return. end. |
| qv =. ev_qembed q |
| ctx =. q ; qv ; reqp |
| ids =. 0 $ a: |
| scores =. 0 $ 0.0 |
| i =. 0 |
| while. i < # EV_CORPUS do. |
| row =. ev_unbox1 i { EV_CORPUS |
| auth =. 6 ev_field row |
| parents =. boxopen 7 ev_field row |
| if. auth >: min_a do. |
| if. (-. reqp) +. (0 < # parents) do. |
| sc =. ctx ev_score_row row |
| ids =. ids , < 0 ev_field row |
| scores =. scores , sc |
| end. |
| end. |
| i =. i + 1 |
| end. |
| if. 0 = # ids do. 0 $ a: return. end. |
| NB. stable: id asc then score desc → ORDER BY score DESC, evidence_id |
| ix =. /: ids |
| ix =. ix \: ix { scores |
| if. top_k < # ix do. ix =. top_k {. ix end. |
| out =. 0 $ a: |
| j =. 0 |
| while. j < # ix do. |
| k =. j { ix |
| out =. out , < ((k { ids) ; k { scores) |
| j =. j + 1 |
| end. |
| out |
| ) |
|
|
| NB. peel nested boxes to open string |
| ev_openstr =: 3 : 0 |
| v =. y |
| while. 32 = 3!:0 v do. v =. > v end. |
| v |
| ) |
|
|
| NB. ids only from retrieve result (single-boxed strings) |
| ev_ids =: 3 : 0 |
| rs =. y |
| out =. 0 $ a: |
| i =. 0 |
| while. i < # rs do. |
| out =. out , < ev_openstr 0 ev_field ev_unbox1 i { rs |
| i =. i + 1 |
| end. |
| out |
| ) |
|
|
| NB. |
| ev_smoke =: 3 : 0 |
| q =. 'cobol account payment balance CALC-BALANCE' |
| NB. 1) COBOL-ish ranks copybook/cobol/db2 above quant; quant not #1 |
| r1 =. ev_retrieve q ; 0.5 ; 5 ; 1 |
| ids1 =. ev_ids r1 |
| assert. 0 < # ids1 |
| assert. -. (< 'ev_quant_pricing_01') -: 0 { ids1 |
| legacy =. 'ev_copybook_01' ; 'ev_cobol_src_01' ; 'ev_db2_schema_01' |
| assert. 0 < # (ids1 -. ids1 -. legacy) |
| top3 =. 3 <. # ids1 |
| top3 =. top3 {. ids1 |
| assert. -. (< 'ev_quant_pricing_01') e. top3 |
| NB. 2) min_authority 0.5 drops junk |
| r2 =. ev_retrieve q ; 0.5 ; 10 ; 0 |
| ids2 =. ev_ids r2 |
| assert. -. (< 'ev_junk_low_auth') e. ids2 |
| r2open =. ev_retrieve q ; 0.0 ; 10 ; 0 |
| ids2o =. ev_ids r2open |
| assert. (< 'ev_junk_low_auth') e. ids2o |
| NB. 3) empty query → empty result |
| r3 =. ev_retrieve '' ; 0.0 ; 10 |
| assert. 0 = # r3 |
| 'ok' ; ids1 |
| ) |
|
|
