From 6acc587dd5c99193977761e01667cd8e6827ffc6 Mon Sep 17 00:00:00 2001 From: Luiz Gustavo Date: Mon, 25 May 2026 05:29:35 -0300 Subject: [PATCH] fix(case-writer): rank doc chunks by richness, not just reading order MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fetchDocChunks took the first N chunks by order_global. On long files the opening chunks are cover pages, routing slips, classification stamps and redaction boxes — administrative front matter, not narrative. A 1001-chunk FBI dossier (doc-65 section-7) handed the narrator 6 form_fields + 4 redaction_blocks + 2 letterheads and only 3 prose paragraphs, so it refused with INSUFFICIENT_ARTEFACTS — burying first-person testimony (e.g. the Flatwoods photographs) that sat deeper in the file. Now select the richest substantive chunks (by content length, redaction boxes last), then re-sort the winners into reading order for narrative coherence. Validated: section-7 goes from 3 prose paragraphs to 22 (avg 1926 chars). 32 wrongly-refused rich documents re-queued. Co-Authored-By: Claude Opus 4.7 (1M context) --- investigator-runtime/src/lib/search.ts | 43 +++++++++++++++++--------- 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/investigator-runtime/src/lib/search.ts b/investigator-runtime/src/lib/search.ts index 15e664c..df8b7f8 100644 --- a/investigator-runtime/src/lib/search.ts +++ b/investigator-runtime/src/lib/search.ts @@ -68,13 +68,20 @@ export interface HybridSearchOpts { } /** - * Fetch a single document's own chunks in reading order — no semantic - * gating. For a per-document case file the narrator wants THIS document's - * substance, not a corpus search; a hybridSearch keyed on the document's - * (often garbage) auto-derived topic returns zero hits even though the - * doc has dozens of embedded chunks. We pull substantive `is_searchable` - * chunks ordered by `order_global` so the document tells its own story in - * sequence. + * Fetch a single document's own chunks — no semantic gating. For a + * per-document case file the narrator wants THIS document's substance, not + * a corpus search; a hybridSearch keyed on the document's (often garbage) + * auto-derived topic returns zero hits even though the doc has dozens of + * embedded chunks. + * + * We pick the most substantive chunks (by content length, deprioritising + * pure redaction boxes) and THEN present them in reading order. Naively + * taking the first N by `order_global` starves the narrator on long files: + * the opening chunks of a 1000-chunk FBI dossier are cover pages, routing + * slips, classification stamps and redaction boxes — administrative front + * matter, not narrative. The substance sits deeper in the file, so a + * length-ranked pick surfaces it regardless of position, while the final + * reading-order sort keeps the story coherent. */ export async function fetchDocChunks( doc_id: string, @@ -83,15 +90,23 @@ export async function fetchDocChunks( ): Promise { if (!doc_id) return []; return await query( - `SELECT chunk_pk, doc_id, chunk_id, page, type, bbox, + `WITH ranked AS ( + SELECT chunk_pk, doc_id, chunk_id, page, type, bbox, + content_en, content_pt, classification, + order_global, order_in_page, + length(COALESCE(content_en,'') || COALESCE(content_pt,'')) AS richness + FROM public.chunks + WHERE doc_id = $1 + AND is_searchable = TRUE + AND length(COALESCE(content_en,'') || COALESCE(content_pt,'')) > 40 + ORDER BY (type = 'redaction') ASC, richness DESC + LIMIT $2 + ) + SELECT chunk_pk, doc_id, chunk_id, page, type, bbox, content_en, content_pt, classification, 1.0::float8 AS score, NULL::int AS bm25_rank, NULL::int AS dense_rank - FROM public.chunks - WHERE doc_id = $1 - AND is_searchable = TRUE - AND length(COALESCE(content_en,'') || COALESCE(content_pt,'')) > 40 - ORDER BY order_global ASC NULLS LAST, page ASC, order_in_page ASC - LIMIT $2`, + FROM ranked + ORDER BY order_global ASC NULLS LAST, page ASC, order_in_page ASC`, [doc_id, limit], ); }