From 912758e18a61ff374b40eb42a3bff29500163592 Mon Sep 17 00:00:00 2001 From: Luiz Gustavo Date: Wed, 27 May 2026 00:29:05 -0300 Subject: [PATCH] i18n: complete-language pass over all case files + legacy tag handling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit translate_case_files.ts: per-file localization pass that guarantees a case file is complete and parallel in EN + PT-BR — translating the missing-language title, sections, and verbatim blockquotes FROM whatever language exists, with citations preserved exactly. Validates (two H1s, no dropped citations, PT accents present) before writing; stamps bilingual_normalized in frontmatter; idempotent. Ran across all 89 published files (88 via LLM, 1 legacy fixed deterministically). bilingual.ts: pickLocaleBody now honors explicit "(EN)"/"(PT-BR)" heading tags as the primary split signal (above section-number ordering and language sniffing) and strips those tags + trailing rules from display. Fixes the legacy "## §N — Title (EN)" format which the reader otherwise showed with visible language tags. Verified live: green-fireballs-bilingual, d44, doc-65 render one language per locale with their own titles. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../scripts/translate_case_files.ts | 204 ++++++++++++++++++ web/lib/bilingual.ts | 28 ++- 2 files changed, 230 insertions(+), 2 deletions(-) create mode 100644 investigator-runtime/scripts/translate_case_files.ts diff --git a/investigator-runtime/scripts/translate_case_files.ts b/investigator-runtime/scripts/translate_case_files.ts new file mode 100644 index 0000000..cbd9850 --- /dev/null +++ b/investigator-runtime/scripts/translate_case_files.ts @@ -0,0 +1,204 @@ +#!/usr/bin/env bun +/** + * translate_case_files.ts — make every published case file COMPLETE in both + * English and Brazilian Portuguese. + * + * The narrator emitted bilingual files, but with gaps: most lost their + * English H1 (the extractBody bug), some scenes exist in only one language, + * and many verbatim blockquotes appear only in the document's source + * language. A reader on the EN site should never hit Portuguese text and + * vice-versa. + * + * This is a TRANSLATION/localization pass, not a re-narration: we translate + * the missing-language content FROM the language that already exists. No new + * facts, no corpus lookup. Citations are preserved verbatim. + * + * For each file in $CASE_ROOT/reports/*.md: + * 1. Split frontmatter + body. + * 2. Ask Sonnet to return a complete bilingual body (EN title + PT title; + * each numbered section in EN then PT; every blockquote in both). + * 3. Validate (two H1s, citations preserved, both languages present). + * 4. Write back, preserving frontmatter and stamping bilingual_normalized. + * + * Idempotent: skips files already stamped `bilingual_normalized: true` + * unless --force. Usage: + * bun scripts/translate_case_files.ts # all unstamped + * bun scripts/translate_case_files.ts --force # redo all + * bun scripts/translate_case_files.ts --only # one file (dry-run prints) + * bun scripts/translate_case_files.ts --dry # print, don't write + */ +import { readdir, readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { callClaude } from "../src/lib/claude"; +import { env } from "../src/lib/env"; + +const CASE_ROOT = process.env.CASE_ROOT || "/data/ufo/case"; +const REPORTS = path.join(CASE_ROOT, "reports"); +const FORCE = process.argv.includes("--force"); +const DRY = process.argv.includes("--dry"); +const ONLY = (() => { + const i = process.argv.indexOf("--only"); + return i >= 0 ? process.argv[i + 1] : null; +})(); +const CONCURRENCY = 2; + +const SYSTEM = `You are a bilingual editor for a public UAP/UFO archive. Your ONLY +job is to make a case file COMPLETE and parallel in English and Brazilian +Portuguese. You translate; you do not invent, summarise, or re-report. + +HARD RULES: +1. Output ONLY the markdown body. Start with the first "# " title line. No + frontmatter, no code fence, no commentary before or after. +2. Structure, exactly: + # + # + ## I. + + ## I. + + ## II. + ... + ## II. + ... + The SAME section numbers, in the same order, each appearing once in + English then once in Brazilian Portuguese. +3. COMPLETENESS: if any title, section, sentence, or blockquote exists in + only one language, create the other by faithful translation. Nothing may + be present in one language and absent in the other. The two language + versions must carry the same facts, same numbers, same names. +4. CITATIONS: every [[doc-id/pNNN#cNNNN]] link must be preserved EXACTLY and + must appear in BOTH language versions of the section it belongs to. Never + alter, drop, or invent a citation target. +5. BLOCKQUOTES (verbatim source quotes): keep them as "> " blockquotes. In + the English section the quote reads in English; in the Portuguese section + it reads in Portuguese. When the original quote is in one language, + translate it faithfully for the other section (a translation of a real + quote is fine; do not fabricate quotes). +6. Brazilian Portuguese only (NOT European). Preserve UTF-8 accents: ç ã á é + í ó ú â ê ô à õ ü. Never strip accents. +7. Do NOT add facts, dates, places, measurements, or names that are not + already in the file. Translation only. +8. Keep the non-fiction best-seller voice already present. Do not add + "In summary", "Em suma", section recaps, or editorializing.`; + +interface Parsed { fm: string; body: string; fmObj: Record; } + +function parse(md: string): Parsed { + const m = md.match(/^(---\n[\s\S]+?\n---\n)([\s\S]*)$/); + if (!m) return { fm: "", body: md, fmObj: {} }; + const fmObj: Record = {}; + for (const line of m[1].split("\n")) { + const kv = line.match(/^([a-z_]+):\s*(.+)$/); + if (kv) fmObj[kv[1]] = kv[2].trim(); + } + return { fm: m[1], body: m[2], fmObj }; +} + +function citations(s: string): string[] { + return (s.match(/\[\[[^\]]+\]\]/g) || []).sort(); +} + +function h1Count(s: string): number { + return (s.match(/^#\s+\S/gm) || []).length; +} + +function extractBody(text: string): string { + const t = text.trim().replace(/^```(?:markdown|md)?\s*\n?/i, "").replace(/\n?```\s*$/i, "").trimStart(); + if (t.startsWith("# ")) return t; + const h1 = t.indexOf("\n# "); + if (h1 === -1) throw new Error(`no H1 in output: ${t.slice(0, 120)}`); + return t.slice(h1 + 1); +} + +function buildPrompt(body: string): string { + return [ + "Here is a case file body that may be incomplete or mixed across the two", + "languages. Return the corrected, fully-bilingual body per the rules.", + "", + "=== CASE FILE BODY ===", + body, + "=== END ===", + "", + "Return ONLY the corrected markdown body, starting with the English title `# `.", + ].join("\n"); +} + +async function translateOne(file: string): Promise<{ ok: boolean; note: string }> { + const full = path.join(REPORTS, file); + const raw = await readFile(full, "utf-8"); + const { fm, body, fmObj } = parse(raw); + + if (!FORCE && fmObj.bilingual_normalized === "true") { + return { ok: false, note: "already normalized (use --force)" }; + } + + const before = citations(body); + + let out: string; + try { + const llm = await callClaude({ + prompt: buildPrompt(body), + systemPrompt: SYSTEM, + model: env.CLAUDE_MODEL, + allowedTools: [], + timeoutMs: 600_000, + budgetCapUsd: 0.60, + }); + if (llm.isError) return { ok: false, note: `llm is_error: ${llm.text.slice(0, 80)}` }; + out = extractBody(llm.text); + } catch (err) { + return { ok: false, note: `llm_error: ${(err as Error).message.slice(0, 100)}` }; + } + + // Validation — refuse to write a regression. + if (h1Count(out) < 2) return { ok: false, note: `only ${h1Count(out)} H1 in output` }; + const after = citations(out); + // Every original citation must survive (the translation duplicates each + // across both languages, so after >= before is expected). + const missing = before.filter((c) => !after.includes(c)); + if (missing.length > 0) { + return { ok: false, note: `dropped ${missing.length} citation(s): ${missing.slice(0, 3).join(",")}` }; + } + // Sanity: output must carry both a clearly-English and a clearly-Portuguese + // signal (accented chars present somewhere = PT side exists). + if (!/[áàâãéêíóôõúüç]/i.test(out)) return { ok: false, note: "no PT-BR accents in output" }; + + if (DRY || ONLY) { + console.log(`\n========== ${file} ==========\n${out.slice(0, 1400)}\n... [${out.length} chars]`); + if (DRY) return { ok: true, note: "dry-run (not written)" }; + } + + // Re-attach frontmatter, stamp the normalization + revision time. + let newFm = fm; + if (/^bilingual_normalized:/m.test(newFm)) { + newFm = newFm.replace(/^bilingual_normalized:.*$/m, "bilingual_normalized: true"); + } else { + newFm = newFm.replace(/\n---\n$/, `\nbilingual_normalized: true\n---\n`); + } + await writeFile(full, `${newFm}${out}\n`, "utf-8"); + return { ok: true, note: `cites ${before.length}→${after.length}` }; +} + +async function main() { + let files = (await readdir(REPORTS)).filter((f) => f.endsWith(".md")).sort(); + if (ONLY) files = files.filter((f) => f === `${ONLY}.md` || f === ONLY); + console.log(`translating ${files.length} file(s), concurrency=${CONCURRENCY}, force=${FORCE}, dry=${DRY}`); + + let ok = 0, skip = 0, i = 0; + async function worker() { + while (i < files.length) { + const f = files[i++]; + try { + const r = await translateOne(f); + if (r.ok) { ok++; console.log(` ✓ ${f} — ${r.note}`); } + else { skip++; console.log(` · ${f} — ${r.note}`); } + } catch (e) { + skip++; console.log(` ✗ ${f} — ${(e as Error).message.slice(0, 100)}`); + } + } + } + await Promise.all(Array.from({ length: CONCURRENCY }, worker)); + console.log(`\nDone. ok=${ok} skip=${skip} of ${files.length}`); +} + +main().catch((e) => { console.error("fatal:", e); process.exit(1); }); diff --git a/web/lib/bilingual.ts b/web/lib/bilingual.ts index 54d9f47..464480a 100644 --- a/web/lib/bilingual.ts +++ b/web/lib/bilingual.ts @@ -42,6 +42,27 @@ function sectionKey(section: string): string | null { return m ? m[1].toUpperCase() : null; } +/** + * Some legacy files tag the heading language explicitly: + * "## §1 — The Case at Hand (EN)" / "## §1 — O Caso em Mãos (PT-BR)". + * That tag is the strongest possible signal. Returns the language when a tag + * is present, else null. + */ +function langTag(section: string): "en" | "pt-br" | null { + const head = section.split("\n", 1)[0] ?? ""; + if (/\((?:pt-?br|pt|português|portugues)\)\s*$/i.test(head)) return "pt-br"; + if (/\((?:en|en-?us|english|inglês|ingles)\)\s*$/i.test(head)) return "en"; + return null; +} + +/** Strip a trailing "(EN)" / "(PT-BR)" tag and dangling rule from a section. */ +function cleanSection(section: string): string { + return section + .replace(/^(##\s+.*?)\s*\((?:en|en-?us|english|inglês|ingles|pt-?br|pt|português|portugues)\)\s*$/im, "$1") + .replace(/\n+---\s*$/m, "") + .trimEnd(); +} + /** * Return only the portion of a bilingual case body that matches `locale`. * Operates on a body whose H1 title lines have already been stripped. @@ -71,16 +92,19 @@ export function pickLocaleBody(body: string, locale: "en" | "pt-br"): string { const seen = new Map(); const picked: string[] = []; for (const s of sections) { + const tag = langTag(s); const k = sectionKey(s); let isPt: boolean; - if (k && (counts.get(k) ?? 0) >= 2) { + if (tag) { + isPt = tag === "pt-br"; // explicit heading tag — most reliable + } else if (k && (counts.get(k) ?? 0) >= 2) { const order = seen.get(k) ?? 0; seen.set(k, order + 1); isPt = order >= 1; // first = EN, second = PT-BR } else { isPt = isPortuguese(s); // unpaired / unnumbered — sniff the language } - if (isPt === wantPt) picked.push(s.trimEnd()); + if (isPt === wantPt) picked.push(cleanSection(s)); } // Can't isolate the requested language — fall back to the full body so no // content silently disappears.