i18n: complete-language pass over all case files + legacy tag handling

translate_case_files.ts: per-file localization pass that guarantees a case
file is complete and parallel in EN + PT-BR — translating the missing-language
title, sections, and verbatim blockquotes FROM whatever language exists, with
citations preserved exactly. Validates (two H1s, no dropped citations, PT
accents present) before writing; stamps bilingual_normalized in frontmatter;
idempotent. Ran across all 89 published files (88 via LLM, 1 legacy fixed
deterministically).

bilingual.ts: pickLocaleBody now honors explicit "(EN)"/"(PT-BR)" heading
tags as the primary split signal (above section-number ordering and language
sniffing) and strips those tags + trailing rules from display. Fixes the
legacy "## §N — Title (EN)" format which the reader otherwise showed with
visible language tags.

Verified live: green-fireballs-bilingual, d44, doc-65 render one language per
locale with their own titles.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

investigator-runtime/scripts/translate_case_files.ts

@@ -0,0 +1,204 @@
+#!/usr/bin/env bun
+/**
+ * translate_case_files.ts — make every published case file COMPLETE in both
+ * English and Brazilian Portuguese.
+ *
+ * The narrator emitted bilingual files, but with gaps: most lost their
+ * English H1 (the extractBody bug), some scenes exist in only one language,
+ * and many verbatim blockquotes appear only in the document's source
+ * language. A reader on the EN site should never hit Portuguese text and
+ * vice-versa.
+ *
+ * This is a TRANSLATION/localization pass, not a re-narration: we translate
+ * the missing-language content FROM the language that already exists. No new
+ * facts, no corpus lookup. Citations are preserved verbatim.
+ *
+ * For each file in $CASE_ROOT/reports/*.md:
+ *   1. Split frontmatter + body.
+ *   2. Ask Sonnet to return a complete bilingual body (EN title + PT title;
+ *      each numbered section in EN then PT; every blockquote in both).
+ *   3. Validate (two H1s, citations preserved, both languages present).
+ *   4. Write back, preserving frontmatter and stamping bilingual_normalized.
+ *
+ * Idempotent: skips files already stamped `bilingual_normalized: true`
+ * unless --force. Usage:
+ *   bun scripts/translate_case_files.ts                # all unstamped
+ *   bun scripts/translate_case_files.ts --force        # redo all
+ *   bun scripts/translate_case_files.ts --only <slug>  # one file (dry-run prints)
+ *   bun scripts/translate_case_files.ts --dry          # print, don't write
+ */
+import { readdir, readFile, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { callClaude } from "../src/lib/claude";
+import { env } from "../src/lib/env";
+
+const CASE_ROOT = process.env.CASE_ROOT || "/data/ufo/case";
+const REPORTS = path.join(CASE_ROOT, "reports");
+const FORCE = process.argv.includes("--force");
+const DRY = process.argv.includes("--dry");
+const ONLY = (() => {
+  const i = process.argv.indexOf("--only");
+  return i >= 0 ? process.argv[i + 1] : null;
+})();
+const CONCURRENCY = 2;
+
+const SYSTEM = `You are a bilingual editor for a public UAP/UFO archive. Your ONLY
+job is to make a case file COMPLETE and parallel in English and Brazilian
+Portuguese. You translate; you do not invent, summarise, or re-report.
+
+HARD RULES:
+1. Output ONLY the markdown body. Start with the first "# " title line. No
+   frontmatter, no code fence, no commentary before or after.
+2. Structure, exactly:
+   # <English title>
+   # <Título em Português Brasileiro>
+   ## I. <English heading>
+   <English body>
+   ## I. <Título em Português>
+   <corpo em português>
+   ## II. <English heading>
+   ...
+   ## II. <Título em Português>
+   ...
+   The SAME section numbers, in the same order, each appearing once in
+   English then once in Brazilian Portuguese.
+3. COMPLETENESS: if any title, section, sentence, or blockquote exists in
+   only one language, create the other by faithful translation. Nothing may
+   be present in one language and absent in the other. The two language
+   versions must carry the same facts, same numbers, same names.
+4. CITATIONS: every [[doc-id/pNNN#cNNNN]] link must be preserved EXACTLY and
+   must appear in BOTH language versions of the section it belongs to. Never
+   alter, drop, or invent a citation target.
+5. BLOCKQUOTES (verbatim source quotes): keep them as "> " blockquotes. In
+   the English section the quote reads in English; in the Portuguese section
+   it reads in Portuguese. When the original quote is in one language,
+   translate it faithfully for the other section (a translation of a real
+   quote is fine; do not fabricate quotes).
+6. Brazilian Portuguese only (NOT European). Preserve UTF-8 accents: ç ã á é
+   í ó ú â ê ô à õ ü. Never strip accents.
+7. Do NOT add facts, dates, places, measurements, or names that are not
+   already in the file. Translation only.
+8. Keep the non-fiction best-seller voice already present. Do not add
+   "In summary", "Em suma", section recaps, or editorializing.`;
+
+interface Parsed { fm: string; body: string; fmObj: Record<string, string>; }
+
+function parse(md: string): Parsed {
+  const m = md.match(/^(---\n[\s\S]+?\n---\n)([\s\S]*)$/);
+  if (!m) return { fm: "", body: md, fmObj: {} };
+  const fmObj: Record<string, string> = {};
+  for (const line of m[1].split("\n")) {
+    const kv = line.match(/^([a-z_]+):\s*(.+)$/);
+    if (kv) fmObj[kv[1]] = kv[2].trim();
+  }
+  return { fm: m[1], body: m[2], fmObj };
+}
+
+function citations(s: string): string[] {
+  return (s.match(/\[\[[^\]]+\]\]/g) || []).sort();
+}
+
+function h1Count(s: string): number {
+  return (s.match(/^#\s+\S/gm) || []).length;
+}
+
+function extractBody(text: string): string {
+  const t = text.trim().replace(/^```(?:markdown|md)?\s*\n?/i, "").replace(/\n?```\s*$/i, "").trimStart();
+  if (t.startsWith("# ")) return t;
+  const h1 = t.indexOf("\n# ");
+  if (h1 === -1) throw new Error(`no H1 in output: ${t.slice(0, 120)}`);
+  return t.slice(h1 + 1);
+}
+
+function buildPrompt(body: string): string {
+  return [
+    "Here is a case file body that may be incomplete or mixed across the two",
+    "languages. Return the corrected, fully-bilingual body per the rules.",
+    "",
+    "=== CASE FILE BODY ===",
+    body,
+    "=== END ===",
+    "",
+    "Return ONLY the corrected markdown body, starting with the English title `# `.",
+  ].join("\n");
+}
+
+async function translateOne(file: string): Promise<{ ok: boolean; note: string }> {
+  const full = path.join(REPORTS, file);
+  const raw = await readFile(full, "utf-8");
+  const { fm, body, fmObj } = parse(raw);
+
+  if (!FORCE && fmObj.bilingual_normalized === "true") {
+    return { ok: false, note: "already normalized (use --force)" };
+  }
+
+  const before = citations(body);
+
+  let out: string;
+  try {
+    const llm = await callClaude({
+      prompt: buildPrompt(body),
+      systemPrompt: SYSTEM,
+      model: env.CLAUDE_MODEL,
+      allowedTools: [],
+      timeoutMs: 600_000,
+      budgetCapUsd: 0.60,
+    });
+    if (llm.isError) return { ok: false, note: `llm is_error: ${llm.text.slice(0, 80)}` };
+    out = extractBody(llm.text);
+  } catch (err) {
+    return { ok: false, note: `llm_error: ${(err as Error).message.slice(0, 100)}` };
+  }
+
+  // Validation — refuse to write a regression.
+  if (h1Count(out) < 2) return { ok: false, note: `only ${h1Count(out)} H1 in output` };
+  const after = citations(out);
+  // Every original citation must survive (the translation duplicates each
+  // across both languages, so after >= before is expected).
+  const missing = before.filter((c) => !after.includes(c));
+  if (missing.length > 0) {
+    return { ok: false, note: `dropped ${missing.length} citation(s): ${missing.slice(0, 3).join(",")}` };
+  }
+  // Sanity: output must carry both a clearly-English and a clearly-Portuguese
+  // signal (accented chars present somewhere = PT side exists).
+  if (!/[áàâãéêíóôõúüç]/i.test(out)) return { ok: false, note: "no PT-BR accents in output" };
+
+  if (DRY || ONLY) {
+    console.log(`\n========== ${file} ==========\n${out.slice(0, 1400)}\n... [${out.length} chars]`);
+    if (DRY) return { ok: true, note: "dry-run (not written)" };
+  }
+
+  // Re-attach frontmatter, stamp the normalization + revision time.
+  let newFm = fm;
+  if (/^bilingual_normalized:/m.test(newFm)) {
+    newFm = newFm.replace(/^bilingual_normalized:.*$/m, "bilingual_normalized: true");
+  } else {
+    newFm = newFm.replace(/\n---\n$/, `\nbilingual_normalized: true\n---\n`);
+  }
+  await writeFile(full, `${newFm}${out}\n`, "utf-8");
+  return { ok: true, note: `cites ${before.length}→${after.length}` };
+}
+
+async function main() {
+  let files = (await readdir(REPORTS)).filter((f) => f.endsWith(".md")).sort();
+  if (ONLY) files = files.filter((f) => f === `${ONLY}.md` || f === ONLY);
+  console.log(`translating ${files.length} file(s), concurrency=${CONCURRENCY}, force=${FORCE}, dry=${DRY}`);
+
+  let ok = 0, skip = 0, i = 0;
+  async function worker() {
+    while (i < files.length) {
+      const f = files[i++];
+      try {
+        const r = await translateOne(f);
+        if (r.ok) { ok++; console.log(`  ✓ ${f} — ${r.note}`); }
+        else { skip++; console.log(`  · ${f} — ${r.note}`); }
+      } catch (e) {
+        skip++; console.log(`  ✗ ${f} — ${(e as Error).message.slice(0, 100)}`);
+      }
+    }
+  }
+  await Promise.all(Array.from({ length: CONCURRENCY }, worker));
+  console.log(`\nDone. ok=${ok} skip=${skip} of ${files.length}`);
+}
+
+main().catch((e) => { console.error("fatal:", e); process.exit(1); });

web/lib/bilingual.ts

@@ -42,6 +42,27 @@ function sectionKey(section: string): string | null {
   return m ? m[1].toUpperCase() : null;
 }
 
+/**
+ * Some legacy files tag the heading language explicitly:
+ * "## §1 — The Case at Hand (EN)" / "## §1 — O Caso em Mãos (PT-BR)".
+ * That tag is the strongest possible signal. Returns the language when a tag
+ * is present, else null.
+ */
+function langTag(section: string): "en" | "pt-br" | null {
+  const head = section.split("\n", 1)[0] ?? "";
+  if (/\((?:pt-?br|pt|português|portugues)\)\s*$/i.test(head)) return "pt-br";
+  if (/\((?:en|en-?us|english|inglês|ingles)\)\s*$/i.test(head)) return "en";
+  return null;
+}
+
+/** Strip a trailing "(EN)" / "(PT-BR)" tag and dangling rule from a section. */
+function cleanSection(section: string): string {
+  return section
+    .replace(/^(##\s+.*?)\s*\((?:en|en-?us|english|inglês|ingles|pt-?br|pt|português|portugues)\)\s*$/im, "$1")
+    .replace(/\n+---\s*$/m, "")
+    .trimEnd();
+}
+
 /**
  * Return only the portion of a bilingual case body that matches `locale`.
  * Operates on a body whose H1 title lines have already been stripped.
@@ -71,16 +92,19 @@ export function pickLocaleBody(body: string, locale: "en" | "pt-br"): string {
   const seen = new Map<string, number>();
   const picked: string[] = [];
   for (const s of sections) {
+    const tag = langTag(s);
     const k = sectionKey(s);
     let isPt: boolean;
-    if (k && (counts.get(k) ?? 0) >= 2) {
+    if (tag) {
+      isPt = tag === "pt-br"; // explicit heading tag — most reliable
+    } else if (k && (counts.get(k) ?? 0) >= 2) {
       const order = seen.get(k) ?? 0;
       seen.set(k, order + 1);
       isPt = order >= 1; // first = EN, second = PT-BR
     } else {
       isPt = isPortuguese(s); // unpaired / unnumbered — sniff the language
     }
-    if (isPt === wantPt) picked.push(s.trimEnd());
+    if (isPt === wantPt) picked.push(cleanSection(s));
   }
   // Can't isolate the requested language — fall back to the full body so no
   // content silently disappears.