disclosure-bureau/investigator-runtime/scripts/translate_case_files.ts

#!/usr/bin/env bun
/**
 * translate_case_files.ts — make every published case file COMPLETE in both
 * English and Brazilian Portuguese.
 *
 * The narrator emitted bilingual files, but with gaps: most lost their
 * English H1 (the extractBody bug), some scenes exist in only one language,
 * and many verbatim blockquotes appear only in the document's source
 * language. A reader on the EN site should never hit Portuguese text and
 * vice-versa.
 *
 * This is a TRANSLATION/localization pass, not a re-narration: we translate
 * the missing-language content FROM the language that already exists. No new
 * facts, no corpus lookup. Citations are preserved verbatim.
 *
 * For each file in $CASE_ROOT/reports/*.md:
 *   1. Split frontmatter + body.
 *   2. Ask Sonnet to return a complete bilingual body (EN title + PT title;
 *      each numbered section in EN then PT; every blockquote in both).
 *   3. Validate (two H1s, citations preserved, both languages present).
 *   4. Write back, preserving frontmatter and stamping bilingual_normalized.
 *
 * Idempotent: skips files already stamped `bilingual_normalized: true`
 * unless --force. Usage:
 *   bun scripts/translate_case_files.ts                # all unstamped
 *   bun scripts/translate_case_files.ts --force        # redo all
 *   bun scripts/translate_case_files.ts --only <slug>  # one file (dry-run prints)
 *   bun scripts/translate_case_files.ts --dry          # print, don't write
 */
import { readdir, readFile, writeFile } from "node:fs/promises";
import path from "node:path";
import { callClaude } from "../src/lib/claude";
import { env } from "../src/lib/env";

const CASE_ROOT = process.env.CASE_ROOT || "/data/ufo/case";
const REPORTS = path.join(CASE_ROOT, "reports");
const FORCE = process.argv.includes("--force");
const DRY = process.argv.includes("--dry");
const ONLY = (() => {
  const i = process.argv.indexOf("--only");
  return i >= 0 ? process.argv[i + 1] : null;
})();
const CONCURRENCY = 2;

const SYSTEM = `You are a bilingual editor for a public UAP/UFO archive. Your ONLY
job is to make a case file COMPLETE and parallel in English and Brazilian
Portuguese. You translate; you do not invent, summarise, or re-report.

HARD RULES:
1. Output ONLY the markdown body. Start with the first "# " title line. No
   frontmatter, no code fence, no commentary before or after.
2. Structure, exactly:
   # <English title>
   # <Título em Português Brasileiro>
   ## I. <English heading>
   <English body>
   ## I. <Título em Português>
   <corpo em português>
   ## II. <English heading>
   ...
   ## II. <Título em Português>
   ...
   The SAME section numbers, in the same order, each appearing once in
   English then once in Brazilian Portuguese.
3. COMPLETENESS: if any title, section, sentence, or blockquote exists in
   only one language, create the other by faithful translation. Nothing may
   be present in one language and absent in the other. The two language
   versions must carry the same facts, same numbers, same names.
4. CITATIONS: every [[doc-id/pNNN#cNNNN]] link must be preserved EXACTLY and
   must appear in BOTH language versions of the section it belongs to. Never
   alter, drop, or invent a citation target.
5. BLOCKQUOTES (verbatim source quotes): keep them as "> " blockquotes. In
   the English section the quote reads in English; in the Portuguese section
   it reads in Portuguese. When the original quote is in one language,
   translate it faithfully for the other section (a translation of a real
   quote is fine; do not fabricate quotes).
6. Brazilian Portuguese only (NOT European). Preserve UTF-8 accents: ç ã á é
   í ó ú â ê ô à õ ü. Never strip accents.
7. Do NOT add facts, dates, places, measurements, or names that are not
   already in the file. Translation only.
8. Keep the non-fiction best-seller voice already present. Do not add
   "In summary", "Em suma", section recaps, or editorializing.`;

interface Parsed { fm: string; body: string; fmObj: Record<string, string>; }

function parse(md: string): Parsed {
  const m = md.match(/^(---\n[\s\S]+?\n---\n)([\s\S]*)$/);
  if (!m) return { fm: "", body: md, fmObj: {} };
  const fmObj: Record<string, string> = {};
  for (const line of m[1].split("\n")) {
    const kv = line.match(/^([a-z_]+):\s*(.+)$/);
    if (kv) fmObj[kv[1]] = kv[2].trim();
  }
  return { fm: m[1], body: m[2], fmObj };
}

function citations(s: string): string[] {
  return (s.match(/\[\[[^\]]+\]\]/g) || []).sort();
}

function h1Count(s: string): number {
  return (s.match(/^#\s+\S/gm) || []).length;
}

function extractBody(text: string): string {
  const t = text.trim().replace(/^```(?:markdown|md)?\s*\n?/i, "").replace(/\n?```\s*$/i, "").trimStart();
  if (t.startsWith("# ")) return t;
  const h1 = t.indexOf("\n# ");
  if (h1 === -1) throw new Error(`no H1 in output: ${t.slice(0, 120)}`);
  return t.slice(h1 + 1);
}

function buildPrompt(body: string): string {
  return [
    "Here is a case file body that may be incomplete or mixed across the two",
    "languages. Return the corrected, fully-bilingual body per the rules.",
    "",
    "=== CASE FILE BODY ===",
    body,
    "=== END ===",
    "",
    "Return ONLY the corrected markdown body, starting with the English title `# `.",
  ].join("\n");
}

async function translateOne(file: string): Promise<{ ok: boolean; note: string }> {
  const full = path.join(REPORTS, file);
  const raw = await readFile(full, "utf-8");
  const { fm, body, fmObj } = parse(raw);

  if (!FORCE && fmObj.bilingual_normalized === "true") {
    return { ok: false, note: "already normalized (use --force)" };
  }

  const before = citations(body);

  let out: string;
  try {
    const llm = await callClaude({
      prompt: buildPrompt(body),
      systemPrompt: SYSTEM,
      model: env.CLAUDE_MODEL,
      allowedTools: [],
      timeoutMs: 600_000,
      budgetCapUsd: 0.60,
    });
    if (llm.isError) return { ok: false, note: `llm is_error: ${llm.text.slice(0, 80)}` };
    out = extractBody(llm.text);
  } catch (err) {
    return { ok: false, note: `llm_error: ${(err as Error).message.slice(0, 100)}` };
  }

  // Validation — refuse to write a regression.
  if (h1Count(out) < 2) return { ok: false, note: `only ${h1Count(out)} H1 in output` };
  const after = citations(out);
  // Every original citation must survive (the translation duplicates each
  // across both languages, so after >= before is expected).
  const missing = before.filter((c) => !after.includes(c));
  if (missing.length > 0) {
    return { ok: false, note: `dropped ${missing.length} citation(s): ${missing.slice(0, 3).join(",")}` };
  }
  // Sanity: output must carry both a clearly-English and a clearly-Portuguese
  // signal (accented chars present somewhere = PT side exists).
  if (!/[áàâãéêíóôõúüç]/i.test(out)) return { ok: false, note: "no PT-BR accents in output" };

  if (DRY || ONLY) {
    console.log(`\n========== ${file} ==========\n${out.slice(0, 1400)}\n... [${out.length} chars]`);
    if (DRY) return { ok: true, note: "dry-run (not written)" };
  }

  // Re-attach frontmatter, stamp the normalization + revision time.
  let newFm = fm;
  if (/^bilingual_normalized:/m.test(newFm)) {
    newFm = newFm.replace(/^bilingual_normalized:.*$/m, "bilingual_normalized: true");
  } else {
    newFm = newFm.replace(/\n---\n$/, `\nbilingual_normalized: true\n---\n`);
  }
  await writeFile(full, `${newFm}${out}\n`, "utf-8");
  return { ok: true, note: `cites ${before.length}→${after.length}` };
}

async function main() {
  let files = (await readdir(REPORTS)).filter((f) => f.endsWith(".md")).sort();
  if (ONLY) files = files.filter((f) => f === `${ONLY}.md` || f === ONLY);
  console.log(`translating ${files.length} file(s), concurrency=${CONCURRENCY}, force=${FORCE}, dry=${DRY}`);

  let ok = 0, skip = 0, i = 0;
  async function worker() {
    while (i < files.length) {
      const f = files[i++];
      try {
        const r = await translateOne(f);
        if (r.ok) { ok++; console.log(`  ✓ ${f} — ${r.note}`); }
        else { skip++; console.log(`  · ${f} — ${r.note}`); }
      } catch (e) {
        skip++; console.log(`  ✗ ${f} — ${(e as Error).message.slice(0, 100)}`);
      }
    }
  }
  await Promise.all(Array.from({ length: CONCURRENCY }, worker));
  console.log(`\nDone. ok=${ok} skip=${skip} of ${files.length}`);
}

main().catch((e) => { console.error("fatal:", e); process.exit(1); });