W5.4 (Phase 3B): sitemap + robots + Article schema + magazine reading view

GEO/SEO surface area:

  app/robots.ts (new — Next.js dynamic robots)
    Explicitly ALLOWS major AI crawlers: GPTBot, OAI-SearchBot,
    ChatGPT-User, ClaudeBot, Claude-Web, anthropic-ai, PerplexityBot,
    Perplexity-User, Google-Extended, Applebot-Extended, CCBot,
    DuckAssistBot, YouBot, Bytespider, Amazonbot. The site exists to
    be cited by LLMs answering UAP/UFO questions — we want them in.
    /api/admin/, /admin/, /auth/ disallowed for everyone.

  app/sitemap.ts (new — Next.js dynamic sitemap)
    Lists 9 top-level routes + every /d/<doc> + every /c/<slug> from
    the filesystem + up to 500 entity URLs per class
    (event, person, uap_object, location, organization),
    sorted with summary-enriched entities first. ~3000 URLs total at
    current corpus size. lastModified honours summary_generated_at so
    crawlers re-index when entities are re-enriched.

  app/c/[slug]/page.tsx (rewritten — magazine reading view)
    - generateMetadata: per-case title, description (auto-extracted
      from the locale-preferred lead paragraph), canonical URL,
      hreflang alternate, OpenGraph article type with publishedTime,
      Twitter card.
    - JSON-LD Article schema embedded at end of page: schema.org
      Article + Organization publisher + inLanguage + isAccessibleForFree.
      This is what makes the case appear as a citable source in
      Google AI Overviews / Perplexity / ChatGPT search.
    - Reading view rewritten: display-serif headline (Fraunces), italic
      blockquotes with gold accent, prose-typography styling, no more
      detective stats line, no more "written by case-writer@detective"
      attribution. Locale-aware: PT-BR pulls topic_pt_br + lead in PT,
      English mirror.

  tailwind.config.ts
    + @tailwindcss/typography plugin
    + font-display family wired to var(--font-display) (Fraunces)

  package.json
    + "@tailwindcss/typography" devDependency

Phase 3A note: bulk entity enrichment hit Claude OAuth weekly quota mid-run.
6 events + 3 uap_objects landed bilingual summaries before the quota
exhausted. UI gracefully splits enriched vs bare entities so /sightings
shows the magazine-grade cards (Kenneth Arnold 1947, Roswell, Maury Island,
Joseph Perry 1960 lunar photo, Civil Defense Director 1966, etc.) on top
of a compact table of the rest. Re-run when quota refreshes.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

web/app/c/[slug]/page.tsx

@@ -1,33 +1,30 @@
 /**
- * /c/[slug] — Case report viewer.
+ * /c/[slug] — Case file reader.
  *
- * Reads /data/ufo/case/reports/<slug>.md, parses frontmatter for metadata,
+ * Renders a single narrated case file from /data/ufo/case/reports/<slug>.md.
- * renders the markdown body via MarkdownBody. The case-writer detective
+ * The reader sees a magazine-style article — title, dateline, body. No
- * writes these files; this page is the reader.
+ * detective attribution, no skeptic framing.
  */
 import { notFound } from "next/navigation";
-import Link from "next/link";
 import { readFile } from "node:fs/promises";
 import path from "node:path";
+import type { Metadata } from "next";
 import { MarkdownBody } from "@/components/markdown-body";
 import { AuthBar } from "@/components/auth-bar";
 import { BureauNav } from "@/components/bureau-nav";
+import { getLocale } from "@/components/locale-toggle";
 
 export const runtime = "nodejs";
 export const dynamic = "force-dynamic";
 
 const CASE_ROOT = process.env.CASE_ROOT || "/data/ufo/case";
+const SITE_URL = process.env.NEXT_PUBLIC_SITE_URL || "https://disclosure.top";
 
 interface Frontmatter {
   topic?: string;
-  created_by?: string;
+  topic_pt_br?: string;
   created_at?: string;
   job_id?: string;
-  n_evidence?: number;
-  n_hypotheses?: number;
-  n_contradictions?: number;
-  n_witnesses?: number;
-  n_outliers?: number;
 }
 
 function parseFrontmatter(md: string): { fm: Frontmatter; body: string } {
@@ -37,74 +34,141 @@ function parseFrontmatter(md: string): { fm: Frontmatter; body: string } {
   for (const line of m[1].split("\n")) {
     const kv = line.match(/^([a-z_]+):\s*(.+)$/);
     if (!kv) continue;
-    const k = kv[1] as keyof Frontmatter;
+    let v = kv[2].trim();
-    let v: string | number = kv[2].trim();
     if (v.startsWith('"') && v.endsWith('"')) v = v.slice(1, -1);
-    if (k === "n_evidence" || k === "n_hypotheses" || k === "n_contradictions"
+    (fm as Record<string, string>)[kv[1]] = v;
-        || k === "n_witnesses" || k === "n_outliers") {
-      const n = Number(v);
-      if (Number.isFinite(n)) (fm[k] as number) = n;
-    } else {
-      (fm[k] as string) = v as string;
-    }
   }
   return { fm, body: m[2] };
 }
 
+async function loadCase(slug: string): Promise<{ fm: Frontmatter; body: string } | null> {
+  try {
+    const md = await readFile(path.join(CASE_ROOT, "reports", `${slug}.md`), "utf-8");
+    return parseFrontmatter(md);
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Extract the first prose paragraph from the body for the meta description
+ * + OG. We pick the locale-preferred sub-section's opener.
+ */
+function pickLead(body: string, locale: "pt-br" | "en"): string {
+  const marker = locale === "pt-br" ? "(PT-BR)" : "(EN)";
+  const idx = body.indexOf(marker);
+  const slice = idx >= 0 ? body.slice(idx) : body;
+  const m = slice.match(/\n\n([^\n#>|`-][^\n]+(?:\n[^\n#>|`-][^\n]+)*)/);
+  return (m?.[1] ?? "").replace(/\s+/g, " ").trim();
+}
+
+export async function generateMetadata(
+  { params }: { params: Promise<{ slug: string }> },
+): Promise<Metadata> {
+  const { slug } = await params;
+  const locale = (await getLocale()) === "en" ? "en" : "pt-br";
+  const c = await loadCase(slug);
+  if (!c) return { title: "Case file not found" };
+
+  const title = locale === "pt-br" ? (c.fm.topic_pt_br ?? c.fm.topic ?? slug) : (c.fm.topic ?? slug);
+  const desc = pickLead(c.body, locale).slice(0, 200);
+  const canonical = `${SITE_URL}/c/${slug}`;
+  return {
+    title,
+    description: desc,
+    alternates: { canonical, languages: { "pt-BR": canonical, "en-US": canonical } },
+    openGraph: {
+      type: "article",
+      title,
+      description: desc,
+      url: canonical,
+      siteName: "The Disclosure Bureau",
+      locale: locale === "pt-br" ? "pt_BR" : "en_US",
+      publishedTime: c.fm.created_at,
+    },
+    twitter: {
+      card: "summary_large_image",
+      title,
+      description: desc,
+    },
+  };
+}
+
 export default async function CaseReportPage({
   params,
 }: { params: Promise<{ slug: string }> }) {
   const { slug } = await params;
   if (!/^[a-z0-9][a-z0-9-]*$/.test(slug)) notFound();
 
-  let md: string;
+  const locale = (await getLocale()) === "en" ? "en" : "pt-br";
-  try {
+  const c = await loadCase(slug);
-    md = await readFile(path.join(CASE_ROOT, "reports", `${slug}.md`), "utf-8");
+  if (!c) notFound();
-  } catch {
+  const { fm, body } = c;
-    notFound();
-  }
-  const { fm, body } = parseFrontmatter(md);
 
-  const stats: Array<{ label: string; value: number | undefined; color: string }> = [
+  const title = locale === "pt-br" ? (fm.topic_pt_br ?? fm.topic ?? slug) : (fm.topic ?? slug);
-    { label: "evidence", value: fm.n_evidence, color: "text-[#06d6a0]" },
+  const dateLabel = fm.created_at ? new Date(fm.created_at).toLocaleDateString(locale === "pt-br" ? "pt-BR" : "en-US", { day: "numeric", month: "long", year: "numeric" }) : null;
-    { label: "hypotheses", value: fm.n_hypotheses, color: "text-[#7fdbff]" },
+  const canonical = `${SITE_URL}/c/${slug}`;
-    { label: "contradictions", value: fm.n_contradictions, color: "text-[#ff8a4d]" },
+  const lead = pickLead(body, locale).slice(0, 280);
-    { label: "witnesses", value: fm.n_witnesses, color: "text-[#9b5de5]" },
-    { label: "outliers", value: fm.n_outliers, color: "text-[#ffd23f]" },
-  ];
 
   return (
     <div className="min-h-screen bg-[#0a0e1a] text-[#e7ecf3]">
       <BureauNav crumbs={[
-        { label: "bureau", href: "/bureau" },
+        { label: locale === "en" ? "case files" : "casos", href: "/bureau" },
-        { label: "reports", href: "/bureau#reports" },
+        { label: title.length > 32 ? title.slice(0, 32) + "…" : title },
-        { label: slug },
       ]} />
       <AuthBar />
-      <div className="mx-auto max-w-3xl px-4 py-6 pt-4">
 
-        <div className="rounded-lg border border-[rgba(224,192,128,0.18)] bg-gradient-to-br from-[rgba(224,192,128,0.06)] to-transparent p-4 mb-6">
+      <article className="mx-auto max-w-3xl px-4 py-10 md:py-14">
-          <div className="text-[10px] font-mono text-[#5a6678] uppercase mb-2">
+        <header className="mb-10 md:mb-14">
-            Case report{fm.created_by && <> · written by <span className="text-[#e0c080]">{fm.created_by}</span></>}
+          <div className="text-[10px] font-mono uppercase tracking-[0.18em] text-[#e0c080] mb-4">
-            {fm.created_at && <> · {fm.created_at}</>}
+            {locale === "en" ? "Declassified case file" : "Arquivo desclassificado"}
+            {dateLabel && <> · {dateLabel}</>}
           </div>
-          {fm.topic && (
+          <h1 className="font-display text-4xl md:text-6xl font-semibold leading-[1.05] tracking-tight text-[#e7ecf3] mb-6">
-            <h1 className="text-xl font-mono text-[#e7ecf3] leading-snug">{fm.topic}</h1>
+            {title}
+          </h1>
+          {lead && (
+            <p className="text-lg md:text-xl text-[#cbd2dd] leading-relaxed font-light max-w-2xl">
+              {lead}
+            </p>
           )}
-          <div className="mt-3 flex flex-wrap gap-3 text-[10px] font-mono">
+        </header>
-            {stats.filter((s) => typeof s.value === "number").map((s) => (
-              <span key={s.label}>
-                <span className={s.color}>{s.value}</span>
-                <span className="text-[#5a6678] ml-1">{s.label}</span>
-              </span>
-            ))}
-          </div>
-        </div>
 
-        <article className="prose prose-invert prose-sm max-w-none">
+        <div className="prose prose-invert prose-lg max-w-none
+                        prose-headings:font-display prose-headings:font-semibold
+                        prose-h2:text-2xl prose-h2:md:text-3xl prose-h2:mt-12 prose-h2:mb-4
+                        prose-h2:border-b prose-h2:border-[rgba(224,192,128,0.15)] prose-h2:pb-3
+                        prose-p:text-[16px] prose-p:leading-relaxed prose-p:text-[#cbd2dd]
+                        prose-blockquote:border-l-[#e0c080] prose-blockquote:bg-[rgba(224,192,128,0.05)]
+                        prose-blockquote:not-italic prose-blockquote:font-display prose-blockquote:text-[#e7ecf3]
+                        prose-blockquote:py-2 prose-blockquote:px-4 prose-blockquote:rounded-r
+                        prose-a:text-[#7fdbff] prose-a:no-underline hover:prose-a:underline
+                        prose-strong:text-[#e7ecf3]">
           <MarkdownBody>{body}</MarkdownBody>
-        </article>
         </div>
+      </article>
+
+      {/* JSON-LD Article — helps Google + AI crawlers parse the case as
+          a citation-bearing piece of journalism */}
+      <script
+        type="application/ld+json"
+        dangerouslySetInnerHTML={{ __html: JSON.stringify({
+          "@context": "https://schema.org",
+          "@type": "Article",
+          headline: title,
+          description: lead,
+          url: canonical,
+          datePublished: fm.created_at,
+          inLanguage: locale === "pt-br" ? "pt-BR" : "en-US",
+          isAccessibleForFree: true,
+          publisher: {
+            "@type": "Organization",
+            name: "The Disclosure Bureau",
+            url: SITE_URL,
+          },
+          about: { "@type": "Thing", name: "UAP/UFO declassified record" },
+        }) }}
+      />
     </div>
   );
 }

web/app/robots.ts

@@ -0,0 +1,43 @@
+/**
+ * robots.txt — generated dynamically by Next.js.
+ *
+ * GEO (Generative Engine Optimization) is a first-class goal here, so we
+ * explicitly allow every notable AI crawler in addition to the standard
+ * search-engine user-agents. The site is a public archive of declassified
+ * documents; we want LLMs to cite it.
+ */
+import type { MetadataRoute } from "next";
+
+const SITE_URL = process.env.NEXT_PUBLIC_SITE_URL ?? "https://disclosure.top";
+
+export default function robots(): MetadataRoute.Robots {
+  return {
+    rules: [
+      // Standard search engines + everyone else: full access.
+      {
+        userAgent: "*",
+        allow: "/",
+        disallow: ["/api/admin/", "/admin/", "/auth/"],
+      },
+      // Major AI / generative crawlers — explicitly allowed so they index
+      // and cite this archive when answering UAP/UFO questions.
+      { userAgent: "GPTBot",          allow: "/", disallow: ["/api/admin/", "/admin/", "/auth/"] },
+      { userAgent: "OAI-SearchBot",   allow: "/", disallow: ["/api/admin/", "/admin/", "/auth/"] },
+      { userAgent: "ChatGPT-User",    allow: "/", disallow: ["/api/admin/", "/admin/", "/auth/"] },
+      { userAgent: "ClaudeBot",       allow: "/", disallow: ["/api/admin/", "/admin/", "/auth/"] },
+      { userAgent: "Claude-Web",      allow: "/", disallow: ["/api/admin/", "/admin/", "/auth/"] },
+      { userAgent: "anthropic-ai",    allow: "/", disallow: ["/api/admin/", "/admin/", "/auth/"] },
+      { userAgent: "PerplexityBot",   allow: "/", disallow: ["/api/admin/", "/admin/", "/auth/"] },
+      { userAgent: "Perplexity-User", allow: "/", disallow: ["/api/admin/", "/admin/", "/auth/"] },
+      { userAgent: "Google-Extended", allow: "/", disallow: ["/api/admin/", "/admin/", "/auth/"] },
+      { userAgent: "Applebot-Extended", allow: "/", disallow: ["/api/admin/", "/admin/", "/auth/"] },
+      { userAgent: "CCBot",           allow: "/", disallow: ["/api/admin/", "/admin/", "/auth/"] },
+      { userAgent: "DuckAssistBot",   allow: "/", disallow: ["/api/admin/", "/admin/", "/auth/"] },
+      { userAgent: "YouBot",          allow: "/", disallow: ["/api/admin/", "/admin/", "/auth/"] },
+      { userAgent: "Bytespider",      allow: "/", disallow: ["/api/admin/", "/admin/", "/auth/"] },
+      { userAgent: "Amazonbot",       allow: "/", disallow: ["/api/admin/", "/admin/", "/auth/"] },
+    ],
+    sitemap: `${SITE_URL}/sitemap.xml`,
+    host: SITE_URL,
+  };
+}

web/app/sitemap.ts

@@ -0,0 +1,122 @@
+/**
+ * sitemap.xml — dynamic, regenerated per request.
+ *
+ * Aggregates:
+ *   - Static pages (home, bureau, sub-pages)
+ *   - Every declassified document (/d/<id>)
+ *   - Every case report (/c/<slug>)
+ *   - Every entity with an AI summary (/e/<class>/<id>) — these get a
+ *     lastModified from summary_generated_at, which helps crawlers
+ *     re-index when we re-enrich.
+ *   - Every top-mentioned entity even without summary (cap at 500 per
+ *     class so the sitemap doesn't balloon past Google's 50k limit).
+ *
+ * Per Next.js the file must export a default function returning a flat
+ * MetadataRoute.Sitemap array. ChangeFreq/priority are honoured by most
+ * crawlers as hints.
+ */
+import type { MetadataRoute } from "next";
+import { listDocuments } from "@/lib/wiki";
+import { pgQuery } from "@/lib/retrieval/db";
+import { readdir } from "node:fs/promises";
+import path from "node:path";
+
+const SITE_URL = process.env.NEXT_PUBLIC_SITE_URL ?? "https://disclosure.top";
+const CASE_ROOT = process.env.CASE_ROOT ?? "/data/ufo/case";
+
+type Url = MetadataRoute.Sitemap[number];
+
+const ENTITY_FOLDER_BY_CLASS: Record<string, string> = {
+  event: "events",
+  person: "people",
+  uap_object: "uap-objects",
+  location: "locations",
+  organization: "organizations",
+};
+
+export default async function sitemap(): Promise<MetadataRoute.Sitemap> {
+  const out: Url[] = [];
+  const now = new Date();
+
+  // 1. Top-level pages
+  const STATIC_PAGES = [
+    { url: "/", priority: 1.0, changeFrequency: "daily" as const },
+    { url: "/bureau", priority: 0.9, changeFrequency: "weekly" as const },
+    { url: "/sightings", priority: 0.9, changeFrequency: "weekly" as const },
+    { url: "/witnesses", priority: 0.8, changeFrequency: "weekly" as const },
+    { url: "/objects", priority: 0.8, changeFrequency: "weekly" as const },
+    { url: "/locations", priority: 0.8, changeFrequency: "weekly" as const },
+    { url: "/operations", priority: 0.8, changeFrequency: "weekly" as const },
+    { url: "/documents", priority: 0.8, changeFrequency: "weekly" as const },
+    { url: "/search", priority: 0.5, changeFrequency: "monthly" as const },
+  ];
+  for (const p of STATIC_PAGES) {
+    out.push({
+      url: `${SITE_URL}${p.url}`,
+      lastModified: now,
+      changeFrequency: p.changeFrequency,
+      priority: p.priority,
+    });
+  }
+
+  // 2. Documents
+  try {
+    const docIds = await listDocuments();
+    for (const id of docIds) {
+      out.push({
+        url: `${SITE_URL}/d/${id}`,
+        lastModified: now,
+        changeFrequency: "monthly",
+        priority: 0.7,
+      });
+    }
+  } catch { /* fs failure — skip docs */ }
+
+  // 3. Case reports — read filesystem for /c/[slug]
+  try {
+    const dir = path.join(CASE_ROOT, "reports");
+    const files = await readdir(dir);
+    for (const f of files.filter((x) => x.endsWith(".md"))) {
+      out.push({
+        url: `${SITE_URL}/c/${f.replace(/\.md$/, "")}`,
+        lastModified: now,
+        changeFrequency: "monthly",
+        priority: 0.95,
+      });
+    }
+  } catch { /* no case files yet */ }
+
+  // 4. Entities — surface those with summaries first (high priority), plus
+  // the top by mention count up to 500/class. Cap per class avoids blowing
+  // past sitemap size limits (Google: 50k urls, 50MB).
+  for (const [klass, folder] of Object.entries(ENTITY_FOLDER_BY_CLASS)) {
+    try {
+      const rows = await pgQuery<{
+        entity_id: string;
+        summary_generated_at: string | null;
+        summary_status: string | null;
+        total_mentions: number;
+      }>(
+        `SELECT entity_id, summary_generated_at, summary_status, total_mentions
+           FROM public.entities
+          WHERE entity_class = $1
+            AND total_mentions >= 1
+          ORDER BY (summary_status IN ('ai_generated','curated')) DESC,
+                   total_mentions DESC, entity_id ASC
+          LIMIT 500`,
+        [klass],
+      );
+      for (const r of rows) {
+        const hasSummary = r.summary_status === "ai_generated" || r.summary_status === "curated";
+        out.push({
+          url: `${SITE_URL}/e/${folder}/${r.entity_id}`,
+          lastModified: r.summary_generated_at ? new Date(r.summary_generated_at) : now,
+          changeFrequency: "monthly",
+          priority: hasSummary ? 0.7 : 0.4,
+        });
+      }
+    } catch { /* db unavailable for this class — skip */ }
+  }
+
+  return out;
+}

web/package-lock.json

@@ -33,6 +33,7 @@
         "sigma": "^3.0.0"
       },
       "devDependencies": {
+        "@tailwindcss/typography": "^0.5.19",
         "@types/node": "^22.7.0",
         "@types/pg": "^8.11.10",
         "@types/react": "^19.0.0",
@@ -5992,6 +5993,33 @@
         "tslib": "^2.8.0"
       }
     },
+    "node_modules/@tailwindcss/typography": {
+      "version": "0.5.19",
+      "resolved": "https://registry.npmjs.org/@tailwindcss/typography/-/typography-0.5.19.tgz",
+      "integrity": "sha512-w31dd8HOx3k9vPtcQh5QHP9GwKcgbMp87j58qi6xgiBnFFtKEAgCWnDw4qUT8aHwkCp8bKvb/KGKWWHedP0AAg==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "postcss-selector-parser": "6.0.10"
+      },
+      "peerDependencies": {
+        "tailwindcss": ">=3.0.0 || insiders || >=4.0.0-alpha.20 || >=4.0.0-beta.1"
+      }
+    },
+    "node_modules/@tailwindcss/typography/node_modules/postcss-selector-parser": {
+      "version": "6.0.10",
+      "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.0.10.tgz",
+      "integrity": "sha512-IQ7TZdoaqbT+LCpShg46jnZVlhWD2w6iQYAcYXfHARZ7X1t/UGhhceQDs5X0cGqKvYlHNOuv7Oa1xmb0oQuA3w==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "cssesc": "^3.0.0",
+        "util-deprecate": "^1.0.2"
+      },
+      "engines": {
+        "node": ">=4"
+      }
+    },
     "node_modules/@types/connect": {
       "version": "3.4.38",
       "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.38.tgz",

web/package.json

@@ -35,6 +35,7 @@
     "sigma": "^3.0.0"
   },
   "devDependencies": {
+    "@tailwindcss/typography": "^0.5.19",
     "@types/node": "^22.7.0",
     "@types/pg": "^8.11.10",
     "@types/react": "^19.0.0",

web/tailwind.config.ts

@@ -1,4 +1,5 @@
 import type { Config } from "tailwindcss";
+import typography from "@tailwindcss/typography";
 
 const config: Config = {
   content: [
@@ -28,10 +29,11 @@ const config: Config = {
       fontFamily: {
         mono:    ["var(--font-mono)", "JetBrains Mono", "Menlo", "monospace"],
         sans:    ["var(--font-sans)", "Inter", "system-ui", "sans-serif"],
+        display: ["var(--font-display)", "Fraunces", "Georgia", "serif"],
       },
     },
   },
-  plugins: [],
+  plugins: [typography],
 };
 
 export default config;