disclosure-bureau/scripts/maintain/46_text_backfill_mentions.py

#!/usr/bin/env python3
"""
Text-based backfill of entity → page references.

The structured pipelines (Sonnet chunks, Haiku page-level events/entities)
miss many entities the corpus actually discusses — they extract only what
they confidently structure into the schema. The vision_description and
narrative_summary fields routinely *talk about* an event/person/place
without listing it in the structured arrays.

This script does a fuzzy back-fill: scans the narrative body of every page
YAML for textual matches of every entity's canonical_name + aliases, and
records the hits as a new signal source `text_refs`. Aho-Corasick is used
so the whole 3k-pages × 34k-entities cross-product collapses to a single
linear scan per page.

Conservative filtering keeps the noise floor low:
  - minimum 5 chars per alias (4 if alias has a digit, to keep e.g. "USS")
  - blacklist of common stopwords / generic terms
  - word-boundary enforcement (\b in regex, manual check after AC scan)
  - skip purely numeric and ASCII-fold-identical-to-id aliases

YAML output (added in-place on each entity file):
  text_mentioned_in: ['[[doc-id/pNNN]]', ...]   # only refs NOT already in mentioned_in
  signal_sources.text_refs: N
  total_mentions = db_chunks + page_refs + cross_refs + text_refs
  signal_strength recomputed using text_refs as a weak signal

Run:
  python3 scripts/maintain/46_text_backfill_mentions.py [--dry-run]
"""
from __future__ import annotations

import argparse
import re
import sys
from collections import defaultdict
from pathlib import Path
from typing import Iterable

import ahocorasick
import yaml

WIKI = Path("/Users/guto/ufo/wiki")
ENTITIES_BASE = WIKI / "entities"
PAGES_BASE = WIKI / "pages"

# Generic / stop-words: never accept these as match patterns even if listed
# as an alias. Lowercased. PT-BR + EN + universally vague terms.
BLACKLIST: set[str] = {
    # english stopwords / common
    "the", "and", "for", "with", "from", "this", "that", "these", "those",
    "report", "reports", "object", "objects", "unknown", "unidentified",
    "anomalous", "aerial", "phenomenon", "phenomena", "sighting", "sightings",
    "case", "cases", "incident", "incidents", "event", "events", "encounter",
    "encounters", "observation", "observations", "document", "documents",
    "memo", "memos", "letter", "letters", "table", "tables", "image", "images",
    "general", "section", "agent", "agents", "subject", "subjects",
    "office", "offices", "field", "fields", "summary", "summaries",
    "true", "false", "type", "types", "data", "name", "names",
    "person", "people", "place", "places", "location", "locations",
    "vehicle", "vehicles", "operation", "operations", "concept", "concepts",
    "page", "pages", "chunk", "chunks", "scan", "scans",
    "north", "south", "east", "west", "central",
    "captain", "major", "colonel", "general", "lieutenant", "sergeant",
    "monday", "tuesday", "wednesday", "thursday", "friday", "saturday", "sunday",
    "january", "february", "march", "april", "june", "july", "august",
    "september", "october", "november", "december",
    # pt-br stopwords / common
    "para", "como", "este", "esta", "esse", "essa", "isso", "aquele",
    "ainda", "outro", "outra", "outros", "outras", "todos", "todas",
    "relatorio", "relatório", "objeto", "objetos", "documento", "documentos",
    "página", "paginas", "páginas", "evento", "eventos", "incidente",
    "incidentes", "pessoa", "pessoas", "lugar", "lugares", "local", "locais",
    "operação", "operacao", "geral", "agente", "agentes", "campo", "campos",
    "norte", "sul", "leste", "oeste",
    "janeiro", "fevereiro", "março", "marco", "abril", "junho", "julho",
    "agosto", "setembro", "outubro", "novembro", "dezembro",
    # generic acronyms widely embedded in unrelated text
    "uap", "ufo", "usaaf", "usaf", "usa", "fbi", "dod", "nasa",
    "atom", "atoms", "atomic",
}


def is_acceptable_alias(name: str) -> bool:
    n = name.strip()
    if not n:
        return False
    nl = n.lower()
    if nl in BLACKLIST:
        return False
    # Must contain at least one letter
    if not re.search(r"[a-zA-ZÀ-ÿ]", n):
        return False
    # Purely numeric or punctuation
    if re.fullmatch(r"[\d\s\-_.,]+", n):
        return False
    # Single-word too short (5 char min unless contains a digit)
    if " " not in n and "-" not in n and len(n) < 5 and not re.search(r"\d", n):
        return False
    return True


def parse_frontmatter(text: str) -> tuple[dict, str]:
    if not text.startswith("---"):
        return {}, text
    parts = text.split("---", 2)
    if len(parts) < 3:
        return {}, text
    try:
        fm = yaml.safe_load(parts[1]) or {}
    except yaml.YAMLError:
        fm = {}
    return fm, parts[2]


def dump_frontmatter_preserving_body(fm: dict, body: str) -> str:
    return "---\n" + yaml.safe_dump(fm, sort_keys=False, allow_unicode=True, width=1000) + "---" + body


def extract_searchable_text(page_fm: dict, page_body: str) -> str:
    """Pick narrative-only fields from a page YAML — avoid YAML keys, IDs, enums."""
    parts: list[str] = []
    for key in (
        "vision_description",
        "vision_description_pt_br",
        "narrative_summary",
        "narrative_summary_pt_br",
        "extracted_text",
    ):
        v = page_fm.get(key)
        if isinstance(v, str):
            parts.append(v)
    parts.append(page_body)
    return "\n".join(parts)


# Map entity_class -> folder name
FOLDER_BY_CLASS = {
    "person": "people",
    "organization": "organizations",
    "location": "locations",
    "event": "events",
    "uap_object": "uap-objects",
    "vehicle": "vehicles",
    "operation": "operations",
    "concept": "concepts",
}


def entity_id_from_fm(fm: dict) -> tuple[str, str] | None:
    cls = fm.get("entity_class")
    if cls:
        eid_key = f"{cls}_id"
        eid = fm.get(eid_key) or fm.get("entity_id")
        if eid:
            return cls, eid
    # legacy fallback
    for k in ("person_id", "organization_id", "location_id", "event_id",
             "uap_object_id", "vehicle_id", "operation_id", "concept_id"):
        if k in fm:
            return k.replace("_id", ""), fm[k]
    return None


def signal_strength(db_chunks: int, page_refs: int, cross_refs: int, text_refs: int) -> str:
    total = db_chunks + page_refs + cross_refs + text_refs
    if total == 0:
        return "orphan"
    if db_chunks >= 3 or page_refs >= 3 or (db_chunks >= 1 and page_refs >= 1) or text_refs >= 5:
        return "strong"
    return "weak"


def main() -> int:
    p = argparse.ArgumentParser()
    p.add_argument("--dry-run", action="store_true")
    p.add_argument("--verbose", action="store_true")
    args = p.parse_args()

    # 1. Load entities + collect (alias_lower → list of (entity_file_path, original_alias))
    print("Loading entities ...")
    automaton = ahocorasick.Automaton()
    entities: dict[Path, dict] = {}
    alias_per_entity_count = 0
    accepted_entities = 0
    for ent_file in ENTITIES_BASE.rglob("*.md"):
        if "_archived" in ent_file.parts:
            continue
        try:
            text = ent_file.read_text(encoding="utf-8")
        except Exception:
            continue
        fm, _body = parse_frontmatter(text)
        if not fm:
            continue
        if entity_id_from_fm(fm) is None:
            continue
        canonical = fm.get("canonical_name") or fm.get("canonical_title")
        aliases = fm.get("aliases") or []
        names = []
        if isinstance(canonical, str):
            names.append(canonical)
        for a in aliases:
            if isinstance(a, str):
                names.append(a)
        accepted = [n for n in names if is_acceptable_alias(n)]
        if not accepted:
            continue
        entities[ent_file] = {"fm": fm, "raw_text": text, "accepted": accepted}
        accepted_entities += 1
        for n in accepted:
            automaton.add_word(n.lower(), (str(ent_file), n))
            alias_per_entity_count += 1

    automaton.make_automaton()
    print(f"  entities considered:  {accepted_entities}")
    print(f"  searchable aliases:   {alias_per_entity_count}")

    # 2. Scan every page YAML
    print("Scanning pages ...")
    hits_by_entity: dict[str, set[str]] = defaultdict(set)
    pages_scanned = 0
    total_hits = 0
    for page_file in PAGES_BASE.rglob("p*.md"):
        try:
            text = page_file.read_text(encoding="utf-8")
        except Exception:
            continue
        fm, body = parse_frontmatter(text)
        if not fm:
            continue
        page_id = fm.get("page_id")
        if not page_id:
            # Derive from filesystem: <doc-id>/p<NNN>
            try:
                rel = page_file.relative_to(PAGES_BASE)
                page_id = f"{rel.parent}/{rel.stem}"
            except ValueError:
                continue
        ref = f"[[{page_id}]]"
        searchable = extract_searchable_text(fm, body).lower()
        pages_scanned += 1
        seen_this_page: set[str] = set()
        for end_idx, (ent_path_str, original) in automaton.iter(searchable):
            pattern = original.lower()
            start_idx = end_idx - len(pattern) + 1
            # Word boundary check
            if start_idx > 0 and (searchable[start_idx - 1].isalnum() or searchable[start_idx - 1] == "_"):
                continue
            after = end_idx + 1
            if after < len(searchable) and (searchable[after].isalnum() or searchable[after] == "_"):
                continue
            if ent_path_str in seen_this_page:
                continue
            seen_this_page.add(ent_path_str)
            hits_by_entity[ent_path_str].add(ref)
            total_hits += 1

    print(f"  pages scanned: {pages_scanned}")
    print(f"  total hits:    {total_hits}")
    print(f"  entities matched: {len(hits_by_entity)}")

    # 3. Write back to entity YAML
    print("Writing back ...")
    promoted = 0
    upgraded = 0
    updated = 0
    for ent_path_str, refs in hits_by_entity.items():
        ent_file = Path(ent_path_str)
        rec = entities.get(ent_file)
        if not rec:
            continue
        fm = rec["fm"]
        raw_text = rec["raw_text"]

        # Don't double-count refs already in mentioned_in (structured page_refs)
        existing_mentioned = set(fm.get("mentioned_in") or [])
        new_text_refs = sorted(refs - existing_mentioned)

        old_sources = (fm.get("signal_sources") or {}).copy()
        db_chunks = int(old_sources.get("db_chunks", 0))
        page_refs = int(old_sources.get("page_refs", len(existing_mentioned)))
        cross_refs = int(old_sources.get("cross_refs", 0))
        text_refs = len(new_text_refs)

        old_strength = fm.get("signal_strength", "unverified")
        new_strength = signal_strength(db_chunks, page_refs, cross_refs, text_refs)
        new_total = db_chunks + page_refs + cross_refs + text_refs

        fm["text_mentioned_in"] = new_text_refs
        sources = old_sources
        sources["db_chunks"] = db_chunks
        sources["page_refs"] = page_refs
        sources["cross_refs"] = cross_refs
        sources["text_refs"] = text_refs
        fm["signal_sources"] = sources
        fm["total_mentions"] = new_total
        fm["signal_strength"] = new_strength

        if old_strength == "orphan" and new_strength != "orphan":
            promoted += 1
        if old_strength == "weak" and new_strength == "strong":
            upgraded += 1
        updated += 1

        if not args.dry_run:
            # Preserve body verbatim
            _, body = parse_frontmatter(raw_text)
            new_text = dump_frontmatter_preserving_body(fm, body)
            ent_file.write_text(new_text, encoding="utf-8")

    # Also: entities not matched at all — they keep their existing state.
    # But ensure their signal_sources.text_refs is at least set to 0 if missing
    # (so the YAML schema is consistent).
    backfill_zeros = 0
    if not args.dry_run:
        for ent_file, rec in entities.items():
            if str(ent_file) in hits_by_entity:
                continue
            fm = rec["fm"]
            sources = (fm.get("signal_sources") or {})
            if "text_refs" not in sources:
                sources["text_refs"] = 0
                fm["signal_sources"] = sources
                _, body = parse_frontmatter(rec["raw_text"])
                ent_file.write_text(dump_frontmatter_preserving_body(fm, body), encoding="utf-8")
                backfill_zeros += 1

    print()
    print(f"  entities updated:        {updated}")
    print(f"  promoted orphan → weak:  {promoted}")
    print(f"  upgraded weak → strong:  {upgraded}")
    print(f"  zero-text-ref backfills: {backfill_zeros}")
    print(f"  dry-run: {args.dry_run}")
    return 0


if __name__ == "__main__":
    sys.exit(main())