44 lines
2.4 KiB
TypeScript
44 lines
2.4 KiB
TypeScript
|
/**
|
||
|
|
* robots.txt — generated dynamically by Next.js.
|
||
|
|
*
|
||
|
|
* GEO (Generative Engine Optimization) is a first-class goal here, so we
|
||
|
|
* explicitly allow every notable AI crawler in addition to the standard
|
||
|
|
* search-engine user-agents. The site is a public archive of declassified
|
||
|
|
* documents; we want LLMs to cite it.
|
||
|
|
*/
|
||
|
|
import type { MetadataRoute } from "next";
|
||
|
|
|
||
|
|
const SITE_URL = process.env.NEXT_PUBLIC_SITE_URL ?? "https://disclosure.top";
|
||
|
|
|
||
|
|
export default function robots(): MetadataRoute.Robots {
|
||
|
|
return {
|
||
|
|
rules: [
|
||
|
|
// Standard search engines + everyone else: full access.
|
||
|
|
{
|
||
|
|
userAgent: "*",
|
||
|
|
allow: "/",
|
||
|
|
disallow: ["/api/admin/", "/admin/", "/auth/"],
|
||
|
|
},
|
||
|
|
// Major AI / generative crawlers — explicitly allowed so they index
|
||
|
|
// and cite this archive when answering UAP/UFO questions.
|
||
|
|
{ userAgent: "GPTBot", allow: "/", disallow: ["/api/admin/", "/admin/", "/auth/"] },
|
||
|
|
{ userAgent: "OAI-SearchBot", allow: "/", disallow: ["/api/admin/", "/admin/", "/auth/"] },
|
||
|
|
{ userAgent: "ChatGPT-User", allow: "/", disallow: ["/api/admin/", "/admin/", "/auth/"] },
|
||
|
|
{ userAgent: "ClaudeBot", allow: "/", disallow: ["/api/admin/", "/admin/", "/auth/"] },
|
||
|
|
{ userAgent: "Claude-Web", allow: "/", disallow: ["/api/admin/", "/admin/", "/auth/"] },
|
||
|
|
{ userAgent: "anthropic-ai", allow: "/", disallow: ["/api/admin/", "/admin/", "/auth/"] },
|
||
|
|
{ userAgent: "PerplexityBot", allow: "/", disallow: ["/api/admin/", "/admin/", "/auth/"] },
|
||
|
|
{ userAgent: "Perplexity-User", allow: "/", disallow: ["/api/admin/", "/admin/", "/auth/"] },
|
||
|
|
{ userAgent: "Google-Extended", allow: "/", disallow: ["/api/admin/", "/admin/", "/auth/"] },
|
||
|
|
{ userAgent: "Applebot-Extended", allow: "/", disallow: ["/api/admin/", "/admin/", "/auth/"] },
|
||
|
|
{ userAgent: "CCBot", allow: "/", disallow: ["/api/admin/", "/admin/", "/auth/"] },
|
||
|
|
{ userAgent: "DuckAssistBot", allow: "/", disallow: ["/api/admin/", "/admin/", "/auth/"] },
|
||
|
|
{ userAgent: "YouBot", allow: "/", disallow: ["/api/admin/", "/admin/", "/auth/"] },
|
||
|
|
{ userAgent: "Bytespider", allow: "/", disallow: ["/api/admin/", "/admin/", "/auth/"] },
|
||
|
|
{ userAgent: "Amazonbot", allow: "/", disallow: ["/api/admin/", "/admin/", "/auth/"] },
|
||
|
|
],
|
||
|
|
sitemap: `${SITE_URL}/sitemap.xml`,
|
||
|
|
host: SITE_URL,
|
||
|
|
};
|
||
|
|
}
|