import type { MetadataRoute } from 'next' const base = process.env.NEXT_PUBLIC_SITE_URL ?? 'https://anwarpi.ai' /* Owner decision: allow all named search/AI crawlers explicitly. The `*` group keeps its `/api` disallow; the named groups opt back into the public content APIs and the llms.txt feeds. */ const AI_BOTS = [ 'GPTBot', 'OAI-SearchBot', 'ChatGPT-User', 'ClaudeBot', 'Claude-User', 'Claude-SearchBot', 'PerplexityBot', 'Google-Extended', 'Applebot', 'Bingbot', ] export default function robots(): MetadataRoute.Robots { return { rules: [ ...AI_BOTS.map(userAgent => ({ userAgent, allow: ['/', '/api/posts', '/llms.txt', '/llms-full.txt'], })), { userAgent: '*', allow: '/', disallow: ['/admin', '/api', '/account-settings', '/onboarding', '/vip-onboarding', '/verify-email', '/payment-success', '/payment-cancelled', '/forgot-password'], }, ], sitemap: `${base}/sitemap.xml`, } }