39 lines
978 B
TypeScript
39 lines
978 B
TypeScript
|
import { SITE_URL } from '@/lib/seo';
|
||
|
|
import type { MetadataRoute } from 'next';
|
||
|
|
|
||
|
|
// AI search/answer crawlers — explicitly allowed so the product surfaces in
|
||
|
|
// ChatGPT, Claude, Perplexity, Google AI and similar answer engines.
|
||
|
|
const AI_CRAWLERS = [
|
||
|
|
'GPTBot',
|
||
|
|
'OAI-SearchBot',
|
||
|
|
'ChatGPT-User',
|
||
|
|
'ClaudeBot',
|
||
|
|
'anthropic-ai',
|
||
|
|
'Claude-Web',
|
||
|
|
'PerplexityBot',
|
||
|
|
'Perplexity-User',
|
||
|
|
'Google-Extended',
|
||
|
|
'CCBot',
|
||
|
|
'Applebot-Extended',
|
||
|
|
'Amazonbot',
|
||
|
|
'meta-externalagent',
|
||
|
|
'DuckAssistBot',
|
||
|
|
];
|
||
|
|
|
||
|
|
export default function robots(): MetadataRoute.Robots {
|
||
|
|
return {
|
||
|
|
rules: [
|
||
|
|
{
|
||
|
|
userAgent: '*',
|
||
|
|
allow: '/',
|
||
|
|
disallow: ['/api/', '/admin', '/dashboard', '/login'],
|
||
|
|
},
|
||
|
|
// Aggressive scraper with no search value.
|
||
|
|
{ userAgent: 'Bytespider', disallow: '/' },
|
||
|
|
{ userAgent: AI_CRAWLERS, allow: '/', disallow: ['/api/', '/admin', '/dashboard'] },
|
||
|
|
],
|
||
|
|
sitemap: `${SITE_URL}/sitemap.xml`,
|
||
|
|
host: SITE_URL,
|
||
|
|
};
|
||
|
|
}
|