import { NextResponse } from 'next/server';

export async function GET() {
  const robotsText = `User-agent: *
Allow: /


# Block common crawling issues

Disallow: /api/
Disallow: /_next/data/
Disallow: /*.json$
Disallow: /*/feed/
Disallow: /*/embed/
Disallow: /sendbtnemail/
Disallow: /resources/
Disallow: /search/
Disallow: /404
Disallow: /500


# Allow important Next.js assets
Allow: /_next/static/css/
Allow: /_next/static/media/
Allow: /_next/static/js/


# AI Crawlers (LLM Bots)

# OpenAI / ChatGPT
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# Google AI (AI Overview / Gemini)
User-agent: Google-Extended
Allow: /

# Anthropic Claude
User-agent: ClaudeBot
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Allow: /

# Amazon AI (Alexa / Rufus)
User-agent: Amazonbot
Allow: /

# Common AI research crawlers
User-agent: CCBot
Allow: /

User-agent: Bytespider
Allow: /

User-agent: Meta-ExternalAgent
Allow: /


# Discourage Known Low-Value Crawlers


User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: MegaIndex
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: SeekportBot
Disallow: /

User-agent: Barkrowler
Disallow: /



# Sitemaps

Sitemap: https://universitiespage.com/sitemap.xml
Sitemap: https://universitiespage.com/university_sitemap.xml
Sitemap: https://universitiespage.com/visa_sitemap.xml
Sitemap: https://universitiespage.com/sitemap_course1.xml
Sitemap: https://universitiespage.com/sitemap_course2.xml
Sitemap: https://universitiespage.com/articles_sitemap.xml
Sitemap: https://universitiespage.com/guides_sitemap.xml;`

  return new NextResponse(robotsText, {
    headers: {
      'Content-Type': 'text/plain',
    },
  });
}