User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Allow: /blog/
Allow: /products/
Allow: /services/

Sitemap: https://www.onpage.com/sitemap_index.xml

# --- AI & Research Crawlers: Explicit Allow ---
User-agent: OAI-SearchBot       # OpenAI's real-time search crawler
Allow: /

User-agent: GPTBot              # OpenAI's training crawler
Allow: /

User-agent: CCBot               # Common Crawl
Allow: /

User-agent: anthropic-ai        # Anthropic's general crawler
Allow: /

User-agent: Claude-Web          # Anthropic's web crawler
Allow: /

User-agent: PerplexityBot       # Perplexity AI crawler
Allow: /

User-agent: GoogleOther         # Google's general-purpose AI crawler
Allow: /