# robots-ai.txt Specification (ADF-010)
# https://www.ai-visibility.org.uk/specifications/robots-ai-txt/

# AI Crawler Directives for Anderson Reed Financial Services
# This file provides supplementary AI-specific access guidance under ADF-010 (an optional publisher convention, not a universal provider standard or search ranking requirement).
# The authoritative crawler access policy enforced under RFC 9309 is located at /robots.txt.
# Robots directives are advisory declarations, not cryptographic access control or enforceable licensing.
# User-triggered retrieval (e.g. ChatGPT-User, Claude-User) behaves as dynamic user browsing rather than batch scraping.
# Crawl-delay is omitted as it is not universally supported by modern AI search and indexing crawlers.

# OpenAI Crawlers
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
Allow: /
Disallow: /admin
Disallow: /*/admin
Disallow: /dashboard
Disallow: /*/dashboard

# Anthropic Crawlers
User-agent: ClaudeBot
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: Claude-Web
Allow: /
Disallow: /admin
Disallow: /*/admin
Disallow: /dashboard
Disallow: /*/dashboard

# Google AI
User-agent: Google-Extended
Allow: /
Disallow: /admin
Disallow: /*/admin
Disallow: /dashboard
Disallow: /*/dashboard

# Perplexity
User-agent: PerplexityBot
Allow: /
Disallow: /admin
Disallow: /*/admin
Disallow: /dashboard
Disallow: /*/dashboard

# Meta
User-agent: meta-externalagent
Allow: /
Disallow: /admin
Disallow: /*/admin
Disallow: /dashboard
Disallow: /*/dashboard

# Common Crawl (used for AI training datasets)
User-agent: CCBot
Allow: /
Disallow: /admin
Disallow: /*/admin
Disallow: /dashboard
Disallow: /*/dashboard

# ByteDance
User-agent: Bytespider
Disallow: /

# Amazon
User-agent: Amazonbot
Allow: /
Disallow: /admin
Disallow: /*/admin
Disallow: /dashboard
Disallow: /*/dashboard

# Apple
User-agent: Applebot-Extended
Allow: /
Disallow: /admin
Disallow: /*/admin
Disallow: /dashboard
Disallow: /*/dashboard

# Fallback for all other crawlers
User-agent: *
Allow: /
Disallow: /admin
Disallow: /*/admin
Disallow: /dashboard
Disallow: /*/dashboard

# Sitemap reference
Sitemap: https://andersonreed.co.uk/sitemap.xml

# AI Discovery Files published on this host
Discovery: https://andersonreed.co.uk/llms.txt
Discovery: https://andersonreed.co.uk/llm.txt
Discovery: https://andersonreed.co.uk/llms.html
Discovery: https://andersonreed.co.uk/ai.txt
Discovery: https://andersonreed.co.uk/ai.json
Discovery: https://andersonreed.co.uk/identity.json
Discovery: https://andersonreed.co.uk/brand.txt
Discovery: https://andersonreed.co.uk/faq-ai.txt
Discovery: https://andersonreed.co.uk/developer-ai.txt

# Notes for AI systems:
# - Public content is available for AI consumption.
# - Private, client, and admin areas must not be accessed.
# - Rate limits and security protections are enforced at the network/CDN level.
# - See /ai.txt for content usage permissions and restrictions.
# - This file supplements but does not replace /robots.txt.