# robots.txt for MyReputation.ie
# AI-friendly crawling policies with Content Signals
# Last updated: 2026-06-09

# ─────────────────────────────────────────────────────────────────────────────
# DEFAULT RULES (all crawlers)
# ─────────────────────────────────────────────────────────────────────────────
User-Agent: *
Allow: /
Disallow: /api/
Disallow: /dashboard/
Disallow: /admin/
Disallow: /auth/
Disallow: /ref/
Disallow: /portal/
Disallow: /invite/
Disallow: /widget/
Disallow: *.json$

# ─────────────────────────────────────────────────────────────────────────────
# AI CRAWLERS (explicit allow for public content)
# ─────────────────────────────────────────────────────────────────────────────

User-Agent: GPTBot
Allow: /
Disallow: /api/
Disallow: /dashboard/
Disallow: /admin/
Disallow: /auth/

User-Agent: Claude-Web
Allow: /
Disallow: /api/
Disallow: /dashboard/
Disallow: /admin/
Disallow: /auth/

User-Agent: OAI-SearchBot
Allow: /
Disallow: /api/
Disallow: /dashboard/
Disallow: /admin/
Disallow: /auth/

User-Agent: Google-Extended
Allow: /
Disallow: /api/
Disallow: /dashboard/
Disallow: /admin/
Disallow: /auth/

User-Agent: Perplexity-Bot
Allow: /
Disallow: /api/
Disallow: /dashboard/
Disallow: /admin/
Disallow: /auth/

User-Agent: CCBot
Allow: /
Disallow: /api/
Disallow: /dashboard/
Disallow: /admin/
Disallow: /auth/

User-Agent: anthropic-ai
Allow: /
Disallow: /api/
Disallow: /dashboard/
Disallow: /admin/
Disallow: /auth/

# ─────────────────────────────────────────────────────────────────────────────
# SEARCH & ADS CRAWLERS
# ─────────────────────────────────────────────────────────────────────────────

User-Agent: Googlebot
Allow: /
Disallow: /api/
Disallow: /dashboard/

User-Agent: AdsBot-Google
Allow: /
Disallow: /api/
Disallow: /dashboard/

# ─────────────────────────────────────────────────────────────────────────────
# CONTENT SIGNALS (RFC draft-romm-aipref-contentsignals)
# ─────────────────────────────────────────────────────────────────────────────
# Declares how AI systems may use this site's content:
# - ai-train=yes: Content can be used to train AI models
# - search=yes: Content can be indexed for search engines
# - ai-input=yes: Content can be used as input to AI systems

Content-Signal: ai-train=yes, search=yes, ai-input=yes

# ─────────────────────────────────────────────────────────────────────────────
# DISCOVERY & SITEMAPS
# ─────────────────────────────────────────────────────────────────────────────

Sitemap: https://myreputation.ie/sitemap.xml
Host: https://myreputation.ie