# RoleWeek wants to be found: search engines, AI search and AI assistants may read every public page.
# Policy and the reasons for each group: docs/SEO.md. Questions: hello@roleweek.com

# Search, answer and fetch-on-request agents: all public pages
# OAI-SearchBot: ChatGPT search
# ChatGPT-User: fetches a page when a ChatGPT user asks
# Claude-SearchBot: Claude search
# Claude-User: fetches a page when a Claude user asks
# PerplexityBot: Perplexity search
# Perplexity-User: fetches a page when a Perplexity user asks
# Googlebot: Google Search, AI Overviews and AI Mode
# Google-Extended: Gemini grounding and training
# Bingbot: Bing and Copilot
# Applebot: Siri, Spotlight and Safari
# DuckAssistBot: DuckDuckGo AI answers
# Meta-WebIndexer: Meta AI search
# Meta-ExternalFetcher: fetches a page when a Meta AI user asks
# Amzn-SearchBot: Amazon and Alexa search
# Amzn-User: fetches a page when an Alexa user asks
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Googlebot
User-agent: Google-Extended
User-agent: Bingbot
User-agent: Applebot
User-agent: DuckAssistBot
User-agent: Meta-WebIndexer
User-agent: Meta-ExternalFetcher
User-agent: Amzn-SearchBot
User-agent: Amzn-User
Disallow: /data/
Disallow: /directions/

# Training-only crawlers: public pages are fine; the employers' own description text on /job/ pages is not
# for model training (it is theirs; RoleWeek shows it to help people apply)
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: Applebot-Extended
User-agent: Meta-ExternalAgent
User-agent: Amazonbot
User-agent: CCBot
User-agent: Bytespider
User-agent: AI2Bot
User-agent: omgili
User-agent: omgilibot
Disallow: /job/
Disallow: /data/
Disallow: /directions/

# Everyone else
User-agent: *
Disallow: /data/
Disallow: /directions/

Sitemap: https://roleweek.com/sitemap.xml
Sitemap: https://roleweek.com/sitemap-recent.xml