# Tailwaters Fly Fishing - robots.txt
# Last updated: May 2026
# Optimized for AEO/GEO visibility (ChatGPT, Claude, Perplexity, Gemini, etc.)
# Shopify store

# ============================================================
# GLOBAL DEFAULT (applies to any bot not specified below)
# ============================================================
User-agent: *
Allow: /products/
Allow: /collections/
Allow: /blogs/
Allow: /pages/
Allow: /policies/
Disallow: /admin
Disallow: /cart
Disallow: /checkout
Disallow: /checkouts/
Disallow: /orders
Disallow: /account
Disallow: /carts
Disallow: /search
Disallow: /*?*oseid=*
Disallow: /*preview_theme_id*
Disallow: /*preview_script_id*
Disallow: /.well-known/shopify/monorail
Disallow: /apple-app-site-association
Disallow: /cdn/wpm/*.js
Disallow: /recommendations/products
Disallow: /*/recommendations/products
Disallow: /collections/*?*sort_by*
Disallow: /collections/*?*filter*
Disallow: /collections/*?*constraint*
Disallow: /collections/*?*pf_*
Disallow: /collections/*?*view*
Disallow: /collections/*?*grid_list*
Disallow: /collections/*+*
Disallow: /collections/*%*
Disallow: /blogs/*+*
Disallow: /blogs/*%*
Disallow: /*/blogs/*+*
Disallow: /*/blogs/*%*

# ============================================================
# TRADITIONAL SEARCH ENGINES
# ============================================================
User-agent: Googlebot
Allow: /

User-agent: Googlebot-Image
Allow: /

User-agent: Googlebot-Mobile
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: adsbot-google
Disallow: /checkouts/
Disallow: /checkout
Disallow: /carts
Disallow: /orders
Disallow: /*?*oseid=*
Disallow: /*preview_theme_id*
Disallow: /*preview_script_id*
Disallow: /cdn/wpm/*.js

# ============================================================
# AI RETRIEVAL & CITATION BOTS (CRITICAL FOR CHATGPT SHOPPING)
# These bots fetch content to answer user questions in real-time
# and cite back to your site. Allow these to appear in AI answers.
# ============================================================

# OpenAI - powers ChatGPT Search and ChatGPT Shopping
User-agent: OAI-SearchBot
Allow: /products/
Allow: /collections/
Allow: /blogs/
Allow: /pages/
Allow: /policies/
Disallow: /admin
Disallow: /cart
Disallow: /checkout
Disallow: /checkouts/
Disallow: /orders
Disallow: /account
Disallow: /carts
Disallow: /search
Disallow: /collections/*?*sort_by*
Disallow: /collections/*?*filter*
Disallow: /collections/*?*pf_*

# OpenAI - user-triggered fetches when someone asks ChatGPT about your products
User-agent: ChatGPT-User
Allow: /

# Anthropic - real-time search for Claude's web tool
User-agent: Claude-SearchBot
Allow: /products/
Allow: /collections/
Allow: /blogs/
Allow: /pages/
Allow: /policies/
Disallow: /admin
Disallow: /cart
Disallow: /checkout
Disallow: /checkouts/
Disallow: /orders
Disallow: /account
Disallow: /carts
Disallow: /search

# Anthropic - user-triggered fetches in Claude conversations
User-agent: Claude-User
Allow: /

# Perplexity - search index for Perplexity answers
User-agent: PerplexityBot
Allow: /products/
Allow: /collections/
Allow: /blogs/
Allow: /pages/
Allow: /policies/
Disallow: /admin
Disallow: /cart
Disallow: /checkout
Disallow: /checkouts/
Disallow: /orders
Disallow: /account
Disallow: /carts
Disallow: /search

# Perplexity - user-triggered fetches
User-agent: Perplexity-User
Allow: /

# You.com search
User-agent: YouBot
Allow: /

# DuckDuckGo AI assistant
User-agent: DuckAssistBot
Allow: /

# Mistral user-triggered fetches
User-agent: MistralAI-User
Allow: /

# ============================================================
# AI TRAINING & INDEX BOTS
# These build long-term AI knowledge. Allowing them widens your
# footprint in baseline AI knowledge across all chat assistants.
# ============================================================

# OpenAI training crawler
User-agent: GPTBot
Allow: /products/
Allow: /collections/
Allow: /blogs/
Allow: /pages/
Allow: /policies/
Disallow: /admin
Disallow: /cart
Disallow: /checkout
Disallow: /checkouts/
Disallow: /orders
Disallow: /account
Disallow: /carts
Disallow: /search

# Anthropic training crawler
User-agent: ClaudeBot
Allow: /products/
Allow: /collections/
Allow: /blogs/
Allow: /pages/
Allow: /policies/
Disallow: /admin
Disallow: /cart
Disallow: /checkout
Disallow: /checkouts/
Disallow: /orders
Disallow: /account
Disallow: /carts
Disallow: /search

# Google AI (Gemini, AI Overviews) - separate from Googlebot
User-agent: Google-Extended
Allow: /

# Google NotebookLM
User-agent: Google-NotebookLM
Allow: /

# Apple Intelligence
User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

# Meta AI (Llama, Meta AI assistant)
User-agent: Meta-ExternalAgent
Allow: /

User-agent: FacebookBot
Allow: /

# Amazon AI (Rufus, Alexa)
User-agent: Amazonbot
Allow: /

# Cohere
User-agent: cohere-ai
Allow: /

User-agent: cohere-training-data-crawler
Allow: /

# ============================================================
# SOCIAL & PARTNER BOTS
# ============================================================
User-agent: Pinterest
Crawl-delay: 1

User-agent: facebookexternalhit
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

# ============================================================
# SEO TOOLS (rate-limited)
# ============================================================
User-agent: AhrefsBot
Crawl-delay: 5
Disallow: /admin
Disallow: /cart
Disallow: /checkout
Disallow: /checkouts/
Disallow: /orders
Disallow: /account
Disallow: /carts
Disallow: /search
Disallow: /*?*oseid=*
Disallow: /*preview_theme_id*

User-agent: AhrefsSiteAudit
Crawl-delay: 5
Disallow: /admin
Disallow: /cart
Disallow: /checkout
Disallow: /checkouts/
Disallow: /orders
Disallow: /account
Disallow: /carts
Disallow: /search

User-agent: SemrushBot
Crawl-delay: 15
Allow: /

User-agent: MJ12bot
Crawl-delay: 10

# ============================================================
# BLOCKED: aggressive scrapers and non-compliant bots
# Bytespider often ignores robots.txt; back this up with WAF rules.
# ============================================================
User-agent: Bytespider
Disallow: /

User-agent: ByteDance
Disallow: /

User-agent: ImagesiftBot
Disallow: /

User-agent: Omgilibot
Disallow: /

User-agent: Omgili
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: Nutch
Disallow: /

# ============================================================
# SITEMAP
# ============================================================
Sitemap: https://tailwatersflyfishing.com/sitemap.xml