# Rock 'n' Roll Radio
# https://rocknroll.radio/
# robots.txt
#
# Goal:
# - Keep public editorial, artist, concert, chart, show, and station pages crawlable.
# - Allow Google, Bing, and major AI search/retrieval crawlers.
# - Keep WordPress administrative/login endpoints out of crawler paths.
# - Keep CSS, JavaScript, images, and /wp-content/ crawlable for rendering.
#
# IMPORTANT:
# Confirm the Sitemap URL before publishing.
# WordPress core normally uses /wp-sitemap.xml.
# If your SEO plugin uses /sitemap_index.xml instead, replace the Sitemap line below.

# ----------------------------------------------------------------------
# Default rules for standards-compliant crawlers
# ----------------------------------------------------------------------
User-agent: *
Allow: /
Allow: /llms.txt
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /xmlrpc.php

# ----------------------------------------------------------------------
# Google Search
# ----------------------------------------------------------------------
User-agent: Googlebot
Allow: /
Allow: /llms.txt
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /xmlrpc.php

# Google-Extended is separate from Google Search.
# This configuration allows eligible Google AI/Gemini uses.
User-agent: Google-Extended
Allow: /
Allow: /llms.txt
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /xmlrpc.php

# ----------------------------------------------------------------------
# Microsoft Bing / Copilot discovery
# ----------------------------------------------------------------------
User-agent: Bingbot
Allow: /
Allow: /llms.txt
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /xmlrpc.php

# ----------------------------------------------------------------------
# OpenAI
# OAI-SearchBot: ChatGPT search discovery
# GPTBot: model-development/training crawler
# ChatGPT-User: user-initiated retrieval; robots behavior may differ
# ----------------------------------------------------------------------
User-agent: OAI-SearchBot
Allow: /
Allow: /llms.txt
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /xmlrpc.php

User-agent: GPTBot
Allow: /
Allow: /llms.txt
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /xmlrpc.php

User-agent: ChatGPT-User
Allow: /
Allow: /llms.txt
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /xmlrpc.php

# ----------------------------------------------------------------------
# Anthropic / Claude
# Claude-SearchBot: search discovery
# Claude-User: user-directed retrieval
# ClaudeBot: model-development/training crawler
# ----------------------------------------------------------------------
User-agent: Claude-SearchBot
Allow: /
Allow: /llms.txt
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /xmlrpc.php

User-agent: Claude-User
Allow: /
Allow: /llms.txt
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /xmlrpc.php

User-agent: ClaudeBot
Allow: /
Allow: /llms.txt
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /xmlrpc.php

# ----------------------------------------------------------------------
# Perplexity
# PerplexityBot: search/index crawler
# Perplexity-User: user-directed retrieval
# ----------------------------------------------------------------------
User-agent: PerplexityBot
Allow: /
Allow: /llms.txt
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /xmlrpc.php

User-agent: Perplexity-User
Allow: /
Allow: /llms.txt
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /xmlrpc.php

# ----------------------------------------------------------------------
# Sitemap
# Replace this URL only if your active WordPress SEO plugin uses a
# different sitemap index, such as:
# https://rocknroll.radio/sitemap_index.xml
# ----------------------------------------------------------------------
Sitemap: https://rocknroll.radio/sitemap_index.xml