# Default: everything is crawlable except Cloudflare's internal endpoints.
User-agent: *
Disallow: /cdn-cgi/

# Perplexity documents two agents and asks sites that want to appear in its
# answers to allow them explicitly. PerplexityBot builds the search index;
# Perplexity-User fetches a page live when a user's question needs it.
# Listing them by name is documentation, not a grant: the "*" group above already
# allows both. It does mean a future tightening of "*" cannot lock them out by
# accident - but a named group replaces "*" wholesale for that agent, which is why
# each repeats the /cdn-cgi/ Disallow.
# NOTE: robots.txt is not the whole story. Cloudflare's "Block AI bots" /
# Bot Fight Mode settings sit in front of this file and will return 403 to
# both agents regardless of what is written here. Verify in the zone settings
# and in the request logs, not by reading this file.
User-agent: PerplexityBot
Allow: /
Disallow: /cdn-cgi/

User-agent: Perplexity-User
Allow: /
Disallow: /cdn-cgi/

Sitemap: https://insightpins.com/sitemap.xml
Host: insightpins.com