# PowerPlatformTip Robots.txt - Optimized for SEO
# Updated: 2026-09-07 (ported from the live site's robots.txt, adapted for Chirpy)
# Purpose: Guide search engines to valuable content, exclude low-value pages

User-agent: *
Allow: /

# Block low-value pages that dilute crawl budget
Disallow: /search
Disallow: /search/
Disallow: /404.html
Disallow: /404
Disallow: /page*/
Disallow: /page/
Disallow: /assets/js/

# NOTE: Internal working folders (CONTENT, DOCS, SOCIALMEDIA, GOOGLESEARCHCONSOLE,
# NEWOPTIMIZEDPOSTS, SEO reports) are no longer published (excluded in _config.yml)
# and now return 404. They are intentionally NOT disallowed here so Googlebot can
# re-crawl, see the 404, and drop them from the index ("Crawled - currently not indexed").

# NOTE: /thank-you/ is set to "noindex, follow" in its front matter and is
# excluded from the sitemap. It is intentionally NOT disallowed here so
# Googlebot can crawl it and honor the noindex directive (a disallowed
# page's noindex is never seen, so it lingers in the index).

# /tags/ IS disallowed here (unlike the live site): it is auto-generated by
# the jekyll-archives plugin with no front matter of its own, so there is no
# reliable way to attach a noindex meta tag to it here. (Categories was
# removed from the site entirely - not needed.)
Disallow: /tags/

# Allow valuable resources
Allow: /assets/css/
Allow: /assets/images/
Allow: /*.css$
Allow: /*.js$
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.png$
Allow: /*.webp$
Allow: /*.svg$

# Sitemap location (generated by jekyll-sitemap plugin)
Sitemap: https://www.powerplatformtip.com/sitemap.xml

# LLM/AI guidance file (curated overview for AI answer engines)
# https://www.powerplatformtip.com/llms.txt

# Bot-specific rules for major search engines
User-agent: Googlebot
Allow: /
Crawl-delay: 0

User-agent: Googlebot-Image
Allow: /
Crawl-delay: 0

User-agent: Bingbot
Allow: /
Crawl-delay: 0

User-agent: Slurp
Allow: /

# AI / LLM crawlers - explicitly allowed (we WANT to be cited in AI answers)
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: cohere-ai
Allow: /

# Block aggressive/problematic crawlers (optional - uncomment if needed)
# User-agent: AhrefsBot
# Crawl-delay: 10

# User-agent: SemrushBot
# Crawl-delay: 10