# START YOAST BLOCK
# ---------------------------
# Allow all search engines, AI crawlers, and social media bots for public pages
User-agent: *
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /wp-includes/
Disallow: /readme.html
Disallow: /xmlrpc.php
Disallow: /?s=            # Blocks internal search results (low-value SEO)
Allow: /wp-admin/admin-ajax.php

# --- Major Search Engines ---
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Baiduspider
Allow: /

User-agent: YandexBot
Allow: /

# --- AI & LLM Crawlers ---
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: GoogleOther
Allow: /

User-agent: CCBot
Allow: /

# --- Social Media Bots (for link previews & sharing) ---
User-agent: facebookexternalhit
Allow: /

User-agent: Facebot
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: Pinterestbot
Allow: /

User-agent: WhatsApp
Allow: /

User-agent: TelegramBot
Allow: /

User-agent: Discordbot
Allow: /

User-agent: Slackbot
Allow: /

# --- Sitemap for All Crawlers ---
Sitemap: https://ripik.ai/sitemap_index.xml
# ---------------------------
# END YOAST BLOCK