# Block API and authentication routes
User-agent: *
Allow: /
Disallow: /api/
Disallow: /oauth2/
Disallow: /login
Disallow: /register

# Standard crawl rules for Googlebot
User-agent: Googlebot
Allow: /
Disallow: /api/
Disallow: /oauth2/
Disallow: /login
Disallow: /register
Crawl-delay: 0.5

# AdsBot Google
User-agent: AdsBot-Google
Allow: /

# Allow social media crawlers
User-agent: facebookexternalhit
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: linkedinbot
Allow: /

# Yandex
User-agent: Yandex
Allow: /
Disallow: /api/
Disallow: /oauth2/
Crawl-delay: 1

# Bing
User-agent: Bingbot
Allow: /
Disallow: /api/
Disallow: /oauth2/

# Sitemap reference
Sitemap: https://checkanddo.com/sitemap.xml