User-agent: *
Allow: /

# Sitemap
Sitemap: https://taskpara.com/sitemap.xml
Sitemap: https://taskpara.com/blogmap.xml

# Important pages to crawl
Allow: /ai-tools/
Allow: /blog/
Allow: /search

# Block admin areas
Disallow: /admin-panel
Disallow: /nosignin
Disallow: /analytics

# Block search result pages with queries to avoid duplicate content
Disallow: /search?*
Disallow: /*?page=*

# Block development/test files
Disallow: /cse-test
Disallow: /*.json$
# Allow XML files for sitemaps
Allow: /sitemap.xml
Allow: /blogmap.xml
Allow: /*.xml

# Crawl delay for aggressive bots
User-agent: SemrushBot
Crawl-delay: 10

User-agent: AhrefsBot
Crawl-delay: 10

User-agent: MJ12bot
Crawl-delay: 30

# Block problematic bots
User-agent: ia_archiver
Disallow: /

User-agent: Wayback
Disallow: /

# Allow Google and Bing priority access
User-agent: Googlebot
Allow: /
Crawl-delay: 1

User-agent: Bingbot
Allow: /
Crawl-delay: 1