# Robots.txt for krutidevtounicode.net
# Optimized for better Google crawling and indexing

User-agent: *
Allow: /

# Allow all important files
Allow: /*.css
Allow: /*.js
Allow: /*.png
Allow: /*.jpg
Allow: /*.gif
Allow: /*.svg
Allow: /*.webp
Allow: /sitemap.xml

# Disallow sensitive or unnecessary directories
Disallow: /admin/
Disallow: /private/
Disallow: /temp/
Disallow: /.git/
Disallow: /node_modules/

# Specific crawler instructions
User-agent: Googlebot
Allow: /
Crawl-delay: 1

User-agent: Bingbot
Allow: /
Crawl-delay: 1

User-agent: facebookexternalhit
Allow: /

User-agent: twitterbot
Allow: /

# Sitemap location
Sitemap: https://krutidevtounicode.net/sitemap.xml

# Performance optimization - prevent crawling of unnecessary files
Disallow: /*.pdf$
Disallow: /*.doc$
Disallow: /*.docx$
Disallow: /*.zip$