# Robots.txt for Delnie.com
# Last updated: 2025-05-13

User-agent: *
Allow: /
Disallow: /*/index.html
Disallow: /*?*
Crawl-delay: 2

# Allow major search engines
User-agent: Googlebot
Allow: /
Crawl-delay: 1

User-agent: Bingbot
Allow: /
Crawl-delay: 1

User-agent: Yandexbot
Allow: /
Crawl-delay: 1

User-agent: DuckDuckBot
Allow: /
Crawl-delay: 1

# Allow SEO and data bots with appropriate rate limits
User-agent: AhrefsBot
Allow: /
Crawl-delay: 5

User-agent: SemrushBot
Allow: /
Crawl-delay: 5

User-agent: MJ12bot
Allow: /
Crawl-delay: 10

User-agent: DotBot
Allow: /
Crawl-delay: 10

# Allow social media bots
User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: LinkedInBot
Allow: /

# Sitemap location
Sitemap: https://www.delnie.com/sitemap.xml