# NOTICE: The collection of content and other data on this
# site through automated means, including any device, tool,
# or process designed to data mine or scrape content, is
# prohibited except (1) for the purpose of search engine indexing or
# artificial intelligence retrieval augmented generation or (2) with express
# written permission from this site’s operator.

# To request permission to license our intellectual
# property and/or other materials, please contact this
# site’s operator directly.

# BEGIN Cloudflare Managed content

User-agent: Amazonbot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: meta-externalagent
Disallow: /

# END Cloudflare Managed Content
# Robots.txt Generated automatically for indexnow.dk
# Allow all crawlers to access all content
User-agent: *
Allow: /

# ===== CRAWL RATE CONTROL =====
# Limit aggressive SEO bots to prevent server overload

User-agent: AhrefsBot
Crawl-delay: 10

User-agent: SemrushBot
Crawl-delay: 10

User-agent: MJ12bot
Crawl-delay: 10

User-agent: BLEXBot
Crawl-delay: 10

User-agent: DotBot
Crawl-delay: 15

# ===== BLOCK HARMFUL/SPAM BOTS =====
# These bots consume bandwidth without providing value

User-agent: SemrushBot-SA
Disallow: /

User-agent: SemrushBot-BA
Disallow: /

User-agent: SemrushBot-SI
Disallow: /

User-agent: SemrushBot-SWA
Disallow: /

User-agent: SemrushBot-CT
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: AhrefsBot
Disallow: /

# Block aggressive scrapers
User-agent: SeznamBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: DotBot
Disallow: /

# Block social media bots that don't respect rate limits
User-agent: facebookexternalhit
Crawl-delay: 5

# ===== SEARCH ENGINES - WELCOME =====
# Explicitly welcome major search engines

User-agent: Googlebot
Allow: /
Crawl-delay: 1

User-agent: Bingbot
Allow: /
Crawl-delay: 1

User-agent: YandexBot
Allow: /
Crawl-delay: 2

User-agent: DuckDuckBot
Allow: /
Crawl-delay: 2

# ===== SITEMAPS =====
Sitemap: https://indexnow.dk/sitemap.xml