# IPtoASN robots.txt
User-agent: *
Allow: /

# Sitemap location
Sitemap: https://iptoasn.com/sitemap.xml

# Crawl-delay for polite crawling
Crawl-delay: 1

# Allow archiving
User-agent: ia_archiver
Allow: /

# Disallow heavy downloads for bots (they should use direct links)
User-agent: *
Disallow: /data/*.gz$