# See http://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file
User-agent: *
Sitemap: https://www.ecfr.gov/sitemaps/sitemap.xml.gz
Disallow: /search
Disallow: /recent-changes
Disallow: /on/
Disallow: /compare/
Disallow: /my/
Disallow: /auth/ofr
# Don't index developer tool links
Disallow: /api/renderer/v1/content/
Disallow: /api/versioner/v1/full/