# See http://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file

User-agent: *

  Sitemap: https://www.ecfr.gov/sitemaps/sitemap.xml.gz
  Disallow: /search
  Disallow: /recent-changes
  Disallow: /on/
  Disallow: /compare/
  Disallow: /my/
  Disallow: /auth/ofr

  # Don't index developer tool links
  Disallow: /api/renderer/v1/content/
  Disallow: /api/versioner/v1/full/