# www.robotstxt.org/ # www.google.com/support/webmasters/bin/answer.py?hl=en&answer=156449 User-agent: * Allow: / # Sitemaps Sitemap: https://www.monsterbevcorp.com/sitemap.xml # Disallow admin and sensitive areas Disallow: /lib/ Disallow: /csp-report.php Disallow: /env.local.php Disallow: /info.php # Crawl-delay for aggressive bots User-agent: * Crawl-delay: 1