User-agent: *
Allow: /

# Block access to sensitive directories
Disallow: /assets/js/
Disallow: /assets/css/
Disallow: /workers/
Disallow: /admin/
Disallow: /private/

# Allow access to all HTML pages and important assets
Allow: /

# Sitemap location
Sitemap: https://randomlygenerate.com/sitemap.xml

# Crawl delay for polite crawling
Crawl-delay: 1