# L-Group International LLP - robots.txt
# Allow Google, Bing, and other major search engines to crawl content
# Block crawling of assets, duplicates, and resource-heavy pages

# Default rules for all user-agents
User-agent: *
Allow: /

# Block static assets to prevent indexing as pages
Disallow: /css/
Disallow: /js/
Disallow: /images/
Disallow: /flipbook_assets/
Disallow: /mp3/
Disallow: /LGROUP_CATALOGUES/

# Block PDF and document files
Disallow: /*.pdf$
Disallow: /*.doc$
Disallow: /*.docx$

# Block JSON data files
Disallow: /languages.json
Disallow: /*.json$

# Block font files and other resources
Disallow: /*.woff$
Disallow: /*.woff2$
Disallow: /*.ttf$
Disallow: /*.eot$

# Block admin and system directories
Disallow: /cgi-bin/
Disallow: /admin/
Disallow: /.well-known/

# Block /index.html (canonical is https://lgroup.in/ without index.html)
Disallow: /index.html

# Block non-existent /collections/ directory pages
Disallow: /collections/

# Block product.html (internal tool page, not for indexing)
Disallow: /product.html

# Block catalogue with query parameters (only base catalogue.html should be indexed)
Disallow: /catalogue.html?*

# Block LGROUP_CATALOGUES internal pages
Disallow: /LGROUP_CATALOGUES/

# Block map utility page
Disallow: /map.html

# Allow Google to crawl specific resources needed for rendering
User-agent: Googlebot
Allow: /css/
Allow: /js/
Allow: /images/

# Allow Bing to crawl resources
User-agent: Bingbot
Allow: /css/
Allow: /js/
Allow: /images/

# Block bad bots and scrapers
User-agent: MJ12bot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: DotBot
Disallow: /

# Crawl delay for resource-intensive bots
User-agent: *
Crawl-delay: 1

# Sitemap location
Sitemap: https://lgroup.in/sitemap.xml