# robots.txt for Namak Mirchi
# Website: https://www.namakmirchi.ca

# Allow all search engines to crawl the entire site
User-agent: *
Allow: /

# Sitemap location
Sitemap: https://www.namakmirchi.ca/sitemap.xml

# LLM-specific information file for AI agents
# This file contains structured information about our restaurant
Allow: /llms.txt

# Main pages - High priority for indexing
Allow: /index.html
Allow: /menu.html
Allow: /about.html
Allow: /our-food.html
Allow: /contact-us.html
Allow: /catering.html

# Legal pages
Allow: /privacy-policy.html
Allow: /terms-of-use.html

# SEO landing pages - Location-specific
Allow: /tags/places/

# SEO landing pages - Dish-specific
Allow: /tags/dishes/

# Service-specific pages
Allow: /tags/indian-food-delivery-calgary.html

# Static assets
Allow: /assets/
Allow: /style.css
Allow: /global.css
Allow: /script.js
Allow: /menu.js
Allow: /components.js
Allow: /components/

# Disallow pages not needed for public access
# Note: reservation.html and enquiry.html are excluded from indexing
Disallow: /reservation.html
Disallow: /enquiry.html

# Disallow development/temporary files
Disallow: /*.bak$
Disallow: /*.tmp$
Disallow: /*~$
Disallow: /*.old$

# Common bot-specific rules
# Google
User-agent: Googlebot
Allow: /
Crawl-delay: 0

# Google Images
User-agent: Googlebot-Image
Allow: /assets/

# Bing
User-agent: Bingbot
Allow: /
Crawl-delay: 1

# Yahoo
User-agent: Slurp
Allow: /
Crawl-delay: 1

# DuckDuckGo
User-agent: DuckDuckBot
Allow: /
Crawl-delay: 1

# Yandex
User-agent: YandexBot
Allow: /
Crawl-delay: 2

# Baidu
User-agent: Baiduspider
Allow: /
Crawl-delay: 2

# AI/LLM Crawlers
User-agent: GPTBot
Allow: /
Allow: /llms.txt

User-agent: ChatGPT-User
Allow: /
Allow: /llms.txt

User-agent: anthropic-ai
Allow: /
Allow: /llms.txt

User-agent: Claude-Web
Allow: /
Allow: /llms.txt

User-agent: Google-Extended
Allow: /
Allow: /llms.txt

User-agent: CCBot
Allow: /
Allow: /llms.txt

User-agent: PerplexityBot
Allow: /
Allow: /llms.txt

# Social Media Crawlers
User-agent: facebookexternalhit
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: Pinterest
Allow: /

# Block malicious/aggressive crawlers (if needed in future)
# User-agent: BadBot
# Disallow: /

# Performance note:
# Crawl-delay is set conservatively to avoid overloading the server
# Most major search engines ignore crawl-delay, but it's here as a courtesy

# Last updated: January 2025