# Knock - AI Trend Detection Platform
# We welcome AI crawlers and search engines

# Default rules for all crawlers
User-agent: *
Allow: /
Disallow: /api/
Disallow: /dashboard/
Disallow: /settings/
Disallow: /checkout/
Disallow: /_next/
Crawl-delay: 1

# OpenAI search, retrieval, and training bots
User-agent: OAI-SearchBot
Allow: /
Disallow: /api/
Disallow: /dashboard/
Disallow: /settings/
Disallow: /checkout/
Crawl-delay: 1

User-agent: GPTBot
Allow: /
Disallow: /api/
Disallow: /dashboard/
Disallow: /settings/
Disallow: /checkout/
Crawl-delay: 1

User-agent: ChatGPT-User
Allow: /
Disallow: /api/
Disallow: /dashboard/
Disallow: /settings/
Disallow: /checkout/
Crawl-delay: 1

# Perplexity search and user retrieval bots
User-agent: PerplexityBot
Allow: /
Disallow: /api/
Disallow: /dashboard/
Disallow: /settings/
Disallow: /checkout/
Crawl-delay: 1

User-agent: Perplexity-User
Allow: /
Disallow: /api/
Disallow: /dashboard/
Disallow: /settings/
Disallow: /checkout/
Crawl-delay: 1

# Anthropic search, retrieval, and training bots
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: Claude-SearchBot
User-agent: Claude-User
Allow: /
Disallow: /api/
Disallow: /dashboard/
Disallow: /settings/
Disallow: /checkout/
Crawl-delay: 1

# Google Extended (Gemini, Bard AI)
User-agent: Google-Extended
Allow: /
Disallow: /api/
Disallow: /dashboard/
Disallow: /settings/
Disallow: /checkout/
Crawl-delay: 1

# Google Gemini
User-agent: GoogleOther
Allow: /
Disallow: /api/
Disallow: /dashboard/
Disallow: /settings/
Disallow: /checkout/
Crawl-delay: 1

# Traditional search engines
User-agent: Googlebot
Allow: /
Disallow: /api/
Disallow: /dashboard/
Disallow: /settings/
Disallow: /checkout/

User-agent: Bingbot
Allow: /
Disallow: /api/
Disallow: /dashboard/
Disallow: /settings/
Disallow: /checkout/

User-agent: Slurp
Allow: /
Disallow: /api/
Disallow: /dashboard/
Disallow: /settings/
Disallow: /checkout/

User-agent: DuckDuckBot
Allow: /
Disallow: /api/
Disallow: /dashboard/
Disallow: /settings/
Disallow: /checkout/

User-agent: Baiduspider
Allow: /
Disallow: /api/
Disallow: /dashboard/
Disallow: /settings/
Disallow: /checkout/

User-agent: YandexBot
Allow: /
Disallow: /api/
Disallow: /dashboard/
Disallow: /settings/
Disallow: /checkout/

# AI Research Crawlers
User-agent: anthropic-ai
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: Omgilibot
Allow: /

User-agent: Omgili
Allow: /

User-agent: FacebookBot
Allow: /

User-agent: CCBot
Allow: /
Disallow: /api/
Disallow: /dashboard/

# Sitemap location
Sitemap: https://knock.fomcode.com/sitemap.xml