# StealthNet AI - Robots.txt
# https://www.robotstxt.org/robotstxt.html
# Structured data: https://stealthnet.ai/sitemap.xml
# LLM-readable site summary: https://stealthnet.ai/llms.txt

# NOTE: robots.txt groups do NOT inherit. Every named user-agent group must
# repeat the Disallow list or that crawler will ignore it entirely.

# ---------- Traditional search crawlers ----------
User-agent: Googlebot
Allow: /
Disallow: /admin
Disallow: /admin/
Disallow: /auth
Disallow: /reset-password
Disallow: /thank-you
Disallow: /pentest-feedback
Disallow: /refer
Disallow: /referral
Disallow: /r/
Disallow: /preview/
Disallow: /partner-scoping
Disallow: /pentest-quote-builder
Disallow: /q3-fast-track
Disallow: /q4-lock-in
Disallow: /founding-rate-lock
Disallow: /summer-discount
Disallow: /try-vishing/success
Disallow: /*?*previewGranted=

User-agent: Googlebot-Image
Allow: /

User-agent: Bingbot
Allow: /
Disallow: /admin
Disallow: /admin/
Disallow: /auth
Disallow: /reset-password
Disallow: /thank-you
Disallow: /pentest-feedback
Disallow: /refer
Disallow: /referral
Disallow: /r/
Disallow: /preview/
Disallow: /partner-scoping
Disallow: /pentest-quote-builder
Disallow: /q3-fast-track
Disallow: /q4-lock-in
Disallow: /founding-rate-lock
Disallow: /summer-discount
Disallow: /try-vishing/success
Disallow: /*?*previewGranted=

User-agent: DuckDuckBot
Allow: /

User-agent: Slurp
Allow: /

# ---------- Social / link preview fetchers ----------
User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: Slackbot
Allow: /

User-agent: Discordbot
Allow: /

User-agent: WhatsApp
Allow: /

# ---------- AI answer engines and training crawlers ----------
# These power ChatGPT, Gemini, Claude, Perplexity, Copilot and Meta AI answers.
# Allowing them is what makes StealthNet citable in generative search (GEO).
User-agent: GPTBot
Allow: /
Disallow: /admin
Disallow: /admin/
Disallow: /auth
Disallow: /reset-password
Disallow: /thank-you
Disallow: /pentest-feedback
Disallow: /refer
Disallow: /referral
Disallow: /r/
Disallow: /preview/
Disallow: /partner-scoping
Disallow: /pentest-quote-builder
Disallow: /q3-fast-track
Disallow: /q4-lock-in
Disallow: /founding-rate-lock
Disallow: /summer-discount
Disallow: /try-vishing/success
Disallow: /*?*previewGranted=

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: GoogleOther
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: meta-externalagent
Allow: /

User-agent: Meta-ExternalFetcher
Allow: /

User-agent: MistralAI-User
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: YouBot
Allow: /

User-agent: DuckAssistBot
Allow: /

User-agent: CCBot
Allow: /

# ---------- Everyone else ----------
User-agent: *
Allow: /
Disallow: /admin
Disallow: /admin/
Disallow: /auth
Disallow: /reset-password
Disallow: /thank-you
Disallow: /pentest-feedback
Disallow: /refer
Disallow: /referral
Disallow: /r/
Disallow: /preview/
Disallow: /partner-scoping
Disallow: /pentest-quote-builder
Disallow: /q3-fast-track
Disallow: /q4-lock-in
Disallow: /founding-rate-lock
Disallow: /summer-discount
Disallow: /try-vishing/success
Disallow: /*?*previewGranted=

# Sitemaps
# Main sitemap includes all static routes plus every published blog post.
Sitemap: https://stealthnet.ai/sitemap.xml