# ai.txt file for useneedle.net
# Purpose: Explicit AI-agent permissions for crawling, indexing, training, and attribution.
# Generated from frontend/lib/seo/aiCrawlerPolicy.mjs (robots.txt is built from the same module).
# DO NOT EDIT BY HAND - run `npm run generate:llms` after policy changes.
# Version: 2.0
# Last Updated: 2026-08-07

# ---------------------------------------------------------
# POLICY (decision D1 - documented in docs/ENGINEERING.md)
# ---------------------------------------------------------
# Public marketing and documentation pages are deliberately available for both
# AI training and AI retrieval: the brand depends on AI visibility, and public
# pages contain no user or product data (see PRIVATE PATHS below, disallowed for
# every agent). Review trigger: if AI-referral share drops materially or crawl
# load becomes abusive, flip training crawlers to block.

AI-Crawling: allow
AI-Training: allow
AI-Indexing: allow
AI-Archiving: allow
AI-Content-Summarization: allow
AI-Commercial-Use: allow
AI-Attribution-Required: yes

# ---------------------------------------------------------
# TRAINING CRAWLERS (allowed - deliberate; see policy note)
# ---------------------------------------------------------
GPTBot — OpenAI model training
CCBot — Common Crawl (feeds multiple labs)
anthropic-ai — Anthropic model training
ClaudeBot — Anthropic model training
Google-Extended — Google Gemini training (does not affect Google Search ranking)
Applebot-Extended — Apple Intelligence training
Bytespider — ByteDance; may ignore robots.txt - WAF backstop
Meta-ExternalAgent — Meta AI / Llama training

# ---------------------------------------------------------
# SEARCH / RETRIEVAL CRAWLERS (allowed - required for AI citations)
# ---------------------------------------------------------
OAI-SearchBot — OpenAI ChatGPT search indexing
ChatGPT-User — OpenAI assistant page fetches
Claude-SearchBot — Anthropic search indexing
Claude-Web — Anthropic assistant page fetches
PerplexityBot — Perplexity indexing
Perplexity-User — Perplexity page fetches
YepBot — Yep search

# ---------------------------------------------------------
# PRIVATE PATHS (disallowed for ALL agents)
# ---------------------------------------------------------
/badges
/conversations/preview
/go/
/dashboard/
/admin
/profile
/settings
/search/
/api/
/fonts/
/features/videos/
/favicon.ico
/test/
/dev/
/staging/
/node_modules/
/src/
/.env
/.env.*
/config/
/logs/
/scripts/
/package.json
/package-lock.json
/next.config.mjs
/tailwind.config.js
/postcss.config.js
/eslint.config.js
/dist/
/build/
/.cache/
/.temp/

# ---------------------------------------------------------
# ATTRIBUTION
# ---------------------------------------------------------
# Please cite Needle and link to the canonical page URL when quoting product facts.
# Support: support@useneedle.net
# Pricing: https://useneedle.net/pricing