# robots.txt — Fast Hook Towing LLC
# https://fasthooktowing.com

# ── Standard crawlers ──────────────────────────────────────────
User-agent: *
Allow: /
Disallow: /404
Disallow: /500

Sitemap: https://fasthooktowing.com/sitemap.xml
Sitemap: https://fasthooktowing.com/sitemap-images.xml

# ── Google ─────────────────────────────────────────────────────
User-agent: Googlebot
Allow: /
Crawl-delay: 0

User-agent: Googlebot-Image
Allow: /img/

# ── Google AI (Gemini training & SGE) ──────────────────────────
User-agent: Google-Extended
Allow: /

# ── Bing ───────────────────────────────────────────────────────
User-agent: Bingbot
Allow: /
Crawl-delay: 1

# ── OpenAI / ChatGPT ───────────────────────────────────────────
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

# ── Anthropic / Claude ─────────────────────────────────────────
User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

# ── Perplexity ─────────────────────────────────────────────────
User-agent: PerplexityBot
Allow: /

# ── Meta AI ────────────────────────────────────────────────────
User-agent: Meta-ExternalAgent
Allow: /

User-agent: Meta-ExternalFetcher
Allow: /

# ── Apple ──────────────────────────────────────────────────────
User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

# ── Amazon / Alexa ─────────────────────────────────────────────
User-agent: Amazonbot
Allow: /

# ── Common Crawl (training data for many LLMs) ─────────────────
User-agent: CCBot
Allow: /

# ── You.com ────────────────────────────────────────────────────
User-agent: YouBot
Allow: /

# ── Cohere ─────────────────────────────────────────────────────
User-agent: cohere-ai
Allow: /

# ── Diffbot ────────────────────────────────────────────────────
User-agent: Diffbot
Allow: /

# ── Semrush / Ahrefs / Moz (SEO tools) ────────────────────────
User-agent: SemrushBot
Allow: /

User-agent: AhrefsBot
Allow: /

User-agent: MJ12bot
Allow: /

User-agent: DotBot
Allow: /

# ── Archive ────────────────────────────────────────────────────
User-agent: ia_archiver
Allow: /