User-agent: *
Allow: /

# Sitemap
Sitemap: https://appfoundry.be/sitemap.xml

# AI Training & Content Licensing
# See also: https://appfoundry.be/ai.txt for detailed AI usage policies

# OpenAI GPT Crawler
User-agent: GPTBot
Allow: /
Crawl-delay: 2

# Anthropic Claude Crawler  
User-agent: Claude-Web
Allow: /
Crawl-delay: 2

# Google Bard/Gemini
User-agent: Google-Extended
Allow: /
Crawl-delay: 2

# ChatGPT Plugin/Browser
User-agent: ChatGPT-User
Allow: /
Crawl-delay: 2

# Common AI Training Crawlers
User-agent: CCBot
Allow: /
Crawl-delay: 2

User-agent: anthropic-ai
Allow: /
Crawl-delay: 2

User-agent: ai2-crawler
Allow: /
Crawl-delay: 2

# Block known scrapers that don't respect AI policies
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

# Crawl-delay for respectful crawling
Crawl-delay: 1

# Block admin and internal paths
Disallow: /admin/
Disallow: /_next/
Disallow: /api/
Disallow: /.well-known/

# Block development and testing paths
Disallow: /test/
Disallow: /dev/
Disallow: /staging/

# Block file types that shouldn't be indexed
Disallow: /*.json$
Disallow: /*.xml$
Disallow: /*.txt$
Disallow: /*.log$

# Allow important files
Allow: /sitemap.xml
Allow: /robots.txt
Allow: /ai.txt