User-agent: *
    Disallow: /api/dropdown-options
    Allow: /

    # Aggressive bot crawl rate limits
    # GPTBot was doing 81K requests/12h - block entirely
    User-agent: GPTBot
    Disallow: /

    # Other aggressive crawlers - moderate rate limiting
    User-agent: meta-externalagent
    Crawl-delay: 30

    User-agent: barkrowler
    Crawl-delay: 30

    User-agent: petalbot
    Crawl-delay: 30

    User-agent: Bytespider
    Crawl-delay: 30

    Sitemap: https://www.sportsjobs.online/sitemap.xml
    
    # LLM-friendly content for AI assistants
    # See https://llmstxt.org for more information
    Allow: /llms.txt