User-agent: * Disallow: /cgi-bin/ Disallow: /tmp/ Disallow: /cache/ Disallow: /logs/ Sitemap: https://calling24o7.com/sitemap.xml # Explicitly allow popular crawlers (including AI crawlers) User-agent: GPTBot Disallow: User-agent: Google-Extended Disallow: User-agent: ClaudeBot Disallow: User-agent: PerplexityBot Disallow: User-agent: CCBot Disallow: User-agent: Bingbot Disallow: User-agent: Applebot Disallow: