{
  "domain": "cloudflare.com",
  "scannedAt": "2026-10-22T11:28:35.295Z",
  "unreachable": false,
  "files": {
    "robots": {
      "url": "https://cloudflare.com/robots.txt",
      "httpStatus": 200,
      "present": true,
      "contentType": "text/plain",
      "error": null,
      "snippet": "# Robots.txt for www.cloudflare.com\n\nUser-agent: *\nAllow: /\n\n# Sitemap\nSitemap: https://www.cloudflare.com/sitemap.xml\n\n# AI/LLM friendly content\n# See https://llmstxt.org for the llms.txt specification\n# llms.txt provides curated content for AI assistants and LLMs\n\n# Allow AI crawlers to access markdown versions of pages\nUser-agent: GPTBot\nAllow: /\n\nUser-agent: ChatGPT-User\nAllow: /\n\nUser-agent: Google-Extended\nAllow: /\n\nUser-agent: Anthropic-AI\nAllow: /\n\nUser-agent: Claude-Web\nAllow: /\n\nUser-agent: CCBot\nAllow: /\n\nUser-agent: PerplexityBot\nAllow: /\n\nUser-agent: Cohere-ai\nAllow: /\n\n# Content Signals — declare AI content usage preferences\n# See https://contentsignals.org/ and https://datatracker.ietf.org/doc/draft-romm-aipref-contentsignals/\nContent-Signal: ai-train=yes, search=yes, ai-input=yes\n\n# AI-friendly content locations\n# - /llms.txt - Curated overview for AI/LLMs (markdown)\n# - /llms-full.txt - Full expanded content for larger context windows\n# - /*.md - Markdown versions of all pages (append .md to any URL)\n# - /.well-known/agents.json - Agent discovery and capabilities"
    },
    "llms": {
      "url": "https://cloudflare.com/llms.txt",
      "httpStatus": 200,
      "present": true,
      "contentType": "text/plain",
      "error": null,
      "snippet": "# Cloudflare\n\n> Cloudflare is a global cloud connectivity company operating in 335+ cities worldwide. It provides a unified platform spanning developer infrastructure (serverless compute, databases, AI inference, and storage), application security (WAF, DDoS mitigation, bot management, and API Shield), Zero Trust / SASE (Access, Gateway, Browser Isolation, DLP, and Email Security), and network ser\n…(truncated)"
    },
    "agentsJson": {
      "url": "https://cloudflare.com/.well-known/agents.json",
      "httpStatus": 200,
      "present": true,
      "error": null
    },
    "aiCatalog": {
      "url": "https://cloudflare.com/.well-known/ai-catalog.json",
      "httpStatus": 404,
      "present": false,
      "error": null
    },
    "botaccess": {
      "url": "https://cloudflare.com/.well-known/botaccess.json",
      "httpStatus": 404,
      "present": false,
      "error": null
    }
  },
  "relevantRobotsGroups": {
    "*": [
      "Allow: /",
      "Sitemap: https://www.cloudflare.com/sitemap.xml"
    ],
    "gptbot": [
      "Allow: /"
    ],
    "chatgpt-user": [
      "Allow: /"
    ],
    "google-extended": [
      "Allow: /"
    ],
    "ccbot": [
      "Allow: /"
    ],
    "perplexitybot": [
      "Allow: /"
    ]
  },
  "bots": {
    "GPTBot": {
      "vendor": "OpenAI",
      "kind": "training-crawler",
      "status": "allowed",
      "explicit": true,
      "effective": "allowed",
      "disallow": [],
      "allow": [
        "/"
      ]
    },
    "OAI-SearchBot": {
      "vendor": "OpenAI",
      "kind": "search-index",
      "status": "unspecified",
      "explicit": false,
      "effective": "allowed",
      "disallow": [],
      "allow": [
        "/"
      ],
      "note": "inherits User-agent: *"
    },
    "ChatGPT-User": {
      "vendor": "OpenAI",
      "kind": "user-action",
      "status": "allowed",
      "explicit": true,
      "effective": "allowed",
      "disallow": [],
      "allow": [
        "/"
      ]
    },
    "ClaudeBot": {
      "vendor": "Anthropic",
      "kind": "training-crawler",
      "status": "unspecified",
      "explicit": false,
      "effective": "allowed",
      "disallow": [],
      "allow": [
        "/"
      ],
      "note": "inherits User-agent: *"
    },
    "Claude-User": {
      "vendor": "Anthropic",
      "kind": "user-action",
      "status": "unspecified",
      "explicit": false,
      "effective": "allowed",
      "disallow": [],
      "allow": [
        "/"
      ],
      "note": "inherits User-agent: *"
    },
    "Claude-SearchBot": {
      "vendor": "Anthropic",
      "kind": "search-index",
      "status": "unspecified",
      "explicit": false,
      "effective": "allowed",
      "disallow": [],
      "allow": [
        "/"
      ],
      "note": "inherits User-agent: *"
    },
    "Google-Extended": {
      "vendor": "Google AI",
      "kind": "training-crawler",
      "status": "allowed",
      "explicit": true,
      "effective": "allowed",
      "disallow": [],
      "allow": [
        "/"
      ]
    },
    "PerplexityBot": {
      "vendor": "Perplexity",
      "kind": "search-index",
      "status": "allowed",
      "explicit": true,
      "effective": "allowed",
      "disallow": [],
      "allow": [
        "/"
      ]
    },
    "Perplexity-User": {
      "vendor": "Perplexity",
      "kind": "user-action",
      "status": "unspecified",
      "explicit": false,
      "effective": "allowed",
      "disallow": [],
      "allow": [
        "/"
      ],
      "note": "inherits User-agent: *"
    },
    "Bytespider": {
      "vendor": "ByteDance",
      "kind": "training-crawler",
      "status": "unspecified",
      "explicit": false,
      "effective": "allowed",
      "disallow": [],
      "allow": [
        "/"
      ],
      "note": "inherits User-agent: *"
    },
    "CCBot": {
      "vendor": "Common Crawl",
      "kind": "training-crawler",
      "status": "allowed",
      "explicit": true,
      "effective": "allowed",
      "disallow": [],
      "allow": [
        "/"
      ]
    },
    "Meta-ExternalAgent": {
      "vendor": "Meta",
      "kind": "training-crawler",
      "status": "unspecified",
      "explicit": false,
      "effective": "allowed",
      "disallow": [],
      "allow": [
        "/"
      ],
      "note": "inherits User-agent: *"
    }
  }
}