{
  "domain": "theguardian.com",
  "scannedAt": "2026-10-22T11:28:35.349Z",
  "unreachable": false,
  "files": {
    "robots": {
      "url": "https://theguardian.com/robots.txt",
      "httpStatus": 200,
      "present": true,
      "contentType": "text/plain; charset=utf-8",
      "error": null,
      "snippet": "# This is the robots.txt file for theguardian.com\n\n# Guardian content is made available under our terms and conditions of use.\n# Any other uses are not permitted, incl. but not limited to: for large language\n# models (LLMs), machine learning and/or artificial intelligence-related\n# purposes; with any of the aforementioned technologies; and/or for any\n# commercial purposes. Contact licensing@theguardian.com for assistance\n\nUser-agent: *\nDisallow: /sendarticle/\nDisallow: /Users/\nDisallow: /users/\nDisallow: /*/print$\nDisallow: /email/\nDisallow: /contactus/\nDisallow: /share/\nDisallow: /websearch\nDisallow: /*?commentpage=\nDisallow: /whsmiths/\nDisallow: /external/overture/\nDisallow: /discussion/report-abuse/*\nDisallow: /discussion/report-abuse-ajax/*\nDisallow: /discussion/comment-permalink/*\nDisallow: /discussion/report-abuse/*\nDisallow: /discussion/user-report-abuse/*\nDisallow: /discussion/handlers/*\nDisallow: /discussion/your-profile\nDisallow: /discussion/your-comments\nDisallow: /discussion/edit-profile\nDisallow: /discussion/search/comments\nDisallow: /discussion/*\nDisallow: /search\nDisallow: /music/artist/*\nDisallow: /music/album/*\nDisallow: /books/data/*\nDisallow: /settings/\nDisallow: /embed/\nDisallow: /*styles/js-on.css$\nDisallow: /sport/olympics/2008/events/*\nDisallow: /sport/olympics/2008/medals/*\nDisallow: /f/healthcheck\nDisallow: /sections\nDisallow: /top-stories\nDisallow: /most-read/sport\nDisallow: /articles\nDisallow: /global$\nDisallow: /*/feedarticle/*\nDisallow: /travel/20\n…(truncated)"
    },
    "llms": {
      "url": "https://theguardian.com/llms.txt",
      "httpStatus": 404,
      "present": false,
      "contentType": "text/html; charset=UTF-8",
      "error": null,
      "snippet": ""
    },
    "agentsJson": {
      "url": "https://theguardian.com/.well-known/agents.json",
      "httpStatus": 404,
      "present": false,
      "error": null
    },
    "aiCatalog": {
      "url": "https://theguardian.com/.well-known/ai-catalog.json",
      "httpStatus": 404,
      "present": false,
      "error": null
    },
    "botaccess": {
      "url": "https://theguardian.com/.well-known/botaccess.json",
      "httpStatus": 404,
      "present": false,
      "error": null
    }
  },
  "relevantRobotsGroups": {
    "*": [
      "Disallow: /sendarticle/",
      "Disallow: /Users/",
      "Disallow: /users/",
      "Disallow: /*/print$",
      "Disallow: /email/",
      "Disallow: /contactus/",
      "Disallow: /share/",
      "Disallow: /websearch",
      "Disallow: /*?commentpage=",
      "Disallow: /whsmiths/",
      "Disallow: /external/overture/",
      "Disallow: /discussion/report-abuse/*",
      "Disallow: /discussion/report-abuse-ajax/*",
      "Disallow: /discussion/comment-permalink/*",
      "Disallow: /discussion/report-abuse/*",
      "Disallow: /discussion/user-report-abuse/*",
      "Disallow: /discussion/handlers/*",
      "Disallow: /discussion/your-profile",
      "Disallow: /discussion/your-comments",
      "Disallow: /discussion/edit-profile",
      "Disallow: /discussion/search/comments",
      "Disallow: /discussion/*",
      "Disallow: /search",
      "Disallow: /music/artist/*",
      "Disallow: /music/album/*",
      "Disallow: /books/data/*",
      "Disallow: /settings/",
      "Disallow: /embed/",
      "Disallow: /*styles/js-on.css$",
      "Disallow: /sport/olympics/2008/events/*",
      "Disallow: /sport/olympics/2008/medals/*",
      "Disallow: /f/healthcheck",
      "Disallow: /sections",
      "Disallow: /top-stories",
      "Disallow: /most-read/sport",
      "Disallow: /articles",
      "Disallow: /global$",
      "Disallow: /*/feedarticle/*",
      "Disallow: /travel/2013/aug/22/been-there-readers-competition?*",
      "Disallow: /preference/*"
    ],
    "ccbot": [
      "Disallow: /",
      "License: https://theguardian.com/license.xml"
    ],
    "bytespider": [
      "Disallow: /",
      "License: https://theguardian.com/license.xml"
    ],
    "perplexitybot": [
      "Disallow: /",
      "License: https://theguardian.com/license.xml"
    ],
    "claudebot": [
      "Disallow: /",
      "License: https://theguardian.com/license.xml"
    ],
    "claude-searchbot": [
      "Disallow: /",
      "License: https://theguardian.com/license.xml"
    ],
    "claude-user": [
      "Disallow: /",
      "License: https://theguardian.com/license.xml"
    ],
    "meta-externalagent": [
      "Disallow: /",
      "License: https://theguardian.com/license.xml"
    ]
  },
  "bots": {
    "GPTBot": {
      "vendor": "OpenAI",
      "kind": "training-crawler",
      "status": "unspecified",
      "explicit": false,
      "effective": "partial",
      "disallow": [
        "/sendarticle/",
        "/Users/",
        "/users/",
        "/*/print$",
        "/email/",
        "/contactus/",
        "/share/",
        "/websearch",
        "/*?commentpage=",
        "/whsmiths/",
        "/external/overture/",
        "/discussion/report-abuse/*",
        "/discussion/report-abuse-ajax/*",
        "/discussion/comment-permalink/*",
        "/discussion/report-abuse/*",
        "/discussion/user-report-abuse/*",
        "/discussion/handlers/*",
        "/discussion/your-profile",
        "/discussion/your-comments",
        "/discussion/edit-profile",
        "/discussion/search/comments",
        "/discussion/*",
        "/search",
        "/music/artist/*",
        "/music/album/*",
        "/books/data/*",
        "/settings/",
        "/embed/",
        "/*styles/js-on.css$",
        "/sport/olympics/2008/events/*",
        "/sport/olympics/2008/medals/*",
        "/f/healthcheck",
        "/sections",
        "/top-stories",
        "/most-read/sport",
        "/articles",
        "/global$",
        "/*/feedarticle/*",
        "/travel/2013/aug/22/been-there-readers-competition?*",
        "/preference/*",
        "/59666047/",
        "/print/",
        "/info/tech-feedback",
        "/production-monitoring/",
        "*.emailjson",
        "*.emailtxt",
        "/headline.txt",
        "*?*dcr=apps*"
      ],
      "allow": [],
      "note": "inherits User-agent: *"
    },
    "OAI-SearchBot": {
      "vendor": "OpenAI",
      "kind": "search-index",
      "status": "unspecified",
      "explicit": false,
      "effective": "partial",
      "disallow": [
        "/sendarticle/",
        "/Users/",
        "/users/",
        "/*/print$",
        "/email/",
        "/contactus/",
        "/share/",
        "/websearch",
        "/*?commentpage=",
        "/whsmiths/",
        "/external/overture/",
        "/discussion/report-abuse/*",
        "/discussion/report-abuse-ajax/*",
        "/discussion/comment-permalink/*",
        "/discussion/report-abuse/*",
        "/discussion/user-report-abuse/*",
        "/discussion/handlers/*",
        "/discussion/your-profile",
        "/discussion/your-comments",
        "/discussion/edit-profile",
        "/discussion/search/comments",
        "/discussion/*",
        "/search",
        "/music/artist/*",
        "/music/album/*",
        "/books/data/*",
        "/settings/",
        "/embed/",
        "/*styles/js-on.css$",
        "/sport/olympics/2008/events/*",
        "/sport/olympics/2008/medals/*",
        "/f/healthcheck",
        "/sections",
        "/top-stories",
        "/most-read/sport",
        "/articles",
        "/global$",
        "/*/feedarticle/*",
        "/travel/2013/aug/22/been-there-readers-competition?*",
        "/preference/*",
        "/59666047/",
        "/print/",
        "/info/tech-feedback",
        "/production-monitoring/",
        "*.emailjson",
        "*.emailtxt",
        "/headline.txt",
        "*?*dcr=apps*"
      ],
      "allow": [],
      "note": "inherits User-agent: *"
    },
    "ChatGPT-User": {
      "vendor": "OpenAI",
      "kind": "user-action",
      "status": "unspecified",
      "explicit": false,
      "effective": "partial",
      "disallow": [
        "/sendarticle/",
        "/Users/",
        "/users/",
        "/*/print$",
        "/email/",
        "/contactus/",
        "/share/",
        "/websearch",
        "/*?commentpage=",
        "/whsmiths/",
        "/external/overture/",
        "/discussion/report-abuse/*",
        "/discussion/report-abuse-ajax/*",
        "/discussion/comment-permalink/*",
        "/discussion/report-abuse/*",
        "/discussion/user-report-abuse/*",
        "/discussion/handlers/*",
        "/discussion/your-profile",
        "/discussion/your-comments",
        "/discussion/edit-profile",
        "/discussion/search/comments",
        "/discussion/*",
        "/search",
        "/music/artist/*",
        "/music/album/*",
        "/books/data/*",
        "/settings/",
        "/embed/",
        "/*styles/js-on.css$",
        "/sport/olympics/2008/events/*",
        "/sport/olympics/2008/medals/*",
        "/f/healthcheck",
        "/sections",
        "/top-stories",
        "/most-read/sport",
        "/articles",
        "/global$",
        "/*/feedarticle/*",
        "/travel/2013/aug/22/been-there-readers-competition?*",
        "/preference/*",
        "/59666047/",
        "/print/",
        "/info/tech-feedback",
        "/production-monitoring/",
        "*.emailjson",
        "*.emailtxt",
        "/headline.txt",
        "*?*dcr=apps*"
      ],
      "allow": [],
      "note": "inherits User-agent: *"
    },
    "ClaudeBot": {
      "vendor": "Anthropic",
      "kind": "training-crawler",
      "status": "blocked",
      "explicit": true,
      "effective": "blocked",
      "disallow": [
        "/"
      ],
      "allow": []
    },
    "Claude-User": {
      "vendor": "Anthropic",
      "kind": "user-action",
      "status": "blocked",
      "explicit": true,
      "effective": "blocked",
      "disallow": [
        "/"
      ],
      "allow": []
    },
    "Claude-SearchBot": {
      "vendor": "Anthropic",
      "kind": "search-index",
      "status": "blocked",
      "explicit": true,
      "effective": "blocked",
      "disallow": [
        "/"
      ],
      "allow": []
    },
    "Google-Extended": {
      "vendor": "Google AI",
      "kind": "training-crawler",
      "status": "unspecified",
      "explicit": false,
      "effective": "partial",
      "disallow": [
        "/sendarticle/",
        "/Users/",
        "/users/",
        "/*/print$",
        "/email/",
        "/contactus/",
        "/share/",
        "/websearch",
        "/*?commentpage=",
        "/whsmiths/",
        "/external/overture/",
        "/discussion/report-abuse/*",
        "/discussion/report-abuse-ajax/*",
        "/discussion/comment-permalink/*",
        "/discussion/report-abuse/*",
        "/discussion/user-report-abuse/*",
        "/discussion/handlers/*",
        "/discussion/your-profile",
        "/discussion/your-comments",
        "/discussion/edit-profile",
        "/discussion/search/comments",
        "/discussion/*",
        "/search",
        "/music/artist/*",
        "/music/album/*",
        "/books/data/*",
        "/settings/",
        "/embed/",
        "/*styles/js-on.css$",
        "/sport/olympics/2008/events/*",
        "/sport/olympics/2008/medals/*",
        "/f/healthcheck",
        "/sections",
        "/top-stories",
        "/most-read/sport",
        "/articles",
        "/global$",
        "/*/feedarticle/*",
        "/travel/2013/aug/22/been-there-readers-competition?*",
        "/preference/*",
        "/59666047/",
        "/print/",
        "/info/tech-feedback",
        "/production-monitoring/",
        "*.emailjson",
        "*.emailtxt",
        "/headline.txt",
        "*?*dcr=apps*"
      ],
      "allow": [],
      "note": "inherits User-agent: *"
    },
    "PerplexityBot": {
      "vendor": "Perplexity",
      "kind": "search-index",
      "status": "blocked",
      "explicit": true,
      "effective": "blocked",
      "disallow": [
        "/"
      ],
      "allow": []
    },
    "Perplexity-User": {
      "vendor": "Perplexity",
      "kind": "user-action",
      "status": "unspecified",
      "explicit": false,
      "effective": "partial",
      "disallow": [
        "/sendarticle/",
        "/Users/",
        "/users/",
        "/*/print$",
        "/email/",
        "/contactus/",
        "/share/",
        "/websearch",
        "/*?commentpage=",
        "/whsmiths/",
        "/external/overture/",
        "/discussion/report-abuse/*",
        "/discussion/report-abuse-ajax/*",
        "/discussion/comment-permalink/*",
        "/discussion/report-abuse/*",
        "/discussion/user-report-abuse/*",
        "/discussion/handlers/*",
        "/discussion/your-profile",
        "/discussion/your-comments",
        "/discussion/edit-profile",
        "/discussion/search/comments",
        "/discussion/*",
        "/search",
        "/music/artist/*",
        "/music/album/*",
        "/books/data/*",
        "/settings/",
        "/embed/",
        "/*styles/js-on.css$",
        "/sport/olympics/2008/events/*",
        "/sport/olympics/2008/medals/*",
        "/f/healthcheck",
        "/sections",
        "/top-stories",
        "/most-read/sport",
        "/articles",
        "/global$",
        "/*/feedarticle/*",
        "/travel/2013/aug/22/been-there-readers-competition?*",
        "/preference/*",
        "/59666047/",
        "/print/",
        "/info/tech-feedback",
        "/production-monitoring/",
        "*.emailjson",
        "*.emailtxt",
        "/headline.txt",
        "*?*dcr=apps*"
      ],
      "allow": [],
      "note": "inherits User-agent: *"
    },
    "Bytespider": {
      "vendor": "ByteDance",
      "kind": "training-crawler",
      "status": "blocked",
      "explicit": true,
      "effective": "blocked",
      "disallow": [
        "/"
      ],
      "allow": []
    },
    "CCBot": {
      "vendor": "Common Crawl",
      "kind": "training-crawler",
      "status": "blocked",
      "explicit": true,
      "effective": "blocked",
      "disallow": [
        "/"
      ],
      "allow": []
    },
    "Meta-ExternalAgent": {
      "vendor": "Meta",
      "kind": "training-crawler",
      "status": "blocked",
      "explicit": true,
      "effective": "blocked",
      "disallow": [
        "/"
      ],
      "allow": []
    }
  }
}