{
  "domain": "nytimes.com",
  "scannedAt": "2026-10-22T11:28:35.153Z",
  "unreachable": false,
  "files": {
    "robots": {
      "url": "https://nytimes.com/robots.txt",
      "httpStatus": 200,
      "present": true,
      "contentType": "text/plain; charset=UTF-8",
      "error": null,
      "snippet": "# New York Times content is made available for your personal, non-commercial\n# use subject to our Terms of Service here:\n# https://help.nytimes.com/hc/en-us/articles/115014893428-Terms-of-Service.\n# Use of any device, tool, or process designed to data mine or scrape the content\n# using automated means is prohibited without prior written permission from\n# The New York Times Company.  Prohibited uses include but are not limited to:\n# (1) text and data mining activities under Art. 4 of the EU Directive on Copyright in\n# the Digital Single Market;\n# (2) the development of any software, machine learning, artificial intelligence (AI),\n# and/or large language models (LLMs);\n# (3) creating or providing archived or cached data sets containing our content to others; and/or\n# (4) any commercial purposes.\n# Contact https://nytlicensing.com/contact/ for assistance.\n\nUser-agent: *\nUser-agent: Googlebot\nDisallow: /ads/\nDisallow: /adx/bin/\nDisallow: /athletic/wp/wp-admin/\nAllow: /athletic/wp/wp-admin/admin-ajax.php\nDisallow: /athletic/async-*\nDisallow: /athletic/search/*\nAllow: /athletic/search/$\nDisallow: /athletic/checkout/\nDisallow: /athletic/checkout?plan_id*\nAllow: /athletic/checkout/$\nDisallow: /athletic/checkout2*\nDisallow: /athletic/login/\nDisallow: /athletic/login?login_source*\nDisallow: /athletic/login?ref_page*\nAllow: /athletic/login/$\nDisallow: /athletic/login2/\nDisallow: /athletic/login2?login_source*\nDisallow: /athletic/login2?ref_page*\nAllow: /athletic/login2/$\nDisallow: /athl\n…(truncated)"
    },
    "llms": {
      "url": "https://nytimes.com/llms.txt",
      "httpStatus": 404,
      "present": false,
      "contentType": "text/html; charset=UTF-8",
      "error": null,
      "snippet": ""
    },
    "agentsJson": {
      "url": "https://nytimes.com/.well-known/agents.json",
      "httpStatus": 404,
      "present": false,
      "error": null
    },
    "aiCatalog": {
      "url": "https://nytimes.com/.well-known/ai-catalog.json",
      "httpStatus": 404,
      "present": false,
      "error": null
    },
    "botaccess": {
      "url": "https://nytimes.com/.well-known/botaccess.json",
      "httpStatus": 404,
      "present": false,
      "error": null
    }
  },
  "relevantRobotsGroups": {
    "*": [
      "Disallow: /ads/",
      "Disallow: /adx/bin/",
      "Disallow: /athletic/wp/wp-admin/",
      "Allow: /athletic/wp/wp-admin/admin-ajax.php",
      "Disallow: /athletic/async-*",
      "Disallow: /athletic/search/*",
      "Allow: /athletic/search/$",
      "Disallow: /athletic/checkout/",
      "Disallow: /athletic/checkout?plan_id*",
      "Allow: /athletic/checkout/$",
      "Disallow: /athletic/checkout2*",
      "Disallow: /athletic/login/",
      "Disallow: /athletic/login?login_source*",
      "Disallow: /athletic/login?ref_page*",
      "Allow: /athletic/login/$",
      "Disallow: /athletic/login2/",
      "Disallow: /athletic/login2?login_source*",
      "Disallow: /athletic/login2?ref_page*",
      "Allow: /athletic/login2/$",
      "Disallow: /athletic/report/",
      "Disallow: /athletic/*/discuss/*",
      "Allow: /athletic/live-blogs/discuss/",
      "Allow: /athletic/nfl/game/discuss/",
      "Disallow: /athletic/register/",
      "Disallow: /athletic/register?welcome_redirect*",
      "Disallow: /athletic/register2/",
      "Disallow: /athletic/register2?welcome_redirect*",
      "Disallow: /athletic/betmgm-redirect*",
      "Disallow: /athletic/cdn-cgi/",
      "Disallow: /athletic/verizon/*",
      "Disallow: /athletic/forgot-password/*",
      "Disallow: /athletic/forgot-password2/*",
      "Disallow: /athletic/amp-social-login*",
      "Disallow: /athletic/track-analytics/",
      "Disallow: /athletic/amp-auth/",
      "Disallow: /athletic/rss-feed/",
      "Disallow: /athletic/*?*rss=1",
      "Disallow: /athletic/global-color-test.php",
      "Disallow: /athletic/global-font-test.php",
      "Disallow: /athletic/graphql*"
    ],
    "bytespider": [
      "Disallow: /"
    ],
    "ccbot": [
      "Disallow: /"
    ],
    "chatgpt-user": [
      "Disallow: /"
    ],
    "claudebot": [
      "Disallow: /"
    ],
    "claude-searchbot": [
      "Disallow: /"
    ],
    "claude-user": [
      "Disallow: /"
    ],
    "google-extended": [
      "Disallow: /"
    ],
    "gptbot": [
      "Disallow: /"
    ],
    "meta-externalagent": [
      "Disallow: /",
      "Disallow: /"
    ],
    "oai-searchbot": [
      "Disallow: /"
    ],
    "perplexitybot": [
      "Disallow: /"
    ],
    "perplexity-user": [
      "Disallow: /"
    ]
  },
  "bots": {
    "GPTBot": {
      "vendor": "OpenAI",
      "kind": "training-crawler",
      "status": "blocked",
      "explicit": true,
      "effective": "blocked",
      "disallow": [
        "/"
      ],
      "allow": []
    },
    "OAI-SearchBot": {
      "vendor": "OpenAI",
      "kind": "search-index",
      "status": "blocked",
      "explicit": true,
      "effective": "blocked",
      "disallow": [
        "/"
      ],
      "allow": []
    },
    "ChatGPT-User": {
      "vendor": "OpenAI",
      "kind": "user-action",
      "status": "blocked",
      "explicit": true,
      "effective": "blocked",
      "disallow": [
        "/"
      ],
      "allow": []
    },
    "ClaudeBot": {
      "vendor": "Anthropic",
      "kind": "training-crawler",
      "status": "blocked",
      "explicit": true,
      "effective": "blocked",
      "disallow": [
        "/"
      ],
      "allow": []
    },
    "Claude-User": {
      "vendor": "Anthropic",
      "kind": "user-action",
      "status": "blocked",
      "explicit": true,
      "effective": "blocked",
      "disallow": [
        "/"
      ],
      "allow": []
    },
    "Claude-SearchBot": {
      "vendor": "Anthropic",
      "kind": "search-index",
      "status": "blocked",
      "explicit": true,
      "effective": "blocked",
      "disallow": [
        "/"
      ],
      "allow": []
    },
    "Google-Extended": {
      "vendor": "Google AI",
      "kind": "training-crawler",
      "status": "blocked",
      "explicit": true,
      "effective": "blocked",
      "disallow": [
        "/"
      ],
      "allow": []
    },
    "PerplexityBot": {
      "vendor": "Perplexity",
      "kind": "search-index",
      "status": "blocked",
      "explicit": true,
      "effective": "blocked",
      "disallow": [
        "/"
      ],
      "allow": []
    },
    "Perplexity-User": {
      "vendor": "Perplexity",
      "kind": "user-action",
      "status": "blocked",
      "explicit": true,
      "effective": "blocked",
      "disallow": [
        "/"
      ],
      "allow": []
    },
    "Bytespider": {
      "vendor": "ByteDance",
      "kind": "training-crawler",
      "status": "blocked",
      "explicit": true,
      "effective": "blocked",
      "disallow": [
        "/"
      ],
      "allow": []
    },
    "CCBot": {
      "vendor": "Common Crawl",
      "kind": "training-crawler",
      "status": "blocked",
      "explicit": true,
      "effective": "blocked",
      "disallow": [
        "/"
      ],
      "allow": []
    },
    "Meta-ExternalAgent": {
      "vendor": "Meta",
      "kind": "training-crawler",
      "status": "blocked",
      "explicit": true,
      "effective": "blocked",
      "disallow": [
        "/",
        "/"
      ],
      "allow": []
    }
  }
}