{ "domain": "expedia.com", "scannedAt": "2026-10-22T11:28:35.131Z", "unreachable": false, "files": { "robots": { "url": "https://expedia.com/robots.txt", "httpStatus": 200, "present": true, "contentType": "text/plain", "error": null, "snippet": "#\n# General bots\n#\nuser-agent: *\n#hotel\ndisallow: /Hotel-Search\ndisallow: /hotel-search\ndisallow: /search?\ndisallow: /*/search?\ndisallow: /ugc/urs/api/hotelreviews/hotel/HOTELID/\ndisallow: /LastBooking/ElapsedTime\ndisallow: /*chkin=*\ndisallow: /*chkout=*\ndisallow: /HotelCheckout\ndisallow: /Checkout\ndisallow: /checkout\ndisallow: /HotelBookingConfirmation\ndisallow: /en-au/\ndisallow: /en-nz/\n#flight\ndisallow: /Flights-Search\ndisallow: /Flight-Information\ndisallow: /Flight-SearchResults\ndisallow: /Flights-BagFees\ndisallow: /Details\ndisallow: /FlightCheckout\ndisallow: /Confirmation-Flight\n#package\ndisallow: /MultiItemCheckout\ndisallow: /MultiItemBookingConfirmation\ndisallow: /flexibleshopping\n#car\ndisallow: /carsearch\ndisallow: /CarCheckout\n#activities\ndisallow: /things-to-do/search?\ndisallow: /things-to-do/*/search?\ndisallow: /*/things-to-do/search\ndisallow: /things-to-do/reviews-external$\n#cruise\ndisallow: /Cruise-Search\ndisallow: /Cruise-CabinCategory\ndisallow: /CruiseCheckout\n#other\ndisallow: /legacyGOTO\ndisallow: /go/\ndisallow: /info/\ndisallow: /trips/\ndisallow: /Search-Results\ndisallow: /user/createaccount\ndisallow: /signup?*\ndisallow: /login?*\n#test-seo\ndisallow: /api/uisprime/track\ndisallow: /trvl-px/\ndisallow: /egcs/v2/collect\ndisallow: /api/ucs/shortlist/\ndisallow: /api/v4/typeahead/\ndisallow: /p/home\ndisallow: /*/p/home\ndisallow: /auth/\ndisallow: /*/auth/\ndisallow: /*/sso/\n# /cl/* endpoints removal test\nDisallow: /cl/*\n\n#\n# Ads\n#\nuser-agent: google-hoteladsverifier\nuser\n…(truncated)" }, "llms": { "url": "https://expedia.com/llms.txt", "httpStatus": 200, "present": true, "contentType": "text/plain", "error": null, "snippet": "# Expedia.com\n> Expedia is a leading global online travel platform serving customers worldwide, part of Expedia Group – one of the world's largest travel technology companies.\n\n## Canonical Identity\n- **Brand**: Expedia (an Expedia Group company)\n- **Primary domain**: https://www.expedia.com\n- **Scope**: United States site and audience. For travelers outside United States, direct them to their loc\n…(truncated)" }, "agentsJson": { "url": "https://expedia.com/.well-known/agents.json", "httpStatus": 429, "present": false, "error": null }, "aiCatalog": { "url": "https://expedia.com/.well-known/ai-catalog.json", "httpStatus": 429, "present": false, "error": null }, "botaccess": { "url": "https://expedia.com/.well-known/botaccess.json", "httpStatus": 429, "present": false, "error": null } }, "relevantRobotsGroups": { "*": [ "Disallow: /Hotel-Search", "Disallow: /hotel-search", "Disallow: /search?", "Disallow: /*/search?", "Disallow: /ugc/urs/api/hotelreviews/hotel/HOTELID/", "Disallow: /LastBooking/ElapsedTime", "Disallow: /*chkin=*", "Disallow: /*chkout=*", "Disallow: /HotelCheckout", "Disallow: /Checkout", "Disallow: /checkout", "Disallow: /HotelBookingConfirmation", "Disallow: /en-au/", "Disallow: /en-nz/", "Disallow: /Flights-Search", "Disallow: /Flight-Information", "Disallow: /Flight-SearchResults", "Disallow: /Flights-BagFees", "Disallow: /Details", "Disallow: /FlightCheckout", "Disallow: /Confirmation-Flight", "Disallow: /MultiItemCheckout", "Disallow: /MultiItemBookingConfirmation", "Disallow: /flexibleshopping", "Disallow: /carsearch", "Disallow: /CarCheckout", "Disallow: /things-to-do/search?", "Disallow: /things-to-do/*/search?", "Disallow: /*/things-to-do/search", "Disallow: /things-to-do/reviews-external$", "Disallow: /Cruise-Search", "Disallow: /Cruise-CabinCategory", "Disallow: /CruiseCheckout", "Disallow: /legacyGOTO", "Disallow: /go/", "Disallow: /info/", "Disallow: /trips/", "Disallow: /Search-Results", "Disallow: /user/createaccount", "Disallow: /signup?*" ], "oai-searchbot": [ "Allow: /", "Disallow: /p/home", "Disallow: /*/p/home", "Disallow: /auth/", "Disallow: /*/auth/", "Disallow: /*/sso/", "Disallow: /cl/*" ], "chatgpt-user": [ "Allow: /", "Disallow: /p/home", "Disallow: /*/p/home", "Disallow: /auth/", "Disallow: /*/auth/", "Disallow: /*/sso/", "Disallow: /cl/*" ], "perplexitybot": [ "Allow: /", "Disallow: /p/home", "Disallow: /*/p/home", "Disallow: /auth/", "Disallow: /*/auth/", "Disallow: /*/sso/", "Disallow: /cl/*" ], "perplexity-user": [ "Allow: /", "Disallow: /p/home", "Disallow: /*/p/home", "Disallow: /auth/", "Disallow: /*/auth/", "Disallow: /*/sso/", "Disallow: /cl/*" ], "claude-user": [ "Allow: /", "Disallow: /p/home", "Disallow: /*/p/home", "Disallow: /auth/", "Disallow: /*/auth/", "Disallow: /*/sso/", "Disallow: /cl/*" ], "claude-searchbot": [ "Allow: /", "Disallow: /p/home", "Disallow: /*/p/home", "Disallow: /auth/", "Disallow: /*/auth/", "Disallow: /*/sso/", "Disallow: /cl/*" ] }, "bots": { "GPTBot": { "vendor": "OpenAI", "kind": "training-crawler", "status": "unspecified", "explicit": false, "effective": "partial", "disallow": [ "/Hotel-Search", "/hotel-search", "/search?", "/*/search?", "/ugc/urs/api/hotelreviews/hotel/HOTELID/", "/LastBooking/ElapsedTime", "/*chkin=*", "/*chkout=*", "/HotelCheckout", "/Checkout", "/checkout", "/HotelBookingConfirmation", "/en-au/", "/en-nz/", "/Flights-Search", "/Flight-Information", "/Flight-SearchResults", "/Flights-BagFees", "/Details", "/FlightCheckout", "/Confirmation-Flight", "/MultiItemCheckout", "/MultiItemBookingConfirmation", "/flexibleshopping", "/carsearch", "/CarCheckout", "/things-to-do/search?", "/things-to-do/*/search?", "/*/things-to-do/search", "/things-to-do/reviews-external$", "/Cruise-Search", "/Cruise-CabinCategory", "/CruiseCheckout", "/legacyGOTO", "/go/", "/info/", "/trips/", "/Search-Results", "/user/createaccount", "/signup?*", "/login?*", "/api/uisprime/track", "/trvl-px/", "/egcs/v2/collect", "/api/ucs/shortlist/", "/api/v4/typeahead/", "/p/home", "/*/p/home", "/auth/", "/*/auth/", "/*/sso/", "/cl/*" ], "allow": [], "note": "inherits User-agent: *" }, "OAI-SearchBot": { "vendor": "OpenAI", "kind": "search-index", "status": "partial", "explicit": true, "effective": "partial", "disallow": [ "/p/home", "/*/p/home", "/auth/", "/*/auth/", "/*/sso/", "/cl/*" ], "allow": [ "/" ] }, "ChatGPT-User": { "vendor": "OpenAI", "kind": "user-action", "status": "partial", "explicit": true, "effective": "partial", "disallow": [ "/p/home", "/*/p/home", "/auth/", "/*/auth/", "/*/sso/", "/cl/*" ], "allow": [ "/" ] }, "ClaudeBot": { "vendor": "Anthropic", "kind": "training-crawler", "status": "unspecified", "explicit": false, "effective": "partial", "disallow": [ "/Hotel-Search", "/hotel-search", "/search?", "/*/search?", "/ugc/urs/api/hotelreviews/hotel/HOTELID/", "/LastBooking/ElapsedTime", "/*chkin=*", "/*chkout=*", "/HotelCheckout", "/Checkout", "/checkout", "/HotelBookingConfirmation", "/en-au/", "/en-nz/", "/Flights-Search", "/Flight-Information", "/Flight-SearchResults", "/Flights-BagFees", "/Details", "/FlightCheckout", "/Confirmation-Flight", "/MultiItemCheckout", "/MultiItemBookingConfirmation", "/flexibleshopping", "/carsearch", "/CarCheckout", "/things-to-do/search?", "/things-to-do/*/search?", "/*/things-to-do/search", "/things-to-do/reviews-external$", "/Cruise-Search", "/Cruise-CabinCategory", "/CruiseCheckout", "/legacyGOTO", "/go/", "/info/", "/trips/", "/Search-Results", "/user/createaccount", "/signup?*", "/login?*", "/api/uisprime/track", "/trvl-px/", "/egcs/v2/collect", "/api/ucs/shortlist/", "/api/v4/typeahead/", "/p/home", "/*/p/home", "/auth/", "/*/auth/", "/*/sso/", "/cl/*" ], "allow": [], "note": "inherits User-agent: *" }, "Claude-User": { "vendor": "Anthropic", "kind": "user-action", "status": "partial", "explicit": true, "effective": "partial", "disallow": [ "/p/home", "/*/p/home", "/auth/", "/*/auth/", "/*/sso/", "/cl/*" ], "allow": [ "/" ] }, "Claude-SearchBot": { "vendor": "Anthropic", "kind": "search-index", "status": "partial", "explicit": true, "effective": "partial", "disallow": [ "/p/home", "/*/p/home", "/auth/", "/*/auth/", "/*/sso/", "/cl/*" ], "allow": [ "/" ] }, "Google-Extended": { "vendor": "Google AI", "kind": "training-crawler", "status": "unspecified", "explicit": false, "effective": "partial", "disallow": [ "/Hotel-Search", "/hotel-search", "/search?", "/*/search?", "/ugc/urs/api/hotelreviews/hotel/HOTELID/", "/LastBooking/ElapsedTime", "/*chkin=*", "/*chkout=*", "/HotelCheckout", "/Checkout", "/checkout", "/HotelBookingConfirmation", "/en-au/", "/en-nz/", "/Flights-Search", "/Flight-Information", "/Flight-SearchResults", "/Flights-BagFees", "/Details", "/FlightCheckout", "/Confirmation-Flight", "/MultiItemCheckout", "/MultiItemBookingConfirmation", "/flexibleshopping", "/carsearch", "/CarCheckout", "/things-to-do/search?", "/things-to-do/*/search?", "/*/things-to-do/search", "/things-to-do/reviews-external$", "/Cruise-Search", "/Cruise-CabinCategory", "/CruiseCheckout", "/legacyGOTO", "/go/", "/info/", "/trips/", "/Search-Results", "/user/createaccount", "/signup?*", "/login?*", "/api/uisprime/track", "/trvl-px/", "/egcs/v2/collect", "/api/ucs/shortlist/", "/api/v4/typeahead/", "/p/home", "/*/p/home", "/auth/", "/*/auth/", "/*/sso/", "/cl/*" ], "allow": [], "note": "inherits User-agent: *" }, "PerplexityBot": { "vendor": "Perplexity", "kind": "search-index", "status": "partial", "explicit": true, "effective": "partial", "disallow": [ "/p/home", "/*/p/home", "/auth/", "/*/auth/", "/*/sso/", "/cl/*" ], "allow": [ "/" ] }, "Perplexity-User": { "vendor": "Perplexity", "kind": "user-action", "status": "partial", "explicit": true, "effective": "partial", "disallow": [ "/p/home", "/*/p/home", "/auth/", "/*/auth/", "/*/sso/", "/cl/*" ], "allow": [ "/" ] }, "Bytespider": { "vendor": "ByteDance", "kind": "training-crawler", "status": "unspecified", "explicit": false, "effective": "partial", "disallow": [ "/Hotel-Search", "/hotel-search", "/search?", "/*/search?", "/ugc/urs/api/hotelreviews/hotel/HOTELID/", "/LastBooking/ElapsedTime", "/*chkin=*", "/*chkout=*", "/HotelCheckout", "/Checkout", "/checkout", "/HotelBookingConfirmation", "/en-au/", "/en-nz/", "/Flights-Search", "/Flight-Information", "/Flight-SearchResults", "/Flights-BagFees", "/Details", "/FlightCheckout", "/Confirmation-Flight", "/MultiItemCheckout", "/MultiItemBookingConfirmation", "/flexibleshopping", "/carsearch", "/CarCheckout", "/things-to-do/search?", "/things-to-do/*/search?", "/*/things-to-do/search", "/things-to-do/reviews-external$", "/Cruise-Search", "/Cruise-CabinCategory", "/CruiseCheckout", "/legacyGOTO", "/go/", "/info/", "/trips/", "/Search-Results", "/user/createaccount", "/signup?*", "/login?*", "/api/uisprime/track", "/trvl-px/", "/egcs/v2/collect", "/api/ucs/shortlist/", "/api/v4/typeahead/", "/p/home", "/*/p/home", "/auth/", "/*/auth/", "/*/sso/", "/cl/*" ], "allow": [], "note": "inherits User-agent: *" }, "CCBot": { "vendor": "Common Crawl", "kind": "training-crawler", "status": "unspecified", "explicit": false, "effective": "partial", "disallow": [ "/Hotel-Search", "/hotel-search", "/search?", "/*/search?", "/ugc/urs/api/hotelreviews/hotel/HOTELID/", "/LastBooking/ElapsedTime", "/*chkin=*", "/*chkout=*", "/HotelCheckout", "/Checkout", "/checkout", "/HotelBookingConfirmation", "/en-au/", "/en-nz/", "/Flights-Search", "/Flight-Information", "/Flight-SearchResults", "/Flights-BagFees", "/Details", "/FlightCheckout", "/Confirmation-Flight", "/MultiItemCheckout", "/MultiItemBookingConfirmation", "/flexibleshopping", "/carsearch", "/CarCheckout", "/things-to-do/search?", "/things-to-do/*/search?", "/*/things-to-do/search", "/things-to-do/reviews-external$", "/Cruise-Search", "/Cruise-CabinCategory", "/CruiseCheckout", "/legacyGOTO", "/go/", "/info/", "/trips/", "/Search-Results", "/user/createaccount", "/signup?*", "/login?*", "/api/uisprime/track", "/trvl-px/", "/egcs/v2/collect", "/api/ucs/shortlist/", "/api/v4/typeahead/", "/p/home", "/*/p/home", "/auth/", "/*/auth/", "/*/sso/", "/cl/*" ], "allow": [], "note": "inherits User-agent: *" }, "Meta-ExternalAgent": { "vendor": "Meta", "kind": "training-crawler", "status": "unspecified", "explicit": false, "effective": "partial", "disallow": [ "/Hotel-Search", "/hotel-search", "/search?", "/*/search?", "/ugc/urs/api/hotelreviews/hotel/HOTELID/", "/LastBooking/ElapsedTime", "/*chkin=*", "/*chkout=*", "/HotelCheckout", "/Checkout", "/checkout", "/HotelBookingConfirmation", "/en-au/", "/en-nz/", "/Flights-Search", "/Flight-Information", "/Flight-SearchResults", "/Flights-BagFees", "/Details", "/FlightCheckout", "/Confirmation-Flight", "/MultiItemCheckout", "/MultiItemBookingConfirmation", "/flexibleshopping", "/carsearch", "/CarCheckout", "/things-to-do/search?", "/things-to-do/*/search?", "/*/things-to-do/search", "/things-to-do/reviews-external$", "/Cruise-Search", "/Cruise-CabinCategory", "/CruiseCheckout", "/legacyGOTO", "/go/", "/info/", "/trips/", "/Search-Results", "/user/createaccount", "/signup?*", "/login?*", "/api/uisprime/track", "/trvl-px/", "/egcs/v2/collect", "/api/ucs/shortlist/", "/api/v4/typeahead/", "/p/home", "/*/p/home", "/auth/", "/*/auth/", "/*/sso/", "/cl/*" ], "allow": [], "note": "inherits User-agent: *" } } }