/v1/web/check$0.005Website check: redirects, headers, SEO, agent readiness
Facts about a URL from a live fetch, never its body: final URL, status and chain after up to 5 validated redirects, response time, content type, server, security headers present/missing (HSTS, CSP, nosniff, framing, Referrer-Policy, Permissions-Policy), title, description, canonical, lang, Open Graph, robots meta, robots.txt rules for GPTBot, ClaudeBot, Google-Extended, CCBot, PerplexityBot, *, llms.txt, agent card, OpenAPI, x402 discovery, 402 terms. DNS via Cloudflare DoH.
Params: url
Sample response
{
"query": {
"url": "https://github.com"
},
"result": {
"input_url": "https://github.com/",
"reachable": true,
"error": null,
"final_url": "https://github.com/",
"final_status": 200,
"redirect_count": 0,
"redirect_chain": [],
"redirect_not_followed": null,
"https": true,
"response_time_ms": 152,
"chain_time_ms": 152,
"content_type": "text/html",
"server": "github.com",
"x_robots_tag": null,
"security_headers": {
"hsts": {
"present": true,
"max_age": 31536000,
"include_subdomains": true,
"preload": true
},
"csp": {
"present": true,
"report_only": false,
"frame_ancestors": true
},
"x_content_type_options": "nosniff",
"frame_protection": {
"present": true,
"via": "x-frame-options",
"x_frame_options": "deny"
},
"referrer_policy": "origin-when-cross-origin, strict-origin-when-cross-origin",
"permissions_policy": false,
"present": [
"strict-transport-security",
"content-security-policy",
"x-content-type-options",
"frame-protection",
"referrer-policy"
],
"missing": [
"permissions-policy"
]
},
"page": {
"title": "GitHub · Change is constant. GitHub keeps you ahead. · GitHub · GitHub",
"meta_description": "Join the world's most widely adopted, AI-powered developer platform where millions of developers, businesses, and the largest open source community build software that advances humanity.",
"canonical": "https://github.com/",
"lang": "en",
"og_title": "GitHub · Change is constant. GitHub keeps you ahead. · GitHub",
"og_image": "https://images.ctfassets.net/8aevphvgewt8/76epZKB2wF998m4e4bLLXn/c505f20b00ec70b88b1be5aec42c4be4/OG_image.jpg",
"robots_meta": null,
"csp_meta": false
},
"agent_readiness": {
"origin": "https://github.com",
"robots_txt": {
"url": "https://github.com/robots.txt",
"status": 200,
"present": true,
"basis": "parsed robots.txt (RFC 9309 longest-match)",
"truncated": false,
"groups": 5,
"sitemaps": [
"https://github.com/sitemap_marketing.xml"
],
"content_signal": null,
"agents": [
{
"agent": "GPTBot",
"allowed_root": true,
"group": "specific",
"matched_rule": null
},
{
"agent": "ClaudeBot",
"allowed_root": true,
"group": "specific",
"matched_rule": null
},
{
"agent": "Google-Extended",
"allowed_root": true,
"group": "*",
"matched_rule": null
},
{
"agent": "CCBot",
"allowed_root": true,
"group": "*",
"matched_rule": null
},
{
"agent": "PerplexityBot",
"allowed_root": true,
"group": "specific",
"matched_rule": null
},
{
"agent": "*",
"allowed_root": true,
"group": "*",
"matched_rule": null
}
]
},
"llms_txt": {
"url": "https://github.com/llms.txt",
"status": 200,
"present": true,
"reason": null,
"title": "GitHub",
"follows_format": true,
"sections": 12,
"links": 117,
"truncated": false
},
"x402_discovery": {
"url": "https://github.com/.well-known/x402",
"status": 404,
"present": false,
"reason": null,
"version": null,
"resources": null,
"ownership_proofs": null
},
"openapi": {
"url": "https://github.com/openapi.json",
"status": 410,
"present": false,
"reason": null,
"spec_version": null,
"title": null,
"api_version": null,
"paths": null,
"truncated": false
},
"agent_card": {
"url": "https://github.com/.well-known/agent.json",
"status": 404,
"present": false,
"reason": null,
"format": null,
"name": null,
"protocol_version": null,
"skills": null
},
"payment_required": {
"status_402": false,
"header_present": false,
"decoded_from": null,
"x402_version": null,
"error": null,
"accepts": []
}
},
"subrequests": 8
},
"sources": [
{
"name": "Cloudflare DNS over HTTPS (JSON API)",
"url": "https://developers.cloudflare.com/1.1.1.1/encryption/dns-over-https/make-api-requests/dns-json/"
},
{
"name": "Live HTTP(S) fetch of the site",
"url": "https://github.com/"
}
],
"notes": [
"Facts describe the response served to the Kit Rx fetcher (User-Agent KitRx/1.0); a site may answer browsers or other crawlers differently."
],
"attribution": [
"DNS resolved via Cloudflare 1.1.1.1 DNS-over-HTTPS."
],
"generated_at": "2026-11-14T22:04:54.031Z"
}https://web.kit-rx.com/v1/web/check?url=https%3A%2F%2Fgithub.com