# SqueezeVid.com - Video Compression and Conversion Tool # Robots.txt for www.squeezevid.com # # NOTE: do NOT add per-bot groups that contain only Crawl-delay. A bot with its # own group ignores the "User-agent: *" group entirely, so a crawl-delay-only # group silently grants that bot access to everything blocked below. # # NOTE: never Disallow /static/ — it holds the CSS and the compressor JS. # Blocking it makes every page render broken/unstyled to crawlers, which reads # as "low value content" in quality reviews (AdSense/Search). User-agent: * # Block admin and private areas Disallow: /admin/ Disallow: /accounts/ Disallow: /api/ Disallow: /processing/ Disallow: /status/ # Block duplicate content from query parameters Disallow: /*?* Disallow: /*& # Block temporary upload directories Disallow: /uploads/ Disallow: /temp/ Disallow: /tmp/ # Block user-uploaded media (privacy); static CSS/JS/images stay crawlable Disallow: /media/ # Allow all language versions of main pages Allow: /*/convert-video/ Allow: /*/convert-to-gif/ Allow: /*/large-video-compress/ Allow: /*/blog/ Allow: /*/about/ Allow: /*/contact/ Allow: /*/privacy-policy/ Allow: /*/terms-of-service/ # Allow high-intent use-case landing pages Allow: /*/compress-video-for-*/ Allow: /*/compress-video-to-*/ Allow: /*/compress-video-without-uploading/ Allow: /*/private-video-compressor/ # Allow all format conversion pages Allow: /*/mp4-to-*/ Allow: /*/mov-to-*/ Allow: /*/avi-to-*/ Allow: /*/webm-to-*/ Allow: /*/mkv-to-*/ # Sitemaps Sitemap: https://www.squeezevid.com/sitemap.xml # Crawl rate (Google ignores this; Bing/Yandex honor it) Crawl-delay: 1 # AI search & answer engines — explicitly allowed so they can cite SqueezeVid. # NOTE: a bot with its own group ignores the "User-agent: *" group entirely, # so the private-area disallows are restated here on purpose. User-agent: GPTBot User-agent: OAI-SearchBot User-agent: ChatGPT-User User-agent: PerplexityBot User-agent: Perplexity-User User-agent: ClaudeBot User-agent: anthropic-ai User-agent: Claude-Web User-agent: Google-Extended User-agent: Applebot-Extended Disallow: /admin/ Disallow: /accounts/ Disallow: /api/ Disallow: /processing/ Disallow: /status/ Disallow: /uploads/ Disallow: /temp/ Disallow: /tmp/ Allow: / # Common Crawl — training-only corpus, drives no citations. Blocked. User-agent: CCBot Disallow: /