User-agent: * Allow: / # Block authenticated app surface — these pages require login and have # no public content for crawlers to index. Disallow: /dashboard Disallow: /onboarding Allow: /calculators Allow: /calculators/ # Block ONLY the authenticated singular /calculator app page. # The $ anchor is essential: an unanchored "/calculator" is a prefix that # also matches the public "/calculators/..." pages (this previously caused # them to be reported "Blocked by robots.txt" in Search Console). Disallow: /calculator$ Disallow: /leave Disallow: /reminders Disallow: /service-history Disallow: /profile Disallow: /tax Disallow: /documents Disallow: /secret-admin-portal Disallow: /auth/ # /govt-news is the public, prerendered news section — crawl freely. Allow: /govt-news Allow: /govt-news/ # /videos, /holidays are public informational pages — allow crawling Allow: /videos Allow: /holidays # AI crawlers — explicitly allow indexing for AI Overviews / Perplexity / ChatGPT. User-agent: GPTBot Allow: / User-agent: PerplexityBot Allow: / User-agent: Google-Extended Allow: / User-agent: ClaudeBot Allow: / User-agent: CCBot Allow: / Sitemap: https://cgseva.com/sitemap.xml Sitemap: https://cgseva.com/news-sitemap.xml