Update robots.ts

Author: dogfrogfog

apps/sanity/src/app/robots.ts

@@ -3,134 +3,6 @@ import { type MetadataRoute } from "next";
 export default function robots(): MetadataRoute.Robots {
   return {
     rules: [
-      {
-        userAgent: "anthropic-ai",
-        disallow: "/",
-      },
-      {
-        userAgent: "Applebot-Extended",
-        disallow: "/",
-      },
-      {
-        userAgent: "Bytespider",
-        disallow: "/",
-      },
-      {
-        userAgent: "CCBot",
-        disallow: "/",
-      },
-      {
-        userAgent: "ChatGPT-User",
-        disallow: "/",
-      },
-      {
-        userAgent: "ClaudeBot",
-        disallow: "/",
-      },
-      {
-        userAgent: "Claude-Web",
-        disallow: "/",
-      },
-      {
-        userAgent: "cohere-ai",
-        disallow: "/",
-      },
-      {
-        userAgent: "Diffbot",
-        disallow: "/",
-      },
-      {
-        userAgent: "Meta-ExternalAgent",
-        disallow: "/",
-      },
-      {
-        userAgent: "Google-Extended",
-        disallow: "/",
-      },
-      {
-        userAgent: "Google-CloudVertexBot",
-        disallow: "/",
-      },
-      {
-        userAgent: "GPTBot",
-        disallow: "/",
-      },
-      {
-        userAgent: "PerplexityBot",
-        disallow: "/",
-      },
-      {
-        userAgent: "Scrapy",
-        disallow: "/",
-      },
-      {
-        userAgent: "Timpibot",
-        disallow: "/",
-      },
-      {
-        userAgent: "Zoominfobot",
-        disallow: "/",
-      },
-      {
-        userAgent: "trovitBot",
-        disallow: "/",
-      },
-      {
-        userAgent: "AhrefsBot",
-        disallow: "/",
-      },
-      {
-        userAgent: "sogou spider",
-        disallow: "/",
-      },
-      {
-        userAgent: "BLEXBot",
-        disallow: "/",
-      },
-      {
-        userAgent: "ShopWiki",
-        disallow: "/",
-      },
-      {
-        userAgent: "MJ12bot",
-        disallow: "/",
-      },
-      {
-        userAgent: "Exabot",
-        disallow: "/",
-      },
-      {
-        userAgent: "CCBot",
-        disallow: "/",
-      },
-      {
-        userAgent: "sistrix",
-        disallow: "/",
-      },
-      {
-        userAgent: "XoviBot",
-        disallow: "/",
-      },
-      {
-        userAgent: "Flamingo_SearchEngine",
-        disallow: "/",
-      },
-      {
-        userAgent: "Nutch",
-        disallow: "/",
-      },
-      {
-        userAgent: "ADmantX",
-        disallow: "/",
-      },
-      {
-        userAgent: "grapeshot",
-        allow: "/",
-      },
-      {
-        userAgent: "CarsnipBot",
-        disallow: "/",
-      },
       {
         userAgent: "*",
         allow: "/",