feat(docs): improve search + LLM discoverability

apps/docs-next/app/not-found.tsx

@@ -1,6 +1,11 @@
 import Link from 'next/link'
 import { AnimatedLogo } from '@/components/brand/animated-logo'
 
+export const metadata = {
+  title: 'Page not found',
+  robots: { index: false, follow: true },
+}
+
 export default function NotFound() {
   const repo = 'AgentsKit-io/agentskit'
   return (

apps/docs-next/app/robots.ts

@@ -5,7 +5,7 @@ export default function robots(): MetadataRoute.Robots {
     rules: [
       {
         userAgent: '*',
-        allow: '/',
+        allow: ['/', '/api/og'],
         disallow: ['/api/'],
       },
       { userAgent: 'GPTBot', allow: '/' },

apps/docs-next/app/sitemap.ts

@@ -1,5 +1,8 @@
 import type { MetadataRoute } from 'next'
 import { source } from '@/lib/source'
+import { slugsOfAll } from '@/lib/blog'
+import { STEPS } from '@/lib/learn-steps'
+import { SHOWCASE } from '@/lib/showcase'
 
 const SITE = 'https://www.agentskit.io'
 
@@ -19,6 +22,33 @@ export default function sitemap(): MetadataRoute.Sitemap {
     { url: `${SITE}/docs/reference`, lastModified: now, changeFrequency: 'weekly', priority: 0.8 },
     { url: `${SITE}/docs/reference/examples`, lastModified: now, changeFrequency: 'weekly', priority: 0.8 },
     { url: `${SITE}/docs/reference/contribute`, lastModified: now, changeFrequency: 'weekly', priority: 0.7 },
+    { url: `${SITE}/learn`, lastModified: now, changeFrequency: 'weekly', priority: 0.85 },
+    { url: `${SITE}/blog`, lastModified: now, changeFrequency: 'weekly', priority: 0.8 },
+    { url: `${SITE}/showcase`, lastModified: now, changeFrequency: 'weekly', priority: 0.75 },
+    { url: `${SITE}/stack`, lastModified: now, changeFrequency: 'weekly', priority: 0.7 },
+    { url: `${SITE}/community`, lastModified: now, changeFrequency: 'monthly', priority: 0.6 },
+    { url: `${SITE}/evals`, lastModified: now, changeFrequency: 'monthly', priority: 0.6 },
+  ]
+
+  const dynamicRoutes: MetadataRoute.Sitemap = [
+    ...slugsOfAll().map((slug) => ({
+      url: `${SITE}/blog/${slug}`,
+      lastModified: now,
+      changeFrequency: 'monthly' as const,
+      priority: 0.6,
+    })),
+    ...STEPS.map((s) => ({
+      url: `${SITE}/learn/${s.slug}`,
+      lastModified: now,
+      changeFrequency: 'weekly' as const,
+      priority: 0.7,
+    })),
+    ...SHOWCASE.map((s) => ({
+      url: `${SITE}/showcase/${s.slug}`,
+      lastModified: now,
+      changeFrequency: 'monthly' as const,
+      priority: 0.6,
+    })),
   ]
 
   const docPages: MetadataRoute.Sitemap = source.getPages().map((page) => {
@@ -33,7 +63,7 @@ export default function sitemap(): MetadataRoute.Sitemap {
   })
 
   const seen = new Set<string>()
-  return [...staticRoutes, ...docPages].filter((r) => {
+  return [...staticRoutes, ...docPages, ...dynamicRoutes].filter((r) => {
     if (seen.has(r.url)) return false
     seen.add(r.url)
     return true

apps/docs-next/components/seo/json-ld.tsx

@@ -1,7 +1,12 @@
 export function JsonLd({ data }: { data: unknown }) {
+  // Trusted, app-controlled structured data only. `<` escaped to < so the
+  // JSON survives in the DOM without React HTML-escaping breaking the markup.
   return (
-    <script type="application/ld+json">
-      {JSON.stringify(data)}
-    </script>
+    <script
+      type="application/ld+json"
+      dangerouslySetInnerHTML={{
+        __html: JSON.stringify(data).replace(/</g, '\\u003c'),
+      }}
+    />
   )
 }