Feat/docs seo discoverability

.github/workflows/indexnow.yml

@@ -0,0 +1,32 @@
+name: Ping search engines
+
+# After a production Vercel deploy succeeds, push the live sitemap URLs to
+# IndexNow (Bing/Yandex/DuckDuckGo/Ecosia). Google is intentionally not
+# pinged: the org blocks service-account keys (iam.disableServiceAccount
+# KeyCreation) and Google recrawls the sitemap on its own anyway.
+
+on:
+  deployment_status:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  ping:
+    # Only on a successful *production* deploy.
+    if: >
+      github.event_name == 'workflow_dispatch' ||
+      (github.event.deployment_status.state == 'success' &&
+       github.event.deployment_status.environment == 'Production')
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
+      - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6
+        with:
+          node-version: 22
+
+      - name: IndexNow
+        env:
+          INDEXNOW_KEY: ${{ secrets.INDEXNOW_KEY }}
+        run: node apps/docs-next/scripts/ping-indexnow.mjs

apps/docs-next/app/not-found.tsx

@@ -1,6 +1,11 @@
 import Link from 'next/link'
 import { AnimatedLogo } from '@/components/brand/animated-logo'
 
+export const metadata = {
+  title: 'Page not found',
+  robots: { index: false, follow: true },
+}
+
 export default function NotFound() {
   const repo = 'AgentsKit-io/agentskit'
   return (

apps/docs-next/app/robots.ts

@@ -5,7 +5,7 @@ export default function robots(): MetadataRoute.Robots {
     rules: [
       {
         userAgent: '*',
-        allow: '/',
+        allow: ['/', '/api/og'],
         disallow: ['/api/'],
       },
       { userAgent: 'GPTBot', allow: '/' },

apps/docs-next/app/sitemap.ts

@@ -1,5 +1,8 @@
 import type { MetadataRoute } from 'next'
 import { source } from '@/lib/source'
+import { slugsOfAll } from '@/lib/blog'
+import { STEPS } from '@/lib/learn-steps'
+import { SHOWCASE } from '@/lib/showcase'
 
 const SITE = 'https://www.agentskit.io'
 
@@ -19,6 +22,33 @@ export default function sitemap(): MetadataRoute.Sitemap {
     { url: `${SITE}/docs/reference`, lastModified: now, changeFrequency: 'weekly', priority: 0.8 },
     { url: `${SITE}/docs/reference/examples`, lastModified: now, changeFrequency: 'weekly', priority: 0.8 },
     { url: `${SITE}/docs/reference/contribute`, lastModified: now, changeFrequency: 'weekly', priority: 0.7 },
+    { url: `${SITE}/learn`, lastModified: now, changeFrequency: 'weekly', priority: 0.85 },
+    { url: `${SITE}/blog`, lastModified: now, changeFrequency: 'weekly', priority: 0.8 },
+    { url: `${SITE}/showcase`, lastModified: now, changeFrequency: 'weekly', priority: 0.75 },
+    { url: `${SITE}/stack`, lastModified: now, changeFrequency: 'weekly', priority: 0.7 },
+    { url: `${SITE}/community`, lastModified: now, changeFrequency: 'monthly', priority: 0.6 },
+    { url: `${SITE}/evals`, lastModified: now, changeFrequency: 'monthly', priority: 0.6 },
+  ]
+
+  const dynamicRoutes: MetadataRoute.Sitemap = [
+    ...slugsOfAll().map((slug) => ({
+      url: `${SITE}/blog/${slug}`,
+      lastModified: now,
+      changeFrequency: 'monthly' as const,
+      priority: 0.6,
+    })),
+    ...STEPS.map((s) => ({
+      url: `${SITE}/learn/${s.slug}`,
+      lastModified: now,
+      changeFrequency: 'weekly' as const,
+      priority: 0.7,
+    })),
+    ...SHOWCASE.map((s) => ({
+      url: `${SITE}/showcase/${s.slug}`,
+      lastModified: now,
+      changeFrequency: 'monthly' as const,
+      priority: 0.6,
+    })),
   ]
 
   const docPages: MetadataRoute.Sitemap = source.getPages().map((page) => {
@@ -33,7 +63,7 @@ export default function sitemap(): MetadataRoute.Sitemap {
   })
 
   const seen = new Set<string>()
-  return [...staticRoutes, ...docPages].filter((r) => {
+  return [...staticRoutes, ...docPages, ...dynamicRoutes].filter((r) => {
     if (seen.has(r.url)) return false
     seen.add(r.url)
     return true

apps/docs-next/components/seo/json-ld.tsx

@@ -1,7 +1,12 @@
 export function JsonLd({ data }: { data: unknown }) {
+  // Trusted, app-controlled structured data only. `<` escaped to < so the
+  // JSON survives in the DOM without React HTML-escaping breaking the markup.
   return (
-    <script type="application/ld+json">
-      {JSON.stringify(data)}
-    </script>
+    <script
+      type="application/ld+json"
+      dangerouslySetInnerHTML={{
+        __html: JSON.stringify(data).replace(/</g, '\\u003c'),
+      }}
+    />
   )
 }

apps/docs-next/next.config.mjs

@@ -1,4 +1,5 @@
 import { createMDX } from 'fumadocs-mdx/next'
+import { LEGACY_404_REDIRECTS } from './legacy-404-redirects.mjs'
 
 const withMDX = createMDX()
 
@@ -67,7 +68,11 @@ const DOC_REDIRECTS = [
 const config = {
   reactStrictMode: true,
   async redirects() {
-    return DOC_REDIRECTS.filter((r) => r.source !== r.destination)
+    // Legacy 404 fixes first — first match wins, so explicit per-URL rules
+    // override the broad wildcard rules that used to chain into dead targets.
+    return [...LEGACY_404_REDIRECTS, ...DOC_REDIRECTS].filter(
+      (r) => r.source !== r.destination,
+    )
   },
 }
 

apps/docs-next/public/d54fbf78f0a84efd44f4a10d00b225ad.txt

@@ -0,0 +1 @@
+d54fbf78f0a84efd44f4a10d00b225ad

apps/docs-next/scripts/ping-indexnow.mjs

@@ -0,0 +1,48 @@
+#!/usr/bin/env node
+// Pushes every sitemap URL to IndexNow (Bing, Yandex, DuckDuckGo, Ecosia, ...).
+// Runs after the Vercel deploy lands so the live sitemap reflects the new build.
+//
+// Env:
+//   SITE          override site origin (default https://www.agentskit.io)
+//   INDEXNOW_KEY  required — must match public/<key>.txt served at the origin
+
+const SITE = (process.env.SITE ?? 'https://www.agentskit.io').replace(/\/$/, '')
+const KEY = process.env.INDEXNOW_KEY
+const HOST = new URL(SITE).host
+
+if (!KEY) {
+  console.error('INDEXNOW_KEY not set — skipping IndexNow ping.')
+  process.exit(0)
+}
+
+async function sitemapUrls() {
+  const res = await fetch(`${SITE}/sitemap.xml`, { headers: { 'user-agent': 'agentskit-indexnow' } })
+  if (!res.ok) throw new Error(`sitemap fetch ${res.status}`)
+  const xml = await res.text()
+  return [...xml.matchAll(/<loc>([^<]+)<\/loc>/g)].map((m) => m[1].trim())
+}
+
+async function main() {
+  const urlList = await sitemapUrls()
+  if (urlList.length === 0) throw new Error('sitemap had no <loc> entries')
+
+  // IndexNow accepts up to 10k URLs per request.
+  for (let i = 0; i < urlList.length; i += 10000) {
+    const batch = urlList.slice(i, i + 10000)
+    const res = await fetch('https://api.indexnow.org/indexnow', {
+      method: 'POST',
+      headers: { 'content-type': 'application/json; charset=utf-8' },
+      body: JSON.stringify({ host: HOST, key: KEY, keyLocation: `${SITE}/${KEY}.txt`, urlList: batch }),
+    })
+    // 200 = accepted, 202 = received/validating. Both fine.
+    if (res.status !== 200 && res.status !== 202) {
+      throw new Error(`IndexNow ${res.status}: ${await res.text()}`)
+    }
+    console.log(`IndexNow: ${batch.length} URLs submitted (HTTP ${res.status}).`)
+  }
+}
+
+main().catch((err) => {
+  console.error(err.message)
+  process.exit(1)
+})