chore: update package-lock.json and remove stale requirements file

README.md

@@ -61,8 +61,10 @@ export const GET = createLlmsFullTxtHandler()
 ```
 
 Set one of:
-- `NEXT_PUBLIC_APP_URL` (recommended), or
-- `VERCEL_URL` (usually already set on Vercel).
+- `NEXT_PUBLIC_APP_URL` (recommended)
+- `VERCEL_URL` (usually already set on Vercel)
+
+> Tip: Set `NEXT_PUBLIC_APP_URL` explicitly in local development and production. `@llmstxt/next` will only fall back to `http://localhost:3000` when `NODE_ENV=development`.
 
 Visit:
 - `/llms.txt`
@@ -237,8 +239,30 @@ Generates `llms-full.txt` as a string (fetches pages and converts HTML→text/Ma
 
 | Variable | Purpose |
 |---|---|
-| `NEXT_PUBLIC_APP_URL` | Your production base URL |
-| `VERCEL_URL` | Automatically set by Vercel |
+| `NEXT_PUBLIC_APP_URL` | Your public app root URL, e.g. `https://example.com` — recommended for production and local dev |
+| `VERCEL_URL` | Automatically set by Vercel; used as a fallback if `NEXT_PUBLIC_APP_URL` is missing |
+
+> Note: When `NODE_ENV=development`, `@llmstxt/next` will default to `http://localhost:3000` if neither variable is set. For predictable results, always set `NEXT_PUBLIC_APP_URL` in `.env.local` or your deployment environment.
+
+---
+
+## Packages included
+
+This repository publishes three npm packages:
+
+- `@llmstxt/core` — framework-agnostic generator for `llms.txt` and `llms-full.txt`
+- `@llmstxt/next` — Next.js App Router route handlers for `llms.txt` and `llms-full.txt`
+- `@llmstxt/middleware` — Next.js middleware that serves Markdown on `Accept: text/markdown`
+
+Install only the packages you need:
+
+```bash
+npm install @llmstxt/core
+npm install @llmstxt/next
+npm install @llmstxt/middleware
+```
+
+See each package README for the full option reference.
 
 ---
 

examples/README.md

@@ -0,0 +1,34 @@
+# Examples
+
+This folder contains two example projects for local testing of the monorepo packages:
+
+- `basic-test` — a simple TypeScript script that imports `@llmstxt/core` and generates `llms.txt` locally.
+- `next-app-test` — a minimal Next.js App Router project that consumes `@llmstxt/next` and exposes `/llms.txt` and `/llms-full.txt`.
+
+## Getting started
+
+1. Build the workspace packages from the repository root:
+
+```bash
+npm install
+npm run build
+```
+
+1. Run the basic example:
+
+```bash
+cd examples/basic-test
+npm install
+npm start
+```
+
+1. Run the Next.js example:
+
+```bash
+cd examples/next-app-test
+npm install
+cp .env.local.example .env.local
+npm run dev
+```
+
+If you add new changes to `packages/core`, `packages/next`, or `packages/middleware`, re-run `npm run build` at the repository root before starting the examples.

examples/nextjs-test/.gitignore

@@ -0,0 +1,41 @@
+# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
+
+# dependencies
+/node_modules
+/.pnp
+.pnp.*
+.yarn/*
+!.yarn/patches
+!.yarn/plugins
+!.yarn/releases
+!.yarn/versions
+
+# testing
+/coverage
+
+# next.js
+/.next/
+/out/
+
+# production
+/build
+
+# misc
+.DS_Store
+*.pem
+
+# debug
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+.pnpm-debug.log*
+
+# env files (can opt-in for committing if needed)
+.env*
+
+# vercel
+.vercel
+
+# typescript
+*.tsbuildinfo
+next-env.d.ts

examples/nextjs-test/AGENTS.md

@@ -0,0 +1,5 @@
+<!-- BEGIN:nextjs-agent-rules -->
+# This is NOT the Next.js you know
+
+This version has breaking changes — APIs, conventions, and file structure may all differ from your training data. Read the relevant guide in `node_modules/next/dist/docs/` before writing any code. Heed deprecation notices.
+<!-- END:nextjs-agent-rules -->

examples/nextjs-test/CLAUDE.md

@@ -0,0 +1 @@
+@AGENTS.md

examples/nextjs-test/README.md

@@ -0,0 +1,45 @@
+# LLMsText Next.js Example Application
+
+This example demonstrates all features of the @llmstxt packages including:
+
+- `@llmstxt/core` for scanning and generating documentation
+- `@llmstxt/next` for Next.js route handlers
+- `@llmstxt/middleware` for markdown content negotiation
+
+## Features Demonstrated
+
+- **Multiple Pages**: Docs, Blog, Features, Pricing, About, API Reference
+- **Fake Widgets & Components**: Reusable UI components
+- **Auto-Generated Documentation**: Visit `/llms.txt` and `/llms-full.txt`
+- **Markdown Middleware**: Request any page with `Accept: text/markdown` header
+
+## Getting Started
+
+1. Install dependencies from project root:
+
+```bash
+npm install
+npm run build
+```
+
+1. Set environment variable:
+
+```bash
+NEXT_PUBLIC_APP_URL=http://localhost:3000
+```
+
+> For local development, put this in `.env.local` so Next.js and `@llmstxt/next` can resolve your public site URL.
+
+1. Run the development server:
+
+```bash
+npm run dev
+```
+
+1. Open [http://localhost:3000](http://localhost:3000)
+
+## Testing
+
+- Visit `/llms.txt` - Auto-generated documentation index
+- Visit `/llms-full.txt` - Full documentation with page content
+- Use `curl -H "Accept: text/markdown" http://localhost:3000/docs/getting-started` - Test markdown middleware

examples/nextjs-test/app/about/page.tsx

@@ -0,0 +1,67 @@
+import { Navigation, Card } from '@/components/widgets';
+
+export const metadata = {
+    title: 'About | LLMsText',
+    description: 'About the llmstxt project',
+};
+
+export default function About() {
+    return (
+        <>
+            <Navigation links={[{ href: '/', label: 'Home' }, { href: '/about', label: 'About' }]} />
+            <div className="max-w-4xl mx-auto px-4 sm:px-6 lg:px-8 py-16">
+                <h1 className="text-4xl font-bold text-gray-900 mb-4">About LLMsText</h1>
+
+                <section className="prose prose-lg max-w-none space-y-8 mt-8">
+                    <div>
+                        <h2 className="text-2xl font-bold text-gray-900 mb-4">Our Mission</h2>
+                        <p className="text-gray-700 leading-relaxed">
+                            We believe that modern websites should be easily readable by both humans and AI systems. LLMsText
+                            provides a simple, standardized way to make your documentation accessible to AI agents and language
+                            models while maintaining excellent SEO and user experience.
+                        </p>
+                    </div>
+
+                    <div>
+                        <h2 className="text-2xl font-bold text-gray-900 mb-4">The Problem We Solve</h2>
+                        <p className="text-gray-700 leading-relaxed mb-4">
+                            Traditional websites and documentation are designed primarily for human readers. When AI systems try to
+                            understand your site structure and content, they often have to parse HTML and navigate complex DOM
+                            structures. This is inefficient and can lead to incomplete or inaccurate understanding.
+                        </p>
+                        <p className="text-gray-700 leading-relaxed">
+                            LLMsText introduces `llms.txt` and `llms-full.txt` formats - simple, standardized text files that
+                            provide AI systems with a clear, comprehensive view of your documentation structure and content.
+                        </p>
+                    </div>
+
+                    <div>
+                        <h2 className="text-2xl font-bold text-gray-900 mb-4">Our Packages</h2>
+                        <div className="grid gap-4">
+                            <Card
+                                title="@llmstxt/core"
+                                description="Framework-agnostic core package for scanning directories and generating documentation"
+                            />
+                            <Card
+                                title="@llmstxt/next"
+                                description="Next.js App Router integration with built-in route handlers for /llms.txt and /llms-full.txt"
+                            />
+                            <Card
+                                title="@llmstxt/middleware"
+                                description="Next.js middleware that serves markdown to clients accepting text/markdown content"
+                            />
+                        </div>
+                    </div>
+
+                    <div>
+                        <h2 className="text-2xl font-bold text-gray-900 mb-4">Get Involved</h2>
+                        <p className="text-gray-700 leading-relaxed">
+                            LLMsText is open source and community-driven. We welcome contributions, bug reports, and feature
+                            requests. Visit our GitHub repository to get started.
+                        </p>
+                    </div>
+                </section>
+            </div>
+        </>
+    );
+}

examples/nextjs-test/app/api-reference/page.tsx

@@ -0,0 +1,134 @@
+import { Navigation, CodeBlock } from '@/components/widgets';
+
+export const metadata = {
+    title: 'API Reference | LLMsText',
+    description: 'API reference for llmstxt packages',
+};
+
+export default function APIReference() {
+    return (
+        <>
+            <Navigation links={[{ href: '/', label: 'Home' }, { href: '/api-reference', label: 'API' }]} />
+            <div className="max-w-4xl mx-auto px-4 sm:px-6 lg:px-8 py-16">
+                <h1 className="text-4xl font-bold text-gray-900 mb-2">API Reference</h1>
+                <p className="text-xl text-gray-600 mb-12">Complete API documentation for all llmstxt packages</p>
+
+                <div className="space-y-12">
+                    <section>
+                        <h2 className="text-2xl font-bold text-gray-900 mb-4">@llmstxt/core</h2>
+                        <div className="space-y-6">
+                            <div>
+                                <h3 className="text-lg font-semibold text-gray-900 mb-2">scanAppDirForPages(options)</h3>
+                                <p className="text-gray-700 mb-3">
+                                    Recursively scans an app directory and returns an array of all pages with metadata.
+                                </p>
+                                <CodeBlock
+                                    code={`interface ScannedPage {
+  filePath: string;
+  routePath: string;
+  url: string;
+  title: string;
+  description?: string;
+}`}
+                                />
+                            </div>
+
+                            <div>
+                                <h3 className="text-lg font-semibold text-gray-900 mb-2">generateLlmsTxt(options)</h3>
+                                <p className="text-gray-700 mb-3">
+                                    Generates a markdown index of all pages organized by route.
+                                </p>
+                                <CodeBlock code={`export async function generateLlmsTxt(
+  options: GenerateLlmsTxtOptions
+): Promise<string>`} />
+                            </div>
+
+                            <div>
+                                <h3 className="text-lg font-semibold text-gray-900 mb-2">generateLlmsFullTxt(options)</h3>
+                                <p className="text-gray-700 mb-3">
+                                    Generates complete documentation with full page content.
+                                </p>
+                                <CodeBlock code={`export async function generateLlmsFullTxt(
+  options: GenerateLlmsFullTxtOptions
+): Promise<string>`} />
+                            </div>
+                        </div>
+                    </section>
+
+                    <section>
+                        <h2 className="text-2xl font-bold text-gray-900 mb-4">@llmstxt/next</h2>
+                        <div className="space-y-6">
+                            <div>
+                                <h3 className="text-lg font-semibold text-gray-900 mb-2">createLlmsTxtHandler()</h3>
+                                <p className="text-gray-700 mb-3">
+                                    Creates a Next.js route handler for the /llms.txt endpoint.
+                                </p>
+                                <CodeBlock
+                                    code={`import { createLlmsTxtHandler } from '@llmstxt/next';
+
+export const GET = createLlmsTxtHandler({
+  title: 'My Documentation',
+  summary: 'Overview of my site',
+});`}
+                                />
+                            </div>
+
+                            <div>
+                                <h3 className="text-lg font-semibold text-gray-900 mb-2">createLlmsFullTxtHandler()</h3>
+                                <p className="text-gray-700 mb-3">
+                                    Creates a Next.js route handler for the /llms-full.txt endpoint.
+                                </p>
+                                <CodeBlock
+                                    code={`import { createLlmsFullTxtHandler } from '@llmstxt/next';
+
+export const GET = createLlmsFullTxtHandler();`}
+                                />
+                            </div>
+                        </div>
+                    </section>
+
+                    <section>
+                        <h2 className="text-2xl font-bold text-gray-900 mb-4">@llmstxt/middleware</h2>
+                        <div className="space-y-6">
+                            <div>
+                                <h3 className="text-lg font-semibold text-gray-900 mb-2">createMarkdownMiddleware(options)</h3>
+                                <p className="text-gray-700 mb-3">
+                                    Creates Next.js middleware that serves markdown for requests with Accept: text/markdown header.
+                                </p>
+                                <CodeBlock
+                                    code={`import { createMarkdownMiddleware } from '@llmstxt/middleware';
+
+export const middleware = createMarkdownMiddleware({
+  exclude: ['_next/*', 'api/*'],
+  cacheControl: 'public, max-age=3600',
+});
+
+export const config = {
+  matcher: ['/((?!_next/static|_next/image).*)'],
+};`}
+                                />
+                            </div>
+                        </div>
+                    </section>
+
+                    <section>
+                        <h2 className="text-2xl font-bold text-gray-900 mb-4">Common Options</h2>
+                        <CodeBlock
+                            code={`interface LlmsTxtOptions {
+  appDir: string;
+  baseUrl: string;
+  title?: string;
+  summary?: string;
+  exclude?: string[];
+  maxDescriptionLength?: number;
+  extractDescription?: (input) => string | undefined;
+  fetchTimeoutMs?: number;
+  htmlToMarkdown?: (html: string) => string;
+}`}
+                        />
+                    </section>
+                </div>
+            </div>
+        </>
+    );
+}