feat(seo): add sitemap.xml and robots.txt generation (#48, #60)

src/squishmark/main.py

@@ -14,7 +14,7 @@
 from squishmark.config import get_settings
 from squishmark.models.content import Config
 from squishmark.models.db import close_db, get_db_session, init_db
-from squishmark.routers import admin, auth, feed, pages, posts, webhooks
+from squishmark.routers import admin, auth, feed, pages, posts, seo, webhooks
 from squishmark.services.analytics import AnalyticsService
 from squishmark.services.github import get_github_service, shutdown_github_service
 from squishmark.services.markdown import get_markdown_service
@@ -296,6 +296,7 @@ async def livereload_ws(websocket: WebSocket) -> None:
     app.include_router(admin.router)
     app.include_router(webhooks.router)
     app.include_router(feed.router)
+    app.include_router(seo.router)
     app.include_router(posts.router)
     app.include_router(pages.router)  # Catch-all for static pages, must be last
 

src/squishmark/models/content.py

@@ -68,6 +68,7 @@ class Page(BaseModel):
 
     slug: str
     title: str
+    date: datetime.date | None = None
     description: str = ""
     content: str = ""  # Raw markdown
     html: str = ""  # Rendered HTML

src/squishmark/routers/feed.py

@@ -8,6 +8,7 @@
 
 from squishmark.models.content import Config, Post
 from squishmark.services.cache import get_cache
+from squishmark.services.content import get_all_posts
 from squishmark.services.github import get_github_service
 from squishmark.services.markdown import get_markdown_service
 
@@ -91,24 +92,8 @@ async def atom_feed() -> Response:
     config = Config.from_dict(config_data)
     markdown_service = get_markdown_service(config)
 
-    # Fetch all published posts
-    post_files = await github_service.list_directory("posts")
-    posts: list[Post] = []
-    for path in post_files:
-        if not path.endswith(".md"):
-            continue
-        file = await github_service.get_file(path)
-        if file is None:
-            continue
-        post = markdown_service.parse_post(path, file.content)
-        if not post.draft:
-            posts.append(post)
-
-    # Newest first
-    posts.sort(key=lambda p: (p.date is not None, p.date), reverse=True)
-
-    # Limit to 20 most recent
-    posts = posts[:20]
+    posts = await get_all_posts(github_service, markdown_service)
+    posts = posts[:20]  # Limit to 20 most recent
 
     xml_bytes = _build_atom_feed(config, posts)
     await cache.set(FEED_CACHE_KEY, xml_bytes)

src/squishmark/routers/seo.py

@@ -0,0 +1,108 @@
+"""SEO routes: sitemap.xml and robots.txt."""
+
+import datetime
+from xml.etree.ElementTree import Element, SubElement, tostring
+
+from fastapi import APIRouter
+from fastapi.responses import Response
+
+from squishmark.models.content import Config, Page, Post
+from squishmark.services.cache import get_cache
+from squishmark.services.content import get_all_pages, get_all_posts
+from squishmark.services.github import get_github_service
+from squishmark.services.markdown import get_markdown_service
+
+router = APIRouter(tags=["seo"])
+
+SITEMAP_NS = "http://www.sitemaps.org/schemas/sitemap/0.9"
+SITEMAP_CACHE_KEY = "seo:sitemap"
+ROBOTS_CACHE_KEY = "seo:robots"
+
+
+def _add_url(urlset: Element, loc: str, lastmod: datetime.date | None = None) -> None:
+    """Append a <url> entry to the sitemap urlset."""
+    url_el = SubElement(urlset, "url")
+    SubElement(url_el, "loc").text = loc
+    if lastmod:
+        SubElement(url_el, "lastmod").text = lastmod.isoformat()
+
+
+def _build_sitemap(config: Config, posts: list[Post], pages: list[Page]) -> bytes:
+    """Build a sitemap.xml from config, posts, and pages."""
+    site_url = config.site.url.rstrip("/") if config.site.url else ""
+    newest_post_date = posts[0].date if posts else None
+
+    urlset = Element("urlset", xmlns=SITEMAP_NS)
+
+    _add_url(urlset, f"{site_url}/", newest_post_date)
+    _add_url(urlset, f"{site_url}/posts", newest_post_date)
+
+    for post in posts:
+        _add_url(urlset, f"{site_url}{post.url}", post.date)
+
+    for page in pages:
+        if page.visibility == "public":
+            _add_url(urlset, f"{site_url}{page.url}", page.date)
+
+    return b'<?xml version="1.0" encoding="utf-8"?>\n' + tostring(urlset, encoding="unicode").encode("utf-8")
+
+
+def _build_robots_txt(config: Config) -> str:
+    """Build robots.txt content."""
+    site_url = config.site.url.rstrip("/") if config.site.url else ""
+
+    lines = [
+        "User-agent: *",
+        "Allow: /",
+        "",
+        "Disallow: /admin/*",
+        "Disallow: /auth/*",
+        "Disallow: /health",
+        "Disallow: /webhooks/*",
+    ]
+
+    if site_url:
+        lines.append("")
+        lines.append(f"Sitemap: {site_url}/sitemap.xml")
+
+    return "\n".join(lines) + "\n"
+
+
+@router.get("/sitemap.xml")
+async def sitemap_xml() -> Response:
+    """Serve the XML sitemap."""
+    cache = get_cache()
+
+    cached = await cache.get(SITEMAP_CACHE_KEY)
+    if cached is not None:
+        return Response(content=cached, media_type="application/xml; charset=utf-8")
+
+    github_service = get_github_service()
+    config_data = await github_service.get_config()
+    config = Config.from_dict(config_data)
+    markdown_service = get_markdown_service(config)
+
+    posts = await get_all_posts(github_service, markdown_service)
+    pages = await get_all_pages(github_service, markdown_service)
+
+    xml_bytes = _build_sitemap(config, posts, pages)
+    await cache.set(SITEMAP_CACHE_KEY, xml_bytes)
+    return Response(content=xml_bytes, media_type="application/xml; charset=utf-8")
+
+
+@router.get("/robots.txt")
+async def robots_txt() -> Response:
+    """Serve robots.txt."""
+    cache = get_cache()
+
+    cached = await cache.get(ROBOTS_CACHE_KEY)
+    if cached is not None:
+        return Response(content=cached, media_type="text/plain; charset=utf-8")
+
+    github_service = get_github_service()
+    config_data = await github_service.get_config()
+    config = Config.from_dict(config_data)
+
+    content = _build_robots_txt(config)
+    await cache.set(ROBOTS_CACHE_KEY, content)
+    return Response(content=content, media_type="text/plain; charset=utf-8")

src/squishmark/services/content.py

@@ -1,6 +1,6 @@
-"""Shared content helpers for fetching and filtering posts."""
+"""Shared content helpers for fetching and filtering posts and pages."""
 
-from squishmark.models.content import Post, SiteConfig
+from squishmark.models.content import Page, Post, SiteConfig
 from squishmark.services.github import GitHubService
 from squishmark.services.markdown import MarkdownService
 
@@ -51,3 +51,31 @@ def get_featured_posts(posts: list[Post], site_config: SiteConfig) -> list[Post]
         ),
     )
     return featured[: site_config.featured_max]
+
+
+async def get_all_pages(
+    github_service: GitHubService,
+    markdown_service: MarkdownService,
+    include_hidden: bool = False,
+) -> list[Page]:
+    """Fetch and parse all pages from the content repository."""
+    page_files = await github_service.list_directory("pages")
+
+    pages: list[Page] = []
+    for path in page_files:
+        if not path.endswith(".md"):
+            continue
+
+        file = await github_service.get_file(path)
+        if file is None:
+            continue
+
+        page = markdown_service.parse_page(path, file.content)
+
+        # Skip hidden pages unless requested
+        if page.visibility == "hidden" and not include_hidden:
+            continue
+
+        pages.append(page)
+
+    return pages

src/squishmark/services/markdown.py

@@ -219,6 +219,7 @@ def parse_page(self, path: str, content: str) -> Page:
         return Page(
             slug=slug,
             title=frontmatter.title,
+            date=frontmatter.date,
             description=description,
             content=markdown_content,
             html=html,