update

Author: Dylan Huang

eval_protocol/cli.py

@@ -34,7 +34,9 @@
 
 def build_parser() -> argparse.ArgumentParser:
     """Build and return the argument parser for the CLI."""
-    parser = argparse.ArgumentParser(description="eval-protocol: Tools for evaluation and reward modeling")
+    parser = argparse.ArgumentParser(
+        description="Inspect evaluation runs locally, upload evaluators, and create reinforcement fine-tuning jobs on Fireworks"
+    )
     return _configure_parser(parser)
 
 
@@ -401,39 +403,52 @@ def _configure_parser(parser: argparse.ArgumentParser) -> argparse.ArgumentParse
     rft_parser.add_argument("--base-model", help="Base model resource id")
     rft_parser.add_argument("--warm-start-from", help="Addon model to warm start from")
     rft_parser.add_argument("--output-model", help="Output model id (defaults from evaluator)")
-    rft_parser.add_argument("--epochs", type=int, default=1)
-    rft_parser.add_argument("--batch-size", type=int, default=128000)
-    rft_parser.add_argument("--learning-rate", type=float, default=3e-5)
-    rft_parser.add_argument("--max-context-length", type=int, default=65536)
-    rft_parser.add_argument("--lora-rank", type=int, default=16)
+    rft_parser.add_argument("--epochs", type=int, default=1, help="Number of training epochs")
+    rft_parser.add_argument("--batch-size", type=int, default=128000, help="Training batch size in tokens")
+    rft_parser.add_argument("--learning-rate", type=float, default=3e-5, help="Learning rate for training")
+    rft_parser.add_argument("--max-context-length", type=int, default=65536, help="Maximum context length in tokens")
+    rft_parser.add_argument("--lora-rank", type=int, default=16, help="LoRA rank for fine-tuning")
     rft_parser.add_argument("--gradient-accumulation-steps", type=int, help="Number of gradient accumulation steps")
-    rft_parser.add_argument("--learning-rate-warmup-steps", type=int, help="Number of LR warmup steps")
-    rft_parser.add_argument("--accelerator-count", type=int)
-    rft_parser.add_argument("--region", help="Fireworks region enum value")
-    rft_parser.add_argument("--display-name", help="RFT job display name")
-    rft_parser.add_argument("--evaluation-dataset", help="Optional separate eval dataset id")
-    rft_parser.add_argument("--eval-auto-carveout", dest="eval_auto_carveout", action="store_true", default=True)
-    rft_parser.add_argument("--no-eval-auto-carveout", dest="eval_auto_carveout", action="store_false")
+    rft_parser.add_argument("--learning-rate-warmup-steps", type=int, help="Number of learning rate warmup steps")
+    rft_parser.add_argument("--accelerator-count", type=int, help="Number of accelerators (GPUs) to use")
+    rft_parser.add_argument("--region", help="Fireworks region for training")
+    rft_parser.add_argument("--display-name", help="Display name for the RFT job")
+    rft_parser.add_argument("--evaluation-dataset", help="Separate dataset id for evaluation")
+    rft_parser.add_argument(
+        "--eval-auto-carveout",
+        dest="eval_auto_carveout",
+        action="store_true",
+        default=True,
+        help="Automatically carve out evaluation data from training set",
+    )
+    rft_parser.add_argument(
+        "--no-eval-auto-carveout",
+        dest="eval_auto_carveout",
+        action="store_false",
+        help="Disable automatic evaluation data carveout",
+    )
     # Rollout chunking
     rft_parser.add_argument("--chunk-size", type=int, default=100, help="Data chunk size for rollout batching")
     # Inference params
-    rft_parser.add_argument("--temperature", type=float)
-    rft_parser.add_argument("--top-p", type=float)
-    rft_parser.add_argument("--top-k", type=int)
-    rft_parser.add_argument("--max-output-tokens", type=int, default=32768)
-    rft_parser.add_argument("--response-candidates-count", type=int, default=8)
+    rft_parser.add_argument("--temperature", type=float, help="Sampling temperature for rollouts")
+    rft_parser.add_argument("--top-p", type=float, help="Top-p (nucleus) sampling parameter")
+    rft_parser.add_argument("--top-k", type=int, help="Top-k sampling parameter")
+    rft_parser.add_argument("--max-output-tokens", type=int, default=32768, help="Maximum output tokens per rollout")
+    rft_parser.add_argument(
+        "--response-candidates-count", type=int, default=8, help="Number of response candidates per prompt"
+    )
     rft_parser.add_argument("--extra-body", help="JSON string for extra inference params")
     # MCP server (optional)
     rft_parser.add_argument(
         "--mcp-server",
-        help="The MCP server resource name to use for the reinforcement fine-tuning job.",
+        help="MCP server resource name for agentic rollouts",
     )
     # Wandb
-    rft_parser.add_argument("--wandb-enabled", action="store_true")
-    rft_parser.add_argument("--wandb-project")
-    rft_parser.add_argument("--wandb-entity")
-    rft_parser.add_argument("--wandb-run-id")
-    rft_parser.add_argument("--wandb-api-key")
+    rft_parser.add_argument("--wandb-enabled", action="store_true", help="Enable Weights & Biases logging")
+    rft_parser.add_argument("--wandb-project", help="Weights & Biases project name")
+    rft_parser.add_argument("--wandb-entity", help="Weights & Biases entity (username or team)")
+    rft_parser.add_argument("--wandb-run-id", help="Weights & Biases run id for resuming")
+    rft_parser.add_argument("--wandb-api-key", help="Weights & Biases API key")
     # Misc
     rft_parser.add_argument("--job-id", help="Specify an explicit RFT job id")
     rft_parser.add_argument("--yes", "-y", action="store_true", help="Non-interactive mode")

eval_protocol/cli_commands/export_docs.py

@@ -59,39 +59,83 @@ def _get_parser_info(parser: argparse.ArgumentParser, subparser_help: str = "")
     return info
 
 
-def _format_argument_row(arg: Dict) -> str:
-    """Format a single argument as a markdown table row."""
-    # Build the flag/argument name
+def _format_argument_item(arg: Dict) -> List[str]:
+    """Format a single argument as a Mintlify ParamField component."""
+    lines = []
+
+    # Build the flag name
     if arg["option_strings"]:
-        name = ", ".join(f"`{opt}`" for opt in arg["option_strings"])
+        long_opts = [o for o in arg["option_strings"] if o.startswith("--")]
+        short_opts = [o for o in arg["option_strings"] if not o.startswith("--")]
+        primary = long_opts[0] if long_opts else arg["option_strings"][0]
     else:
-        name = f"`{arg['dest']}`"
+        primary = arg["dest"]
+        short_opts = []
 
-    # Build type info
+    # Map Python types to ParamField types
     type_str = ""
     if arg["type"]:
-        type_str = getattr(arg["type"], "__name__", str(arg["type"]))
-    if arg["choices"]:
-        type_str = f"choices: {arg['choices']}"
-
-    # Format default value
+        python_type = getattr(arg["type"], "__name__", str(arg["type"]))
+        type_map = {"int": "number", "float": "number", "str": "string", "bool": "boolean"}
+        type_str = type_map.get(python_type, python_type)
+    elif arg["default"] is not None:
+        # Infer type from default
+        if isinstance(arg["default"], bool):
+            type_str = "boolean"
+        elif isinstance(arg["default"], int):
+            type_str = "number"
+        elif isinstance(arg["default"], float):
+            type_str = "number"
+        elif isinstance(arg["default"], str):
+            type_str = "string"
+
+    # Build ParamField attributes
+    attrs = [f'path="{primary}"']
+
+    if type_str:
+        attrs.append(f'type="{type_str}"')
+
+    # Default value
     default = arg["default"]
-    if default is None:
-        default_str = "-"
-    elif default == argparse.SUPPRESS:
-        default_str = "-"
-    elif isinstance(default, bool):
-        default_str = str(default).lower()
-    else:
-        default_str = f"`{default}`"
+    if default is not None and default != argparse.SUPPRESS:
+        if isinstance(default, bool):
+            default_str = str(default).lower()
+        elif isinstance(default, str):
+            # Escape quotes in string defaults
+            default_str = default.replace('"', '\\"')
+        else:
+            default_str = str(default)
+        attrs.append(f'default="{default_str}"')
+
+    if arg["required"]:
+        attrs.append("required")
+
+    # Build description with short alias mention
+    help_text = (arg["help"] or "").replace("<", "&lt;").replace(">", "&gt;")
+    if short_opts:
+        alias_note = f"Short: `{short_opts[0]}`"
+        if help_text:
+            help_text = f"{help_text} ({alias_note})"
+        else:
+            help_text = alias_note
 
-    # Help text (escape pipe characters for markdown tables)
-    help_text = (arg["help"] or "-").replace("|", "\\|")
+    # Add choices info to description
+    if arg["choices"]:
+        choices_str = ", ".join(f"`{c}`" for c in arg["choices"])
+        choices_note = f"Choices: {choices_str}"
+        if help_text:
+            help_text = f"{help_text}. {choices_note}"
+        else:
+            help_text = choices_note
 
-    # Required indicator
-    required = "Yes" if arg["required"] else "No"
+    # Generate ParamField
+    lines.append(f"<ParamField {' '.join(attrs)}>")
+    if help_text:
+        lines.append(f"  {help_text}")
+    lines.append("</ParamField>")
+    lines.append("")
 
-    return f"| {name} | {type_str} | {default_str} | {required} | {help_text} |"
+    return lines
 
 
 def _generate_command_section(
@@ -105,6 +149,21 @@ def _generate_command_section(
     full_command = f"{parent_command} {name}".strip()
     heading = "#" * heading_level
 
+    # Skip commands that have no arguments and only subparsers (like "ep create")
+    # Instead, just render the subcommands directly at the same level
+    if not info["arguments"] and info["subparsers"]:
+        # Skip this level, render subcommands directly
+        for subname, subinfo in info["subparsers"].items():
+            lines.extend(
+                _generate_command_section(
+                    subname,
+                    subinfo,
+                    full_command,
+                    heading_level,  # Keep same heading level
+                )
+            )
+        return lines
+
     lines.append(f"{heading} `{full_command}`")
     lines.append("")
 
@@ -114,13 +173,10 @@ def _generate_command_section(
         lines.append(description)
         lines.append("")
 
-    # Arguments table
+    # Arguments (no extra heading to keep TOC clean)
     if info["arguments"]:
-        lines.append("| Option | Type | Default | Required | Description |")
-        lines.append("|--------|------|---------|----------|-------------|")
         for arg in info["arguments"]:
-            lines.append(_format_argument_row(arg))
-        lines.append("")
+            lines.extend(_format_argument_item(arg))
 
     # Handle nested subparsers recursively
     if info["subparsers"]:
@@ -162,22 +218,30 @@ def generate_cli_docs(parser: argparse.ArgumentParser, output_path: str) -> int:
         if name != "export-docs"  # Don't document the hidden command
     }
 
-    # Generate single page
+    # Generate single page with Mintlify frontmatter
     lines = []
-    lines.append("# CLI Reference")
+    lines.append("---")
+    lines.append("title: CLI")
+    lines.append("icon: terminal")
+    lines.append("---")
     lines.append("")
-    lines.append(f"**{info['prog']}** - {info['description']}")
+    lines.append(
+        f"The `{info['prog']}` command-line interface can {info['description'][0].lower()}{info['description'][1:]}."
+    )
+    lines.append("")
+    lines.append("```bash")
+    lines.append(f"{info['prog']} [global options] <command> [command options]")
+    lines.append("```")
     lines.append("")
 
     # Global options
     if info["arguments"]:
         lines.append("## Global Options")
         lines.append("")
-        lines.append("| Option | Type | Default | Required | Description |")
-        lines.append("|--------|------|---------|----------|-------------|")
-        for arg in info["arguments"]:
-            lines.append(_format_argument_row(arg))
+        lines.append("These options can be used with any command:")
         lines.append("")
+        for arg in info["arguments"]:
+            lines.extend(_format_argument_item(arg))
 
     # Commands section
     if visible_subparsers: