Generate a list of NIM models in pre-commit

.pre-commit-config.yaml

@@ -61,5 +61,15 @@ repos:
         args: ["-q", "--config", "ci/markdown-link-check-config.json"]
         exclude: "^(packages/nvidia_nat_core/src/nat/meta/pypi\\.md|CHANGELOG\\.md)$"
 
+  - repo: local
+    hooks:
+      - id: generate-models-list
+        name: Generate Models List
+        entry: python ./ci/scripts/model_health_check.py --log-level=WARNING --dry-run --output-json ci/.nim_models_used.json
+        language: python
+        files: "^examples/.*\\.(yml|yaml)$"
+        pass_filenames: false
+        additional_dependencies: ["pyyaml"]
+
 default_language_version:
   python: python3

ci/.nim_models_used.json

@@ -0,0 +1,58 @@
+{
+  "llms": [
+    {
+      "model": "meta/llama-3.1-70b-instruct",
+      "usage_count": 65
+    },
+    {
+      "model": "nvidia/nemotron-3-nano-30b-a3b",
+      "usage_count": 62
+    },
+    {
+      "model": "meta/llama-3.3-70b-instruct",
+      "usage_count": 41
+    },
+    {
+      "model": "nvidia/nemotron-3-super-120b-a12b",
+      "usage_count": 24
+    },
+    {
+      "model": "meta/llama-3.1-8b-instruct",
+      "usage_count": 9
+    },
+    {
+      "model": "nvidia/llama-3.3-nemotron-super-49b-v1",
+      "usage_count": 4
+    },
+    {
+      "model": "nvidia/llama-3.3-nemotron-super-49b-v1.5",
+      "usage_count": 3
+    },
+    {
+      "model": "nvidia/nvidia-nemotron-nano-9b-v2",
+      "usage_count": 1
+    },
+    {
+      "model": "nvidia/llama-3.1-nemotron-safety-guard-8b-v3",
+      "usage_count": 1
+    },
+    {
+      "model": "mistralai/mistral-small-4-119b-2603",
+      "usage_count": 1
+    },
+    {
+      "model": "mistralai/mistral-large-3-675b-instruct-2512",
+      "usage_count": 1
+    }
+  ],
+  "embedders": [
+    {
+      "model": "nvidia/nv-embedqa-e5-v5",
+      "usage_count": 23
+    },
+    {
+      "model": "nvidia/llama-nemotron-embed-1b-v2",
+      "usage_count": 1
+    }
+  ]
+}

ci/scripts/gitlab/report_test_results.py

@@ -178,8 +178,9 @@ def build_model_health_messages(health_data: dict[str, typing.Any]) -> ReportMes
 
     removed = health_data.get("removed", [])
     down = health_data.get("down", [])
+    deprecated = health_data.get("deprecated", [])
     ok = health_data.get("ok", [])
-    num_failures = len(removed) + len(down)
+    num_failures = len(removed) + len(down) + len(deprecated)
 
     plain_text: list[str] = []
     blocks: list[dict] = []
@@ -196,8 +197,9 @@ def build_model_health_messages(health_data: dict[str, typing.Any]) -> ReportMes
     stats = "\n".join([
         get_error_string(len(removed), "Removed"),
         get_error_string(len(down), "Down"),
+        get_error_string(len(deprecated), "Deprecated"),
         f"OK: {len(ok)}",
-        f"Total models: {len(removed) + len(down) + len(ok)}",
+        f"Total models: {len(removed) + len(down) + len(deprecated) + len(ok)}",
     ])
     add_text(stats, blocks, plain_text)
 
@@ -228,6 +230,16 @@ def build_model_health_messages(health_data: dict[str, typing.Any]) -> ReportMes
                 msg = f"`{model_name}` ({model_type}) HTTP {status}: {detail}\n{configs}"
                 add_text(msg, failure_blocks, failure_text)
 
+        if deprecated:
+            add_text(f"*Deprecated ({len(deprecated)}):*", failure_blocks, failure_text)
+            for entry in deprecated:
+                detail = entry.get("detail", "")
+                configs = "\n".join(f"  - {c}" for c in entry.get("configs", []))
+                model_name = entry.get('model', 'unknown')
+                model_type = entry.get('type', 'unknown')
+                msg = f"`{model_name}` ({model_type}) Deprecated: {detail}\n{configs}"
+                add_text(msg, failure_blocks, failure_text)
+
         job_url = os.environ.get("CI_JOB_URL")
         if job_url is not None:
             failure_blocks.append({"type": "divider"})