[RHDHPAI-1170] Add Question Validation Prompt Template Sync Script

.github/workflows/validation.yml

@@ -0,0 +1,21 @@
+name: Question Validation Sync Check
+
+on:
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  validate-prompt:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    defaults:
+      run:
+        working-directory: ./scripts/python-scripts
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Setup Environment
+        run: pip3 install -r requirements.txt
+      - name: Validate prompt
+        run: python3 sync.py -t validate

.gitignore

@@ -4,4 +4,6 @@
 embeddings_model
 vector_db
 
-values.env
+values.env
+
+.venv

Makefile

@@ -1,4 +1,7 @@
 RAG_CONTENT_IMAGE ?= quay.io/redhat-ai-dev/rag-content:release-1.7-lcs
+VENV := $(CURDIR)/scripts/python-scripts/.venv
+PYTHON := $(VENV)/bin/python3
+PIP := $(VENV)/bin/pip3
 
 default: help
 
@@ -20,4 +23,21 @@ help: ## Show this help screen
 
 # TODO (Jdubrick): Replace reference to lightspeed-core/lightspeed-providers once bug is addressed.
 update-question-validation:
-	curl -o ./config/providers.d/inline/safety/lightspeed_question_validity.yaml https://raw.githubusercontent.com/Jdubrick/lightspeed-providers/refs/heads/devai/resources/external_providers/inline/safety/lightspeed_question_validity.yaml
+	curl -o ./config/providers.d/inline/safety/lightspeed_question_validity.yaml https://raw.githubusercontent.com/Jdubrick/lightspeed-providers/refs/heads/devai/resources/external_providers/inline/safety/lightspeed_question_validity.yaml
+
+$(VENV)/bin/activate: ./scripts/python-scripts/requirements.txt
+	python3 -m venv $(VENV)
+	$(PIP) install -r scripts/python-scripts/requirements.txt
+	touch $(VENV)/bin/activate
+
+define run_sync
+	cd ./scripts/python-scripts && \
+	$(PYTHON) sync.py -t $(1)
+endef
+
+.PHONY: validate-prompt-templates update-prompt-templates
+validate-prompt-templates: $(VENV)/bin/activate
+	$(call run_sync,validate)
+
+update-prompt-templates: $(VENV)/bin/activate
+	$(call run_sync,update)

README.md

@@ -141,6 +141,8 @@ To deploy on a cluster see [DEPLOYMENT.md](./docs/DEPLOYMENT.md).
 | ---- | ----|
 | **get-rag** | Gets the RAG data and the embeddings model from the rag-content image registry to your local project directory |
 | **update-question-validation** | Updates the question validation content in `providers.d` |
+| **validate-prompt-templates** | Validates prompt values in run.yaml. **Requires Python >= 3.11** |
+| **update-prompt-templates** | Updates the prompt values in run.yaml. **Requires Python >= 3.11** |
 
 ## Contributing
 

run.yaml

@@ -2,32 +2,32 @@ version: '2'
 image_name: minimal-viable-llama-stack-configuration
 
 apis:
-  - agents
-  - datasetio
-  - eval
-  - inference
-  - post_training
-  - safety
-  - scoring
-  - telemetry
-  - tool_runtime
-  - vector_io
+- agents
+- datasetio
+- eval
+- inference
+- post_training
+- safety
+- scoring
+- telemetry
+- tool_runtime
+- vector_io
 benchmarks: []
-container_image: null
+container_image:
 datasets: []
 external_providers_dir: "/app-root/config/providers.d"
 inference_store:
   db_path: .llama/distributions/ollama/inference_store.db
   type: sqlite
-logging: null
+logging:
 metadata_store:
   db_path: .llama/distributions/ollama/registry.db
-  namespace: null
+  namespace:
   type: sqlite
 models:
 - model_id: sentence-transformers/all-mpnet-base-v2
   metadata:
-      embedding_dimension: 768
+    embedding_dimension: 768
   model_type: embedding
   provider_id: sentence-transformers
   provider_model_id: "/app-root/embeddings_model"
@@ -36,7 +36,7 @@ providers:
   - config:
       persistence_store:
         db_path: .llama/distributions/ollama/agents_store.db
-        namespace: null
+        namespace:
         type: sqlite
       responses_store:
         db_path: .llama/distributions/ollama/responses_store.db
@@ -47,22 +47,22 @@ providers:
   - config:
       kvstore:
         db_path: .llama/distributions/ollama/huggingface_datasetio.db
-        namespace: null
+        namespace:
         type: sqlite
     provider_id: huggingface
     provider_type: remote::huggingface
   - config:
       kvstore:
         db_path: .llama/distributions/ollama/localfs_datasetio.db
-        namespace: null
+        namespace:
         type: sqlite
     provider_id: localfs
     provider_type: inline::localfs
   eval:
   - config:
       kvstore:
         db_path: .llama/distributions/ollama/meta_reference_eval.db
-        namespace: null
+        namespace:
         type: sqlite
     provider_id: meta-reference
     provider_type: inline::meta-reference
@@ -94,80 +94,80 @@ providers:
   - config:
       checkpoint_format: huggingface
       device: cpu
-      distributed_backend: null
+      distributed_backend:
       dpo_output_dir: "."
     provider_id: huggingface
     provider_type: inline::huggingface
   safety:
-    - config:
-        excluded_categories: []
-      provider_id: llama-guard
-      provider_type: inline::llama-guard
-    - provider_id: lightspeed_question_validity
-      provider_type: inline::lightspeed_question_validity
-      config:
-        model_id: ${env.VALIDATION_PROVIDER:=vllm}/${env.VALIDATION_MODEL_NAME}
-        model_prompt: |-
-          Instructions:
+  - config:
+      excluded_categories: []
+    provider_id: llama-guard
+    provider_type: inline::llama-guard
+  - provider_id: lightspeed_question_validity
+    provider_type: inline::lightspeed_question_validity
+    config:
+      model_id: ${env.VALIDATION_PROVIDER:=vllm}/${env.VALIDATION_MODEL_NAME}
+      model_prompt: |-
+        Instructions:
+
+        You area question classification tool. You are an expert in the following categories:
+        - Backstage
+        - Red Hat Developer Hub (RHDH)
+        - Kubernetes
+        - Openshift
+        - CI/CD
+        - GitOps
+        - Pipelines
+        - Developer Portals
+        - Deployments
+        - Software Catalogs
+        - Software Templates
+        - Tech Docs
 
-          You area question classification tool. You are an expert in the following categories:
-          - Backstage
-          - Red Hat Developer Hub (RHDH)
-          - Kubernetes
-          - Openshift
-          - CI/CD
-          - GitOps
-          - Pipelines
-          - Developer Portals
-          - Deployments
-          - Software Catalogs
-          - Software Templates
-          - Tech Docs
-
-          Your job is to determine if a user's question is related to the categories you are an expert in. If the question is related to those categories, \
-          or any features that may be related to those categories, you will answer with ${allowed}.
+        Your job is to determine if a user's question is related to the categories you are an expert in. If the question is related to those categories, \
+        or any features that may be related to those categories, you will answer with ${allowed}.
 
-          If a question is not related to your expert categories, answer with ${rejected}.
+        If a question is not related to your expert categories, answer with ${rejected}.
 
-          You do not need to explain your answer.
+        You do not need to explain your answer.
 
-          Below are some example questions:
-          Example Question:
-          Why is the sky blue?
-          Example Response:
-          ${rejected}
+        Below are some example questions:
+        Example Question:
+        Why is the sky blue?
+        Example Response:
+        ${rejected}
 
-          Example Question:
-          Can you help configure my cluster to automatically scale?
-          Example Response:
-          ${allowed}
+        Example Question:
+        Can you help configure my cluster to automatically scale?
+        Example Response:
+        ${allowed}
 
-          Example Question:
-          How do I create import an existing software template in Backstage?
-          Example Response:
-          ${allowed}
+        Example Question:
+        How do I create import an existing software template in Backstage?
+        Example Response:
+        ${allowed}
 
-          Example Question:
-          How do I accomplish a task in RHDH?
-          Example Response:
-          ${allowed}
+        Example Question:
+        How do I accomplish a task in RHDH?
+        Example Response:
+        ${allowed}
 
-          Example Question:
-          How do I explore a component in RHDH catalog?
-          Example Response:
-          ${allowed}
+        Example Question:
+        How do I explore a component in RHDH catalog?
+        Example Response:
+        ${allowed}
 
-          Example Question:
-          How can I integrate GitOps into my pipeline?
-          Example Response:
-          ${allowed}
+        Example Question:
+        How can I integrate GitOps into my pipeline?
+        Example Response:
+        ${allowed}
 
-          Question:
-          ${message}
-          Response:
-        invalid_question_response: |-
-          Hi, I'm the Red Hat Developer Hub Lightspeed assistant, I can help you with questions about Red Hat Developer Hub or Backstage.
-          Please ensure your question is about these topics, and feel free to ask again!
+        Question:
+        ${message}
+        Response:
+      invalid_question_response: |-
+        Hi, I'm the Red Hat Developer Hub Lightspeed assistant, I can help you with questions about Red Hat Developer Hub or Backstage.
+        Please ensure your question is about these topics, and feel free to ask again!
   scoring:
   - config: {}
     provider_id: basic
@@ -190,36 +190,36 @@ providers:
   - provider_id: model-context-protocol
     provider_type: remote::model-context-protocol
     config: {}
-  - provider_id: rag-runtime 
+  - provider_id: rag-runtime
     provider_type: inline::rag-runtime
     config: {}
   vector_io:
   - config:
       kvstore:
         db_path: .llama/distributions/ollama/faiss_store.db
-        namespace: null
+        namespace:
         type: sqlite
     provider_id: faiss
     provider_type: inline::faiss
-  - provider_id: rhdh-docs 
+  - provider_id: rhdh-docs
     provider_type: inline::faiss
     config:
       kvstore:
         type: sqlite
-        namespace: null
+        namespace:
         db_path: /app-root/vector_db/rhdh_product_docs/1.7/faiss_store.db
 scoring_fns: []
 server:
-  auth: null
-  host: null
+  auth:
+  host:
   port: 8321
-  quota: null
-  tls_cafile: null
-  tls_certfile: null
-  tls_keyfile: null
+  quota:
+  tls_cafile:
+  tls_certfile:
+  tls_keyfile:
 shields:
-  - shield_id: lightspeed_question_validity-shield
-    provider_id: lightspeed_question_validity
+- shield_id: lightspeed_question_validity-shield
+  provider_id: lightspeed_question_validity
 tool_groups:
 - provider_id: rag-runtime
   toolgroup_id: builtin::rag

scripts/python-scripts/requirements.txt

@@ -0,0 +1,2 @@
+ruamel.yaml>=0.17.0
+requests>=2.25.0