https://raw.githubusercontent.com/MicrosoftLearning/mslearn-genaiops/refs/heads/main/docs/04-automated-evaluation.md - Issues #13
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Train model in prod (PR comment) | |
| on: | |
| issue_comment: | |
| types: [created] | |
| permissions: | |
| contents: read | |
| issues: write | |
| jobs: | |
| train-prod: | |
| if: >- | |
| github.event.issue.pull_request != null && | |
| contains(github.event.comment.body, '/train-prod') | |
| runs-on: ubuntu-latest | |
| environment: prod | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@v4 | |
| - name: Sign in to Azure | |
| uses: azure/login@v2 | |
| with: | |
| creds: ${{ secrets.AZURE_CREDENTIALS }} | |
| - name: Install Azure ML CLI | |
| run: | | |
| az extension add -n ml -y | |
| - name: Detect Azure ML workspace | |
| id: detect | |
| run: | | |
| RG_NAME=$(az group list --query "[?starts_with(name,'rg-ai300-l')].name | [0]" -o tsv) | |
| if [ -z "$RG_NAME" ]; then | |
| echo "No resource group starting with rg-ai300-l found." >&2 | |
| exit 1 | |
| fi | |
| WS_NAME=$(az ml workspace list --resource-group "$RG_NAME" --query "[?starts_with(name,'mlw-ai300-l')].name | [0]" -o tsv) | |
| if [ -z "$WS_NAME" ]; then | |
| echo "No workspace starting with mlw-ai300-l found in $RG_NAME." >&2 | |
| exit 1 | |
| fi | |
| az configure --defaults group="$RG_NAME" workspace="$WS_NAME" | |
| echo "resource_group=$RG_NAME" >> "$GITHUB_OUTPUT" | |
| echo "workspace=$WS_NAME" >> "$GITHUB_OUTPUT" | |
| - name: Run training job in prod and capture logs | |
| id: train | |
| run: | | |
| JOB_NAME="diabetes-train-prod-${{ github.run_id }}" | |
| echo "Running Azure ML job in prod: $JOB_NAME" | |
| az ml job create \ | |
| -f src/job.yml \ | |
| --name "$JOB_NAME" \ | |
| --set inputs.training_data.path=azureml:diabetes-prod-folder@latest \ | |
| --stream | tee training_output.log | |
| - name: Extract prod metrics from logs | |
| id: parse-metrics | |
| run: | | |
| if [ ! -f training_output.log ]; then | |
| echo "No training_output.log found to parse metrics." >&2 | |
| exit 1 | |
| fi | |
| ACC=$(grep -o "Accuracy: .*" training_output.log | tail -n 1 | awk '{print $2}') || true | |
| AUC=$(grep -o "AUC: .*" training_output.log | tail -n 1 | awk '{print $2}') || true | |
| echo "Prod accuracy: $ACC" | |
| echo "Prod AUC: $AUC" | |
| echo "prod_accuracy=$ACC" >> "$GITHUB_OUTPUT" | |
| echo "prod_auc=$AUC" >> "$GITHUB_OUTPUT" | |
| - name: Comment prod metrics on pull request | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const acc = '${{ steps.parse-metrics.outputs.prod_accuracy }}'; | |
| const auc = '${{ steps.parse-metrics.outputs.prod_auc }}'; | |
| const prNumber = context.issue.number; | |
| let body = '✅ Prod training workflow completed.'; | |
| if (acc || auc) { | |
| body += '\n\n**Prod evaluation metrics**'; | |
| if (acc) body += `\n- Accuracy: ${acc}`; | |
| if (auc) body += `\n- AUC: ${auc}`; | |
| } else { | |
| body += '\n\nMetrics could not be parsed from logs. Check the Azure Machine Learning job run for details.'; | |
| } | |
| await github.rest.issues.createComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: prNumber, | |
| body, | |
| }); |