feat: annotate automatic tools and exclude from fluency scoring (#529) #78
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Sync Tool Names from vscode-copilot-chat | |
| on: | |
| workflow_dispatch: # Manual trigger | |
| schedule: | |
| - cron: '30 17 * * 1' # Run every Monday at 17:30 UTC | |
| push: | |
| paths: | |
| - .github/workflows/sync-toolnames.yml | |
| - .github/prompts/sync-toolnames.prompt.md | |
| permissions: | |
| contents: read | |
| jobs: | |
| sync-toolnames: | |
| runs-on: ubuntu-latest | |
| if: github.actor != 'dependabot[bot]' && !startsWith(github.ref_name, 'dependabot/') | |
| permissions: | |
| contents: write # Need write permission to create branches and push changes | |
| pull-requests: write # Need write permission to create PRs | |
| steps: | |
| - name: Harden the runner (Audit all outbound calls) | |
| uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 | |
| with: | |
| egress-policy: audit | |
| - name: Checkout current repository | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| path: current-repo | |
| fetch-depth: 0 | |
| - name: Checkout vscode-copilot-chat repository | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| repository: microsoft/vscode-copilot-chat | |
| path: vscode-copilot-chat | |
| fetch-depth: 1 | |
| - name: Setup Node.js | |
| uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 | |
| with: | |
| node-version: '20.x' | |
| - name: Authenticate GitHub CLI | |
| env: | |
| # Try to use COPILOT_CLI_TOKEN first, fall back to GITHUB_TOKEN | |
| GH_TOKEN: ${{ secrets.GH_PAT }} | |
| run: | | |
| echo "=== Authenticating GitHub CLI ===" | |
| if [ -z "$GH_TOKEN" ]; then | |
| echo "❌ Error: No authentication token available" | |
| echo "Environment variable GH_TOKEN is empty" | |
| exit 1 | |
| fi | |
| echo "" | |
| echo "=== Checking authentication status ===" | |
| if gh auth status 2>&1; then | |
| echo "✅ Authentication verified" | |
| else | |
| echo "❌ Authentication check failed" | |
| echo "Note: This may indicate an issue with token permissions or GitHub CLI configuration" | |
| exit 1 | |
| fi | |
| - name: Install dependencies | |
| working-directory: current-repo | |
| run: npm ci | |
| - name: Check Copilot CLI version | |
| env: | |
| GH_TOKEN: ${{ secrets.GH_PAT }} | |
| COPILOT_GITHUB_TOKEN: ${{ secrets.GH_PAT }} | |
| run: | | |
| npx @github/copilot --version | |
| - name: Run Copilot CLI with prompt | |
| id: copilot_run | |
| working-directory: current-repo | |
| env: | |
| GH_TOKEN: ${{ secrets.GH_PAT }} | |
| COPILOT_GITHUB_TOKEN: ${{ secrets.GH_PAT }} | |
| run: | | |
| # Read the prompt from the file | |
| PROMPT=$(cat .github/prompts/sync-toolnames.prompt.md) | |
| # Get the path to the vscode-copilot-chat repo | |
| UPSTREAM_PATH="${GITHUB_WORKSPACE}/vscode-copilot-chat" | |
| # Get the path to toolNames.json | |
| TOOLNAMES_PATH="${GITHUB_WORKSPACE}/current-repo/src/toolNames.json" | |
| # Create the full prompt with context - append context to the base prompt | |
| { | |
| echo "$PROMPT" | |
| echo "" | |
| echo "## Context Paths" | |
| echo "" | |
| echo "- Upstream repository path: ${UPSTREAM_PATH}" | |
| echo "- Current repository toolNames.json path: ${TOOLNAMES_PATH}" | |
| echo "- Upstream toolNames.ts path: ${UPSTREAM_PATH}/src/extension/tools/common/toolNames.ts" | |
| echo "" | |
| echo "## Additional Instructions" | |
| echo "" | |
| echo "Please check if the upstream file exists at the specified path. If it doesn't, search for toolNames.ts or similar files in the upstream repository and use the correct path." | |
| } > /tmp/copilot-prompt.md | |
| echo "=== Running Copilot CLI with prompt ===" | |
| cat /tmp/copilot-prompt.md | |
| echo "" | |
| echo "=== Copilot CLI Output ===" | |
| cd "${GITHUB_WORKSPACE}/current-repo/" | |
| # Run gh copilot with the prompt using non-interactive mode | |
| # Use -s/--silent for script-friendly output, -p for prompt, --allow-all for permissions | |
| # Read the prompt into a variable for safer handling | |
| COPILOT_PROMPT_TEXT=$(cat /tmp/copilot-prompt.md) | |
| # > /tmp/copilot-output.txt | |
| npx @github/copilot -p "$COPILOT_PROMPT_TEXT" --silent --allow-all --add-dir "${UPSTREAM_PATH}" --add-dir "${GITHUB_WORKSPACE}/current-repo" --no-ask-user 2>&1 || { | |
| EXIT_CODE=$? | |
| echo "Warning: gh copilot command exited with code $EXIT_CODE, but continuing to check output" | |
| cat /tmp/copilot-output.txt | |
| } | |
| - name: Check for changes | |
| id: changes | |
| working-directory: current-repo | |
| run: | | |
| if git diff --quiet; then | |
| echo "No changes detected in toolNames.json" | |
| echo "changed=false" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "Changes detected in toolNames.json" | |
| echo "changed=true" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Create Pull Request | |
| if: steps.changes.outputs.changed == 'true' | |
| uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0 # v8.1.0 | |
| with: | |
| path: current-repo | |
| branch: sync-toolnames | |
| add-paths: | | |
| src/toolNames.json | |
| commit-message: | | |
| chore: sync toolNames.json with vscode-copilot-chat | |
| This commit automatically updates src/toolNames.json with new tool | |
| identifiers from microsoft/vscode-copilot-chat repository. | |
| Source: microsoft/vscode-copilot-chat | |
| File: src/extension/tools/common/toolNames.ts | |
| title: "chore: sync toolNames.json with vscode-copilot-chat" | |
| body: | | |
| This pull request updates `src/toolNames.json` with new tool identifiers | |
| found in the `microsoft/vscode-copilot-chat` repository. | |
| **Source Repository:** microsoft/vscode-copilot-chat | |
| **Source File:** src/extension/tools/common/toolNames.ts | |
| The changes were automatically generated by scanning the upstream repository | |
| and comparing tool IDs with our existing mapping. | |
| Please review the new entries to ensure the friendly names are appropriate. | |
| labels: | | |
| autogenerated | |
| - name: Summary | |
| working-directory: current-repo | |
| run: | | |
| if [ "${{ steps.changes.outputs.changed }}" == "true" ]; then | |
| echo "✅ toolNames.json has been successfully updated with new tool names from vscode-copilot-chat" | |
| echo "" | |
| echo "A pull request has been created for review." | |
| else | |
| echo "ℹ️ toolNames.json was already up to date with vscode-copilot-chat" | |
| fi |