Test ci 62

.github/workflows/ci.yml

@@ -89,8 +89,12 @@ jobs:
           repository: ${{ github.event.pull_request.head.repo.full_name }}
           submodules: true
 
-      - id: checkout_cassettes
-        name: Check out cassettes
+      - name: Setup git username and email
+        run: |
+          git config --global user.name "Auto-GPT-Bot"
+          git config --global user.email "github-bot@agpt.co"
+
+      - name: Check out cassettes
         if: ${{ startsWith(github.event_name, 'pull_request') }}
         run: |
           cassette_branch="${{ github.event.pull_request.user.login }}-${{ github.event.pull_request.head.ref }}"
@@ -101,38 +105,45 @@ jobs:
             git fetch origin ${{ github.event.pull_request.base.ref }}
 
             git checkout $cassette_branch
-
-            if git merge --no-commit --no-ff ${{ github.event.pull_request.base.ref }}; then
-              echo "Using cassettes from mirror branch, synced to upstream branch '${{ github.event.pull_request.base.ref }}'"
-            else
-              echo "Could not merge upstream changes to cassettes. Using cassettes from ${{ github.event.pull_request.base.ref }}."
-              git merge --abort
-              git checkout ${{ github.event.pull_request.base.ref }}
-
-              # Delete branch to prevent conflict when re-creating it
-              git branch -D $cassette_branch
-            fi
-            echo "cassette_branch=$(git branch --show-current)" >> $GITHUB_OUTPUT
+            git merge --no-commit --strategy-option ours origin/${{ github.event.pull_request.base.ref }}
+            echo "Using cassettes from mirror branch, synced to upstream branch '${{ github.event.pull_request.base.ref }}' if no conflicts."
           else
+            git checkout -b $cassette_branch
             echo "Branch '$cassette_branch' does not exist in cassette submodule."\
               "Using cassettes from ${{ github.event.pull_request.base.ref }}."
-            echo "cassette_branch=${{ github.event.pull_request.base.ref }}" >> $GITHUB_OUTPUT
           fi
 
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: Install dependencies
+      - name: Run pytest tests with coverage
         run: |
-          python -m pip install --upgrade pip
-          pip install -r requirements.txt
+          echo "test" >> tests/Auto-GPT-test-cassettes/test_write_file/test_write_file.yaml
+          echo "test" >> tests/integration/challenges/current_score.json
+        env:
+          CI: true
+          PROXY: ${{ secrets.PROXY }}
+          AGENT_MODE: ${{ vars.AGENT_MODE }}
+          AGENT_TYPE: ${{ vars.AGENT_TYPE }}
 
-      - name: Run pytest tests with coverage
+      - id: get_cassettes_diff
+        name: Identify differences between cassettes of current PR and target branch
+        if: ${{ startsWith(github.event_name, 'pull_request') }}
+        run: |
+          cd tests/Auto-GPT-test-cassettes
+          cassette_diff=$(git diff --name-only --diff-filter=M -- '**/*.yaml')
+
+          if [ -n "$cassette_diff" ]; then
+            echo "updated=true" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Attempt to beat Challenges if cassettes changed
+        if: ${{ startsWith(github.event_name, 'pull_request') }} && "${{ steps.push_cassettes.outputs.updated }}" == "true"
         run: |
-          pytest -n auto --cov=autogpt --cov-report term-missing --cov-branch --cov-report xml --cov-report term
-          python tests/integration/challenges/utils/build_current_score.py
+          echo "test" >> tests/Auto-GPT-test-cassettes/test_write_file/test_write_file.yaml
+          echo "test" >> tests/integration/challenges/current_score.json
         env:
           CI: true
           PROXY: ${{ secrets.PROXY }}
@@ -145,9 +156,6 @@ jobs:
       - id: setup_git_auth
         name: Set up git token authentication
         run: |
-          git config --global user.name "Auto-GPT-Bot"
-          git config --global user.email "github-bot@agpt.co"
-
           config_key="http.${{ github.server_url }}/.extraheader"
           base64_pat=$(echo -n "pat:${{ secrets.PAT_REVIEW }}" | base64 -w0)
 
@@ -173,41 +181,29 @@ jobs:
             echo "The challenge scores didn't change."
           fi
 
-      - id: push_cassettes
-        name: Push updated cassettes
+      - name: Push updated cassettes
         run: |
           if [ "${{ startsWith(github.event_name, 'pull_request') }}" = "true" ]; then
             is_pull_request=true
             cassette_branch="${{ github.event.pull_request.user.login }}-${{ github.event.pull_request.head.ref }}"
-            cassette_source_branch="${{ steps.checkout_cassettes.outputs.cassette_branch }}"
-            base_branch="${{ github.event.pull_request.base.ref }}"
           else
             current_branch=$(echo ${{ github.ref }} | sed -e "s/refs\/heads\///g")
             cassette_branch=$current_branch
           fi
 
           cd tests/Auto-GPT-test-cassettes
-          git fetch origin $cassette_source_branch:$cassette_source_branch
 
           # Commit & push changes to cassettes if any
-          if ! git diff --quiet $cassette_source_branch --; then
-            if [ "$cassette_branch" != "$cassette_source_branch" ]; then
-              git checkout -b $cassette_branch
-            fi
+          if ! git diff --quiet; then
             git add .
             git commit -m "Auto-update cassettes"
 
             if [ $is_pull_request ]; then
-              git push --force origin HEAD:$cassette_branch
+              git push -f origin $cassette_branch
             else
+              git pull --rebase origin $cassette_branch
               git push origin HEAD:$cassette_branch
-            fi
-
-            cd ../..
-            if [ $is_pull_request ]; then
-              git fetch origin $base_branch
-              cassette_diff=$(git diff origin/$base_branch)
-            else
+              cd ../..
               git add tests/Auto-GPT-test-cassettes
               git commit -m "Update cassette submodule"
               git push origin HEAD:$current_branch
@@ -216,10 +212,51 @@ jobs:
             echo "No cassette changes to commit"
           fi
 
-          if [ -n "$cassette_diff" ]; then
-            echo "updated=true" >> $GITHUB_OUTPUT
+      - id: create_pr
+        name: Create Pull Request if score changed
+        if: ${{ startsWith(github.event_name, 'pull_request') }}
+        run: |
+          base_repo_owner=${{ github.event.pull_request.base.repo.owner.login }}
+          base_repo_name=${{ github.event.pull_request.base.repo.name }}
+          updated_score_branch="update-score-${{ github.event.pull_request.number }}-$(date +'%Y%m%d%H%M%S')"
+
+          if ! git diff --quiet -- tests/integration/challenges/current_score.json; then          
+            git add tests/integration/challenges/current_score.json
+            git commit -m "Update score"
+            git checkout -b $updated_score_branch
+
+            git remote add base_repo https://github.com/$base_repo_owner/$base_repo_name.git
+            echo ${{ secrets.PAT_REVIEW }} | gh auth login --with-token
+            branches_to_delete=$(gh api repos/$base_repo_owner/$base_repo_name/git/refs --jq ".[] | select(.ref | startswith(\"refs/heads/update-score-${{ github.event.pull_request.number }}\")) | .ref")
+
+            if [ ! -z "$branches_to_delete" ]; then
+              for branch in $branches_to_delete
+              do
+                branch_short=${branch#refs/heads/}
+                prs=$(gh api repos/${{ github.repository }}/pulls -q ".[] | select(.head.ref == \"$branch_short\") | .number")
+
+                # iterate through all previous PRs created, close them and delete the branch
+                for pr in $prs
+                do
+                  gh api repos/${{ github.repository }}/pulls/$pr -X PATCH -F state="closed"
+                done
+                gh api -X DELETE /repos/$base_repo_owner/$base_repo_name/git/$branch
+              done
+            else
+              echo "No matching branches found to delete."
+            fi
+
+            git push base_repo $updated_score_branch
+
+            pr_url=$(gh pr create --title "Update Score In Pull Request Number ${{ github.event.pull_request.number }}" \
+            --head "$base_repo_owner:$updated_score_branch" \
+            --base "${{ github.event.pull_request.head.ref }}" \
+            --body "This pull request updates the current score of Auto-GPT. Please check the files changed and merge the pull request." \
+            --repo ${{ github.event.pull_request.head.repo.full_name }})
+
+            echo "pr_url=$pr_url" >> $GITHUB_OUTPUT
           else
-            echo "updated=false" >> $GITHUB_OUTPUT
+            echo "The current score didn't change."
           fi
 
       - name: Post Set up git token auth
@@ -228,27 +265,34 @@ jobs:
           git config --unset-all '${{ steps.setup_git_auth.outputs.config_key }}'
           git submodule foreach git config --unset-all '${{ steps.setup_git_auth.outputs.config_key }}'
 
-      - name: Apply or remove behaviour change label and comment on PR
+      - name: Apply or remove behaviour change label and comment
         if: ${{ startsWith(github.event_name, 'pull_request') }}
         run: |
-          PR_NUMBER=${{ github.event.pull_request.number }}
-          TOKEN=${{ secrets.PAT_REVIEW }}
           REPO=${{ github.repository }}
-
+          echo ${{ secrets.PAT_REVIEW }} | gh auth login --with-token
+          COMMENT_BODY=""
+          SCORE_CHANGED_MESSAGE="The pipeline will fail until you merge the updated score: ${{ steps.create_pr.outputs.pr_url }}"
+
           if [[ "${{ steps.push_cassettes.outputs.updated }}" == "true" ]]; then
             echo "Adding label and comment..."
             curl -X POST \
-            -H "Authorization: Bearer $TOKEN" \
+            -H "Authorization: Bearer ${{ secrets.PAT_REVIEW }}" \
             -H "Accept: application/vnd.github.v3+json" \
-            https://api.github.com/repos/$REPO/issues/$PR_NUMBER/labels \
+            https://api.github.com/repos/$REPO/issues/${{ github.event.pull_request.number }}/labels \
             -d '{"labels":["behaviour change"]}'
-
-            echo $TOKEN | gh auth login --with-token
-            gh api repos/$REPO/issues/$PR_NUMBER/comments -X POST -F body="You changed AutoGPT's behaviour. The cassettes have been updated and will be merged to the submodule when this Pull Request gets merged."
-          else
-            echo "Removing label..."
-            curl -X DELETE \
-            -H "Authorization: Bearer $TOKEN" \
-            -H "Accept: application/vnd.github.v3+json" \
-            https://api.github.com/repos/$REPO/issues/$PR_NUMBER/labels/behaviour%20change
+
+            COMMENT_BODY="You changed AutoGPT's behaviour. The cassettes have been updated and will be merged to the submodule when this Pull Request gets merged. "
+          fi
+
+          if [[ -n "${{ steps.create_pr.outputs.pr_url }}" ]]; then
+            COMMENT_BODY="$COMMENT_BODY$SCORE_CHANGED_MESSAGE"
+          fi
+
+          if [ ! -z "$COMMENT_BODY" ]; then
+            gh api repos/$REPO/issues/${{ github.event.pull_request.number }}/comments -X POST -F body="$COMMENT_BODY"
+          fi
+
+          if [[ -n "${{ steps.create_pr.outputs.pr_url }}" ]]; then
+            echo "$SCORE_CHANGED_MESSAGE"
+            exit 1
           fi

tests/integration/challenges/current_score.json

@@ -46,3 +46,5 @@
         }
     }
 }
+test
+test