diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 43ab7d423..163249f99 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -3,7 +3,7 @@ name: CI
 on:
   push:
     branches:
-      - master
+      - "**"
     tags:
       - v*
   pull_request:
@@ -419,41 +419,61 @@ jobs:
 
   build-and-deploy-docs:
     runs-on: ubuntu-latest
-    needs: [check-python-style, build-python-wheels]
+    needs: [check-python-style, build-python-wheels, build-python-wheels-rocm]
 
+    permissions:
+      contents: write
     steps:
       - uses: actions/checkout@v4
-
       - name: Set up Python 3.10.11
         uses: actions/setup-python@v5
         with:
           python-version: "3.10.11"
-
       - name: Download CTranslate2 wheels
         uses: actions/download-artifact@v4
         with:
           pattern: python-wheels-${{ runner.os }}-*
           merge-multiple: true
           path: .
-
       - name: Install CTranslate2 wheel
         run: |
           pip install *cp310*manylinux*x86_64.whl
-
       - name: Install dependencies to build docs
         working-directory: docs
         run: |
           python -m pip install -r requirements.txt
-
       - name: Build docs
         working-directory: docs
         run: |
           python generate.py python
           sphinx-build . build
-
       - name: Deploy docs
         if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
         uses: JamesIves/github-pages-deploy-action@v4
         with:
           folder: docs/build
           clean: true
+      - name: Download ROCm wheels and zip for release
+        if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          mkdir -p release-zips
+
+          gh run download ${{ github.run_id }} \
+            --name rocm-python-wheels-Linux \
+            --dir temp-linux
+          zip -r release-zips/rocm-python-wheels-Linux.zip temp-linux/*
+          rm -rf temp-linux
+
+          gh run download ${{ github.run_id }} \
+            --name rocm-python-wheels-Windows \
+            --dir temp-windows
+          zip -r release-zips/rocm-python-wheels-Windows.zip temp-windows/*
+          rm -rf temp-windows
+
+      - name: Create GitHub Release
+        if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
+        uses: softprops/action-gh-release@v2
+        with:
+          files: release-zips/*.zip