don't add optional dependencies to requirements.txt

Signed-off-by: Vikrant Puppala <vikrant.puppala@databricks.com>

Author: vikrantpuppala

.github/workflows/code-quality-checks.yml

@@ -152,7 +152,7 @@ jobs:
       -   name: Generate requirements file with pyarrow
           if: matrix.dependency-version != 'default'
           run: |
-            poetry run python scripts/dependency_manager.py ${{ matrix.dependency-version }} --include-optional --output requirements-${{ matrix.dependency-version }}-arrow.txt
+            poetry run python scripts/dependency_manager.py ${{ matrix.dependency-version }} --output requirements-${{ matrix.dependency-version }}-arrow.txt
             echo "Generated requirements for ${{ matrix.dependency-version }} versions with PyArrow:"
             cat requirements-${{ matrix.dependency-version }}-arrow.txt
 

.github/workflows/integration.yml

@@ -67,7 +67,7 @@ jobs:
       - name: Generate requirements file for e2e
         if: matrix.dependency-version != 'default'
         run: |
-          poetry run python scripts/dependency_manager.py ${{ matrix.dependency-version }} --include-optional --output requirements-${{ matrix.dependency-version }}-e2e.txt
+          poetry run python scripts/dependency_manager.py ${{ matrix.dependency-version }} --output requirements-${{ matrix.dependency-version }}-e2e.txt
           echo "Generated requirements for ${{ matrix.dependency-version }} versions (E2E):"
           cat requirements-${{ matrix.dependency-version }}-e2e.txt