feat: Modernize precommit hooks and optimize test performance

.github/workflows/linter.yml

@@ -22,6 +22,14 @@ jobs:
       - name: Install dependencies
         run: |
           make install-python-dependencies-ci
+      - name: Cache MyPy
+        uses: actions/cache@v4
+        with:
+          path: sdk/python/.mypy_cache
+          key: mypy-${{ runner.os }}-py${{ env.PYTHON }}-${{ hashFiles('pyproject.toml', 'uv.lock', 'requirements*.txt', 'mypy.ini', 'setup.cfg') }}
+          restore-keys: |
+            mypy-${{ runner.os }}-py${{ env.PYTHON }}-
+            mypy-${{ runner.os }}-
       - name: Lint python
         run: make lint-python
       - name: Minimize uv cache

.github/workflows/registry-rest-api-tests.yml

@@ -146,7 +146,7 @@ jobs:
         run: |
           echo "Running Registry REST API tests..."
           cd sdk/python
-          pytest tests/integration/registration/rest_api/test_registry_rest_api.py --integration -s
+          uv run pytest tests/integration/registration/rest_api/test_registry_rest_api.py --integration -s
 
       - name: Clean up docker images
         if: always()

.github/workflows/smoke_tests.yml

@@ -29,6 +29,6 @@ jobs:
       - name: Install dependencies
         run: make install-python-dependencies-ci
       - name: Test Imports
-        run: python -c "from feast.cli import cli"
+        run: uv run python -c "from feast.cli import cli"
       - name: Minimize uv cache
         run: uv cache prune --ci

.github/workflows/unit_tests.yml

@@ -33,10 +33,40 @@ jobs:
         uses: astral-sh/setup-uv@v5
         with:
           enable-cache: true
+      - name: Install system dependencies
+        run: |
+          if [ "$RUNNER_OS" = "Linux" ]; then
+            sudo apt-get update
+            sudo apt-get install -y make
+          elif [ "$RUNNER_OS" = "macOS" ]; then
+            # make is already installed on macOS runners
+            which make || brew install make
+          fi
       - name: Install dependencies
         run: make install-python-dependencies-ci
       - name: Test Python
-        run: make test-python-unit
+        run: |
+          # Set up environment for Ray workers to access packages
+          export PATH="${{ github.workspace }}/.venv/bin:$PATH"
+
+          # Dynamically detect site-packages for uv env
+          SITE_PACKAGES=$(uv run python -c "import site; print(site.getsitepackages()[0])")
+
+          # Preserve any existing PYTHONPATH and add repo + site-packages
+          export PYTHONPATH="${{ github.workspace }}/sdk/python:${PYTHONPATH}:$SITE_PACKAGES"
+
+          echo "Using PYTHONPATH: $PYTHONPATH"
+          echo "Using PATH: $PATH"
+
+          # Ray macOS workarounds
+          if [[ "$RUNNER_OS" == "macOS" ]]; then
+            echo "=== Applying macOS Ray compatibility workarounds ==="
+            export RAY_DISABLE_RUNTIME_ENV_HOOK=1
+            export PYTHONDONTWRITEBYTECODE=1
+            echo "Applied macOS workarounds for Python ${{ matrix.python-version }}"
+          fi
+
+          make test-python-unit
       - name: Minimize uv cache
         run: uv cache prune --ci
 

.gitignore

@@ -239,3 +239,5 @@ infra/website/dist/
 
 # offline builds
 offline_build/
+feast_profile_demo/
+scripts/perf-monitor.py

.pre-commit-config.yaml

@@ -1,23 +1,23 @@
-default_stages:
-  - push
+default_stages: [commit]
+
 repos:
   - repo: local
     hooks:
       - id: format
         name: Format
-        stages: [ push ]
+        stages: [commit]
         language: system
         entry: make format-python
         pass_filenames: false
       - id: lint
         name: Lint
-        stages: [ push ]
+        stages: [commit]
         language: system
         entry: make lint-python
         pass_filenames: false
       - id: template
         name: Build Templates
-        stages: [ commit ]
+        stages: [commit]
         language: system
         entry: make build-templates
-        pass_filenames: false
+        pass_filenames: false

Makefile

@@ -55,14 +55,35 @@ protos: compile-protos-python compile-protos-docs ## Compile protobufs for Pytho
 build: protos build-docker ## Build protobufs and Docker images
 
 format-python: ## Format Python code
-	cd ${ROOT_DIR}/sdk/python; python -m ruff check --fix feast/ tests/
-	cd ${ROOT_DIR}/sdk/python; python -m ruff format feast/ tests/
+	uv run ruff check --fix sdk/python/feast/ sdk/python/tests/
+	uv run ruff format sdk/python/feast/ sdk/python/tests/
 
 lint-python: ## Lint Python code
-	cd ${ROOT_DIR}/sdk/python; python -m mypy feast
-	cd ${ROOT_DIR}/sdk/python; python -m ruff check feast/ tests/
-	cd ${ROOT_DIR}/sdk/python; python -m ruff format --check feast/ tests
-
+	uv run ruff check sdk/python/feast/ sdk/python/tests/
+	uv run bash -c "cd sdk/python && mypy feast"
+
+# New combined target
+precommit-check: format-python lint-python ## Run all precommit checks
+	@echo "✅ All precommit checks passed"
+
+# Install precommit hooks with correct stages
+install-precommit: ## Install precommit hooks (runs on commit, not push)
+	pip install pre-commit
+	pre-commit install --hook-type pre-commit
+	@echo "✅ Precommit hooks installed (will run on commit, not push)"
+
+# Manual full type check
+mypy-full: ## Full MyPy type checking with all files
+	uv run bash -c "cd sdk/python && mypy feast tests"
+
+# Run precommit on all files
+precommit-all: ## Run all precommit hooks on all files
+	pre-commit run --all-files
+
+# Make scripts executable
+setup-scripts: ## Make helper scripts executable
+	chmod +x scripts/uv-run.sh scripts/check-init-py.sh scripts/mypy-daemon.sh
+
 ##@ Python SDK - local
 # formerly install-python-ci-dependencies-uv-venv
 # editable install
@@ -74,23 +95,22 @@ install-python-dependencies-minimal: ## Install minimal Python dependencies usin
 	uv pip sync --require-hashes sdk/python/requirements/py$(PYTHON_VERSION)-minimal-requirements.txt
 	uv pip install --no-deps -e .[minimal]
 
-##@ Python SDK - system
-# the --system flag installs dependencies in the global python context
-# instead of a venv which is useful when working in a docker container or ci.
+##@ Python SDK - CI (uses uv with virtualenv)
+# Uses uv pip sync with virtualenv for CI environments
 
 # Used in github actions/ci
-# formerly install-python-ci-dependencies-uv
-install-python-dependencies-ci: ## Install Python CI dependencies in system environment using uv
-	# Install CPU-only torch first to prevent CUDA dependency issues
-	pip uninstall torch torchvision -y || true
+install-python-dependencies-ci: ## Install Python CI dependencies using uv pip sync
+	# Create virtualenv if it doesn't exist
+	uv venv .venv
+	# Install CPU-only torch first to prevent CUDA dependency issues (Linux only)
 	@if [ "$$(uname -s)" = "Linux" ]; then \
 		echo "Installing dependencies with torch CPU index for Linux..."; \
-		uv pip sync --system --extra-index-url https://download.pytorch.org/whl/cpu --index-strategy unsafe-best-match sdk/python/requirements/py$(PYTHON_VERSION)-ci-requirements.txt; \
+		uv pip sync --extra-index-url https://download.pytorch.org/whl/cpu --index-strategy unsafe-best-match sdk/python/requirements/py$(PYTHON_VERSION)-ci-requirements.txt; \
 	else \
 		echo "Installing dependencies from PyPI for macOS..."; \
-		uv pip sync --system sdk/python/requirements/py$(PYTHON_VERSION)-ci-requirements.txt; \
+		uv pip sync sdk/python/requirements/py$(PYTHON_VERSION)-ci-requirements.txt; \
 	fi
-	uv pip install --system --no-deps -e .
+	uv pip install --no-deps -e .
 
 # Used in github actions/ci
 install-hadoop-dependencies-ci: ## Install Hadoop dependencies
@@ -151,22 +171,44 @@ benchmark-python-local: ## Run integration + benchmark tests for Python (local d
 ##@ Tests
 
 test-python-unit: ## Run Python unit tests (use pattern=<pattern> to filter tests, e.g., pattern=milvus, pattern=test_online_retrieval.py, pattern=test_online_retrieval.py::test_get_online_features_milvus)
-	python -m pytest -n 8 --color=yes $(if $(pattern),-k "$(pattern)") sdk/python/tests
+	uv run python -m pytest -n 8 --color=yes $(if $(pattern),-k "$(pattern)") sdk/python/tests
+
+# Fast unit tests only
+test-python-unit-fast: ## Run fast unit tests only (no external dependencies)
+	uv run python -m pytest sdk/python/tests/unit -n auto -x --tb=short
+
+# Changed files only (requires pytest-testmon)
+test-python-changed: ## Run tests for changed files only
+	uv run python -m pytest --testmon -n 8 --tb=short sdk/python/tests
+
+# Quick smoke test for PRs
+test-python-smoke: ## Quick smoke test for development
+	uv run python -m pytest \
+		sdk/python/tests/unit/test_unit_feature_store.py \
+		sdk/python/tests/unit/test_repo_operations_validate_feast_project_name.py \
+		-n 4 --tb=short
 
 test-python-integration: ## Run Python integration tests (CI)
-	python -m pytest --tb=short -v -n 8 --integration --color=yes --durations=10 --timeout=1200 --timeout_method=thread --dist loadgroup \
+	uv run python -m pytest --tb=short -v -n 8 --integration --color=yes --durations=10 --timeout=1200 --timeout_method=thread --dist loadgroup \
 		-k "(not snowflake or not test_historical_features_main)" \
 		-m "not rbac_remote_integration_test" \
 		--log-cli-level=INFO -s \
 		sdk/python/tests
 
+# Integration tests with better parallelization
+test-python-integration-parallel: ## Run integration tests with enhanced parallelization
+	uv run python -m pytest sdk/python/tests/integration \
+		-n auto --dist loadscope \
+		--timeout=300 --tb=short -v \
+		--integration --color=yes --durations=20
+
 test-python-integration-local: ## Run Python integration tests (local dev mode)
 	FEAST_IS_LOCAL_TEST=True \
 	FEAST_LOCAL_ONLINE_CONTAINER=True \
 	HADOOP_HOME=$$HOME/hadoop \
 	CLASSPATH="$$( $$HADOOP_HOME/bin/hadoop classpath --glob ):$$CLASSPATH" \
 	HADOOP_USER_NAME=root \
-	python -m pytest --tb=short -v -n 8 --color=yes --integration --durations=10 --timeout=1200 --timeout_method=thread --dist loadgroup \
+	uv run python -m pytest --tb=short -v -n 8 --color=yes --integration --durations=10 --timeout=1200 --timeout_method=thread --dist loadgroup \
 		-k "not test_lambda_materialization and not test_snowflake_materialization" \
 		-m "not rbac_remote_integration_test" \
 		--log-cli-level=INFO -s \
@@ -175,7 +217,7 @@ test-python-integration-local: ## Run Python integration tests (local dev mode)
 test-python-integration-rbac-remote: ## Run Python remote RBAC integration tests
 	FEAST_IS_LOCAL_TEST=True \
 	FEAST_LOCAL_ONLINE_CONTAINER=True \
-	python -m pytest --tb=short -v -n 8 --color=yes --integration --durations=10 --timeout=1200 --timeout_method=thread --dist loadgroup \
+	uv run python -m pytest --tb=short -v -n 8 --color=yes --integration --durations=10 --timeout=1200 --timeout_method=thread --dist loadgroup \
 		-k "not test_lambda_materialization and not test_snowflake_materialization" \
 		-m "rbac_remote_integration_test" \
 		--log-cli-level=INFO -s \
@@ -184,7 +226,7 @@ test-python-integration-rbac-remote: ## Run Python remote RBAC integration tests
 test-python-integration-container: ## Run Python integration tests using Docker
 	@(docker info > /dev/null 2>&1 && \
 		FEAST_LOCAL_ONLINE_CONTAINER=True \
-		python -m pytest -n 8 --integration sdk/python/tests \
+		uv run python -m pytest -n 8 --integration sdk/python/tests \
 	) || echo "This script uses Docker, and it isn't running - please start the Docker Daemon and try again!";
 
 test-python-universal-spark: ## Run Python Spark integration tests
@@ -220,7 +262,7 @@ test-python-historical-retrieval:
 			test_historical_features_persisting or \
 			test_historical_retrieval_fails_on_validation" \
  	 sdk/python/tests
-	 
+
 test-python-universal-trino: ## Run Python Trino integration tests
 	PYTHONPATH='.' \
 	FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.offline_stores.contrib.trino_repo_configuration \
@@ -556,7 +598,7 @@ test-python-universal-couchbase-online:	## Run Python Couchbase online store int
 		sdk/python/tests
 
 test-python-universal: ## Run all Python integration tests
-	python -m pytest -n 8 --integration sdk/python/tests
+	uv run python -m pytest -n 8 --integration sdk/python/tests
 
 ##@ Java
 
@@ -622,7 +664,7 @@ build-feature-transformation-server-docker: ## Build Feature Transformation Serv
 push-feature-server-java-docker: ## Push Feature Server Java Docker image
 	docker push $(REGISTRY)/feature-server-java:$(VERSION)
 
-build-feature-server-java-docker: ## Build Feature Server Java Docker image	
+build-feature-server-java-docker: ## Build Feature Server Java Docker image
 	docker buildx build --build-arg VERSION=$(VERSION) \
 		-t $(REGISTRY)/feature-server-java:$(VERSION) \
 		-f java/infra/docker/feature-server/Dockerfile --load .
@@ -727,12 +769,12 @@ build-ui-local: ## Build Feast UI locally
 	cd $(ROOT_DIR)/ui && yarn install && npm run build --omit=dev
 	rm -rf $(ROOT_DIR)/sdk/python/feast/ui/build
 	cp -r $(ROOT_DIR)/ui/build $(ROOT_DIR)/sdk/python/feast/ui/
-	
+
 format-ui: ## Format Feast UI
 	cd $(ROOT_DIR)/ui && NPM_TOKEN= yarn install && NPM_TOKEN= yarn format
 
 
-##@ Go SDK 
+##@ Go SDK
 PB_REL = https://github.com/protocolbuffers/protobuf/releases
 PB_VERSION = 30.2
 PB_ARCH := $(shell uname -m)
@@ -798,4 +840,3 @@ build-go-docker-dev: ## Build Go Docker image for development
 	docker buildx build --build-arg VERSION=dev \
 		-t feastdev/feature-server-go:dev \
 		-f go/infra/docker/feature-server/Dockerfile --load .
-

scripts/check-init-py.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+# Check for missing __init__.py files in Python packages
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ROOT_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
+
+# Find Python package directories missing __init__.py
+missing_init_files=()
+
+while IFS= read -r -d '' dir; do
+    # Skip .ipynb_checkpoints directories and other unwanted directories
+    if [[ "${dir}" == *".ipynb_checkpoints"* ]] || [[ "${dir}" == *"__pycache__"* ]]; then
+        continue
+    fi
+
+    if [[ ! -f "${dir}/__init__.py" ]] && [[ -n "$(find "${dir}" -maxdepth 1 -name "*.py" -print -quit)" ]]; then
+        missing_init_files+=("${dir}")
+    fi
+done < <(find "${ROOT_DIR}/sdk/python/feast" -type d -print0)
+
+if [[ ${#missing_init_files[@]} -gt 0 ]]; then
+    echo "❌ Missing __init__.py files in:"
+    printf "   %s\n" "${missing_init_files[@]}"
+    echo ""
+    echo "Run: touch ${missing_init_files[*]/%//__init__.py}"
+    exit 1
+fi
+
+echo "✅ All Python packages have __init__.py files"

scripts/mypy-daemon.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+# MyPy daemon for sub-second type checking
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ROOT_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
+
+MYPY_CACHE_DIR="${ROOT_DIR}/sdk/python/.mypy_cache"
+PID_FILE="$MYPY_CACHE_DIR/dmypy.pid"
+
+case "$1" in
+  start)
+    echo "🚀 Starting MyPy daemon..."
+    cd ${ROOT_DIR}/sdk/python
+    uv run dmypy start -- --config-file=pyproject.toml
+    echo "✅ MyPy daemon started"
+    ;;
+  check)
+    echo "🔍 Running MyPy daemon check..."
+    cd ${ROOT_DIR}/sdk/python
+    time uv run dmypy check feast tests
+    ;;
+  stop)
+    echo "🛑 Stopping MyPy daemon..."
+    cd ${ROOT_DIR}/sdk/python
+    uv run dmypy stop
+    echo "✅ MyPy daemon stopped"
+    ;;
+  restart)
+    echo "🔄 Restarting MyPy daemon..."
+    $0 stop
+    $0 start
+    ;;
+  status)
+    echo "📊 MyPy daemon status:"
+    cd ${ROOT_DIR}/sdk/python
+    if uv run dmypy status; then
+      echo "✅ MyPy daemon is running"
+    else
+      echo "❌ MyPy daemon is not running"
+    fi
+    ;;
+  *)
+    echo "Usage: $0 {start|check|stop|restart|status}"
+    echo ""
+    echo "Commands:"
+    echo "  start   - Start the MyPy daemon"
+    echo "  check   - Run type checking with the daemon"
+    echo "  stop    - Stop the MyPy daemon"
+    echo "  restart - Restart the daemon"
+    echo "  status  - Check daemon status"
+    exit 1
+    ;;
+esac