upgrade to python 3.11 and with uv support

.devcontainer/devcontainer.json

@@ -1,23 +1,24 @@
 // For format details, see https://aka.ms/devcontainer.json. For config options, see the
 // README at: https://github.com/devcontainers/templates/tree/main/src/python
 {
-	"name": "Python 3",
+	"name": "Data Formulator Dev",
 	// Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
-	"image": "mcr.microsoft.com/devcontainers/python:1-3.12-bullseye",
+	"image": "mcr.microsoft.com/devcontainers/python:1-3.11-bullseye",
 
 	// Features to add to the dev container. More info: https://containers.dev/features.
-    "features": {
-        "ghcr.io/devcontainers/features/node:1": {
-            "version": "18"
-        },
-	"ghcr.io/devcontainers/features/azure-cli:1": {}
-    },
+	"features": {
+		"ghcr.io/devcontainers/features/node:1": {
+			"version": "18"
+		},
+		"ghcr.io/devcontainers/features/azure-cli:1": {},
+		"ghcr.io/astral-sh/uv:1": {}
+	},
 
 	// Use 'forwardPorts' to make a list of ports inside the container available locally.
-	// "forwardPorts": [],
+	"forwardPorts": [5000, 5173],
 
 	// Use 'postCreateCommand' to run commands after the container is created.
-	"postCreateCommand": "cd /workspaces/data-formulator && npm install && npm run build && python3 -m venv /workspaces/data-formulator/venv && . /workspaces/data-formulator/venv/bin/activate && pip install -e /workspaces/data-formulator --verbose && data_formulator"
+	"postCreateCommand": "cd /workspaces/data-formulator && npm install && npm run build && uv sync && uv run data_formulator"
 
 	// Configure tool-specific properties.
 	// "customizations": {},

.env.template

@@ -3,6 +3,4 @@
 #   python -m data_formulator -p 5000 --exec-python-in-subprocess true --disable-display-keys true
 
 DISABLE_DISPLAY_KEYS=false # if true, the display keys will not be shown in the frontend
-EXEC_PYTHON_IN_SUBPROCESS=false # if true, the python code will be executed in a subprocess to avoid crashing the main app, but it will increase the time of response
-
-LOCAL_DB_DIR= # the directory to store the local database, if not provided, the app will use the temp directory
+EXEC_PYTHON_IN_SUBPROCESS=false # if true, the python code will be executed in a subprocess to avoid crashing the main app, but it will increase the time of response

.github/workflows/python-build.yml

@@ -9,6 +9,11 @@ on:
   pull_request:
     branches: [ "main" ]
 
+# Global permissions required for OIDC authentication
+permissions:
+  id-token: write
+  contents: read
+
 jobs:
   build:
     runs-on: ubuntu-latest
@@ -21,42 +26,37 @@ jobs:
       with:
         node-version: 20
         cache: 'yarn'
-    - name: Set up Python 3.12
-      uses: actions/setup-python@v5
+    - name: Set up uv
+      uses: astral-sh/setup-uv@v4
       with:
         python-version: 3.12
     - name: Install node dependencies
       run: yarn install
-    - name: Install python dependencies
-      run: |
-        python -m pip install --upgrade pip
-        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-        python -m pip install build
     - name: Build frontend
       run: yarn build
     - name: Build python artifact
-      run: python -m build
+      run: uv build
     - name: Archive production artifacts
       uses: actions/upload-artifact@v4
       with:
         name: release-dist
         path: dist
 
-  # pypi-publish:
-  #   runs-on: ubuntu-latest
-  #   needs:
-  #     - build
-  #   if: github.event_name == 'push' && contains(github.event.head_commit.message, '[deploy]')
-  #   environment:
-  #     name: pypi
-  #     url: https://pypi.org/p/data-formulator
-  #   permissions:
-  #     id-token: write
-  #   steps:
-  #     - name: Retrieve release distributions
-  #       uses: actions/download-artifact@v4
-  #       with:
-  #         name: release-dist
-  #         path: dist/
-  #     - name: Publish package distributions to PyPI
-  #       uses: pypa/gh-action-pypi-publish@release/v1
+  pypi-publish:
+    runs-on: ubuntu-latest
+    needs:
+      - build
+    if: github.event_name == 'push' && contains(github.event.head_commit.message, '[deploy]')
+    environment:
+      name: pypi
+      url: https://pypi.org/p/data-formulator
+    permissions:
+      id-token: write
+    steps:
+      - name: Retrieve release distributions
+        uses: actions/download-artifact@v4
+        with:
+          name: release-dist
+          path: dist/
+      - name: Publish package distributions to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

.gitignore

@@ -1,5 +1,6 @@
 
 *env
+.venv/
 *api-keys.env
 **/*.ipynb_checkpoints/
 .DS_Store

.python-version

@@ -0,0 +1 @@
+3.11

DEVELOPMENT.md

@@ -2,16 +2,34 @@
 How to set up your local machine.
 
 ## Prerequisites
-* Python > 3.11
+* Python >= 3.11
 * Node.js
 * Yarn
+* [uv](https://docs.astral.sh/uv/) (recommended) or pip
 
 ## Backend (Python)
 
+### Option 1: With uv (recommended)
+
+uv is faster and provides reproducible builds via lockfile.
+
+```bash
+uv sync                        # Creates .venv and installs all dependencies
+uv run data_formulator         # Run app (opens browser automatically)
+uv run data_formulator --dev   # Run backend only (for frontend development)
+```
+
+**Which command to use:**
+- **End users / testing the full app**: `uv run data_formulator` - starts server and opens browser to http://localhost:5000
+- **Frontend development**: `uv run data_formulator --dev` - starts backend server only, then run `yarn start` separately for the Vite dev server on http://localhost:5173
+
+### Option 2: With pip (fallback)
+
 - **Create a Virtual Environment**  
     ```bash
     python -m venv venv
-    .\venv\Scripts\activate
+    source venv/bin/activate  # Unix
+    # or .\venv\Scripts\activate  # Windows
     ```
 
 - **Install Dependencies**  
@@ -29,7 +47,6 @@ How to set up your local machine.
         - configure settings as needed:
             - DISABLE_DISPLAY_KEYS: if true, API keys will not be shown in the frontend
             - EXEC_PYTHON_IN_SUBPROCESS: if true, Python code runs in a subprocess (safer but slower), you may consider setting it true when you are hosting Data Formulator for others
-            - LOCAL_DB_DIR: directory to store the local database (uses temp directory if not set)
             - External database settings (when USE_EXTERNAL_DB=true):
                 - DB_NAME: name to refer to this database connection
                 - DB_TYPE: mysql or postgresql (currently only these two are supported)
@@ -41,14 +58,16 @@ How to set up your local machine.
 
 
 - **Run the app**
-    - **Windows**
-    ```bash
-    .\local_server.bat
-    ```
-
-    - **Unix-based**
     ```bash
+    # Unix
     ./local_server.sh
+
+    # Windows
+    .\local_server.bat
+
+    # Or directly
+    data_formulator         # Opens browser automatically
+    data_formulator --dev   # Backend only (for frontend development)
     ```
 
 ## Frontend (TypeScript)
@@ -61,7 +80,12 @@ How to set up your local machine.
 
 - **Development mode**
 
-    Run the front-end in development mode using, allowing real-time edits and previews:
+    First, start the backend server (in a separate terminal):
+    ```bash
+    uv run data_formulator --dev   # or ./local_server.sh
+    ```
+
+    Then, run the frontend in development mode with hot reloading:
     ```bash
     yarn start
     ```
@@ -81,6 +105,10 @@ How to set up your local machine.
     Then, build python package:
 
     ```bash
+    # With uv
+    uv build
+
+    # Or with pip
     pip install build
     python -m build
     ```
@@ -112,23 +140,23 @@ How to set up your local machine.
 
 When deploying Data Formulator to production, please be aware of the following security considerations:
 
-### Database Storage Security
+### Database and Data Storage Security
 
-1. **Local DuckDB Files**: When database functionality is enabled (default), Data Formulator stores DuckDB database files locally on the server. These files contain user data and are stored in the system's temporary directory or a configured `LOCAL_DB_DIR`.
+1. **Workspace and table data**: Table data is stored in per-identity workspaces (e.g. parquet files). DuckDB is used only in-memory per request when needed (e.g. for SQL mode); no persistent DuckDB database files are created by the app.
 
-2. **Session Management**: 
-   - When database is **enabled**: Session IDs are stored in Flask sessions (cookies) and linked to local DuckDB files
-   - When database is **disabled**: No persistent storage is used, and no cookies are set. Session IDs are generated per request for API consistency
+2. **Identity Management**: 
+   - Each user's data is isolated by a namespaced identity key (e.g., `user:alice@example.com` or `browser:550e8400-...`)
+   - Anonymous users get a browser-based UUID stored in localStorage
+   - Authenticated users get their verified user ID from the auth provider
 
-3. **Data Persistence**: User data processed through Data Formulator may be temporarily stored in these local DuckDB files, which could be a security risk in multi-tenant environments.
+3. **Data persistence**: User data may be written to workspace storage (e.g. parquet) on the server. In multi-tenant deployments, ensure workspace directories are isolated and access-controlled.
 
 ### Recommended Security Measures
 
 For production deployment, consider:
 
-1. **Use `--disable-database` flag** for stateless deployments where no data persistence is needed
+1. **Use `--disable-database` flag** to disable table-connector routes when you do not need external or uploaded table support
 2. **Implement proper authentication, authorization, and other security measures** as needed for your specific use case, for example:
-   - Store DuckDB file in a database
    - User authentication (OAuth, JWT tokens, etc.)
    - Role-based access control
    - API rate limiting
@@ -142,5 +170,90 @@ For production deployment, consider:
 python -m data_formulator.app --disable-database
 ```
 
+## Authentication Architecture
+
+Data Formulator supports a **hybrid identity system** that supports both anonymous and authenticated users.
+
+### Identity Flow Overview
+
+```
+┌─────────────────────────────────────────────────────────────────────┐
+│                         Frontend Request                             │
+├─────────────────────────────────────────────────────────────────────┤
+│  Headers:                                                            │
+│    X-Identity-Id: "browser:550e8400-..." (namespace sent by client) │
+│    Authorization: Bearer <jwt>  (if custom auth implemented)         │
+│    (Azure also adds X-MS-CLIENT-PRINCIPAL-ID automatically)          │
+└─────────────────────────────────────────────────────────────────────┘
+                                │
+                                ▼
+┌─────────────────────────────────────────────────────────────────────┐
+│                    Backend Identity Resolution                       │
+│                       (auth.py: get_identity_id)                    │
+├─────────────────────────────────────────────────────────────────────┤
+│  Priority 1: Azure X-MS-CLIENT-PRINCIPAL-ID → "user:<azure_id>"     │
+│  Priority 2: JWT Bearer token (if implemented) → "user:<jwt_sub>"    │
+│  Priority 3: X-Identity-Id header → ALWAYS "browser:<id>"           │
+│              (client-provided namespace is IGNORED for security)     │
+└─────────────────────────────────────────────────────────────────────┘
+                                │
+                                ▼
+┌─────────────────────────────────────────────────────────────────────┐
+│                      Storage Isolation                               │
+├─────────────────────────────────────────────────────────────────────┤
+│  "user:alice@example.com"  → alice's DuckDB file (ONLY via auth)     │
+│  "browser:550e8400-..."    → anonymous user's DuckDB file            │
+└─────────────────────────────────────────────────────────────────────┘
+```
+
+### Security Model
+
+**Critical Security Rule:** The backend NEVER trusts the namespace prefix from the client-provided `X-Identity-Id` header. Even if a client sends `X-Identity-Id: "user:alice@..."`, the backend strips the prefix and forces `browser:alice@...`. Only verified authentication (Azure headers or JWT) can result in a `user:` prefixed identity.
+
+The key security principle is **namespaced isolation with forced prefixing**:
+
+| Scenario | X-Identity-Id Sent | Backend Resolution | Storage Key |
+|----------|-------------------|-------------------|-------------|
+| Anonymous user | `browser:550e8400-...` | Strips prefix, forces `browser:` | `browser:550e8400-...` |
+| Azure logged-in user | `browser:550e8400-...` | Uses Azure header (priority 1) | `user:alice@...` |
+| Attacker spoofing | `user:alice@...` (forged) | No valid auth, strips & forces `browser:` | `browser:alice@...` |
+
+**Why this is secure:** An attacker sending `X-Identity-Id: user:alice@...` gets `browser:alice@...` as their storage key, which is completely separate from the real `user:alice@...` that only authenticated Alice can access.
+
+### Implementing Custom Authentication
+
+To add JWT-based authentication:
+
+1. **Backend** (`tables_routes.py`): Uncomment and configure the JWT verification code in `get_identity_id()`
+2. **Frontend** (`utils.tsx`): Implement `getAuthToken()` to retrieve the JWT from your auth context
+3. **Add JWT secret** to Flask config: `current_app.config['JWT_SECRET']`
+
+### Azure App Service Authentication
+
+When deployed to Azure with EasyAuth enabled:
+- Azure automatically adds `X-MS-CLIENT-PRINCIPAL-ID` header to authenticated requests
+- The backend reads this header first (highest priority)
+- No frontend changes needed - Azure handles the auth flow
+
+### Frontend Identity Management
+
+The frontend (`src/app/identity.ts`) manages identity as follows:
+
+```typescript
+// Identity is always initialized with browser ID
+identity: { type: 'browser', id: getBrowserId() }
+
+// If user logs in (e.g., via Azure), it's updated to:
+identity: { type: 'user', id: userInfo.userId }
+
+// All API requests send namespaced identity:
+// X-Identity-Id: "browser:550e8400-..." or "user:alice@..."
+```
+
+This ensures:
+1. **Anonymous users**: Work immediately with localStorage-based browser ID
+2. **Logged-in users**: Get their verified user ID from the auth provider
+3. **Cross-tab consistency**: Browser ID is shared via localStorage across all tabs
+
 ## Usage
 See the [Usage section on the README.md page](README.md#usage).