support pdf and docx format for rag uploads

.github/workflows/ci.yml

@@ -18,16 +18,19 @@ jobs:
         with:
           python-version: "3.11"
 
-      - name: Install dependencies
+      - name: Install Poetry
         run: |
           python -m pip install --upgrade pip
-          pip install ruff
+          pip install poetry
+
+      - name: Install dev dependencies
+        run: poetry install --only dev
 
       - name: Ruff lint
-        run: ruff check .
+        run: poetry run ruff check .
 
       - name: Ruff format check
-        run: ruff format --check .
+        run: poetry run ruff format --check .
 
   test:
     name: Unit Tests

Makefile

@@ -0,0 +1,46 @@
+.PHONY: lint format test check all clean ui help
+
+# Default target
+help:
+	@echo "Available commands:"
+	@echo "  make lint     - Run ruff linter"
+	@echo "  make format   - Auto-format code with ruff"
+	@echo "  make test     - Run unit tests (mocked, no API keys)"
+	@echo "  make check    - Run lint + format check (CI equivalent)"
+	@echo "  make all      - Run check + test"
+	@echo "  make ui       - Start Streamlit UI"
+	@echo "  make clean    - Remove cache and build artifacts"
+
+# Linting
+lint:
+	poetry run ruff check .
+
+# Auto-format code
+format:
+	poetry run ruff format .
+
+# Run unit tests (mocked only)
+test:
+	poetry run pytest tests/ -v --tb=short -m "not requires_api"
+
+# Run all tests including integration (requires API keys)
+test-all:
+	poetry run pytest tests/ -v --tb=short
+
+# CI-equivalent check (lint + format check)
+check: lint
+	poetry run ruff format --check .
+
+# Run everything
+all: check test
+
+# Start Streamlit UI
+ui:
+	poetry run streamlit run ui/app.py
+
+# Clean up
+clean:
+	rm -rf __pycache__ */__pycache__ */*/__pycache__
+	rm -rf .pytest_cache .ruff_cache
+	rm -rf *.egg-info dist build
+	rm -rf .coverage htmlcov

graph_flow/nodes/nodes.py

@@ -143,11 +143,17 @@ def executor_node(state: TraceFlowState) -> dict:
 
     context_str = ""
     if state.context:
-        context_str = "\n".join([f"- {chunk.content}" for chunk in state.context])
+        context_str = "\n".join(
+            [f"[{i + 1}] {chunk.content}" for i, chunk in enumerate(state.context)]
+        )
         context_str = f"\n\nContext:\n{context_str}"
-    system_prompt = (
-        f"You are a helpful assistant. Answer the user's question concisely.{context_str}"
-    )
+
+    # Include revision instructions if this is a retry
+    revision_str = ""
+    if state.revisions > 0 and state.eval_report and state.eval_report.revision_instructions:
+        revision_str = f"\n\nIMPORTANT (Attempt {state.revisions + 1}): {state.eval_report.revision_instructions}"
+
+    system_prompt = f"You are a helpful assistant. Answer the user's question concisely.{context_str}{revision_str}"
 
     response = provider.chat_complete(
         model=state.config.model,
@@ -214,7 +220,7 @@ def evaluator_node(state: TraceFlowState) -> dict:
 
     cost_exceeded = config.max_cost_usd and draft and draft.cost_usd > config.max_cost_usd
     latency_exceeded = config.max_latency_ms and draft and draft.latency_ms > config.max_latency_ms
-    revision_limit_reached = state.revisions >= config.max_revisions
+    revision_limit_reached = state.revisions + 1 >= config.max_revisions
 
     # Mode-specific checks
     citation_coverage_ok = True

pyproject.toml

@@ -34,11 +34,13 @@ opentelemetry-sdk = "^1.39.0"
 openai = "^2.14.0"
 anthropic = "^0.18.0"
 tiktoken = "^0.7.0"
+pypdf = "^4.0.0"
+python-docx = "^1.1.0"
 
 [tool.poetry.group.dev.dependencies]
 pytest = "^8.2"
 pytest-cov = "^5.0"
-ruff = "^0.5.0"
+ruff = "^0.14.0"
 mypy = "^1.10"
 
 [build-system]

requirements.txt

@@ -14,3 +14,5 @@ opentelemetry-sdk>=1.39.0
 openai>=2.14.0
 anthropic>=0.18.0
 tiktoken>=0.7.0
+pypdf>=4.0.0
+python-docx>=1.1.0

ui/app.py

@@ -24,6 +24,21 @@
 except ImportError:
     RAG_AVAILABLE = False
 
+# Optional document parsing imports
+try:
+    from pypdf import PdfReader
+
+    PDF_AVAILABLE = True
+except ImportError:
+    PDF_AVAILABLE = False
+
+try:
+    from docx import Document as DocxDocument
+
+    DOCX_AVAILABLE = True
+except ImportError:
+    DOCX_AVAILABLE = False
+
 # Page config
 st.set_page_config(page_title="TraceFlow Lite", layout="wide", initial_sidebar_state="expanded")
 
@@ -346,14 +361,51 @@ def render_new_run_page():
                     # Split by double newlines or treat as single doc
                     documents = [d.strip() for d in doc_text.split("\n\n") if d.strip()]
             else:
+                # Build allowed file types based on available parsers
+                allowed_types = ["txt", "md"]
+                if PDF_AVAILABLE:
+                    allowed_types.append("pdf")
+                if DOCX_AVAILABLE:
+                    allowed_types.append("docx")
+
                 uploaded_files = st.file_uploader(
-                    "Upload text files",
-                    type=["txt", "md"],
+                    "Upload documents",
+                    type=allowed_types,
                     accept_multiple_files=True,
+                    help=f"Supported: {', '.join(t.upper() for t in allowed_types)}",
                 )
                 for file in uploaded_files:
-                    content = file.read().decode("utf-8")
-                    documents.append(content)
+                    file_ext = file.name.lower().split(".")[-1]
+
+                    if file_ext == "pdf" and PDF_AVAILABLE:
+                        # Extract text from PDF
+                        try:
+                            pdf_reader = PdfReader(file)
+                            text_parts = []
+                            for page in pdf_reader.pages:
+                                page_text = page.extract_text()
+                                if page_text:
+                                    text_parts.append(page_text)
+                            content = "\n\n".join(text_parts)
+                        except Exception as e:
+                            st.warning(f"Failed to parse PDF {file.name}: {e}")
+                            continue
+                    elif file_ext == "docx" and DOCX_AVAILABLE:
+                        # Extract text from DOCX
+                        try:
+                            doc = DocxDocument(file)
+                            content = "\n\n".join(
+                                [para.text for para in doc.paragraphs if para.text.strip()]
+                            )
+                        except Exception as e:
+                            st.warning(f"Failed to parse DOCX {file.name}: {e}")
+                            continue
+                    else:
+                        # Plain text files (txt, md)
+                        content = file.read().decode("utf-8")
+
+                    if content.strip():
+                        documents.append(content)
 
             # Vector store settings
             col1, col2 = st.columns(2)