diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..a2c256e --- /dev/null +++ b/.env.example @@ -0,0 +1,18 @@ +# Server Configuration +PORT=8000 +DEBUG=True +HOST=0.0.0.0 + +# Database Configuration +DB_PATH=~/.grepify_history.db +MAX_HISTORY=200 + +# Semantic Search Configuration +SPACY_MODEL=en_core_web_md +SEMANTIC_THRESHOLD=0.45 +SEMANTIC_TOP_N=10 + +# Rate Limiting +RATE_LIMIT_ENABLED=True +RATE_LIMIT_REQUESTS=100 +RATE_LIMIT_WINDOW=60 diff --git a/README.md b/README.md index a0b5b01..d6c6c2b 100644 --- a/README.md +++ b/README.md @@ -65,24 +65,24 @@ @@ -248,6 +248,6 @@ This project is licensed under the MIT License - see the [LICENSE](LICENSE) file
-[Report Bug](https://github.com/vivekkdagar/deepgrep/issues) · [Request Feature](https://github.com/vivekkdagar/deepgrep/issues) +[Report Bug](https://github.com/alwaysvivek/deepgrep/issues) · [Request Feature](https://github.com/alwaysvivek/deepgrep/issues)
diff --git a/deepgrep/core/history.py b/deepgrep/core/history.py index 0f26b9c..61ee88b 100644 --- a/deepgrep/core/history.py +++ b/deepgrep/core/history.py @@ -1,10 +1,16 @@ import sqlite3, json +import logging from datetime import datetime, timezone from pathlib import Path from typing import List, Tuple +from decouple import config -DB_PATH = Path.home() / ".grepify_history.db" -MAX_HISTORY = 200 +# Configure logging +logger = logging.getLogger(__name__) + +# Load configuration from environment variables +DB_PATH = Path(config('DB_PATH', default='~/.grepify_history.db')).expanduser() +MAX_HISTORY = config('MAX_HISTORY', default=200, cast=int) class SearchHistoryDB: """SQLite-backed search history logger.""" @@ -14,6 +20,7 @@ def __init__(self, db_path: Path = DB_PATH): self._init_db() def _init_db(self): + logger.info(f"Initializing search history database at {self.db_path}") with sqlite3.connect(self.db_path) as conn: conn.execute(""" CREATE TABLE IF NOT EXISTS search_logs ( @@ -24,6 +31,7 @@ def _init_db(self): files TEXT ); """) + logger.info("Database initialized successfully") def log_search(self, pattern: str, match_count: int, files: List[str]): with sqlite3.connect(self.db_path) as conn: @@ -31,12 +39,17 @@ def log_search(self, pattern: str, match_count: int, files: List[str]): "INSERT INTO search_logs (pattern, timestamp, match_count, files) VALUES (?, ?, ?, ?)", (pattern, datetime.now(timezone.utc).isoformat(), match_count, json.dumps(files)), ) - conn.execute(""" + # Cleanup old records + result = conn.execute(""" DELETE FROM search_logs WHERE id NOT IN ( SELECT id FROM search_logs ORDER BY id DESC LIMIT ? ); """, (MAX_HISTORY,)) + deleted_count = result.rowcount conn.commit() + logger.info(f"Logged search: pattern='{pattern}', matches={match_count}, files={len(files)}") + if deleted_count > 0: + logger.debug(f"Cleaned up {deleted_count} old records") def get_recent(self, limit: int = 5) -> List[Tuple]: with sqlite3.connect(self.db_path) as conn: @@ -68,6 +81,7 @@ def export_to_json(self, file_path: Path) -> int: for r in rows ] file_path.write_text(json.dumps(data, indent=2), encoding="utf-8") + logger.info(f"Exported {len(data)} search records to {file_path}") return len(data) def import_from_json(self, file_path: Path) -> int: @@ -92,4 +106,5 @@ def import_from_json(self, file_path: Path) -> int: ); """, (MAX_HISTORY,)) conn.commit() + logger.info(f"Imported {len(data)} search records from {file_path}") return len(data) \ No newline at end of file diff --git a/deepgrep/core/semantic_engine.py b/deepgrep/core/semantic_engine.py index 75cb87e..5d4937e 100644 --- a/deepgrep/core/semantic_engine.py +++ b/deepgrep/core/semantic_engine.py @@ -1,6 +1,12 @@ import spacy from nltk.corpus import wordnet from typing import List, Tuple +from decouple import config + +# Load configuration from environment variables +SPACY_MODEL = config('SPACY_MODEL', default='en_core_web_md') +SEMANTIC_THRESHOLD = config('SEMANTIC_THRESHOLD', default=0.45, cast=float) +SEMANTIC_TOP_N = config('SEMANTIC_TOP_N', default=10, cast=int) def get_antonyms(word: str) -> set: @@ -15,7 +21,9 @@ def get_antonyms(word: str) -> set: class SemanticEngine: """Semantic search avoiding antonyms and irrelevant words.""" - def __init__(self, model_name: str = "en_core_web_md"): + def __init__(self, model_name: str = None): + if model_name is None: + model_name = SPACY_MODEL try: self.nlp = spacy.load(model_name) except OSError: @@ -23,7 +31,9 @@ def __init__(self, model_name: str = "en_core_web_md"): f"SpaCy model '{model_name}' not found. Run: python -m spacy download {model_name}" ) - def find_semantic_matches(self, text: str, keyword: str, top_n: int = 10) -> List[Tuple[str, float]]: + def find_semantic_matches(self, text: str, keyword: str, top_n: int = None) -> List[Tuple[str, float]]: + if top_n is None: + top_n = SEMANTIC_TOP_N doc = self.nlp(text) key_token = self.nlp(keyword)[0] antonyms = get_antonyms(keyword.lower()) @@ -43,5 +53,5 @@ def find_semantic_matches(self, text: str, keyword: str, top_n: int = 10) -> Lis # Sort descending and keep top N results = sorted(results, key=lambda x: x[1], reverse=True)[:top_n] # Filter out weak matches - results = [(w, s) for w, s in results if s > 0.45] + results = [(w, s) for w, s in results if s > SEMANTIC_THRESHOLD] return results \ No newline at end of file diff --git a/deepgrep/web/app.py b/deepgrep/web/app.py index d05afa7..5f30689 100644 --- a/deepgrep/web/app.py +++ b/deepgrep/web/app.py @@ -1,28 +1,70 @@ -# app.py (small changes to init components and avoid nltk download in runtime) +# app.py - Production-ready Flask application with logging and rate limiting +import logging from flask import Flask, request, jsonify, render_template from flask_cors import CORS +from flask_limiter import Limiter +from flask_limiter.util import get_remote_address +from decouple import config from deepgrep.core.engine import find_matches from deepgrep.core.history import SearchHistoryDB from deepgrep.core.semantic_engine import SemanticEngine -import threading + +# Configure logging +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' +) +logger = logging.getLogger(__name__) + +# Load configuration from environment variables +PORT = config('PORT', default=8000, cast=int) +DEBUG = config('DEBUG', default=True, cast=bool) +HOST = config('HOST', default='0.0.0.0') +RATE_LIMIT_ENABLED = config('RATE_LIMIT_ENABLED', default=True, cast=bool) +RATE_LIMIT_REQUESTS = config('RATE_LIMIT_REQUESTS', default=100, cast=int) app = Flask(__name__, template_folder="templates", static_folder="static") CORS(app) -# Initialize engines once (heavy models loaded at import time) -semantic_engine = SemanticEngine() -history_db = SearchHistoryDB() +# Initialize rate limiter +limiter = Limiter( + app=app, + key_func=get_remote_address, + default_limits=[f"200 per minute"], + enabled=RATE_LIMIT_ENABLED +) + +# Initialize engines (lazy initialization to support testing) +semantic_engine = None +history_db = None + +def init_engines(): + """Initialize semantic engine and history database (called lazily on first use).""" + global semantic_engine, history_db + if semantic_engine is None: + logger.info("Initializing semantic engine and history database...") + semantic_engine = SemanticEngine() + history_db = SearchHistoryDB() + logger.info("Application initialization complete") @app.route("/") def home(): + logger.info("GET / - Home page requested") return render_template("index.html") @app.route("/search", methods=["POST"]) +@limiter.limit(f"{RATE_LIMIT_REQUESTS}/minute") def search_regex(): + # Only initialize history_db, not semantic_engine (not needed for regex) + if history_db is None: + init_engines() + logger.info("POST /search - Regex search request received") data = request.get_json() pattern = data.get("pattern") text = data.get("text") + if not pattern or not text: + logger.warning("POST /search - Missing pattern or text in request") return jsonify({"error": "Missing pattern or text"}), 400 matches = [] @@ -32,6 +74,8 @@ def search_regex(): history_db.log_search(pattern, len(matches), ["web_input"]) all_history = history_db.list_all(limit=50) + + logger.info(f"POST /search - Successfully found {len(matches)} matches for pattern '{pattern}'") return jsonify({ "matches": matches, @@ -41,16 +85,27 @@ def search_regex(): }) @app.route("/semantic", methods=["POST"]) +@limiter.limit(f"{RATE_LIMIT_REQUESTS}/minute") def search_semantic(): + logger.info("POST /semantic - Semantic search request received") data = request.get_json() keyword = data.get("keyword") text = data.get("text") + + # Validate input first, before initializing engines if not keyword or not text: + logger.warning("POST /semantic - Missing keyword or text in request") return jsonify({"error": "Missing keyword or text"}), 400 + + # Only initialize engines after validation passes + if semantic_engine is None or history_db is None: + init_engines() matches = semantic_engine.find_semantic_matches(text, keyword) history_db.log_search(keyword, len(matches), ["web_input"]) all_history = history_db.list_all(limit=50) + + logger.info(f"POST /semantic - Successfully found {len(matches)} semantic matches for keyword '{keyword}'") return jsonify({ "matches": [{"word": w, "similarity": round(s, 3)} for w, s in matches], @@ -63,4 +118,6 @@ def search_semantic(): # Remove in-production NLTK downloads; expect preinstalled corpora # import nltk # nltk.download('wordnet') - app.run(port=8000, debug=True) \ No newline at end of file + init_engines() # Ensure engines are initialized when running directly + logger.info(f"Starting Flask server on {HOST}:{PORT} (debug={DEBUG})") + app.run(host=HOST, port=PORT, debug=DEBUG) \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 6011bfd..70025f2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,5 @@ spacy nltk flask-cors flask +python-decouple +flask-limiter diff --git a/tests/test_api.py b/tests/test_api.py new file mode 100644 index 0000000..98ce21e --- /dev/null +++ b/tests/test_api.py @@ -0,0 +1,292 @@ +""" +Comprehensive API tests for DeepGrep web application. +Tests cover regex search, semantic search, and home route endpoints. +""" +import pytest +import json +from deepgrep.web.app import app +from unittest.mock import patch + + +@pytest.fixture +def client(): + """Create a test client for the Flask app.""" + from unittest.mock import MagicMock + import deepgrep.web.app as app_module + + app.config['TESTING'] = True + # Disable rate limiting for tests + app.config['RATELIMIT_ENABLED'] = False + + # Mock history_db to avoid database operations during tests + mock_history_db = MagicMock() + mock_history_db.log_search.return_value = None + mock_history_db.list_all.return_value = [] + app_module.history_db = mock_history_db + + with app.test_client() as client: + yield client + + +class TestHomeRoute: + """Tests for the home route.""" + + def test_home_returns_200(self, client): + """Test that the home route returns status 200.""" + response = client.get('/') + assert response.status_code == 200 + + def test_home_returns_html(self, client): + """Test that the home route returns HTML content.""" + response = client.get('/') + assert b'html' in response.data.lower() or response.content_type == 'text/html; charset=utf-8' + + +class TestRegexSearchEndpoint: + """Tests for the /search endpoint (regex matching).""" + + def test_regex_search_with_valid_pattern(self, client): + """Test regex search with a valid pattern that matches.""" + data = { + "pattern": r"\d+", + "text": "User logged in at 14:32, error code 404" + } + response = client.post('/search', + data=json.dumps(data), + content_type='application/json') + + assert response.status_code == 200 + json_data = response.get_json() + assert 'matches' in json_data + assert 'history' in json_data + assert isinstance(json_data['matches'], list) + assert len(json_data['matches']) > 0 + assert '14' in json_data['matches'] or '32' in json_data['matches'] + + def test_regex_search_empty_pattern(self, client): + """Test that empty pattern returns 400 error.""" + data = { + "pattern": "", + "text": "Some text here" + } + response = client.post('/search', + data=json.dumps(data), + content_type='application/json') + + assert response.status_code == 400 + json_data = response.get_json() + assert 'error' in json_data + + def test_regex_search_empty_text(self, client): + """Test that empty text returns 400 error.""" + data = { + "pattern": r"\d+", + "text": "" + } + response = client.post('/search', + data=json.dumps(data), + content_type='application/json') + + assert response.status_code == 400 + json_data = response.get_json() + assert 'error' in json_data + + def test_regex_search_missing_pattern(self, client): + """Test that missing pattern field returns 400 error.""" + data = { + "text": "Some text here" + } + response = client.post('/search', + data=json.dumps(data), + content_type='application/json') + + assert response.status_code == 400 + + def test_regex_search_missing_text(self, client): + """Test that missing text field returns 400 error.""" + data = { + "pattern": r"\d+" + } + response = client.post('/search', + data=json.dumps(data), + content_type='application/json') + + assert response.status_code == 400 + + def test_regex_search_complex_email_pattern(self, client): + """Test complex regex pattern for email matching.""" + data = { + "pattern": r"\w+@\w+\.\w+", + "text": "Contact us at support@example.com or admin@test.org" + } + response = client.post('/search', + data=json.dumps(data), + content_type='application/json') + + assert response.status_code == 200 + json_data = response.get_json() + assert 'matches' in json_data + assert len(json_data['matches']) > 0 + + def test_regex_search_date_pattern(self, client): + """Test complex regex pattern for date matching.""" + data = { + "pattern": r"\d{4}-\d{2}-\d{2}", + "text": "Event dates: 2024-01-15 and 2024-12-31" + } + response = client.post('/search', + data=json.dumps(data), + content_type='application/json') + + assert response.status_code == 200 + json_data = response.get_json() + assert 'matches' in json_data + assert len(json_data['matches']) == 2 + + def test_regex_search_response_structure(self, client): + """Test that response has correct structure with matches and history arrays.""" + data = { + "pattern": r"\w+", + "text": "hello world" + } + response = client.post('/search', + data=json.dumps(data), + content_type='application/json') + + assert response.status_code == 200 + json_data = response.get_json() + assert 'matches' in json_data + assert 'history' in json_data + assert isinstance(json_data['matches'], list) + assert isinstance(json_data['history'], list) + + +class TestSemanticSearchEndpoint: + """Tests for the /semantic endpoint.""" + + @patch('deepgrep.web.app.semantic_engine') + def test_semantic_search_with_valid_keyword(self, mock_engine, client): + """Test semantic search with a valid keyword.""" + # Mock the semantic engine to return predictable results + mock_engine.find_semantic_matches.return_value = [ + ("joyful", 0.88), + ("delighted", 0.82) + ] + + data = { + "keyword": "happy", + "text": "She felt joyful and delighted after the announcement" + } + response = client.post('/semantic', + data=json.dumps(data), + content_type='application/json') + + assert response.status_code == 200 + json_data = response.get_json() + assert 'matches' in json_data + assert 'history' in json_data + assert isinstance(json_data['matches'], list) + assert len(json_data['matches']) == 2 + + # Verify match structure + first_match = json_data['matches'][0] + assert 'word' in first_match + assert 'similarity' in first_match + + def test_semantic_search_missing_keyword(self, client): + """Test that missing keyword returns 400 error.""" + data = { + "text": "Some text here" + } + response = client.post('/semantic', + data=json.dumps(data), + content_type='application/json') + + assert response.status_code == 400 + json_data = response.get_json() + assert 'error' in json_data + + def test_semantic_search_empty_keyword(self, client): + """Test that empty keyword returns 400 error.""" + data = { + "keyword": "", + "text": "Some text here" + } + response = client.post('/semantic', + data=json.dumps(data), + content_type='application/json') + + assert response.status_code == 400 + + def test_semantic_search_missing_text(self, client): + """Test that missing text returns 400 error.""" + data = { + "keyword": "happy" + } + response = client.post('/semantic', + data=json.dumps(data), + content_type='application/json') + + assert response.status_code == 400 + + def test_semantic_search_empty_text(self, client): + """Test that empty text returns 400 error.""" + data = { + "keyword": "happy", + "text": "" + } + response = client.post('/semantic', + data=json.dumps(data), + content_type='application/json') + + assert response.status_code == 400 + + @patch('deepgrep.web.app.semantic_engine') + def test_semantic_search_similarity_scores_present(self, mock_engine, client): + """Test that similarity scores are present in the response.""" + mock_engine.find_semantic_matches.return_value = [ + ("happy", 0.95), + ("glad", 0.78) + ] + + data = { + "keyword": "joyful", + "text": "I feel happy and glad today" + } + response = client.post('/semantic', + data=json.dumps(data), + content_type='application/json') + + assert response.status_code == 200 + json_data = response.get_json() + + for match in json_data['matches']: + assert 'similarity' in match + assert isinstance(match['similarity'], (int, float)) + assert 0 <= match['similarity'] <= 1 + + @patch('deepgrep.web.app.semantic_engine') + def test_semantic_search_response_structure(self, mock_engine, client): + """Test that response includes matches with word and similarity fields.""" + mock_engine.find_semantic_matches.return_value = [ + ("excited", 0.85) + ] + + data = { + "keyword": "happy", + "text": "I am excited about the news" + } + response = client.post('/semantic', + data=json.dumps(data), + content_type='application/json') + + assert response.status_code == 200 + json_data = response.get_json() + assert 'matches' in json_data + assert 'history' in json_data + assert isinstance(json_data['matches'], list) + + if len(json_data['matches']) > 0: + match = json_data['matches'][0] + assert 'word' in match + assert 'similarity' in match