Add production-ready features: logging, environment config, rate limiting, and comprehensive tests

.env.example

@@ -0,0 +1,18 @@
+# Server Configuration
+PORT=8000
+DEBUG=True
+HOST=0.0.0.0
+
+# Database Configuration
+DB_PATH=~/.grepify_history.db
+MAX_HISTORY=200
+
+# Semantic Search Configuration
+SPACY_MODEL=en_core_web_md
+SEMANTIC_THRESHOLD=0.45
+SEMANTIC_TOP_N=10
+
+# Rate Limiting
+RATE_LIMIT_ENABLED=True
+RATE_LIMIT_REQUESTS=100
+RATE_LIMIT_WINDOW=60

README.md

@@ -65,24 +65,24 @@
     <ul>
         <li style="list-style-type: none;">
             <b>Regex Usage</b>: Shows a complex Regex pattern matching date/time data in a log snippet. <br>
-            <img src="https://github.com/vivekkdagar/deepgrep/blob/77532f804aa766a01abe6d0375965aa20c8623b2/assets/outputs/regex.jpeg" alt="Regex Matching Output" style="height: 300px; max-width: 100%; display: block; margin: 10px auto;">
+            <img src="https://github.com/alwaysvivek/deepgrep/blob/main/assets/outputs/regex.jpeg" alt="Regex Matching Output" style="height: 300px; max-width: 100%; display: block; margin: 10px auto;">
         </li>
      <br>
         <li style="list-style-type: none;">
             <b>Semantic Search Usage</b>: Shows a Semantic search for "happy" returning "proud" with a similarity score. <br>
-            <img src="https://github.com/vivekkdagar/deepgrep/blob/main/assets/outputs/semantic.jpeg" alt="Semantic Search Result" style="height: 300px; max-width: 100%; display: block; margin: 10px auto;">
+            <img src="https://github.com/alwaysvivek/deepgrep/blob/main/assets/outputs/semantic.jpeg" alt="Semantic Search Result" style="height: 300px; max-width: 100%; display: block; margin: 10px auto;">
         </li> <br>
         <li style="list-style-type: none;">
             <b>Server Log</b>: The server terminal output showing multiple successful POST /semantic calls. <br>
-            <img src="https://github.com/vivekkdagar/deepgrep/blob/77532f804aa766a01abe6d0375965aa20c8623b2/assets/outputs/api-running.jpeg" alt="API Server Log" style="height: 200px; max-width: 100%; display: block; margin: 10px auto;">
+            <img src="https://github.com/alwaysvivek/deepgrep/blob/main/assets/outputs/api-running.jpeg" alt="API Server Log" style="height: 200px; max-width: 100%; display: block; margin: 10px auto;">
         </li> <br>
         <li style="list-style-type: none;">
             <b>History</b>: The history table view, displaying matches, pattern, and timestamp columns. <br>
-            <img src="https://github.com/vivekkdagar/deepgrep/blob/77532f804aa766a01abe6d0375965aa20c8623b2/assets/outputs/history.jpeg" alt="History Database View" style="height: 300px; max-width: 100%; display: block; margin: 10px auto;">
+            <img src="https://github.com/alwaysvivek/deepgrep/blob/main/assets/outputs/history.jpeg" alt="History Database View" style="height: 300px; max-width: 100%; display: block; margin: 10px auto;">
         </li> <br>
         <li style="list-style-type: none;">
             <b>Postman Output</b>: Shows the Postman interface with the JSON history response from the /search endpoint. <br>
-            <img src="https://github.com/vivekkdagar/deepgrep/blob/77532f804aa766a01abe6d0375965aa20c8623b2/assets/outputs/postman%20output.jpeg" alt="Postman API History JSON" style="height: 300px; max-width: 100%; display: block; margin: 10px auto;">
+            <img src="https://github.com/alwaysvivek/deepgrep/blob/main/assets/outputs/postman%20output.jpeg" alt="Postman API History JSON" style="height: 300px; max-width: 100%; display: block; margin: 10px auto;">
         </li> <br>
     </ul>
 </div>
@@ -248,6 +248,6 @@ This project is licensed under the MIT License - see the [LICENSE](LICENSE) file
 
 <div align="center">
 
-[Report Bug](https://github.com/vivekkdagar/deepgrep/issues) · [Request Feature](https://github.com/vivekkdagar/deepgrep/issues)
+[Report Bug](https://github.com/alwaysvivek/deepgrep/issues) · [Request Feature](https://github.com/alwaysvivek/deepgrep/issues)
 
 </div>

deepgrep/core/history.py

@@ -1,10 +1,16 @@
 import sqlite3, json
+import logging
 from datetime import datetime, timezone
 from pathlib import Path
 from typing import List, Tuple
+from decouple import config
 
-DB_PATH = Path.home() / ".grepify_history.db"
-MAX_HISTORY = 200
+# Configure logging
+logger = logging.getLogger(__name__)
+
+# Load configuration from environment variables
+DB_PATH = Path(config('DB_PATH', default='~/.grepify_history.db')).expanduser()
+MAX_HISTORY = config('MAX_HISTORY', default=200, cast=int)
 
 class SearchHistoryDB:
     """SQLite-backed search history logger."""
@@ -14,6 +20,7 @@ def __init__(self, db_path: Path = DB_PATH):
         self._init_db()
 
     def _init_db(self):
+        logger.info(f"Initializing search history database at {self.db_path}")
         with sqlite3.connect(self.db_path) as conn:
             conn.execute("""
                 CREATE TABLE IF NOT EXISTS search_logs (
@@ -24,19 +31,25 @@ def _init_db(self):
                     files TEXT
                 );
             """)
+        logger.info("Database initialized successfully")
 
     def log_search(self, pattern: str, match_count: int, files: List[str]):
         with sqlite3.connect(self.db_path) as conn:
             conn.execute(
                 "INSERT INTO search_logs (pattern, timestamp, match_count, files) VALUES (?, ?, ?, ?)",
                 (pattern, datetime.now(timezone.utc).isoformat(), match_count, json.dumps(files)),
             )
-            conn.execute("""
+            # Cleanup old records
+            result = conn.execute("""
                 DELETE FROM search_logs WHERE id NOT IN (
                     SELECT id FROM search_logs ORDER BY id DESC LIMIT ?
                 );
             """, (MAX_HISTORY,))
+            deleted_count = result.rowcount
             conn.commit()
+        logger.info(f"Logged search: pattern='{pattern}', matches={match_count}, files={len(files)}")
+        if deleted_count > 0:
+            logger.debug(f"Cleaned up {deleted_count} old records")
 
     def get_recent(self, limit: int = 5) -> List[Tuple]:
         with sqlite3.connect(self.db_path) as conn:
@@ -68,6 +81,7 @@ def export_to_json(self, file_path: Path) -> int:
             for r in rows
         ]
         file_path.write_text(json.dumps(data, indent=2), encoding="utf-8")
+        logger.info(f"Exported {len(data)} search records to {file_path}")
         return len(data)
 
     def import_from_json(self, file_path: Path) -> int:
@@ -92,4 +106,5 @@ def import_from_json(self, file_path: Path) -> int:
                 );
             """, (MAX_HISTORY,))
             conn.commit()
+        logger.info(f"Imported {len(data)} search records from {file_path}")
         return len(data)

deepgrep/core/semantic_engine.py

@@ -1,6 +1,12 @@
 import spacy
 from nltk.corpus import wordnet
 from typing import List, Tuple
+from decouple import config
+
+# Load configuration from environment variables
+SPACY_MODEL = config('SPACY_MODEL', default='en_core_web_md')
+SEMANTIC_THRESHOLD = config('SEMANTIC_THRESHOLD', default=0.45, cast=float)
+SEMANTIC_TOP_N = config('SEMANTIC_TOP_N', default=10, cast=int)
 
 
 def get_antonyms(word: str) -> set:
@@ -15,15 +21,19 @@ def get_antonyms(word: str) -> set:
 class SemanticEngine:
     """Semantic search avoiding antonyms and irrelevant words."""
 
-    def __init__(self, model_name: str = "en_core_web_md"):
+    def __init__(self, model_name: str = None):
+        if model_name is None:
+            model_name = SPACY_MODEL
         try:
             self.nlp = spacy.load(model_name)
         except OSError:
             raise RuntimeError(
                 f"SpaCy model '{model_name}' not found. Run: python -m spacy download {model_name}"
             )
 
-    def find_semantic_matches(self, text: str, keyword: str, top_n: int = 10) -> List[Tuple[str, float]]:
+    def find_semantic_matches(self, text: str, keyword: str, top_n: int = None) -> List[Tuple[str, float]]:
+        if top_n is None:
+            top_n = SEMANTIC_TOP_N
         doc = self.nlp(text)
         key_token = self.nlp(keyword)[0]
         antonyms = get_antonyms(keyword.lower())
@@ -43,5 +53,5 @@ def find_semantic_matches(self, text: str, keyword: str, top_n: int = 10) -> Lis
         # Sort descending and keep top N
         results = sorted(results, key=lambda x: x[1], reverse=True)[:top_n]
         # Filter out weak matches
-        results = [(w, s) for w, s in results if s > 0.45]
+        results = [(w, s) for w, s in results if s > SEMANTIC_THRESHOLD]
         return results

deepgrep/web/app.py

@@ -1,28 +1,70 @@
-# app.py (small changes to init components and avoid nltk download in runtime)
+# app.py - Production-ready Flask application with logging and rate limiting
+import logging
 from flask import Flask, request, jsonify, render_template
 from flask_cors import CORS
+from flask_limiter import Limiter
+from flask_limiter.util import get_remote_address
+from decouple import config
 from deepgrep.core.engine import find_matches
 from deepgrep.core.history import SearchHistoryDB
 from deepgrep.core.semantic_engine import SemanticEngine
-import threading
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+)
+logger = logging.getLogger(__name__)
+
+# Load configuration from environment variables
+PORT = config('PORT', default=8000, cast=int)
+DEBUG = config('DEBUG', default=True, cast=bool)
+HOST = config('HOST', default='0.0.0.0')
+RATE_LIMIT_ENABLED = config('RATE_LIMIT_ENABLED', default=True, cast=bool)
+RATE_LIMIT_REQUESTS = config('RATE_LIMIT_REQUESTS', default=100, cast=int)
 
 app = Flask(__name__, template_folder="templates", static_folder="static")
 CORS(app)
 
-# Initialize engines once (heavy models loaded at import time)
-semantic_engine = SemanticEngine()
-history_db = SearchHistoryDB()
+# Initialize rate limiter
+limiter = Limiter(
+    app=app,
+    key_func=get_remote_address,
+    default_limits=[f"200 per minute"],
+    enabled=RATE_LIMIT_ENABLED
+)
+
+# Initialize engines (lazy initialization to support testing)
+semantic_engine = None
+history_db = None
+
+def init_engines():
+    """Initialize semantic engine and history database (called lazily on first use)."""
+    global semantic_engine, history_db
+    if semantic_engine is None:
+        logger.info("Initializing semantic engine and history database...")
+        semantic_engine = SemanticEngine()
+        history_db = SearchHistoryDB()
+        logger.info("Application initialization complete")
 
 @app.route("/")
 def home():
+    logger.info("GET / - Home page requested")
     return render_template("index.html")
 
 @app.route("/search", methods=["POST"])
+@limiter.limit(f"{RATE_LIMIT_REQUESTS}/minute")
 def search_regex():
+    # Only initialize history_db, not semantic_engine (not needed for regex)
+    if history_db is None:
+        init_engines()
+    logger.info("POST /search - Regex search request received")
     data = request.get_json()
     pattern = data.get("pattern")
     text = data.get("text")
+
     if not pattern or not text:
+        logger.warning("POST /search - Missing pattern or text in request")
         return jsonify({"error": "Missing pattern or text"}), 400
 
     matches = []
@@ -32,6 +74,8 @@ def search_regex():
 
     history_db.log_search(pattern, len(matches), ["web_input"])
     all_history = history_db.list_all(limit=50)
+
+    logger.info(f"POST /search - Successfully found {len(matches)} matches for pattern '{pattern}'")
 
     return jsonify({
         "matches": matches,
@@ -41,16 +85,27 @@ def search_regex():
     })
 
 @app.route("/semantic", methods=["POST"])
+@limiter.limit(f"{RATE_LIMIT_REQUESTS}/minute")
 def search_semantic():
+    logger.info("POST /semantic - Semantic search request received")
     data = request.get_json()
     keyword = data.get("keyword")
     text = data.get("text")
+
+    # Validate input first, before initializing engines
     if not keyword or not text:
+        logger.warning("POST /semantic - Missing keyword or text in request")
         return jsonify({"error": "Missing keyword or text"}), 400
+
+    # Only initialize engines after validation passes
+    if semantic_engine is None or history_db is None:
+        init_engines()
 
     matches = semantic_engine.find_semantic_matches(text, keyword)
     history_db.log_search(keyword, len(matches), ["web_input"])
     all_history = history_db.list_all(limit=50)
+
+    logger.info(f"POST /semantic - Successfully found {len(matches)} semantic matches for keyword '{keyword}'")
 
     return jsonify({
         "matches": [{"word": w, "similarity": round(s, 3)} for w, s in matches],
@@ -63,4 +118,6 @@ def search_semantic():
     # Remove in-production NLTK downloads; expect preinstalled corpora
     # import nltk
     # nltk.download('wordnet')
-    app.run(port=8000, debug=True)
+    init_engines()  # Ensure engines are initialized when running directly
+    logger.info(f"Starting Flask server on {HOST}:{PORT} (debug={DEBUG})")
+    app.run(host=HOST, port=PORT, debug=DEBUG)

requirements.txt

@@ -3,3 +3,5 @@ spacy
 nltk
 flask-cors
 flask
+python-decouple
+flask-limiter