filter.py

import os
import re
import json
import pickle
import random
import time
import numpy as np
from collections import defaultdict, Counter
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity
from sklearn.decomposition import TruncatedSVD
import sqlite3

class FilteredTrainingAI:
    def __init__(self, data_folder="dataset_clean", model_name="filtered_ai_model"):
        """
        Initialize Filtered Training AI with content filtering
        """
        self.data_folder = data_folder
        self.db_path = f"{model_name}_data.db"
        self.model_path = f"{model_name}.pkl"
        self.embedding_dim = 100
        
        # Content filters
        self.harmful_keywords = [
            'misinformation', 'conspiracy', 'racist', 'lizard people', 
            'extremist', 'malicious', 'harmful', 'unethical', 'illegal',
            'hack', 'exploit', 'virus', 'malware', 'scam', 'fraud',
            'hate speech', 'discrimination', 'violence', 'terrorism'
        ]
        
        self.spam_patterns = [
            r'(.)\1{10,}',  # Repeated characters
            r'misinformation' * 5,  # Repeated words
            r'[A-Z]{20,}',  # All caps spam
            r'[^\w\s]{10,}',  # Special character spam
        ]
        
        # Model components
        self.vectorizer = None
        self.svd_model = None
        self.document_embeddings = None
        self.documents = []
        self.word_vectors = {}
        self.response_templates = {}
        
        # Initialize database
        self.init_database()
        
        print("🧠 Initializing Filtered Training AI...")
        print("🛡️ Content filtering enabled...")
        print("📁 Loading data from dataset folder...")
        
        # Load and process data
        self.load_and_filter_data()
        self.train_model()
    
    def init_database(self):
        """Initialize SQLite database"""
        self.conn = sqlite3.connect(self.db_path)
        self.cursor = self.conn.cursor()
        
        self.cursor.execute('''
            CREATE TABLE IF NOT EXISTS clean_documents (
                id INTEGER PRIMARY KEY,
                content TEXT,
                source_file TEXT,
                quality_score REAL,
                processed_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
            )
        ''')
        
        self.cursor.execute('''
            CREATE TABLE IF NOT EXISTS filtered_content (
                id INTEGER PRIMARY KEY,
                original_content TEXT,
                reason TEXT,
                source_file TEXT
            )
        ''')
        
        self.conn.commit()
    
    def is_content_harmful(self, text):
        """Check if content contains harmful material"""
        text_lower = text.lower()
        
        # Check for harmful keywords
        for keyword in self.harmful_keywords:
            if keyword in text_lower:
                return True, f"Contains harmful keyword: {keyword}"
        
        # Check for spam patterns
        for pattern in self.spam_patterns:
            if re.search(pattern, text):
                return True, f"Matches spam pattern: {pattern}"
        
        # Check for excessive repetition
        words = text.split()
        if len(words) > 10:
            word_counts = Counter(words)
            most_common = word_counts.most_common(1)[0]
            if most_common[1] > len(words) * 0.3:  # More than 30% repetition
                return True, f"Excessive word repetition: {most_common[0]}"
        
        # Check for gibberish
        if re.search(r'[a-z]{20,}', text_lower):  # Very long words without spaces
            return True, "Contains gibberish text"
        
        return False, None
    
    def calculate_quality_score(self, text):
        """Calculate quality score for content"""
        score = 1.0
        
        # Length factor
        length = len(text)
        if 50 <= length <= 1000:
            score += 0.2
        elif length < 20:
            score -= 0.5
        elif length > 2000:
            score -= 0.3
        
        # Sentence structure
        sentences = re.split(r'[.!?]+', text)
        valid_sentences = [s.strip() for s in sentences if len(s.strip()) > 5]
        if len(valid_sentences) >= 2:
            score += 0.2
        
        # Word diversity
        words = re.findall(r'\b[a-zA-Z]{3,}\b', text.lower())
        if words:
            unique_ratio = len(set(words)) / len(words)
            score += unique_ratio * 0.3
        
        # Proper capitalization
        if re.search(r'^[A-Z]', text) and not text.isupper():
            score += 0.1
        
        # Contains meaningful content indicators
        meaningful_patterns = [
            r'\b(is|are|was|were|can|could|will|would|should|may|might)\b',
            r'\b(the|this|that|these|those)\b',
            r'\b(and|but|or|because|since|although|however)\b'
        ]
        
        for pattern in meaningful_patterns:
            if re.search(pattern, text.lower()):
                score += 0.05
        
        return min(score, 2.0)  # Cap at 2.0
    
    def load_and_filter_data(self):
        """Load data with content filtering"""
        txt_files = self.find_txt_files()
        
        if not txt_files:
            print("❌ No .txt files found in dataset folder!")
            return
        
        print(f"📊 Found {len(txt_files)} .txt files")
        
        total_chunks = 0
        filtered_chunks = 0
        clean_chunks = 0
        
        for file_path in txt_files:
            print(f"📖 Processing: {os.path.basename(file_path)}")
            
            try:
                with open(file_path, 'r', encoding='utf-8') as f:
                    content = f.read()
                
                # Split into chunks
                chunks = self.split_into_chunks(content, file_path)
                total_chunks += len(chunks)
                
                file_clean = 0
                file_filtered = 0
                
                for chunk in chunks:
                    chunk_content = chunk['content']
                    
                    # Check if harmful
                    is_harmful, reason = self.is_content_harmful(chunk_content)
                    
                    if is_harmful:
                        # Log filtered content
                        self.cursor.execute(
                            'INSERT INTO filtered_content (original_content, reason, source_file) VALUES (?, ?, ?)',
                            (chunk_content[:500], reason, file_path)
                        )
                        file_filtered += 1
                        filtered_chunks += 1
                    else:
                        # Calculate quality score
                        quality_score = self.calculate_quality_score(chunk_content)
                        
                        # Only keep high-quality content
                        if quality_score >= 0.8:
                            self.documents.append(chunk)
                            
                            # Save to database
                            self.cursor.execute(
                                'INSERT INTO clean_documents (content, source_file, quality_score) VALUES (?, ?, ?)',
                                (chunk_content, file_path, quality_score)
                            )
                            
                            file_clean += 1
                            clean_chunks += 1
                
                print(f"   ✅ Clean: {file_clean:,} | ❌ Filtered: {file_filtered:,}")
                
            except Exception as e:
                print(f"   ❌ Error: {e}")
        
        self.conn.commit()
        
        print(f"\n📊 FILTERING RESULTS:")
        print(f"• Total chunks processed: {total_chunks:,}")
        print(f"• Clean chunks kept: {clean_chunks:,} ({clean_chunks/total_chunks*100:.1f}%)")
        print(f"• Harmful chunks filtered: {filtered_chunks:,} ({filtered_chunks/total_chunks*100:.1f}%)")
        
        if clean_chunks == 0:
            print("❌ No clean content found! Please check your dataset.")
            return
        
        # Extract response templates from clean data
        self.extract_clean_templates()
    
    def find_txt_files(self):
        """Find all .txt files in dataset folder"""
        txt_files = []
        
        if os.path.exists(self.data_folder):
            for root, dirs, files in os.walk(self.data_folder):
                for file in files:
                    if file.endswith('.txt'):
                        txt_files.append(os.path.join(root, file))
        
        return txt_files
    
    def split_into_chunks(self, content, source_file, chunk_size=300):
        """Split content into chunks"""
        # Clean content first
        content = self.clean_text(content)
        
        # Split by paragraphs
        paragraphs = re.split(r'\n\s*\n', content)
        
        chunks = []
        current_chunk = ""
        
        for para in paragraphs:
            para = para.strip()
            if not para:
                continue
            
            # Skip very short paragraphs
            if len(para) < 20:
                continue
            
            if len(current_chunk) + len(para) < chunk_size:
                current_chunk += para + " "
            else:
                if current_chunk:
                    chunks.append({
                        'content': current_chunk.strip(),
                        'source': source_file
                    })
                current_chunk = para + " "
        
        # Add the last chunk
        if current_chunk:
            chunks.append({
                'content': current_chunk.strip(),
                'source': source_file
            })
        
        return chunks
    
    def clean_text(self, text):
        """Clean and normalize text"""
        # Remove excessive whitespace
        text = re.sub(r'\s+', ' ', text)
        
        # Remove special characters but keep basic punctuation
        text = re.sub(r'[^\w\s\.\?\!,\'\-$$$$]', ' ', text)
        
        # Remove repeated punctuation
        text = re.sub(r'([.!?]){2,}', r'\1', text)
        
        # Remove lines that are mostly special characters
        lines = text.split('\n')
        clean_lines = []
        for line in lines:
            if len(re.findall(r'[a-zA-Z]', line)) > len(line) * 0.5:
                clean_lines.append(line)
        
        return '\n'.join(clean_lines).strip()
    
    def extract_clean_templates(self):
        """Extract response templates from clean data"""
        print("🔍 Extracting clean response templates...")
        
        template_patterns = {
            'definition': [
                r'([A-Z][a-z]+(?:\s+[a-z]+)*)\s+is\s+([^.]+\.)',
                r'([A-Z][a-z]+(?:\s+[a-z]+)*)\s+refers\s+to\s+([^.]+\.)',
                r'([A-Z][a-z]+(?:\s+[a-z]+)*)\s+means\s+([^.]+\.)'
            ],
            'explanation': [
                r'([A-Z][a-z]+(?:\s+[a-z]+)*)\s+works\s+by\s+([^.]+\.)',
                r'The\s+process\s+of\s+([^,]+)\s+involves\s+([^.]+\.)',
                r'To\s+([^,]+),\s+([^.]+\.)'
            ],
            'example': [
                r'For\s+example,\s+([^.]+\.)',
                r'An\s+example\s+of\s+([^,]+)\s+is\s+([^.]+\.)',
                r'Such\s+as\s+([^.]+\.)'
            ]
        }
        
        templates_found = defaultdict(list)
        
        for doc in self.documents:
            content = doc['content']
            
            for template_type, patterns in template_patterns.items():
                for pattern in patterns:
                    matches = re.findall(pattern, content)
                    for match in matches:
                        if isinstance(match, tuple) and len(match) >= 1:
                            # Only keep clean matches
                            match_text = ' '.join(match) if isinstance(match, tuple) else match
                            is_harmful, _ = self.is_content_harmful(match_text)
                            
                            if not is_harmful and len(match_text) > 10:
                                templates_found[template_type].append(match_text)
        
        self.response_templates = dict(templates_found)
        
        print(f"✅ Extracted clean templates: {sum(len(t) for t in templates_found.values()):,}")
        for t_type, templates in templates_found.items():
            print(f"   • {t_type}: {len(templates):,}")
    
    def train_model(self):
        """Train the AI model on clean data"""
        print("🧠 Training AI model on clean data...")
        
        if not self.documents:
            print("❌ No clean documents to train on!")
            return
        
        # Create TF-IDF vectorizer
        print("   ⚙️ Creating TF-IDF vectors...")
        self.vectorizer = TfidfVectorizer(
            max_features=8000,
            ngram_range=(1, 2),
            stop_words='english',
            min_df=2,  # Ignore terms that appear in less than 2 documents
            max_df=0.95  # Ignore terms that appear in more than 95% of documents
        )
        
        # Extract document content
        doc_contents = [doc['content'] for doc in self.documents]
        
        # Fit and transform documents
        tfidf_matrix = self.vectorizer.fit_transform(doc_contents)
        
        # Apply dimensionality reduction
        print("   ⚙️ Creating semantic embeddings...")
        self.svd_model = TruncatedSVD(n_components=self.embedding_dim)
        self.document_embeddings = self.svd_model.fit_transform(tfidf_matrix)
        
        # Save model
        self.save_model()
        
        print("✅ Clean model training complete!")
    
    def save_model(self):
        """Save trained model"""
        model_data = {
            'vectorizer': self.vectorizer,
            'svd_model': self.svd_model,
            'document_embeddings': self.document_embeddings,
            'documents': self.documents,
            'response_templates': self.response_templates
        }
        
        with open(self.model_path, 'wb') as f:
            pickle.dump(model_data, f)
        
        print(f"💾 Clean model saved to {self.model_path}")
    
    def load_model(self):
        """Load trained model"""
        if os.path.exists(self.model_path):
            with open(self.model_path, 'rb') as f:
                model_data = pickle.load(f)
                
                self.vectorizer = model_data['vectorizer']
                self.svd_model = model_data['svd_model']
                self.document_embeddings = model_data['document_embeddings']
                self.documents = model_data['documents']
                self.response_templates = model_data.get('response_templates', {})
            
            return True
        return False
    
    def answer_question(self, question):
        """Answer question using clean model"""
        if not self.load_model():
            return {
                'answer': "Model not trained yet. Please run training first.",
                'confidence': 0,
                'sources': []
            }
        
        # Check if question itself is harmful
        is_harmful, reason = self.is_content_harmful(question)
        if is_harmful:
            return {
                'answer': "I can't provide information on that topic. Please ask about something else.",
                'confidence': 0,
                'sources': []
            }
        
        # Analyze question
        question_type, focus_words = self.analyze_question(question)
        
        # Find relevant documents
        relevant_docs = self.find_relevant_documents(question, top_k=3)
        
        if not relevant_docs:
            return {
                'answer': "I don't have enough information to answer this question accurately.",
                'confidence': 0,
                'sources': []
            }
        
        # Generate clean answer
        answer = self.generate_clean_answer(question, question_type, focus_words, relevant_docs)
        
        # Calculate confidence
        confidence = self.calculate_confidence(question, answer, relevant_docs)
        
        # Format sources
        sources = [os.path.basename(doc['source']) for doc in relevant_docs]
        
        return {
            'answer': answer,
            'confidence': confidence,
            'sources': sources
        }
    
    def analyze_question(self, question):
        """Analyze question type and focus"""
        question = question.lower()
        
        # Determine question type
        if re.search(r'\b(what|who|which)\s+(is|are|was|were)\b', question):
            question_type = 'definition'
        elif re.search(r'\b(how)\s+(does|do|can|could|would|should)\b', question):
            question_type = 'explanation'
        elif re.search(r'\b(why)\b', question):
            question_type = 'reasoning'
        elif re.search(r'\b(when|where)\b', question):
            question_type = 'factual'
        else:
            question_type = 'general'
        
        # Extract focus words
        cleaned_question = re.sub(r'\b(what|who|how|why|when|where|which|is|are|do|does|can|could|would|should|the|a|an)\b', '', question)
        focus_words = [word for word in re.findall(r'\b[a-z]{3,}\b', cleaned_question) 
                      if word not in ['this', 'that', 'these', 'those', 'there', 'here', 'from', 'with', 'about']]
        
        return question_type, focus_words
    
    def find_relevant_documents(self, question, top_k=3):
        """Find relevant documents using semantic search"""
        # Convert question to vector
        question_vector = self.vectorizer.transform([question])
        
        # Calculate similarities
        doc_vectors = self.vectorizer.transform([doc['content'] for doc in self.documents])
        similarities = cosine_similarity(question_vector, doc_vectors)[0]
        
        # Get top documents
        top_indices = np.argsort(similarities)[::-1][:top_k]
        
        relevant_docs = []
        for idx in top_indices:
            if similarities[idx] > 0.1:  # Minimum similarity threshold
                relevant_docs.append({
                    'content': self.documents[idx]['content'],
                    'source': self.documents[idx]['source'],
                    'similarity': similarities[idx]
                })
        
        return relevant_docs
    
    def generate_clean_answer(self, question, question_type, focus_words, relevant_docs):
        """Generate clean, helpful answer"""
        # Extract relevant sentences
        relevant_sentences = []
        for doc in relevant_docs:
            sentences = re.split(r'(?<=[.!?])\s+', doc['content'])
            for sentence in sentences:
                # Check if sentence is relevant and clean
                if any(word.lower() in sentence.lower() for word in focus_words):
                    is_harmful, _ = self.is_content_harmful(sentence)
                    if not is_harmful and len(sentence) > 20:
                        relevant_sentences.append(sentence)
        
        if not relevant_sentences:
            return "I don't have enough clean information to answer this question accurately."
        
        # Generate answer based on question type
        if question_type == 'definition':
            return self.generate_definition_answer(focus_words, relevant_sentences)
        elif question_type == 'explanation':
            return self.generate_explanation_answer(focus_words, relevant_sentences)
        elif question_type == 'reasoning':
            return self.generate_reasoning_answer(focus_words, relevant_sentences)
        else:
            return self.generate_general_clean_answer(focus_words, relevant_sentences)
    
    def generate_definition_answer(self, focus_words, relevant_sentences):
        """Generate definition answer"""
        # Look for definition patterns
        for sentence in relevant_sentences:
            if re.search(r'\b(is|are|refers to|means|defined as)\b', sentence.lower()):
                return sentence
        
        # Fallback to first relevant sentence
        if relevant_sentences:
            topic = ' '.join(focus_words).title()
            return f"{topic} can be described as follows: {relevant_sentences[0]}"
        
        return "I don't have a clear definition for this topic."
    
    def generate_explanation_answer(self, focus_words, relevant_sentences):
        """Generate explanation answer"""
        # Look for process/method descriptions
        explanation_sentences = []
        for sentence in relevant_sentences:
            if re.search(r'\b(works|process|method|technique|approach|steps|involves)\b', sentence.lower()):
                explanation_sentences.append(sentence)
        
        if explanation_sentences:
            return ' '.join(explanation_sentences[:2])  # Combine up to 2 sentences
        elif relevant_sentences:
            return f"Here's how it works: {relevant_sentences[0]}"
        
        return "I don't have enough information to explain this process."
    
    def generate_reasoning_answer(self, focus_words, relevant_sentences):
        """Generate reasoning answer"""
        # Look for causal relationships
        reasoning_sentences = []
        for sentence in relevant_sentences:
            if re.search(r'\b(because|since|due to|reason|cause|result|leads to)\b', sentence.lower()):
                reasoning_sentences.append(sentence)
        
        if reasoning_sentences:
            return ' '.join(reasoning_sentences[:2])
        elif relevant_sentences:
            return f"The reason is: {relevant_sentences[0]}"
        
        return "I don't have enough information to explain the reasoning."
    
    def generate_general_clean_answer(self, focus_words, relevant_sentences):
        """Generate general clean answer"""
        # Select best sentences (up to 3)
        selected_sentences = relevant_sentences[:3]
        answer = ' '.join(selected_sentences)
        
        # Add context if needed
        if focus_words:
            topic = ' '.join(focus_words).title()
            answer = f"Regarding {topic}: {answer}"
        
        return answer
    
    def calculate_confidence(self, question, answer, relevant_docs):
        """Calculate confidence score"""
        if not relevant_docs:
            return 0.0
        
        # Base confidence on document similarity
        base_confidence = max([doc['similarity'] for doc in relevant_docs])
        
        # Adjust for answer quality
        quality_score = 0.0
        
        # Length factor
        answer_length = len(answer)
        if 50 <= answer_length <= 300:
            quality_score += 0.3
        elif answer_length < 20:
            quality_score -= 0.5
        
        # Check if answer is clean
        is_harmful, _ = self.is_content_harmful(answer)
        if is_harmful:
            return 0.0  # Zero confidence for harmful content
        
        # Focus word coverage
        question_focus = self.analyze_question(question)[1]
        if question_focus:
            focus_coverage = sum(1 for word in question_focus if word.lower() in answer.lower()) / len(question_focus)
            quality_score += focus_coverage * 0.4
        
        return min(base_confidence * 0.6 + quality_score, 1.0)
    
    def get_stats(self):
        """Get model statistics"""
        if not self.load_model():
            return {'status': 'Model not trained'}
        
        # Get filtering stats from database
        self.cursor.execute('SELECT COUNT(*) FROM clean_documents')
        clean_count = self.cursor.fetchone()[0]
        
        self.cursor.execute('SELECT COUNT(*) FROM filtered_content')
        filtered_count = self.cursor.fetchone()[0]
        
        return {
            'status': 'Model trained',
            'clean_documents': clean_count,
            'filtered_documents': filtered_count,
            'vocabulary': len(self.vectorizer.get_feature_names_out()) if self.vectorizer else 0,
            'response_templates': sum(len(templates) for templates in self.response_templates.values()),
            'filtering_ratio': f"{filtered_count/(clean_count+filtered_count)*100:.1f}%" if (clean_count+filtered_count) > 0 else "0%"
        }

def main():
    """Main function"""
    print("🛡️ FILTERED TRAINING AI - SAFE & CLEAN")
    print("="*60)
    
    try:
        # Initialize AI with filtering
        ai = FilteredTrainingAI(data_folder="dataset")
        
        # Show statistics
        stats = ai.get_stats()
        print(f"\n📊 MODEL STATISTICS:")
        for key, value in stats.items():
            print(f"• {key}: {value}")
        
        print("\n" + "="*60)
        print("🎯 Clean AI ready to answer questions!")
        print("Type 'quit' to exit")
        print("Type 'stats' to see statistics")
        print("="*60)
        
        while True:
            question = input("\n❓ Your question: ").strip()
            
            if question.lower() == 'quit':
                print("👋 Goodbye!")
                break
            elif question.lower() == 'stats':
                stats = ai.get_stats()
                print(f"\n📊 CURRENT STATISTICS:")
                for key, value in stats.items():
                    print(f"• {key}: {value}")
                continue
            
            if not question:
                continue
            
            print("🤔 Thinking...")
            
            # Answer question
            start_time = time.time()
            result = ai.answer_question(question)
            processing_time = time.time() - start_time
            
            print(f"\n🧠 **Answer:** (generated in {processing_time:.2f}s)")
            print(result['answer'])
            print(f"\n🎯 **Confidence:** {result['confidence']:.2f}")
            
            if result['sources']:
                print(f"📚 **Sources:** {', '.join(result['sources'])}")
    
    except Exception as e:
        print(f"❌ Error: {e}")
        print("\nPlease check:")
        print("1. 'dataset' folder exists and contains .txt files")
        print("2. Required libraries are installed")

if __name__ == "__main__":
    main()