camera.py

# camera.py
import time
import threading
import numpy as np
import cv2
import mediapipe as mp
from tensorflow.keras.models import load_model
import queue
import os

# ===================== SETTINGS =====================
MAIN_MODEL_PATH = "project/Model/landmark_mlp_model_dropZ.h5"
RUV_MODEL_PATH = "project/Model/mlp_model_RUV.h5"
MAIN_LABELS_PATH = "project/Model/label_classes.npy"
RUV_LABELS_PATH = "project/Model/label_classes_RUV.npy"

CONF_THRESHOLD = 0.0    # Threshold removed
STABILITY_SECONDS = 3.0
SMOOTHING = 0.5

LANDMARK_COLOR = (0, 255, 0)
LANDMARK_RADIUS = 3

# ===================== MODEL LOADING =====================
try:
    main_model = load_model(MAIN_MODEL_PATH)
    main_labels = np.load(MAIN_LABELS_PATH, allow_pickle=True)

    ruv_model = None
    ruv_labels = None
    if os.path.exists(RUV_MODEL_PATH) and os.path.exists(RUV_LABELS_PATH):
        ruv_model = load_model(RUV_MODEL_PATH)
        ruv_labels = np.load(RUV_LABELS_PATH, allow_pickle=True)
except Exception as e:
    print(f"ERROR loading models in camera.py: {e}")
    main_model, main_labels = None, None # Prevent app start

# ===================== MEDIA PIPE SETUP =====================
mp_hands = mp.solutions.hands
hands_detector = mp_hands.Hands(
    max_num_hands=2,
    min_detection_confidence=0.6,
    min_tracking_confidence=0.6
)

# ===================== UTIL FUNCTIONS =====================
def is_open_palm(landmarks, handedness_label, img_w, img_h):
    # Implementation remains the same as your original script
    pts = [(int(lm.x * img_w), int(lm.y * img_h)) for lm in landmarks]
    finger_tips = [8, 12, 16, 20]
    finger_pips = [6, 10, 14, 18]
    for tip, pip in zip(finger_tips, finger_pips):
        if pts[tip][1] > pts[pip][1] + 8: return False

    thumb_tip_x = pts[4][0]
    thumb_ip_x = pts[3][0]

    if handedness_label == "Left":
        if thumb_tip_x < thumb_ip_x - 8: return False
    else:
        if thumb_tip_x > thumb_ip_x + 8: return False
    return True

def predict_label_from_landmarks(landmarks_np):
    if main_model is None: return "", 0.0

    input_main = landmarks_np[:, :2].flatten()[np.newaxis, :]
    preds = main_model.predict(input_main, verbose=0)[0]
    top_idx = int(np.argmax(preds))
    top_prob = float(preds[top_idx])
    main_label = str(main_labels[top_idx])

    if main_label in ['R', 'U', 'V'] and ruv_model is not None:
        input_ruv = landmarks_np.flatten()[np.newaxis, :]
        preds2 = ruv_model.predict(input_ruv, verbose=0)[0]
        top2 = int(np.argmax(preds2))
        final_label = str(ruv_labels[top2])
        final_conf = max(top_prob, float(preds2[top2]))
        return final_label, final_conf
    else:
        return main_label, top_prob

# ===================== VIDEO CAMERA CLASS =====================
class VideoCamera:
    def __init__(self):
        self.cap = cv2.VideoCapture(0, cv2.CAP_DSHOW if os.name == "nt" else 0)
        if not self.cap.isOpened():
             raise IOError("Cannot open webcam")
             
        self.frame = None
        self.output_queue = queue.Queue() # Thread-safe output queue for characters
        self.lock = threading.Lock()
        self.stop_event = threading.Event()
        self.thread = threading.Thread(target=self._update, daemon=True)
        self.thread.start()

        # Prediction state
        self.last_pred = None
        self.pred_start = None
        self.prev_landmarks = {}
        self.last_space_time = 0
        self.space_cooldown = 0.5

    def __del__(self):
        self.stop_event.set()
        self.cap.release()
        if self.thread.is_alive():
            self.thread.join(timeout=2)

    def get_frame(self):
        # Read the latest processed frame
        with self.lock:
            if self.frame is None:
                return np.zeros((480, 640, 3), dtype=np.uint8) # Return black frame if not ready
            ret, jpeg = cv2.imencode('.jpg', self.frame)
            return jpeg.tobytes()

    def _update(self):
        """Worker thread loop for video capture and ASL prediction."""
        while not self.stop_event.is_set():
            ret, frame = self.cap.read()
            if not ret:
                time.sleep(0.1)
                continue

            frame = cv2.flip(frame, 1)
            img_h, img_w = frame.shape[:2]
            rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
            results = hands_detector.process(rgb)

            prediction_text = ""
            highest_confidence = 0.0
            left_hand_open = False
            now = time.time()

            if results.multi_hand_landmarks:
                handedness_list = [h.classification[0].label for h in results.multi_handedness] if results.multi_handedness else []

                for idx, hand_landmarks in enumerate(results.multi_hand_landmarks):
                    landmarks = np.array([[lm.x, lm.y, lm.z] for lm in hand_landmarks.landmark])

                    # Smoothing
                    if idx in self.prev_landmarks:
                        landmarks = SMOOTHING * self.prev_landmarks[idx] + (1 - SMOOTHING) * landmarks
                    self.prev_landmarks[idx] = landmarks

                    # Draw landmarks
                    for (x, y, z) in landmarks:
                        cv2.circle(frame, (int(x*img_w), int(y*img_h)), LANDMARK_RADIUS, LANDMARK_COLOR, -1)

                    handed_label = handedness_list[idx] if idx < len(handedness_list) else ("Right" if idx == 0 else "Left")

                    if handed_label == "Left" and is_open_palm(hand_landmarks.landmark, handed_label, img_w, img_h):
                        left_hand_open = True

                    label, conf = predict_label_from_landmarks(landmarks)
                    if conf > highest_confidence:
                        highest_confidence = conf
                        prediction_text = label

            # Stability / Commit Logic
            if left_hand_open and (now - self.last_space_time) > self.space_cooldown:
                self.output_queue.put(" ") # Put space into queue
                self.last_space_time = now

            if prediction_text != "":
                if prediction_text != self.last_pred:
                    self.last_pred = prediction_text
                    self.pred_start = now
                else:
                    elapsed = now - (self.pred_start or now)
                    conf_ok = highest_confidence >= CONF_THRESHOLD
                    if elapsed >= STABILITY_SECONDS and conf_ok:
                        self.output_queue.put(prediction_text) # Put character into queue
                        self.last_pred = None
                        self.pred_start = None
            else:
                self.last_pred = None
                self.pred_start = None

            # Annotate frame
            status = f"Pred: {prediction_text} ({highest_confidence:.2f})"
            cv2.putText(frame, status, (10, 30), cv2.FONT_HERSHEY_SIMPLEX, 0.9, (0, 200, 200), 2)
            if left_hand_open:
                cv2.putText(frame, "LEFT OPEN → space", (10, 60), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (180,180,80), 2)

            # Write processed frame (thread-safe)
            with self.lock:
                self.frame = frame.copy()