Merge pull request #148 from GAchuzia/feature/confidence-limits

feat: improve food scanning UX for low-confidence and non-food images

Author: Indecisive613

backend/endpoints/ml_endpoints.py

@@ -35,9 +35,10 @@ def predict_food_image():
     - Body: image file (form field name: 'image')
 
     Responses:
-    200 OK - Successfully predicted food
-        Response Body (JSON):
+    200 OK - Prediction result (either success or low confidence)
+        If confidence >=80%:
         {
+            "success": true,
             "food_name": string,
             "confidence": float,
             "calories": int
@@ -60,6 +61,13 @@ def predict_food_image():
             "has_treenuts": bool,
             "has_wheat": bool,
         }
+        If confidence < 80% (unclear or non-food image):
+        {
+            "success": false,
+            "reason": "low_confidence",
+            "confidence": float,
+            "message": string,
+        }
 
     400 Bad Request - No file provided or invalid file
     500 Internal Server Error - Prediction failed

backend/services/ml_service.py

@@ -24,6 +24,9 @@
 MODEL_PATH = MODEL_DIR / "best_model.pth"
 CLASS_NAMES_PATH = MODEL_DIR / "class_names.json"
 
+# Minimum confidence to accept a prediction (handles both unclear photos and non-food images).
+MIN_CONFIDENCE_THRESHOLD = 0.80
+
 # Global variables to cache the model
 _model = None
 _class_names = None
@@ -115,6 +118,16 @@ def predict_food(image_file):
 
         predicted_class = class_names[predicted_idx.item()]
         confidence_score = confidence.item()
+        confidence_pct = round(confidence_score * 100, 2)
+
+        # Reject low-confidence predictions: unclear photo or non-food (model is food-only).
+        if confidence_score < MIN_CONFIDENCE_THRESHOLD:
+            return {
+                "success": False,
+                "reason": "low_confidence",
+                "confidence": confidence_pct,
+                "message": "We couldn't identify the food with enough confidence. Please retake a clear photo of your food.",
+            }
 
         # Extract food name (remove dataset prefix like "food101:")
         food_name = predicted_class.split(":")[-1].replace("_", " ").title().strip()
@@ -125,8 +138,9 @@ def predict_food(image_file):
 
             # Return unknowns for everything besides the name
             return {
+                "success": True,
                 "food_name": food_name,
-                "confidence": round(confidence_score * 100, 2),
+                "confidence": confidence_pct,
                 "calories": -1,
                 "fat_g": -1,
                 "carbs_g": -1,
@@ -152,8 +166,9 @@ def predict_food(image_file):
 
         # Fill in the object attributes based on the generic category
         obj = {
+            "success": True,
             "food_name": food_name,
-            "confidence": round(confidence_score * 100, 2),
+            "confidence": confidence_pct,
             "calories": food_category.calories,
             "fat_g": food_category.fat_g,
             "carbs_g": food_category.carbs_g,

frontend/cu-bytes/app/_styles/style-scan.tsx

@@ -23,6 +23,36 @@ export const styles = StyleSheet.create({
     width: w(100),
   },
 
+  lowConfidenceBanner: {
+    alignItems: 'center',
+    backgroundColor: '#AB0006',
+    borderColor: '#FFFFFF',
+    borderWidth: 2,
+    borderRadius: 12,
+    marginTop: h(1.5),
+    marginBottom: h(1.0),
+    paddingVertical: h(2.0),
+    paddingHorizontal: w(4.0),
+    width: w(80),
+  },
+
+  lowConfidenceTitle: {
+    color: '#FFFFFF',
+    fontFamily: 'arial',
+    fontSize: font(175),
+    fontWeight: '700',
+    marginBottom: h(0.8),
+    textAlign: 'center',
+  },
+
+  lowConfidenceMessage: {
+    color: '#FFFFFF',
+    fontFamily: 'arial',
+    fontSize: font(140),
+    fontWeight: '600',
+    textAlign: 'center',
+  },
+
   savedFoodItemMessageContainer: {
     alignItems: 'center',
     backgroundColor: '#AB0006',

frontend/cu-bytes/app/scan.tsx

@@ -10,6 +10,7 @@ import { useUser } from './_context';
 import { apiService, API_BASE_URL } from '../services/api';
 
 interface PredictionResult {
+    success?: true;
     food_name: string;
     confidence: number;
     calories: number;
@@ -33,12 +34,21 @@ interface PredictionResult {
     has_wheat: boolean;
 }
 
+/** Returned when confidence is below threshold (e.g. unclear photo or non-food). */
+interface LowConfidenceResponse {
+    success: false;
+    reason: 'low_confidence';
+    confidence: number;
+    message: string;
+}
+
 export default function ScanScreen() {
 
     const [loading, setLoading] = useState(false);
     const [modalVisible, setModalVisible] = useState(false);
     const [selectedImage, setSelectedImage] = useState<string | null>(null);
     const [prediction, setPrediction] = useState<PredictionResult | null>(null);
+    const [lowConfidenceMessage, setLowConfidenceMessage] = useState<string | null>(null);
     const [isBackPressed, setIsBackPressed] = useState(false);
     const [isLoginLogoutPressed, setIsLoginLogoutPressed] = useState(false);
     const [isUploadPhotoPressed, setIsUploadPhotoPressed] = useState(false);
@@ -356,8 +366,9 @@ export default function ScanScreen() {
             if (!result.canceled && result.assets[0]) {
 
                 setSelectedImage(result.assets[0].uri);
-                // Clear previous prediction
+                // Clear previous prediction and low-confidence message
                 setPrediction(null);
+                setLowConfidenceMessage(null);
             }
         } catch (err) {
             console.error('Error picking image:', err);
@@ -387,6 +398,7 @@ export default function ScanScreen() {
             if (!result.canceled && result.assets[0]) {
                 setSelectedImage(result.assets[0].uri);
                 setPrediction(null);
+                setLowConfidenceMessage(null);
             }
         } catch (err) {
             console.error('Error taking photo:', err);
@@ -409,7 +421,17 @@ export default function ScanScreen() {
         try {
             const result = await apiService.predictFood(selectedImage);
             console.log('Prediction result received:', result);
-            setPrediction(result);
+
+            const lowConf = result as LowConfidenceResponse;
+            if (lowConf.success === false && lowConf.reason === 'low_confidence') {
+                setLowConfidenceMessage(lowConf.message);
+                setPrediction(null);
+                Alert.alert('Please retake the photo', lowConf.message, [{ text: 'OK' }]);
+                return;
+            }
+
+            setLowConfidenceMessage(null);
+            setPrediction(result as PredictionResult);
         } catch (err: any) {
             console.error('Prediction error:', err);
             Alert.alert('Error', err.response?.data?.error || 'Failed to predict food. Please try again.');
@@ -531,6 +553,16 @@ export default function ScanScreen() {
                     </Text>
                 )}
 
+                {/* Display message when confidence was too low (e.g. unclear or non-food image) */}
+                {lowConfidenceMessage && (
+                    <View id="lowConfidenceBanner" style={styles.lowConfidenceBanner}>
+                        <Text style={styles.lowConfidenceTitle}>No food detected</Text>
+                        <Text style={styles.lowConfidenceMessage}>
+                            We couldn&apos;t detect any food in this photo. Please take a clear photo of the food item.
+                        </Text>
+                    </View>
+                )}
+
                 {/* Display information about the selected food item */}
                 {prediction && (
                     <View id="foodItemOuterView" style={styles.selectedFoodItemContainer}>