Skip to content

Commit 5910d28

Browse files
GAchuziaGAchuzia
authored
Feature/ml tests (#164)
* feat: script to rename image files based on file count in a given folder - default folder is hardcoded but can be changed - base name index_baseName is also hardcoded but can be changed - you can run a dry run to see if the file name results match what you want * chore: ran blake/flake8 pre-commit changes * feat: add script to check which food categories are covered by food101, uec256, and arent in the generic food categories list - this will allow me to remove the overlapping categories in the uec256 dataset so they don't conflict with the food101 list - find out which categories dont have dataset images to represent them * feat: add try except clause so one can run the scrip directly * feat: add ml prediction test coverage for ml service and endpoint - runs tests on carleton foods, non foods, and generic foods * feat: update script to only directly match food categories
1 parent a7bb3df commit 5910d28

4 files changed

Lines changed: 489 additions & 0 deletions

File tree

‎backend/tests/test_ml_predict_endpoint.py‎

Lines changed: 189 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,189 @@
1+
import os
2+
from pathlib import Path
3+
4+
import pytest
5+
6+
import backend.services.ml_service as ml_service
7+
8+
9+
REPO_ROOT = Path(__file__).resolve().parents[2]
10+
11+
12+
def _list_images(folder: Path):
13+
exts = {".jpg", ".jpeg", ".png", ".webp"}
14+
if not folder.exists():
15+
return []
16+
return sorted(
17+
[p for p in folder.iterdir() if p.is_file() and p.suffix.lower() in exts]
18+
)
19+
20+
21+
def _take_subset(images, max_images: int):
22+
# max_images <= 0 means "all"
23+
if max_images <= 0:
24+
return images
25+
return images[:max_images]
26+
27+
28+
@pytest.fixture(scope="session")
29+
def ml_images_root():
30+
"""
31+
Default test images:
32+
machine-learning/data/evaluation_data
33+
34+
Override with env var ML_TEST_IMAGES_DIR if you move the data.
35+
"""
36+
root = os.getenv("ML_TEST_IMAGES_DIR")
37+
if root:
38+
return Path(root)
39+
return REPO_ROOT / "machine-learning" / "data" / "evaluation_data"
40+
41+
42+
@pytest.fixture(scope="session")
43+
def require_model_files():
44+
if not (ml_service.MODEL_PATH.exists() and ml_service.CLASS_NAMES_PATH.exists()):
45+
pytest.skip(
46+
"Model files missing. Expected "
47+
f"{ml_service.MODEL_PATH} and {ml_service.CLASS_NAMES_PATH}"
48+
)
49+
50+
51+
@pytest.fixture(scope="session")
52+
def max_images():
53+
# Keep tests fast. Set ML_TEST_MAX_IMAGES=0 to include all images.
54+
return int(os.getenv("ML_TEST_MAX_IMAGES", "100"))
55+
56+
57+
@pytest.fixture(scope="session")
58+
def non_food_min_low_conf_ratio():
59+
# Non-food images should usually be rejected by the confidence threshold.
60+
return float(os.getenv("ML_TEST_NONFOOD_MIN_LOW_CONF_RATIO", "0.7"))
61+
62+
63+
def _post_image(client, image_path: Path):
64+
with open(image_path, "rb") as f:
65+
response = client.post(
66+
"/ml/predict",
67+
data={"image": (f, image_path.name)},
68+
content_type="multipart/form-data",
69+
)
70+
return response
71+
72+
73+
def _assert_common_fields(payload: dict):
74+
assert "success" in payload
75+
assert "confidence" in payload
76+
assert 0 <= float(payload["confidence"]) <= 100
77+
78+
79+
def _log_prediction(stage: str, image_path: Path, payload: dict):
80+
confidence = payload.get("confidence")
81+
success = payload.get("success")
82+
food_name = payload.get("food_name")
83+
reason = payload.get("reason")
84+
message = payload.get("message")
85+
86+
print(
87+
f"[{stage}] {image_path.name} | success={success} confidence={confidence} "
88+
f"food_name={food_name} reason={reason} message={message}"
89+
)
90+
91+
92+
def test_ml_predict_carleton_food_images_return_valid_response(
93+
client, ml_images_root: Path, require_model_files, max_images: int
94+
):
95+
folder = ml_images_root / "hey_chef_carleton"
96+
images = _take_subset(_list_images(folder), max_images)
97+
if not images:
98+
pytest.skip(f"No images found in {folder}")
99+
100+
print(f"[ENDPOINT] Carleton food folder: {folder} | images={len(images)}")
101+
102+
success_true = 0
103+
for image_path in images:
104+
response = _post_image(client, image_path)
105+
assert response.status_code == 200
106+
107+
payload = response.get_json()
108+
_log_prediction("ENDPOINT", image_path, payload)
109+
_assert_common_fields(payload)
110+
111+
if payload["success"] is True:
112+
success_true += 1
113+
assert "food_name" in payload, f"Missing food_name for {image_path}"
114+
assert "calories" in payload, f"Missing calories for {image_path}"
115+
else:
116+
assert (
117+
payload.get("reason") == "low_confidence"
118+
), f"Expected low_confidence for {image_path} but got {payload}"
119+
assert "message" in payload, f"Missing message for {image_path}"
120+
121+
print(f"[ENDPOINT] Accepted (success=True): {success_true}/{len(images)}")
122+
assert success_true >= 1
123+
124+
125+
def test_ml_predict_non_food_images_are_low_confidence(
126+
client,
127+
ml_images_root: Path,
128+
require_model_files,
129+
max_images: int,
130+
non_food_min_low_conf_ratio: float,
131+
):
132+
folder = ml_images_root / "non_food"
133+
images = _take_subset(_list_images(folder), max_images)
134+
if not images:
135+
pytest.skip(f"No images found in {folder}")
136+
137+
print(f"[ENDPOINT] Non-food folder: {folder} | images={len(images)}")
138+
139+
low_conf_count = 0
140+
for image_path in images:
141+
response = _post_image(client, image_path)
142+
assert response.status_code == 200
143+
144+
payload = response.get_json()
145+
_log_prediction("ENDPOINT", image_path, payload)
146+
_assert_common_fields(payload)
147+
148+
if payload["success"] is False:
149+
assert (
150+
payload.get("reason") == "low_confidence"
151+
), f"Expected low_confidence for {image_path} but got {payload}"
152+
low_conf_count += 1
153+
154+
low_conf_ratio = low_conf_count / len(images)
155+
print(f"[ENDPOINT] Low-confidence (rejected): {low_conf_count}/{len(images)} ratio={low_conf_ratio}")
156+
assert low_conf_ratio >= non_food_min_low_conf_ratio
157+
158+
159+
def test_ml_predict_general_food_images_return_valid_response(
160+
client, ml_images_root: Path, require_model_files, max_images: int
161+
):
162+
folder = ml_images_root / "UPMC-Food101"
163+
images = _take_subset(_list_images(folder), max_images)
164+
if not images:
165+
pytest.skip(f"No images found in {folder}")
166+
167+
print(f"[ENDPOINT] General foods folder: {folder} | images={len(images)}")
168+
169+
success_true = 0
170+
for image_path in images:
171+
response = _post_image(client, image_path)
172+
assert response.status_code == 200
173+
174+
payload = response.get_json()
175+
_log_prediction("ENDPOINT", image_path, payload)
176+
_assert_common_fields(payload)
177+
178+
if payload["success"] is True:
179+
success_true += 1
180+
assert "food_name" in payload, f"Missing food_name for {image_path}"
181+
assert "calories" in payload, f"Missing calories for {image_path}"
182+
else:
183+
assert (
184+
payload.get("reason") == "low_confidence"
185+
), f"Expected low_confidence for {image_path} but got {payload}"
186+
assert "message" in payload, f"Missing message for {image_path}"
187+
188+
print(f"[ENDPOINT] Accepted (success=True): {success_true}/{len(images)}")
189+
assert success_true >= 1

‎backend/tests/test_ml_predict_service.py‎

Lines changed: 176 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,176 @@
1+
import os
2+
from pathlib import Path
3+
4+
import pytest
5+
6+
import backend.services.ml_service as ml_service
7+
from backend.services.ml_service import predict_food
8+
9+
10+
REPO_ROOT = Path(__file__).resolve().parents[2]
11+
12+
13+
def _list_images(folder: Path):
14+
exts = {".jpg", ".jpeg", ".png", ".webp"}
15+
if not folder.exists():
16+
return []
17+
return sorted(
18+
[p for p in folder.iterdir() if p.is_file() and p.suffix.lower() in exts]
19+
)
20+
21+
22+
def _take_subset(images, max_images: int):
23+
# max_images <= 0 means "all"
24+
if max_images <= 0:
25+
return images
26+
return images[:max_images]
27+
28+
29+
@pytest.fixture(scope="session")
30+
def ml_images_root():
31+
"""
32+
Default test images:
33+
machine-learning/data/evaluation_data
34+
35+
Override with env var ML_TEST_IMAGES_DIR if you move the data.
36+
"""
37+
root = os.getenv("ML_TEST_IMAGES_DIR")
38+
if root:
39+
return Path(root)
40+
return REPO_ROOT / "machine-learning" / "data" / "evaluation_data"
41+
42+
43+
@pytest.fixture(scope="session")
44+
def require_model_files():
45+
if not (ml_service.MODEL_PATH.exists() and ml_service.CLASS_NAMES_PATH.exists()):
46+
pytest.skip(
47+
"Model files missing. Expected "
48+
f"{ml_service.MODEL_PATH} and {ml_service.CLASS_NAMES_PATH}"
49+
)
50+
51+
52+
@pytest.fixture(scope="session")
53+
def max_images():
54+
# Keep tests fast. Set ML_TEST_MAX_IMAGES=0 to include all images.
55+
return int(os.getenv("ML_TEST_MAX_IMAGES", "100"))
56+
57+
58+
@pytest.fixture(scope="session")
59+
def non_food_min_low_conf_ratio():
60+
# Non-food images should usually be rejected by the confidence threshold.
61+
return float(os.getenv("ML_TEST_NONFOOD_MIN_LOW_CONF_RATIO", "0.7"))
62+
63+
64+
def _assert_common_fields(result: dict):
65+
assert isinstance(result, dict)
66+
assert "success" in result
67+
assert "confidence" in result
68+
assert 0 <= float(result["confidence"]) <= 100
69+
70+
71+
def _log_prediction(stage: str, image_path: Path, result: dict):
72+
confidence = result.get("confidence")
73+
success = result.get("success")
74+
food_name = result.get("food_name")
75+
reason = result.get("reason")
76+
message = result.get("message")
77+
78+
print(
79+
f"[{stage}] {image_path.name} | success={success} confidence={confidence} "
80+
f"food_name={food_name} reason={reason} message={message}"
81+
)
82+
83+
84+
def test_predict_food_carleton_food_images_return_valid_response(
85+
app, ml_images_root: Path, require_model_files, max_images: int
86+
):
87+
folder = ml_images_root / "hey_chef_carleton"
88+
images = _take_subset(_list_images(folder), max_images)
89+
if not images:
90+
pytest.skip(f"No images found in {folder}")
91+
92+
print(f"[SERVICE] Carleton food folder: {folder} | images={len(images)}")
93+
94+
success_true = 0
95+
with app.app_context():
96+
for image_path in images:
97+
result = predict_food(str(image_path))
98+
_log_prediction("SERVICE", image_path, result)
99+
_assert_common_fields(result)
100+
101+
if result["success"] is True:
102+
success_true += 1
103+
assert "food_name" in result, f"Missing food_name for {image_path}"
104+
assert "calories" in result, f"Missing calories for {image_path}"
105+
else:
106+
assert (
107+
result.get("reason") == "low_confidence"
108+
), f"Expected low_confidence for {image_path} but got {result}"
109+
assert "message" in result, f"Missing message for {image_path}"
110+
111+
# At least one should be accepted as food.
112+
print(f"[SERVICE] Accepted (success=True): {success_true}/{len(images)}")
113+
assert success_true >= 1
114+
115+
116+
def test_predict_food_non_food_images_are_low_confidence(
117+
app,
118+
ml_images_root: Path,
119+
require_model_files,
120+
max_images: int,
121+
non_food_min_low_conf_ratio: float,
122+
):
123+
folder = ml_images_root / "non_food"
124+
images = _take_subset(_list_images(folder), max_images)
125+
if not images:
126+
pytest.skip(f"No images found in {folder}")
127+
128+
print(f"[SERVICE] Non-food folder: {folder} | images={len(images)}")
129+
130+
low_conf_count = 0
131+
with app.app_context():
132+
for image_path in images:
133+
result = predict_food(str(image_path))
134+
_log_prediction("SERVICE", image_path, result)
135+
_assert_common_fields(result)
136+
137+
if result["success"] is False:
138+
assert (
139+
result.get("reason") == "low_confidence"
140+
), f"Expected low_confidence for {image_path} but got {result}"
141+
low_conf_count += 1
142+
143+
low_conf_ratio = low_conf_count / len(images)
144+
print(f"[SERVICE] Low-confidence (rejected): {low_conf_count}/{len(images)} ratio={low_conf_ratio}")
145+
assert low_conf_ratio >= non_food_min_low_conf_ratio
146+
147+
148+
def test_predict_food_general_food_images_return_valid_response(
149+
app, ml_images_root: Path, require_model_files, max_images: int
150+
):
151+
folder = ml_images_root / "UPMC-Food101"
152+
images = _take_subset(_list_images(folder), max_images)
153+
if not images:
154+
pytest.skip(f"No images found in {folder}")
155+
156+
print(f"[SERVICE] General foods folder: {folder} | images={len(images)}")
157+
158+
success_true = 0
159+
with app.app_context():
160+
for image_path in images:
161+
result = predict_food(str(image_path))
162+
_log_prediction("SERVICE", image_path, result)
163+
_assert_common_fields(result)
164+
165+
if result["success"] is True:
166+
success_true += 1
167+
assert "food_name" in result, f"Missing food_name for {image_path}"
168+
assert "calories" in result, f"Missing calories for {image_path}"
169+
else:
170+
assert (
171+
result.get("reason") == "low_confidence"
172+
), f"Expected low_confidence for {image_path} but got {result}"
173+
assert "message" in result, f"Missing message for {image_path}"
174+
175+
print(f"[SERVICE] Accepted (success=True): {success_true}/{len(images)}")
176+
assert success_true >= 1

0 commit comments

Comments
 (0)