HW-Lab-Spring23/detector.py at main · mofayezi/HW-Lab-Spring23 · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
import cv2
import numpy as np
import importlib.util
# Import TensorFlow libraries
# If tflite_runtime is installed, import interpreter from tflite_runtime, else import from regular tensorflow
pkg = importlib.util.find_spec('tflite_runtime')
if pkg:
    from tflite_runtime.interpreter import Interpreter
else:
    from tensorflow.lite.python.interpreter import Interpreter


class Detector:
    def __init__(self, model_checkpoint, input_size=224) -> None:
        self.model = Interpreter(model_path=model_checkpoint)
        self.model.allocate_tensors()
        self.input_size = input_size
        self.input_details = self.model.get_input_details()
        self.output_details = self.model.get_output_details()
        self.num_fingers = 5


    def process_hands(self, image, draw=False):
        keypoints = self.get_keypoints(image)

        for keypoint in keypoints:
            cx, cy = int(keypoint[0]), int(keypoint[1])
            if draw:
                cv2.circle(image, (cx, cy), 5, (255, 0, 255), cv2.FILLED)

        bbox = []
        if keypoints.any():
            xmin, xmax = min(keypoints[:, 0]), max(keypoints[:, 0])
            ymin, ymax = min(keypoints[:, 1]), max(keypoints[:, 1])
            bbox = xmin, ymin, xmax, ymax

        if draw:
            if keypoints.any():
                cv2.rectangle(image, (int(xmin) - 20, int(ymin) - 20), (int(xmax) + 20, int(ymax) + 20), (0, 255, 0), 2)
            # cv2.imwrite('./results/test.jpg', image)
            cv2.imshow("My Hand", image)
            cv2.waitKey(1)

        return keypoints, bbox


    def get_keypoints(self, image):
        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB).astype(np.float32)
        image = cv2.resize(image, (self.input_size, self.input_size))
        # image = image / 127.5
        # image -= 1
        image = np.expand_dims(image, axis=0) / 255.0

        self.model.set_tensor(self.input_details[0]['index'], image)
        self.model.invoke()
        y_pred = self.model.get_tensor(self.output_details[0]['index'])
        confidence = self.model.get_tensor(self.output_details[1]['index'])
        if confidence < 0.5:
            return np.array([])
        # y_pred = np.squeeze(y_pred, axis=0)
        # return self.find_fingers(y_pred)
        return np.squeeze(y_pred, axis=0).reshape(21, 3)[:, :2]


    def find_finger(self, P, X, Y, img_size=224, k=7):
        ind = np.unravel_index(np.argmax(P, axis=None), P.shape)
        P = P[ind]

        thresh = -10
        if P < thresh:
            points = [-1, -1]
            return points

        X, Y = X[ind], Y[ind]

        cell_size = img_size // k

        X = (ind[1] + X) * cell_size
        Y = (ind[0] + Y) * cell_size

        points = [X, Y]

        return points


    def find_fingers(self, y_pred):
        p_pred = y_pred[:, :, :self.num_fingers]

        C = 7
        xy_pred = y_pred[:, :, self.num_fingers:].reshape(C, C, 2, self.num_fingers)
        xy_pred = np.transpose(xy_pred, (0, 1, 3, 2))

        # sigmoid
        xy_pred = 1 / (1 + np.exp(-xy_pred))

        fingers = []
        for i in range(self.num_fingers):
            fingers.append(self.find_finger(p_pred[:, :, i], xy_pred[:, :, i, 0], xy_pred[:, :, i, 1]))

        return np.array(fingers)


    def check_fingers(self, points):
        finger_tips = [4, 8, 12, 16, 20]
        fingers = []

        if len(points) == 0:
            return [0, 0, 0, 0, 0]

        if points[finger_tips[0]][0] > points[finger_tips[0] - 1][0]:
            fingers.append(1)
        else:
            fingers.append(0)

        for i in range(1, 5):
            if points[finger_tips[i]][1] < points[finger_tips[i] - 2][1]:
                fingers.append(1)
            else:
                fingers.append(0)

        return fingers