trackmultiplefaces.py 16 KB
Newer Older
klorydryk's avatar
klorydryk committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14
#!/usr/bin/python
'''
    Author: Guido Diepen <gdiepen@deloitte.nl>
'''

#Import the OpenCV and dlib libraries
import cv2
import dlib


import threading
import time
import random

15 16
import randomnames

klorydryk's avatar
klorydryk committed
17 18 19 20 21
#Initialize a face cascade using the frontal face haar cascade provided with
#the OpenCV library
#Make sure that you copy this file from the opencv project to the root of this
#project folder
faceCascade = cv2.CascadeClassifier('haarcascade_frontalface_default.xml')
22

klorydryk's avatar
klorydryk committed
23 24 25 26 27 28 29 30 31 32 33 34 35 36 37
#The deisred output width and height
OUTPUT_SIZE_WIDTH = 775
OUTPUT_SIZE_HEIGHT = 600

RED = (0, 0, 255)
BLUE = (255, 0, 0)
PURPLE = (127, 0, 255)
GREEN = (0, 255, 0)
suspiciousColors = {0:(0,0,0), 1:GREEN, 2:PURPLE, 3:RED}

averageSuspicious = 0

#We are not doing really face recognition
def doRecognizePerson(faceNames, fid):
    time.sleep(2)
cb's avatar
cb committed
38
    faceNames[ fid ] = randomnames.rand_name(charset) #"Person " + str(fid)
klorydryk's avatar
klorydryk committed
39 40 41 42 43 44 45 46 47


def calculateAverageSuspicious(faceSuspicion):
    global averageSuspicious
    currentFaces = len(faceSuspicion) - faceSuspicion.count(0)
    if(currentFaces>0):
        averageSuspicious = sum(faceSuspicion)/currentFaces
        print(f"Average suspicion: {averageSuspicious}")

48 49 50 51 52
def detectAndTrackMultipleFaces(start_thread, 
                                charset, 
                                draw_person_dangerosity,
                                add_warning_message):

klorydryk's avatar
klorydryk committed
53 54 55
    #Open the first webcame device
    capture = cv2.VideoCapture(0)

klorydryk's avatar
klorydryk committed
56 57
    FULLSCREEN = False

klorydryk's avatar
klorydryk committed
58 59
    #Create two opencv named windows
    cv2.namedWindow("base-image", cv2.WINDOW_AUTOSIZE)
klorydryk's avatar
klorydryk committed
60
    cv2.namedWindow("result-image", cv2.WINDOW_NORMAL)
klorydryk's avatar
klorydryk committed
61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88

    #Position the windows next to eachother
    cv2.moveWindow("base-image",0,100)
    cv2.moveWindow("result-image",400,100)

    #Start the window thread for the two windows we are using
    cv2.startWindowThread()

    #The color of the rectangle we draw around the face
    rectangleColor = (0,165,255)

    #variables holding the current frame number and the current faceid
    frameCounter = 0
    currentFaceID = 0

    #Variables holding the correlation trackers and the name per faceid
    faceTrackers = {}
    faceNames = {}
    faceSuspicion = list() # 0 is neutral/non existent person. 1 is ok, 2 is suspect, 3 is dangerous

    try:
        while True:
            #Retrieve the latest image from the webcam
            rc,fullSizeBaseImage = capture.read()

            #Check if a key was pressed and if it was Q, then break
            #from the infinite loop
            pressedKey = cv2.waitKey(2)
klorydryk's avatar
klorydryk committed
89 90 91
            if pressedKey == -1:
                pass
            elif pressedKey == 113:   # Q
klorydryk's avatar
klorydryk committed
92
                break
klorydryk's avatar
klorydryk committed
93 94
            elif pressedKey == 194:   # F5
                FULLSCREEN = not FULLSCREEN
klorydryk's avatar
klorydryk committed
95

klorydryk's avatar
klorydryk committed
96 97
            #Resize the image to 320x240
            baseImage = cv2.resize( fullSizeBaseImage, ( 320, 240))
klorydryk's avatar
klorydryk committed
98 99 100 101

            #Result image is the image we will show the user, which is a
            #combination of the original image from the webcam and the
            #overlayed rectangle for the largest face
klorydryk's avatar
klorydryk committed
102 103
            if FULLSCREEN ==  True:
                resultImage = fullSizeBaseImage.copy()
cb's avatar
cb committed
104 105 106
                cv2.setWindowProperty('result-image', 
                                      cv2.WND_PROP_FULLSCREEN,
                                      cv2.WINDOW_FULLSCREEN)
klorydryk's avatar
klorydryk committed
107 108
            else:
                resultImage = baseImage.copy()
cb's avatar
cb committed
109 110 111
                cv2.setWindowProperty('result-image', 
                                      cv2.WND_PROP_FULLSCREEN,
                                      cv2.WINDOW_NORMAL)
klorydryk's avatar
klorydryk committed
112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131

            #STEPS:
            # * Update all trackers and remove the ones that are not
            #   relevant anymore
            # * Every 10 frames:
            #       + Use face detection on the current frame and look
            #         for faces.
            #       + For each found face, check if centerpoint is within
            #         existing tracked box. If so, nothing to do
            #       + If centerpoint is NOT in existing tracked box, then
            #         we add a new tracker with a new face-id


            #Increase the framecounter
            frameCounter += 1

            #Update all the trackers and remove the ones for which the update
            #indicated the quality was not good enough
            fidsToDelete = []
            for fid in faceTrackers.keys():
klorydryk's avatar
klorydryk committed
132
                trackingQuality = faceTrackers[ fid ].update( resultImage )
klorydryk's avatar
klorydryk committed
133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152

                #If the tracking quality is good enough, we must delete
                #this tracker
                if trackingQuality < 7:
                    fidsToDelete.append( fid )

            for fid in fidsToDelete:
                print("Removing fid " + str(fid) + " from list of trackers")
                faceTrackers.pop( fid , None )
                faceSuspicion[fid] = 0

                calculateAverageSuspicious(faceSuspicion)

            #Every 10 frames, we will have to determine which faces
            #are present in the frame
            if (frameCounter % 10) == 0:

                #For the face detection, we need to make use of a gray
                #colored image so we will convert the baseImage to a
                #gray-based image
klorydryk's avatar
klorydryk committed
153
                gray = cv2.cvtColor(resultImage, cv2.COLOR_BGR2GRAY)
klorydryk's avatar
klorydryk committed
154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213
                #Now use the haar cascade detector to find all faces
                #in the image
                faces = faceCascade.detectMultiScale(gray, 1.3, 5)

                #Loop over all faces and check if the area for this
                #face is the largest so far
                #We need to convert it to int here because of the
                #requirement of the dlib tracker. If we omit the cast to
                #int here, you will get cast errors since the detector
                #returns numpy.int32 and the tracker requires an int
                for (_x,_y,_w,_h) in faces:
                    x = int(_x)
                    y = int(_y)
                    w = int(_w)
                    h = int(_h)


                    #calculate the centerpoint
                    x_bar = x + 0.5 * w
                    y_bar = y + 0.5 * h

                    #Variable holding information which faceid we
                    #matched with
                    matchedFid = None

                    #Now loop over all the trackers and check if the
                    #centerpoint of the face is within the box of a
                    #tracker
                    for fid in faceTrackers.keys():
                        tracked_position =  faceTrackers[fid].get_position()

                        t_x = int(tracked_position.left())
                        t_y = int(tracked_position.top())
                        t_w = int(tracked_position.width())
                        t_h = int(tracked_position.height())


                        #calculate the centerpoint
                        t_x_bar = t_x + 0.5 * t_w
                        t_y_bar = t_y + 0.5 * t_h

                        #check if the centerpoint of the face is within the
                        #rectangleof a tracker region. Also, the centerpoint
                        #of the tracker region must be within the region
                        #detected as a face. If both of these conditions hold
                        #we have a match
                        if ( ( t_x <= x_bar   <= (t_x + t_w)) and
                             ( t_y <= y_bar   <= (t_y + t_h)) and
                             ( x   <= t_x_bar <= (x   + w  )) and
                             ( y   <= t_y_bar <= (y   + h  ))):
                            matchedFid = fid

                    #If no matched fid, then we have to create a new tracker
                    if matchedFid is None:

                        suspicionLevel = random.randint(1,3)
                        faceSuspicion.append(suspicionLevel)
                        print("Creating new tracker " + str(currentFaceID) + " with suspicion level: " + str(suspicionLevel))
                        #Create and store the tracker
                        tracker = dlib.correlation_tracker()
klorydryk's avatar
klorydryk committed
214
                        tracker.start_track(resultImage,
klorydryk's avatar
klorydryk committed
215 216 217 218 219 220 221
                                            dlib.rectangle( x-10,
                                                            y-20,
                                                            x+w+10,
                                                            y+h+20))

                        faceTrackers[ currentFaceID ] = tracker

222 223
                        # Pick up random name: Do we want a thread for that? 
                        # Advantage is that we can pop an "Identifying..." Message
224
                        # Disadvantage is that is more "lourd"
225 226 227 228 229 230 231 232
                        
                        if start_thread :
                            #Start a new thread that is used to simulate face recognition. 
                            t = threading.Thread( target = doRecognizePerson ,
                                                   args=(faceNames, currentFaceID))
                            t.start()

                        else:
cb's avatar
cb committed
233 234 235
                            # Directly add name + pass a charset to which name must belong 
                            # "Person " + str(fid)
                            faceNames[ currentFaceID ] = randomnames.rand_name(charset) 
klorydryk's avatar
klorydryk committed
236

237
                        calculateAverageSuspicious(faceSuspicion)
klorydryk's avatar
klorydryk committed
238 239 240 241 242 243 244 245 246 247

                        #Increase the currentFaceID counter
                        currentFaceID += 1

            #Now loop over all the trackers we have and draw the rectangle
            #around the detected faces. If we 'know' the name for this person
            #(i.e. the recognition thread is finished), we print the name
            #of the person, otherwise the message indicating we are detecting
            #the name of the person
            for fid in faceTrackers.keys():
248 249 250 251 252

                # Draw a rectangle around people's face
                rectangle_around_face(fid, 
                                      faceTrackers, 
                                      faceSuspicion,
253
                                      # If False, colors will not change with suspicion level
254 255 256 257
                                      draw_person_dangerosity, 
                                      resultImage)

                # Deal with the two others rectangles
klorydryk's avatar
klorydryk committed
258 259 260 261 262 263 264
                tracked_position =  faceTrackers[fid].get_position()

                t_x = int(tracked_position.left())
                t_y = int(tracked_position.top())
                t_w = int(tracked_position.width())
                t_h = int(tracked_position.height())

cb's avatar
cb committed
265 266 267 268 269
                cv2.rectangle(resultImage, 
                             (t_x+t_w, t_y), 
                             (t_x+t_w+30, t_y+80), 
                             suspiciousColors[faceSuspicion[fid]], 
                             -1)
klorydryk's avatar
klorydryk committed
270 271 272 273 274 275 276 277 278 279

                if averageSuspicious>2.5:
                    color = RED
                elif averageSuspicious>1.5:
                    color = PURPLE
                else:
                    color = GREEN

                width = resultImage.shape[1]
                cv2.rectangle(resultImage, (int(width-50), 0), (int(width), 120), color, -1)
280 281
                
                # If Known -> print Name 
klorydryk's avatar
klorydryk committed
282
                if fid in faceNames.keys():
283

cb's avatar
cb committed
284
                    cv2.putText(resultImage, 
285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302
                        faceNames[fid] ,
                        # Name's alignment
                        (t_x+ int((t_w - len(faceNames[fid])*10)/2 + 10), int(t_y) - 10),
                        cv2.FONT_HERSHEY_SIMPLEX,
                        0.5, 
                        (255, 255, 255), 
                        1, 
                        cv2.LINE_AA)
                    
                    # Add a warning message depending on suspicious level
                    if add_warning_message:
                        put_warning_message(fid, 
                                            faceTrackers, 
                                            faceSuspicion, 
                                            resultImage, 
                                            threshold=2)
                
                # Else print Identifying
klorydryk's avatar
klorydryk committed
303
                else:
cb's avatar
cb committed
304
                    text = "Identifying"
cb's avatar
cb committed
305 306
                    cv2.putText(resultImage, 
                                text ,
307
                                (t_x + int((t_w - len(text)*10)/2 + 10), int(t_y)),
klorydryk's avatar
klorydryk committed
308
                                cv2.FONT_HERSHEY_SIMPLEX,
cb's avatar
cb committed
309 310 311 312
                                0.5, 
                                (255, 255, 255), 
                                1, 
                                cv2.LINE_AA)
klorydryk's avatar
klorydryk committed
313 314 315 316 317 318 319 320

            #Since we want to show something larger on the screen than the
            #original 320x240, we resize the image again
            #
            #Note that it would also be possible to keep the large version
            #of the baseimage and make the result image a copy of this large
            #base image and use the scaling factor to draw the rectangle
            #at the right coordinates.
klorydryk's avatar
klorydryk committed
321
            # largeResult = cv2.resize(resultImage, OUTPUT_SIZE_WIDTH,OUTPUT_SIZE_HEIGHT))
klorydryk's avatar
klorydryk committed
322 323 324

            #Finally, we want to show the images on the screen
            cv2.imshow("base-image", baseImage)
klorydryk's avatar
klorydryk committed
325
            cv2.imshow("result-image", resultImage) #largeResult)
klorydryk's avatar
klorydryk committed
326 327 328 329 330 331 332 333 334 335 336

    #To ensure we can also deal with the user pressing Ctrl-C in the console
    #we have to check for the KeyboardInterrupt exception and break out of
    #the main loop
    except KeyboardInterrupt as e:
        pass

    #Destroy any OpenCV windows and exit the application
    cv2.destroyAllWindows()
    exit(0)

337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362
def put_warning_message(fid, faceTrackers, faceSuspicion, resultImage, threshold=2):
    '''
    Print a Warning message depending on suspicion level
    '''
    tracked_position =  faceTrackers[fid].get_position()

    t_x = int(tracked_position.left())
    t_y = int(tracked_position.top())
    t_w = int(tracked_position.width())
    t_h = int(tracked_position.height())
    
    suspicion_level = faceSuspicion[fid]
    
    if suspicion_level > threshold:
        text = "ATTENTION! INDIVIDU DANGEREUX!"
        cv2.putText(resultImage, 
                    text ,
                    # Name's alignment
                    (t_x+ int((t_w - len(text)*10)/2 + 10), int(t_y) + 20 + t_h),
                    cv2.FONT_HERSHEY_SIMPLEX,
                    0.5, 
                    RED, 
                    1, 
                    cv2.LINE_AA)


363 364 365 366 367 368 369 370 371 372 373 374 375 376 377
def rectangle_around_face(fid, faceTrackers, faceSuspicion, draw_person_dangerosity, resultImage):
    '''
    Draw a rectangle around's people face
    If draw_person_dangerosity = True -> Add a color based on suspicionLevel
    '''
    tracked_position =  faceTrackers[fid].get_position()

    t_x = int(tracked_position.left())
    t_y = int(tracked_position.top())
    t_w = int(tracked_position.width())
    t_h = int(tracked_position.height())

    # If we want to have a dangerosity color per personn  
    if draw_person_dangerosity :
        suspicionLevel = faceSuspicion[fid]
378 379
        person_color = pick_person_dangerosity_color(suspicionLevel)
    
380 381 382 383
    # Else default to rectangleColor
    else:
        person_color = rectangleColor

384 385 386 387 388 389
    # Draw Rectangle
    cv2.rectangle(resultImage, 
                  (t_x, t_y),
                  (t_x + t_w , t_y + t_h),
                  person_color,
                  1)
390

391
def pick_person_dangerosity_color(suspicionLevel, threshold=2):
392 393 394 395
    '''
    Compute person level suspicion -> simplified to 2 levels
    Can reintroduce the faceSuspicious dictionnary lated, was more clean
    '''
396
    if suspicionLevel> threshold:
397 398 399 400 401
        color = RED
    else:
        color = GREEN

    return color
klorydryk's avatar
klorydryk committed
402 403

if __name__ == '__main__':
404
    detectAndTrackMultipleFaces(
cb's avatar
cb committed
405
            # False: Names are directly printed on screen, no new thread
406
            start_thread             = False, 
cb's avatar
cb committed
407
            # Select only names with letters and whitespace. Any other value will select printable.
408
            charset                  = "letters",
409
            # If True, then rectangle color around people' face will turn red if dangerous
410 411 412
            draw_person_dangerosity  = True,
            # Add a warning message if dangerous
            add_warning_message      = True
413
            )