stream function

deepface/DeepFace.py

@@ -8,14 +8,15 @@ import numpy as np
 import pandas as pd
 from tqdm import tqdm
 import json
+import cv2
 
 #from basemodels import VGGFace, OpenFace, Facenet, FbDeepFace
 #from extendedmodels import Age, Gender, Race, Emotion
-#from commons import functions, distance as dst
+#from commons import functions, realtime, distance as dst
 
 from deepface.basemodels import VGGFace, OpenFace, Facenet, FbDeepFace
 from deepface.extendedmodels import Age, Gender, Race, Emotion
-from deepface.commons import functions, distance as dst
+from deepface.commons import functions, realtime, distance as dst
 
 def verify(img1_path, img2_path=''
 	, model_name ='VGG-Face', distance_metric = 'cosine', plot = False):
@@ -293,8 +294,12 @@ def detectFace(img_path):
 	img = functions.detectFace(img_path)[0] #detectFace returns (1, 224, 224, 3)
 	return img[:, :, ::-1] #bgr to rgb
 
+def stream(db_path, model_name ='VGG-Face', distance_metric = 'cosine', enable_face_analysis = True):
+	realtime.analysis(db_path, model_name, distance_metric, enable_face_analysis)
+
 #---------------------------
 
 functions.initializeFolder()
 
 #---------------------------
+

deepface/commons/functions.py

@@ -11,6 +11,7 @@ import gdown
 import hashlib
 import math
 from PIL import Image
+import copy
 
 def distance(a, b):
 	x1 = a[0]; y1 = a[1]
@@ -108,8 +109,7 @@ def findThreshold(model_name, distance_metric):
 	
 	return threshold
 
-def detectFace(image_path, target_size=(224, 224), grayscale = False):
+def get_opencv_path():
-	
 	opencv_home = cv2.__file__
 	folders = opencv_home.split(os.path.sep)[0:-1]
 	
@@ -120,6 +120,25 @@ def detectFace(image_path, target_size=(224, 224), grayscale = False):
 	face_detector_path = path+"/data/haarcascade_frontalface_default.xml"
 	eye_detector_path = path+"/data/haarcascade_eye.xml"
 	
+	if os.path.isfile(face_detector_path) != True:
+		raise ValueError("Confirm that opencv is installed on your environment! Expected path ",face_detector_path," violated.")
+	
+	return path+"/data/"
+
+def detectFace(img, target_size=(224, 224), grayscale = False):
+	
+	#-----------------------
+	
+	exact_image = False
+	if type(img).__module__ == np.__name__:
+		exact_image = True
+	
+	#-----------------------
+	
+	opencv_path = get_opencv_path()
+	face_detector_path = opencv_path+"haarcascade_frontalface_default.xml"
+	eye_detector_path = opencv_path+"haarcascade_eye.xml"
+	
 	if os.path.isfile(face_detector_path) != True:
 		raise ValueError("Confirm that opencv is installed on your environment! Expected path ",face_detector_path," violated.")
 	
@@ -128,7 +147,8 @@ def detectFace(image_path, target_size=(224, 224), grayscale = False):
 	face_detector = cv2.CascadeClassifier(face_detector_path)
 	eye_detector = cv2.CascadeClassifier(eye_detector_path)
 	
-	img = cv2.imread(image_path)
+	if exact_image != True: #image path passed as input
+		img = cv2.imread(img)
 	
 	img_raw = img.copy()
 	
@@ -241,4 +261,16 @@ def detectFace(image_path, target_size=(224, 224), grayscale = False):
 		return img_pixels
 		
 	else:
-		raise ValueError("Face could not be detected in ", image_path,". Please confirm that the picture is a face photo.")
+		
+		if exact_image == True:
+			
+			if grayscale == True:
+				img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
+			
+			img = cv2.resize(img, target_size)
+			img_pixels = image.img_to_array(img)
+			img_pixels = np.expand_dims(img_pixels, axis = 0)
+			img_pixels /= 255
+			return img_pixels
+		else:
+			raise ValueError("Face could not be detected in ", img,". Please confirm that the picture is a face photo.")

deepface/commons/realtime.py

@@ -0,0 +1,387 @@
+import os
+from tqdm import tqdm
+import numpy as np
+import pandas as pd
+import cv2
+import time
+
+import os
+os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
+
+#from basemodels import VGGFace, OpenFace, Facenet, FbDeepFace
+#from extendedmodels import Age, Gender, Race, Emotion
+#from commons import functions, realtime, distance as dst
+
+from deepface.basemodels import VGGFace, OpenFace, Facenet, FbDeepFace
+from deepface.extendedmodels import Age, Gender, Race, Emotion
+from deepface.commons import functions, realtime, distance as dst
+
+def analysis(db_path, model_name, distance_metric, enable_face_analysis = True):
+	
+	input_shape = (224, 224)
+	text_color = (255,255,255)
+	
+	employees = []
+	#check passed db folder exists
+	if os.path.isdir(db_path) == True:
+		for r, d, f in os.walk(db_path): # r=root, d=directories, f = files
+			for file in f:
+				if ('.jpg' in file):
+					employees.append(file)
+	
+	#------------------------
+	
+	if len(employees) > 0:
+		if model_name == 'VGG-Face':
+			print("Using VGG-Face model backend and", distance_metric,"distance.")
+			model = VGGFace.loadModel()
+			input_shape = (224, 224)	
+		
+		elif model_name == 'OpenFace':
+			print("Using OpenFace model backend", distance_metric,"distance.")
+			model = OpenFace.loadModel()
+			input_shape = (96, 96)
+		
+		elif model_name == 'Facenet':
+			print("Using Facenet model backend", distance_metric,"distance.")
+			model = Facenet.loadModel()
+			input_shape = (160, 160)
+		
+		elif model_name == 'DeepFace':
+			print("Using FB DeepFace model backend", distance_metric,"distance.")
+			model = FbDeepFace.loadModel()
+			input_shape = (152, 152)
+		
+		else:
+			raise ValueError("Invalid model_name passed - ", model_name)
+		#------------------------
+		
+		#tuned thresholds for model and metric pair
+		threshold = functions.findThreshold(model_name, distance_metric)
+		
+	#------------------------
+	#facial attribute analysis models
+		
+	if enable_face_analysis == True:
+		
+		tic = time.time()
+		
+		#emotion_model = Emotion.loadModel()
+		#print("Emotion model loaded")
+		
+		age_model = Age.loadModel()
+		print("Age model loaded")
+		
+		gender_model = Gender.loadModel()
+		print("Gender model loaded")
+		
+		toc = time.time()
+		
+		print("Facial attibute analysis models loaded in ",toc-tic," seconds")
+	
+	#------------------------
+	
+	#find embeddings for employee list
+	
+	tic = time.time()
+	
+	pbar = tqdm(range(0, len(employees)), desc='Finding embeddings')
+	
+	embeddings = []
+	#for employee in employees:
+	for index in pbar:
+		employee = employees[index]
+		pbar.set_description("Finding embedding for %s" % (employee))
+		embedding = []
+		img = functions.detectFace(db_path+"/"+employee, input_shape)
+		img_representation = model.predict(img)[0,:]
+		
+		embedding.append(employee)
+		embedding.append(img_representation)
+		embeddings.append(embedding)
+	
+	df = pd.DataFrame(embeddings, columns = ['employee', 'embedding'])
+	df['distance_metric'] = distance_metric
+	
+	toc = time.time()
+	
+	print("Embeddings found for given data set in ", toc-tic," seconds")
+	
+	#-----------------------
+
+	time_threshold = 5; frame_threshold = 5
+	pivot_img_size = 112 #face recognition result image
+
+	#-----------------------
+	
+	opencv_path = functions.get_opencv_path()
+	face_detector_path = opencv_path+"haarcascade_frontalface_default.xml"
+	face_cascade = cv2.CascadeClassifier(face_detector_path)
+	
+	#-----------------------
+
+	freeze = False
+	face_detected = False
+	face_included_frames = 0 #freeze screen if face detected sequantially 5 frames
+	freezed_frame = 0
+	tic = time.time()
+
+	cap = cv2.VideoCapture(0) #webcam
+
+	while(True):
+		ret, img = cap.read()
+		
+		#cv2.namedWindow('img', cv2.WINDOW_FREERATIO)
+		#cv2.setWindowProperty('img', cv2.WND_PROP_FULLSCREEN, cv2.WINDOW_FULLSCREEN)
+		
+		raw_img = img.copy()
+		resolution = img.shape
+		
+		resolution_x = img.shape[1]; resolution_y = img.shape[0]
+
+		if freeze == False: 
+			faces = face_cascade.detectMultiScale(img, 1.3, 5)
+			
+			if len(faces) == 0:
+				face_included_frames = 0
+		else: 
+			faces = []
+		
+		detected_faces = []
+		face_index = 0
+		for (x,y,w,h) in faces:
+			if w > 130: #discard small detected faces
+				
+				face_detected = True
+				if face_index == 0:
+					face_included_frames = face_included_frames + 1 #increase frame for a single face
+				
+				cv2.rectangle(img, (x,y), (x+w,y+h), (67,67,67), 1) #draw rectangle to main image
+				
+				cv2.putText(img, str(frame_threshold - face_included_frames), (int(x+w/4),int(y+h/1.5)), cv2.FONT_HERSHEY_SIMPLEX, 4, (255, 255, 255), 2)
+				
+				detected_face = img[int(y):int(y+h), int(x):int(x+w)] #crop detected face
+				
+				#-------------------------------------
+				
+				detected_faces.append((x,y,w,h))
+				face_index = face_index + 1
+				
+				#-------------------------------------
+				
+		if face_detected == True and face_included_frames == frame_threshold and freeze == False:
+			freeze = True
+			#base_img = img.copy()
+			base_img = raw_img.copy()
+			detected_faces_final = detected_faces.copy()
+			tic = time.time()
+		
+		if freeze == True:
+
+			toc = time.time()
+			if (toc - tic) < time_threshold:
+				
+				if freezed_frame == 0:
+					freeze_img = base_img.copy()
+					#freeze_img = np.zeros(resolution, np.uint8) #here, np.uint8 handles showing white area issue
+					
+					for detected_face in detected_faces_final:
+						x = detected_face[0]; y = detected_face[1]
+						w = detected_face[2]; h = detected_face[3]
+						
+						cv2.rectangle(freeze_img, (x,y), (x+w,y+h), (67,67,67), 1) #draw rectangle to main image
+						
+						#-------------------------------
+						
+						#apply deep learning for custom_face
+						
+						custom_face = base_img[y:y+h, x:x+w]
+						
+						#-------------------------------
+						#facial attribute analysis
+						
+						if enable_face_analysis == True:
+							
+							"""
+							gray_img = functions.detectFace(custom_face, (48, 48), True)
+							emotion_labels = ['Angry', 'Disgust', 'Fear', 'Happy', 'Sad', 'Surprise', 'Neutral']
+							emotion_predictions = emotion_model.predict(gray_img)[0,:]
+							sum_of_predictions = emotion_predictions.sum()
+							
+							mood_items = []
+							for i in range(0, len(emotion_labels)):
+								mood_item = []
+								emotion_label = emotion_labels[i]
+								emotion_prediction = 100 * emotion_predictions[i] / sum_of_predictions
+								mood_item.append(emotion_label)
+								mood_item.append(emotion_prediction)
+								mood_items.append(mood_item)
+							
+							emotion_df = pd.DataFrame(mood_items, columns = ["emotion", "score"])
+							emotion_df = emotion_df.sort_values(by = ["score"], ascending=False)
+							dominant_emotion = emotion_df.iloc[0].emotion
+							emotion_score = emotion_df.iloc[0].score
+							"""
+							
+							#-------------------------------
+							
+							face_224 = functions.detectFace(custom_face, (224, 224), False)
+							
+							age_predictions = age_model.predict(face_224)[0,:]
+							apparent_age = Age.findApparentAge(age_predictions)
+						
+							#-------------------------------
+							
+							gender_prediction = gender_model.predict(face_224)[0,:]
+							
+							if np.argmax(gender_prediction) == 0:
+								gender = "W"
+							elif np.argmax(gender_prediction) == 1:
+								gender = "M"
+							
+							#print(str(int(apparent_age))," years old ", dominant_emotion, " ", gender)
+							
+							analysis_report = str(int(apparent_age))+" "+gender
+							
+							#-------------------------------
+							
+							info_box_color = (46,200,255)
+							
+							#top
+							if y - pivot_img_size + int(pivot_img_size/5) > 0:
+								
+								triangle_coordinates = np.array( [
+									(x+int(w/2), y)
+									, (x+int(w/2)-int(w/10), y-int(pivot_img_size/3))
+									, (x+int(w/2)+int(w/10), y-int(pivot_img_size/3))
+								] )
+								
+								cv2.drawContours(freeze_img, [triangle_coordinates], 0, info_box_color, -1)
+								
+								cv2.rectangle(freeze_img, (x+int(w/5), y-pivot_img_size+int(pivot_img_size/5)), (x+w-int(w/5), y-int(pivot_img_size/3)), info_box_color, cv2.FILLED)
+								
+								cv2.putText(freeze_img, analysis_report, (x+int(w/3.5), y - int(pivot_img_size/2.1)), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 111, 255), 2)
+							
+							#bottom
+							elif y + h + pivot_img_size - int(pivot_img_size/5) < resolution_y:
+							
+								triangle_coordinates = np.array( [
+									(x+int(w/2), y+h)
+									, (x+int(w/2)-int(w/10), y+h+int(pivot_img_size/3))
+									, (x+int(w/2)+int(w/10), y+h+int(pivot_img_size/3))
+								] )
+								
+								cv2.drawContours(freeze_img, [triangle_coordinates], 0, info_box_color, -1)
+								
+								cv2.rectangle(freeze_img, (x+int(w/5), y + h + int(pivot_img_size/3)), (x+w-int(w/5), y+h+pivot_img_size-int(pivot_img_size/5)), info_box_color, cv2.FILLED)
+								
+								cv2.putText(freeze_img, analysis_report, (x+int(w/3.5), y + h + int(pivot_img_size/1.5)), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 111, 255), 2)
+								
+						#-------------------------------
+						#face recognition
+						
+						custom_face = functions.detectFace(custom_face, input_shape)
+						
+						#check detectFace function handled
+						if custom_face.shape[1:3] == input_shape:
+							if df.shape[0] > 0: #if there are images to verify, apply face recognition
+								img1_representation = model.predict(custom_face)[0,:]
+								
+								#print(freezed_frame," - ",img1_representation[0:5])
+								
+								def findDistance(row):
+									distance_metric = row['distance_metric']
+									img2_representation = row['embedding']
+									
+									distance = 1000 #initialize very large value
+									if distance_metric == 'cosine':
+										distance = dst.findCosineDistance(img1_representation, img2_representation)
+									elif distance_metric == 'euclidean':
+										distance = dst.findEuclideanDistance(img1_representation, img2_representation)
+									elif distance_metric == 'euclidean_l2':
+										distance = dst.findEuclideanDistance(dst.l2_normalize(img1_representation), dst.l2_normalize(img2_representation))
+										
+									return distance
+								
+								df['distance'] = df.apply(findDistance, axis = 1)
+								df = df.sort_values(by = ["distance"])
+								
+								candidate = df.iloc[0]
+								employee_name = candidate['employee']
+								best_distance = candidate['distance']
+								#employee_name = employee_name.replace("_", "")
+								
+								if best_distance <= threshold:
+									#print(employee_name)
+									display_img = cv2.imread("%s/%s" % (db_path, employee_name))
+									
+									display_img = cv2.resize(display_img, (pivot_img_size, pivot_img_size))
+																		
+									label = employee_name.replace("_", "").replace(".jpg", "")+" ("+"{0:.2f}".format(best_distance)+")"
+									
+									try:
+										if y - pivot_img_size > 0 and x + w + pivot_img_size < resolution_x:
+											#top right
+											freeze_img[y - pivot_img_size:y, x+w:x+w+pivot_img_size] = display_img
+											cv2.putText(freeze_img, label, (x+w, y+10), cv2.FONT_HERSHEY_SIMPLEX, 0.5, text_color, 1)
+											
+											#connect face and text
+											cv2.line(freeze_img,(x+int(w/2), y), (x+3*int(w/4), y-int(pivot_img_size/2)),(67,67,67),1)
+											cv2.line(freeze_img, (x+3*int(w/4), y-int(pivot_img_size/2)), (x+w, y - int(pivot_img_size/2)), (67,67,67),1)
+											
+										elif y + h + pivot_img_size < resolution_y and x - pivot_img_size > 0:
+											#bottom left
+											freeze_img[y+h:y+h+pivot_img_size, x-pivot_img_size:x] = display_img
+											cv2.putText(freeze_img, label, (x - pivot_img_size, y+h-10), cv2.FONT_HERSHEY_SIMPLEX, 0.5, text_color, 1)
+											
+											#connect face and text
+											cv2.line(freeze_img,(x+int(w/2), y+h), (x+int(w/2)-int(w/4), y+h+int(pivot_img_size/2)),(67,67,67),1)
+											cv2.line(freeze_img, (x+int(w/2)-int(w/4), y+h+int(pivot_img_size/2)), (x, y+h+int(pivot_img_size/2)), (67,67,67),1)
+											
+										elif y - pivot_img_size > 0 and x - pivot_img_size > 0:
+											#top left
+											freeze_img[y-pivot_img_size:y, x-pivot_img_size:x] = display_img
+											cv2.putText(freeze_img, label, (x - pivot_img_size, y+10), cv2.FONT_HERSHEY_SIMPLEX, 0.5, text_color, 1)
+											
+											#connect face and text
+											cv2.line(freeze_img,(x+int(w/2), y), (x+int(w/2)-int(w/4), y-int(pivot_img_size/2)),(67,67,67),1)
+											cv2.line(freeze_img, (x+int(w/2)-int(w/4), y-int(pivot_img_size/2)), (x, y - int(pivot_img_size/2)), (67,67,67),1)
+											
+										elif x+w+pivot_img_size < resolution_x and y + h + pivot_img_size < resolution_y:
+											#bottom righ
+											freeze_img[y+h:y+h+pivot_img_size, x+w:x+w+pivot_img_size] = display_img
+											cv2.putText(freeze_img, label, (x+w, y+h-10), cv2.FONT_HERSHEY_SIMPLEX, 0.5, text_color, 1)
+											
+											#connect face and text
+											cv2.line(freeze_img,(x+int(w/2), y+h), (x+int(w/2)+int(w/4), y+h+int(pivot_img_size/2)),(67,67,67),1)
+											cv2.line(freeze_img, (x+int(w/2)+int(w/4), y+h+int(pivot_img_size/2)), (x+w, y+h+int(pivot_img_size/2)), (67,67,67),1)
+									except Exception as err:
+										print(str(err))
+						
+						tic = time.time() #in this way, freezed image can show 5 seconds
+						
+						#-------------------------------
+				
+				time_left = int(time_threshold - (toc - tic) + 1)
+				
+				cv2.rectangle(freeze_img, (10, 10), (90, 50), (67,67,67), -10)
+				cv2.putText(freeze_img, str(time_left), (40, 40), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 1)
+				
+				cv2.imshow('img', freeze_img)
+				
+				freezed_frame = freezed_frame + 1
+			else:
+				face_detected = False
+				face_included_frames = 0
+				freeze = False
+				freezed_frame = 0
+			
+		else:
+			cv2.imshow('img',img)
+		
+		if cv2.waitKey(1) & 0xFF == ord('q'): #press q to quit
+			break
+		
+	#kill open cv things		
+	cap.release()
+	cv2.destroyAllWindows()