diff --git a/.gitignore b/.gitignore index cde5928..dc5c9b8 100644 --- a/.gitignore +++ b/.gitignore @@ -11,4 +11,5 @@ Pipfile.lock deepface.egg-info/ deepface/__pycache__/* deepface/commons/__pycache__/* -deepface/basemodels/__pycache__/* \ No newline at end of file +deepface/basemodels/__pycache__/* +deepface/subsidiarymodels/__pycache__/* \ No newline at end of file diff --git a/README.md b/README.md index 4c0357a..06cd396 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,19 @@ # deepface -**deepface** is a lightweight python based face recognition framework. You can verify faces with just a few lines of codes. +[![Downloads](https://pepy.tech/badge/deepface)](https://pepy.tech/project/deepface) + +**deepface** is a lightweight python based facial analysis framework including face recognition and demography. You can use the framework with a just few lines of codes. + +# Face Recognition + +Verify function under the DeepFace interface is used for face recognition. ```python from deepface import DeepFace result = DeepFace.verify("img1.jpg", "img2.jpg") ``` -# Face recognition models +## Face recognition models Face recognition can be handled by different models. Currently, [`VGG-Face`](https://sefiks.com/2018/08/06/deep-face-recognition-with-keras/) , [`Facenet`](https://sefiks.com/2018/09/03/face-recognition-with-facenet-in-keras/) and [`OpenFace`](https://sefiks.com/2019/07/21/face-recognition-with-openface-in-keras/) models are supported in deepface. The default configuration verifies faces with **VGG-Face** model. You can set the base model while verification as illustared below. Accuracy and speed show difference based on the performing model. @@ -18,7 +24,7 @@ facenet_result = DeepFace.verify("img1.jpg", "img2.jpg", model_name = "Facenet") openface_result = DeepFace.verify("img1.jpg", "img2.jpg", model_name = "OpenFace") ``` -# Similarity +## Similarity These models actually find the vector embeddings of faces. Decision of verification is based on the distance between vectors. Distance could be found by different metrics such as [`Cosine Similarity`](https://sefiks.com/2018/08/13/cosine-similarity-in-machine-learning/), Euclidean Distance and L2 form. The default configuration finds the **cosine similarity**. You can alternatively set the similarity metric while verification as demostratred below. @@ -30,7 +36,7 @@ result = DeepFace.verify("img1.jpg", "img2.jpg", model_name = "VGG-Face", distan VGG-Face has the highest accuracy score but it is not convenient for real time studies because of its complex structure. Facenet is a complex model as well. On the other hand, OpenFace has a close accuracy score but it performs the fastest. That's why, OpenFace is much more convenient for real time studies. -# Verification +## Verification Verification function returns a tuple including boolean verification result, distance between two faces and max threshold to identify. @@ -50,9 +56,54 @@ Instead of using pre-tuned threshold values, you can alternatively check the dis distance = result[1] #the less the better threshold = 0.30 #threshold for VGG-Face and Cosine Similarity if distance < threshold: - return True + return True else: - return False + return False +``` + +# Facial Attribute Analysis + +Deepface also offers facial attribute analysis including [`age`](https://sefiks.com/2019/02/13/apparent-age-and-gender-prediction-in-keras/), [`gender`](https://sefiks.com/2019/02/13/apparent-age-and-gender-prediction-in-keras/), [`emotion`](https://sefiks.com/2018/01/01/facial-expression-recognition-with-keras/) and [`race`](https://sefiks.com/2019/11/11/race-and-ethnicity-prediction-in-keras/) predictions. Analysis function under the DeepFace interface is used to find demography of a face. + +```python +from deepface import DeepFace +demography = DeepFace.analyze("img.zip") #passing nothing as 2nd argument will find everything +#demography = DeepFace.analyze("img.zip", ['age', 'gender', 'race', 'emotion']) #identical to above line +``` + +Analysis function returns a json object. + +``` +{ + "age": 31.940666721338523 + , "gender": "Woman" + , "race": { + "asian": 11.314528435468674, + "indian": 17.498773336410522, + "black": 3.541698679327965, + "white": 21.96589708328247, + "middle eastern": 19.87851709127426, + "latino hispanic": 25.800585746765137 + } + , "dominant_race": "latino hispanic" + , "emotion": { + "angry": 6.004959843039945e-16, + "disgust": 4.9082449499136944e-34, + "fear": 4.7907148065142067e-23, + "happy": 100.0, + "sad": 4.8685008000541987e-14, + "surprise": 5.66862615875019e-10, + "neutral": 3.754812086254056e-09 + } + , "dominant_emotion": "happy" +} +``` + +Then, you can retrieve the fields of the response object easily in Python. + +```python +import json +print("Age: ",demography["age"]) ``` # Installation @@ -63,7 +114,7 @@ The easiest way to install deepface is to download it from [PyPI](https://pypi.o pip install deepface ``` -Alternatively, you can directly download the source code from this repository. GitHub repo might be newer than the PyPI version. +Alternatively, you can directly download the source code from this repository. **GitHub repo might be newer than the PyPI version**. ``` git clone https://github.com/serengil/deepface.git @@ -75,11 +126,13 @@ Initial tests are run for Python 3.5.5 on Windows 10 but this is an OS-independe ``` pip install numpy==1.14.0 +pip install pandas==0.23.4 pip install matplotlib==2.2.2 pip install gdown==3.10.1 pip install opencv-python==3.4.4 pip install tensorflow==1.9.0 pip install keras==2.2.0 +pip install tqdm==4.30.0 ``` # Disclaimer diff --git a/deepface/DeepFace.py b/deepface/DeepFace.py index 5749f04..f3135e4 100644 --- a/deepface/DeepFace.py +++ b/deepface/DeepFace.py @@ -4,8 +4,14 @@ import warnings warnings.filterwarnings("ignore") import time import os +import numpy as np +import pandas as pd +from tqdm import tqdm -from deepface.basemodels import VGGFace, OpenFace, Facenet +#from basemodels import VGGFace, OpenFace, Facenet, Age, Gender, Race, Emotion +#from commons import functions, distance as dst + +from deepface.basemodels import VGGFace, OpenFace, Facenet, Age, Gender, Race, Emotion from deepface.commons import functions, distance as dst def verify(img1_path, img2_path @@ -104,6 +110,103 @@ def verify(img1_path, img2_path #Second item is the threshold. You might want to customize this threshold to identify faces. return (identified, distance, threshold) +def analyze(img_path, actions= []): + + resp_obj = "{\n " + + #if a specific target is not passed, then find them all + if len(actions) == 0: + actions= ['emotion', 'age', 'gender', 'race'] + + print("Actions to do: ", actions) + + img = functions.detectFace(img_path, (224, 224)) + + #TO-DO: do this in parallel + + pbar = tqdm(range(0,len(actions)), desc='Finding actions') + + action_idx = 0 + #for action in actions: + for index in pbar: + action = actions[index] + pbar.set_description("Action: %s" % (action)) + + if action_idx > 0: + resp_obj += "\n , " + + if action == 'emotion': + emotion_labels = ['angry', 'disgust', 'fear', 'happy', 'sad', 'surprise', 'neutral'] + img = functions.detectFace(img_path, (48, 48), True) + + model = Emotion.loadModel() + emotion_predictions = model.predict(img)[0,:] + + sum_of_predictions = emotion_predictions.sum() + + emotion_obj = "\"emotion\": {" + for i in range(0, len(emotion_labels)): + emotion_label = emotion_labels[i] + emotion_prediction = 100 * emotion_predictions[i] / sum_of_predictions + + if i > 0: emotion_obj += ", " + + emotion_obj += "\n " + emotion_obj += "\"%s\": %s" % (emotion_label, emotion_prediction) + + emotion_obj += "\n }" + + emotion_obj += "\n , \"dominant_emotion\": \"%s\"" % (emotion_labels[np.argmax(emotion_predictions)]) + + resp_obj += emotion_obj + + elif action == 'age': + #print("age prediction") + model = Age.loadModel() + age_predictions = model.predict(img)[0,:] + apparent_age = Age.findApparentAge(age_predictions) + + resp_obj += "\"age\": %s" % (apparent_age) + + elif action == 'gender': + #print("gender prediction") + + model = Gender.loadModel() + gender_prediction = model.predict(img)[0,:] + + if np.argmax(gender_prediction) == 0: + gender = "Woman" + elif np.argmax(gender_prediction) == 1: + gender = "Man" + + resp_obj += "\"gender\": \"%s\"" % (gender) + + elif action == 'race': + model = Race.loadModel() + race_predictions = model.predict(img)[0,:] + race_labels = ['asian', 'indian', 'black', 'white', 'middle eastern', 'latino hispanic'] + + sum_of_predictions = race_predictions.sum() + + race_obj = "\"race\": {" + for i in range(0, len(race_labels)): + race_label = race_labels[i] + race_prediction = 100 * race_predictions[i] / sum_of_predictions + + if i > 0: race_obj += ", " + + race_obj += "\n " + race_obj += "\"%s\": %s" % (race_label, race_prediction) + + race_obj += "\n }" + race_obj += "\n , \"dominant_race\": \"%s\"" % (race_labels[np.argmax(race_predictions)]) + + resp_obj += race_obj + + action_idx = action_idx + 1 + + resp_obj += "\n}" + return resp_obj #--------------------------- functions.initializeFolder() diff --git a/deepface/basemodels/Age.py b/deepface/basemodels/Age.py new file mode 100644 index 0000000..42f7bb5 --- /dev/null +++ b/deepface/basemodels/Age.py @@ -0,0 +1,49 @@ +#from basemodels import VGGFace +from deepface.basemodels import VGGFace + +import os +from pathlib import Path +import gdown +import numpy as np +from keras.models import Model, Sequential +from keras.layers import Convolution2D, Flatten, Activation + +def loadModel(): + + model = VGGFace.baseModel() + + #-------------------------- + + classes = 101 + base_model_output = Sequential() + base_model_output = Convolution2D(classes, (1, 1), name='predictions')(model.layers[-4].output) + base_model_output = Flatten()(base_model_output) + base_model_output = Activation('softmax')(base_model_output) + + #-------------------------- + + age_model = Model(inputs=model.input, outputs=base_model_output) + + #-------------------------- + + #load weights + + home = str(Path.home()) + + if os.path.isfile(home+'/.deepface/weights/age_model_weights.h5') != True: + print("age_model_weights.h5 will be downloaded...") + + url = 'https://drive.google.com/uc?id=1YCox_4kJ-BYeXq27uUbasu--yz28zUMV' + output = home+'/.deepface/weights/age_model_weights.h5' + gdown.download(url, output, quiet=False) + + age_model.load_weights(home+'/.deepface/weights/age_model_weights.h5') + + return age_model + + #-------------------------- + +def findApparentAge(age_predictions): + output_indexes = np.array([i for i in range(0, 101)]) + apparent_age = np.sum(age_predictions * output_indexes) + return apparent_age \ No newline at end of file diff --git a/deepface/basemodels/Emotion.py b/deepface/basemodels/Emotion.py new file mode 100644 index 0000000..6936219 --- /dev/null +++ b/deepface/basemodels/Emotion.py @@ -0,0 +1,62 @@ +import os +import gdown +from pathlib import Path +from keras.models import Model, Sequential +from keras.layers import Conv2D, MaxPooling2D, AveragePooling2D, Flatten, Dense, Dropout +import zipfile + +def loadModel(): + + num_classes = 7 + + model = Sequential() + + #1st convolution layer + model.add(Conv2D(64, (5, 5), activation='relu', input_shape=(48,48,1))) + model.add(MaxPooling2D(pool_size=(5,5), strides=(2, 2))) + + #2nd convolution layer + model.add(Conv2D(64, (3, 3), activation='relu')) + model.add(Conv2D(64, (3, 3), activation='relu')) + model.add(AveragePooling2D(pool_size=(3,3), strides=(2, 2))) + + #3rd convolution layer + model.add(Conv2D(128, (3, 3), activation='relu')) + model.add(Conv2D(128, (3, 3), activation='relu')) + model.add(AveragePooling2D(pool_size=(3,3), strides=(2, 2))) + + model.add(Flatten()) + + #fully connected neural networks + model.add(Dense(1024, activation='relu')) + model.add(Dropout(0.2)) + model.add(Dense(1024, activation='relu')) + model.add(Dropout(0.2)) + + model.add(Dense(num_classes, activation='softmax')) + + #---------------------------- + + home = str(Path.home()) + + if os.path.isfile(home+'/.deepface/weights/facial_expression_model_weights.h5') != True: + print("facial_expression_model_weights.h5 will be downloaded...") + + #TO-DO: upload weights to google drive + + #zip + url = 'https://drive.google.com/uc?id=13iUHHP3SlNg53qSuQZDdHDSDNdBP9nwy' + output = home+'/.deepface/weights/facial_expression_model_weights.zip' + gdown.download(url, output, quiet=False) + + #unzip facial_expression_model_weights.zip + with zipfile.ZipFile(output, 'r') as zip_ref: + zip_ref.extractall(home+'/.deepface/weights/') + + model.load_weights(home+'/.deepface/weights/facial_expression_model_weights.h5') + + return model + + #---------------------------- + + return 0 \ No newline at end of file diff --git a/deepface/basemodels/Facenet.py b/deepface/basemodels/Facenet.py index 7b01fa7..8222191 100644 --- a/deepface/basemodels/Facenet.py +++ b/deepface/basemodels/Facenet.py @@ -540,7 +540,7 @@ def loadModel(): if os.path.isfile(home+'/.deepface/weights/facenet_weights.h5') != True: print("facenet_weights.h5 will be downloaded...") - url = 'https://drive.google.com/file/d/1971Xk5RwedbudGgTIrGAL4F7Aifu7id1/view?usp=sharing' + url = 'https://drive.google.com/uc?id=1971Xk5RwedbudGgTIrGAL4F7Aifu7id1' output = home+'/.deepface/weights/facenet_weights.h5' gdown.download(url, output, quiet=False) diff --git a/deepface/basemodels/Gender.py b/deepface/basemodels/Gender.py new file mode 100644 index 0000000..cb19cca --- /dev/null +++ b/deepface/basemodels/Gender.py @@ -0,0 +1,44 @@ +#from basemodels import VGGFace +from deepface.basemodels import VGGFace + +import os +from pathlib import Path +import gdown +import numpy as np +from keras.models import Model, Sequential +from keras.layers import Convolution2D, Flatten, Activation + +def loadModel(): + + model = VGGFace.baseModel() + + #-------------------------- + + classes = 2 + base_model_output = Sequential() + base_model_output = Convolution2D(classes, (1, 1), name='predictions')(model.layers[-4].output) + base_model_output = Flatten()(base_model_output) + base_model_output = Activation('softmax')(base_model_output) + + #-------------------------- + + gender_model = Model(inputs=model.input, outputs=base_model_output) + + #-------------------------- + + #load weights + + home = str(Path.home()) + + if os.path.isfile(home+'/.deepface/weights/gender_model_weights.h5') != True: + print("gender_model_weights.h5 will be downloaded...") + + url = 'https://drive.google.com/uc?id=1wUXRVlbsni2FN9-jkS_f4UTUrm1bRLyk' + output = home+'/.deepface/weights/gender_model_weights.h5' + gdown.download(url, output, quiet=False) + + gender_model.load_weights(home+'/.deepface/weights/gender_model_weights.h5') + + return gender_model + + #-------------------------- \ No newline at end of file diff --git a/deepface/basemodels/OpenFace.py b/deepface/basemodels/OpenFace.py index 830d113..08eae7c 100644 --- a/deepface/basemodels/OpenFace.py +++ b/deepface/basemodels/OpenFace.py @@ -237,7 +237,7 @@ def loadModel(): if os.path.isfile(home+'/.deepface/weights/openface_weights.h5') != True: print("openface_weights.h5 will be downloaded...") - url = 'https://drive.google.com/file/d/1LSe1YCV1x-BfNnfb7DFZTNpv_Q9jITxn' + url = 'https://drive.google.com/uc?id=1LSe1YCV1x-BfNnfb7DFZTNpv_Q9jITxn' output = home+'/.deepface/weights/openface_weights.h5' gdown.download(url, output, quiet=False) diff --git a/deepface/basemodels/Race.py b/deepface/basemodels/Race.py new file mode 100644 index 0000000..a1f6e93 --- /dev/null +++ b/deepface/basemodels/Race.py @@ -0,0 +1,50 @@ +#from basemodels import VGGFace +from deepface.basemodels import VGGFace + +import os +from pathlib import Path +import gdown +import numpy as np +from keras.models import Model, Sequential +from keras.layers import Convolution2D, Flatten, Activation +import zipfile + +def loadModel(): + + model = VGGFace.baseModel() + + #-------------------------- + + classes = 6 + base_model_output = Sequential() + base_model_output = Convolution2D(classes, (1, 1), name='predictions')(model.layers[-4].output) + base_model_output = Flatten()(base_model_output) + base_model_output = Activation('softmax')(base_model_output) + + #-------------------------- + + race_model = Model(inputs=model.input, outputs=base_model_output) + + #-------------------------- + + #load weights + + home = str(Path.home()) + + if os.path.isfile(home+'/.deepface/weights/race_model_single_batch.h5') != True: + print("race_model_single_batch.h5 will be downloaded...") + + #zip + url = 'https://drive.google.com/file/d/1nz-WDhghGQBC4biwShQ9kYjvQMpO6smj' + output = home+'/.deepface/weights/race_model_single_batch.zip' + gdown.download(url, output, quiet=False) + + #unzip race_model_single_batch.zip + with zipfile.ZipFile(output, 'r') as zip_ref: + zip_ref.extractall(home+'/.deepface/weights/') + + race_model.load_weights(home+'/.deepface/weights/race_model_single_batch.h5') + + return race_model + + #-------------------------- \ No newline at end of file diff --git a/deepface/basemodels/VGGFace.py b/deepface/basemodels/VGGFace.py index 7a02ef6..f354526 100644 --- a/deepface/basemodels/VGGFace.py +++ b/deepface/basemodels/VGGFace.py @@ -6,7 +6,7 @@ import gdown #--------------------------------------- -def loadModel(): +def baseModel(): model = Sequential() model.add(ZeroPadding2D((1,1),input_shape=(224,224, 3))) model.add(Convolution2D(64, (3, 3), activation='relu')) @@ -52,6 +52,12 @@ def loadModel(): model.add(Flatten()) model.add(Activation('softmax')) + return model + +def loadModel(): + + model = baseModel() + #----------------------------------- home = str(Path.home()) @@ -69,6 +75,7 @@ def loadModel(): #----------------------------------- + #TO-DO: why? vgg_face_descriptor = Model(inputs=model.layers[0].input, outputs=model.layers[-2].output) return vgg_face_descriptor \ No newline at end of file diff --git a/deepface/commons/functions.py b/deepface/commons/functions.py index b2c9c9e..42351aa 100644 --- a/deepface/commons/functions.py +++ b/deepface/commons/functions.py @@ -19,10 +19,6 @@ def initializeFolder(): if not os.path.exists(home+"/.deepface/weights"): os.mkdir(home+"/.deepface/weights") print("Directory ",home,"/.deepface/weights created") - - if not os.path.exists(home+"/.deepface/config"): - os.mkdir(home+"/.deepface/config") - print("Directory ",home,"/.deepface/config created") def validateInputs(model_name, distance_metric): @@ -67,10 +63,11 @@ def findThreshold(model_name, distance_metric): return threshold -def detectFace(image_path, target_size=(224, 224)): +def detectFace(image_path, target_size=(224, 224), grayscale = False): opencv_home = cv2.__file__ - folders = opencv_home.split("\\")[0:-1] + folders = opencv_home.split(os.path.sep)[0:-1] + path = folders[0] for folder in folders[1:]: path = path + "/" + folder @@ -84,7 +81,10 @@ def detectFace(image_path, target_size=(224, 224)): detector = cv2.CascadeClassifier(detector_path) - img = cv2.imread(image_path) + if grayscale != True: + img = cv2.imread(image_path) + else: #gray scale + img = cv2.imread(image_path, 0) faces = detector.detectMultiScale(img, 1.3, 5) diff --git a/setup.py b/setup.py index 0195dac..4f0c1fb 100644 --- a/setup.py +++ b/setup.py @@ -5,10 +5,10 @@ with open("README.md", "r", encoding="utf-8") as fh: setuptools.setup( name="deepface", - version="0.0.1", + version="0.0.2", author="Sefik Ilkin Serengil", author_email="serengil@gmail.com", - description="Deep Face Recognition Framework", + description="Deep Face Anaylsis Framework for Face Recognition and Demography", long_description=long_description, long_description_content_type="text/markdown", url="https://github.com/serengil/deepface", @@ -19,5 +19,5 @@ setuptools.setup( "Operating System :: OS Independent", ], python_requires='>=3.5.5', - install_requires=["numpy>=1.14.0", "matplotlib>=2.2.2", "opencv-python>=3.4.4", "tensorflow>=1.9.0", "keras>=2.2.0", "gdown>=3.10.1"] + install_requires=["numpy>=1.14.0", "pandas>=0.23.4", "tqdm>=4.30.0", "gdown>=3.10.1", "matplotlib>=2.2.2", "opencv-python>=3.4.4", "tensorflow>=1.9.0", "keras>=2.2.0"] ) diff --git a/tests/unit_tests.py b/tests/unit_tests.py index feb5908..cc76e0d 100644 --- a/tests/unit_tests.py +++ b/tests/unit_tests.py @@ -1,6 +1,24 @@ from deepface import DeepFace +import json #----------------------------------------- +print("Facial analysis tests") + +img = "dataset/img1.jpg" +demography = DeepFace.analyze(img, ['age', 'gender', 'race', 'emotion']) + +print("Demography:") +print(demography) + +#check response is a valid json +print("Age: ", demography["age"]) +print("Gender: ", demography["gender"]) +print("Race: ", demography["dominant_race"]) +print("Emotion: ", demography["dominant_emotion"]) + +print("-----------------------------------------") + +print("Face recognition tests") dataset = [ ['dataset/img1.jpg', 'dataset/img2.jpg', True],