From 888545692259248d5b0b59f2c1e3cb61583b9367 Mon Sep 17 00:00:00 2001 From: Onur Atakan ULUSOY Date: Wed, 11 May 2022 16:20:53 +0000 Subject: [PATCH 01/10] Added tests workflow --- .github/workflows/tests.yml | 69 +++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 .github/workflows/tests.yml diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..6c5b4e3 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,69 @@ +name: Tests + +on: + push: + paths: + - '.github/workflows/tests.yml' + - 'deepface/**' + - 'tests/**' + - 'api/**' + - 'requirements.txt' + - '.gitignore' + - 'setup.py' + pull_request: + paths: + - '.github/workflows/tests.yml' + - 'deepface/**' + - 'tests/**' + - 'api/**' + - 'requirements.txt' + - '.gitignore' + - 'setup.py' + +jobs: + lint_with_flake8: + + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.8] + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install flake8 + - name: Lint with flake8 + run: | + # stop the build if there are Python syntax errors + flake8 . --count --select=E9,F63,F7 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + unit-test-ubuntu-latest: + needs: lint_with_flake8 + + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.8] + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pytest + pip install . + - name: Test with pytest + run: | + cd tests + pytest unit_tests.py \ No newline at end of file From 2f684ba3f810833861009c59426e96cdde4afe2e Mon Sep 17 00:00:00 2001 From: Onur Atakan ULUSOY Date: Wed, 11 May 2022 16:21:57 +0000 Subject: [PATCH 02/10] Typo fix --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 6c5b4e3..ea83056 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -44,7 +44,7 @@ jobs: flake8 . --count --select=E9,F63,F7 --show-source --statistics # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - unit-test-ubuntu-latest: + unit-tests-ubuntu-latest: needs: lint_with_flake8 runs-on: ubuntu-latest From 6d0f29991e1d1930b434991593396ad69c1d8117 Mon Sep 17 00:00:00 2001 From: Onur Atakan ULUSOY Date: Wed, 11 May 2022 16:28:08 +0000 Subject: [PATCH 03/10] Added a gitignore for ignoring .pyc files --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index fadaca5..64a5195 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,4 @@ deepface/detectors/__pycache__/* tests/dataset/*.pkl .DS_Store deepface/.DS_Store +*.pyc From 37ab3ffcbb012392b93575d54c5a03d746d928cf Mon Sep 17 00:00:00 2001 From: Onur Atakan ULUSOY Date: Wed, 11 May 2022 16:28:12 +0000 Subject: [PATCH 04/10] Test --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ea83056..03f93f5 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -63,7 +63,7 @@ jobs: python -m pip install --upgrade pip pip install pytest pip install . + cd tests - name: Test with pytest run: | - cd tests pytest unit_tests.py \ No newline at end of file From 41e968bd1500e1249f66cc490465c568c9159a18 Mon Sep 17 00:00:00 2001 From: Onur Atakan ULUSOY Date: Wed, 11 May 2022 16:32:09 +0000 Subject: [PATCH 05/10] Fix --- .github/workflows/tests.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 03f93f5..b5ee5ae 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -63,7 +63,8 @@ jobs: python -m pip install --upgrade pip pip install pytest pip install . - cd tests + - name: Test with pytest run: | - pytest unit_tests.py \ No newline at end of file + cd tests + pytest deepface/tests/unit_tests.py \ No newline at end of file From 6622d84a916f1a96ccd6c1d77c3ff29e9df5ce14 Mon Sep 17 00:00:00 2001 From: Onur Atakan ULUSOY Date: Wed, 11 May 2022 16:32:39 +0000 Subject: [PATCH 06/10] Fix --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b5ee5ae..d7f4e9d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -67,4 +67,4 @@ jobs: - name: Test with pytest run: | cd tests - pytest deepface/tests/unit_tests.py \ No newline at end of file + pytest /home/runner/work/deepface/tests/unit_tests.py \ No newline at end of file From 34c2dbef1707c55ab5c7f3faf7a8952a0554208d Mon Sep 17 00:00:00 2001 From: Onur Atakan ULUSOY Date: Wed, 11 May 2022 16:44:48 +0000 Subject: [PATCH 07/10] Converted to unittest --- .github/workflows/tests.yml | 2 +- tests/unit_tests.py | 471 ++++++++++++++++++------------------ 2 files changed, 241 insertions(+), 232 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d7f4e9d..f1b3836 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -67,4 +67,4 @@ jobs: - name: Test with pytest run: | cd tests - pytest /home/runner/work/deepface/tests/unit_tests.py \ No newline at end of file + pytest unit_tests.py \ No newline at end of file diff --git a/tests/unit_tests.py b/tests/unit_tests.py index c83e065..c2f3223 100644 --- a/tests/unit_tests.py +++ b/tests/unit_tests.py @@ -9,336 +9,345 @@ from deepface import DeepFace from deepface.commons import functions import json import time +import unittest #----------------------------------------- import tensorflow as tf -tf_version = int(tf.__version__.split(".")[0]) -if tf_version == 2: - import logging - tf.get_logger().setLevel(logging.ERROR) +class deepface_unit_tests(unittest.TestCase): -print("Running unit tests for TF ", tf.__version__) + def test_deepface(self): + tf_version = int(tf.__version__.split(".")[0]) -from deepface.basemodels import VGGFace, OpenFace, Facenet, FbDeepFace -from deepface.extendedmodels import Age, Gender, Race, Emotion + if tf_version == 2: + import logging + tf.get_logger().setLevel(logging.ERROR) -print("-----------------------------------------") -#----------------------------------------- + print("Running unit tests for TF ", tf.__version__) -print("DeepFace.detectFace test") -#detectors = ['opencv', 'ssd', 'dlib', 'mtcnn', 'retinaface'] -detectors = ['opencv', 'ssd', 'mtcnn', 'retinaface'] + from deepface.basemodels import VGGFace, OpenFace, Facenet, FbDeepFace + from deepface.extendedmodels import Age, Gender, Race, Emotion -for detector in detectors: - img = DeepFace.detectFace("dataset/img11.jpg", detector_backend = detector) - print(detector," test is done") + print("-----------------------------------------") + #----------------------------------------- -#import matplotlib.pyplot as plt -#plt.imshow(img) -#plt.show() + print("DeepFace.detectFace test") + #detectors = ['opencv', 'ssd', 'dlib', 'mtcnn', 'retinaface'] + detectors = ['opencv', 'ssd', 'mtcnn', 'retinaface'] -#----------------------------------------- -print("-----------------------------------------") + for detector in detectors: + img = DeepFace.detectFace("dataset/img11.jpg", detector_backend = detector) + print(detector," test is done") -img_path = "dataset/img1.jpg" -embedding = DeepFace.represent(img_path) -print("Function returned ", len(embedding), "dimensional vector") + #import matplotlib.pyplot as plt + #plt.imshow(img) + #plt.show() -model_name = "VGG-Face" -model = DeepFace.build_model(model_name) -print(model_name," is built") -embedding = DeepFace.represent(img_path, model = model) -print("Represent function returned ", len(embedding), "dimensional vector") + #----------------------------------------- + print("-----------------------------------------") -#----------------------------------------- + img_path = "dataset/img1.jpg" + embedding = DeepFace.represent(img_path) + print("Function returned ", len(embedding), "dimensional vector") -dataset = [ - ['dataset/img1.jpg', 'dataset/img2.jpg', True], - ['dataset/img1.jpg', 'dataset/img6.jpg', True] -] + model_name = "VGG-Face" + model = DeepFace.build_model(model_name) + print(model_name," is built") + embedding = DeepFace.represent(img_path, model = model) + print("Represent function returned ", len(embedding), "dimensional vector") -print("-----------------------------------------") + #----------------------------------------- -print("Face detectors test") + dataset = [ + ['dataset/img1.jpg', 'dataset/img2.jpg', True], + ['dataset/img1.jpg', 'dataset/img6.jpg', True] + ] -print("retinaface detector") -res = DeepFace.verify(dataset, detector_backend = 'retinaface') -print(res) + print("-----------------------------------------") -print("ssd detector") -res = DeepFace.verify(dataset, detector_backend = 'ssd') -print(res) + print("Face detectors test") -print("opencv detector") -res = DeepFace.verify(dataset, detector_backend = 'opencv') -print(res) + print("retinaface detector") + res = DeepFace.verify(dataset, detector_backend = 'retinaface') + print(res) -if False: - print("dlib detector") - res = DeepFace.verify(dataset, detector_backend = 'dlib') - print(res) + print("ssd detector") + res = DeepFace.verify(dataset, detector_backend = 'ssd') + print(res) -print("mtcnn detector") -res = DeepFace.verify(dataset, detector_backend = 'mtcnn') -print(res) + print("opencv detector") + res = DeepFace.verify(dataset, detector_backend = 'opencv') + print(res) -print("-----------------------------------------") + if False: + print("dlib detector") + res = DeepFace.verify(dataset, detector_backend = 'dlib') + print(res) -print("Single find function test") + print("mtcnn detector") + res = DeepFace.verify(dataset, detector_backend = 'mtcnn') + print(res) -df = DeepFace.find(img_path = "dataset/img1.jpg", db_path = "dataset" - #, model_name = 'Dlib' -) -print(df.head()) + print("-----------------------------------------") -print("-----------------------------------------") + print("Single find function test") -print("Pre-built model for single find function test") + df = DeepFace.find(img_path = "dataset/img1.jpg", db_path = "dataset" + #, model_name = 'Dlib' + ) + print(df.head()) -#model_name = "VGG-Face" -#model = DeepFace.build_model(model_name) -#print(model_name," is built") + print("-----------------------------------------") -df = DeepFace.find(img_path = "dataset/img1.jpg", db_path = "dataset" - , model_name = model_name, model = model -) -print(df.head()) + print("Pre-built model for single find function test") -print("-----------------------------------------") + #model_name = "VGG-Face" + #model = DeepFace.build_model(model_name) + #print(model_name," is built") -print("Bulk find function tests") + df = DeepFace.find(img_path = "dataset/img1.jpg", db_path = "dataset" + , model_name = model_name, model = model + ) + print(df.head()) -dfs = DeepFace.find(img_path = ["dataset/img1.jpg", "dataset/img2.jpg"], db_path = "dataset" - #, model_name = 'Dlib' -) -print(dfs[0].head()) -print(dfs[1].head()) + print("-----------------------------------------") -print("-----------------------------------------") + print("Bulk find function tests") -print("Bulk verification tests") + dfs = DeepFace.find(img_path = ["dataset/img1.jpg", "dataset/img2.jpg"], db_path = "dataset" + #, model_name = 'Dlib' + ) + print(dfs[0].head()) + print(dfs[1].head()) -resp_obj = DeepFace.verify(dataset) -print(resp_obj) -print(resp_obj["pair_1"]["verified"] == True) -print(resp_obj["pair_2"]["verified"] == True) + print("-----------------------------------------") -print("-----------------------------------------") + print("Bulk verification tests") -print("Bulk facial analysis tests") + resp_obj = DeepFace.verify(dataset) + print(resp_obj) + print(resp_obj["pair_1"]["verified"] == True) + print(resp_obj["pair_2"]["verified"] == True) -dataset = [ - 'dataset/img1.jpg', - 'dataset/img2.jpg', - 'dataset/img5.jpg', - 'dataset/img6.jpg' -] + print("-----------------------------------------") -resp_obj = DeepFace.analyze(dataset) -print(resp_obj["instance_1"]["age"]," years old ", resp_obj["instance_1"]["dominant_emotion"], " ",resp_obj["instance_1"]["gender"]) -print(resp_obj["instance_2"]["age"]," years old ", resp_obj["instance_2"]["dominant_emotion"], " ",resp_obj["instance_2"]["gender"]) -print(resp_obj["instance_3"]["age"]," years old ", resp_obj["instance_3"]["dominant_emotion"], " ",resp_obj["instance_3"]["gender"]) -print(resp_obj["instance_4"]["age"]," years old ", resp_obj["instance_4"]["dominant_emotion"], " ",resp_obj["instance_4"]["gender"]) + print("Bulk facial analysis tests") -print("-----------------------------------------") + dataset = [ + 'dataset/img1.jpg', + 'dataset/img2.jpg', + 'dataset/img5.jpg', + 'dataset/img6.jpg' + ] -print("Facial analysis test. Passing nothing as an action") + resp_obj = DeepFace.analyze(dataset) + print(resp_obj["instance_1"]["age"]," years old ", resp_obj["instance_1"]["dominant_emotion"], " ",resp_obj["instance_1"]["gender"]) + print(resp_obj["instance_2"]["age"]," years old ", resp_obj["instance_2"]["dominant_emotion"], " ",resp_obj["instance_2"]["gender"]) + print(resp_obj["instance_3"]["age"]," years old ", resp_obj["instance_3"]["dominant_emotion"], " ",resp_obj["instance_3"]["gender"]) + print(resp_obj["instance_4"]["age"]," years old ", resp_obj["instance_4"]["dominant_emotion"], " ",resp_obj["instance_4"]["gender"]) -img = "dataset/img4.jpg" -demography = DeepFace.analyze(img) -print(demography) + print("-----------------------------------------") -print("-----------------------------------------") + print("Facial analysis test. Passing nothing as an action") -print("Facial analysis test. Passing all to the action") -demography = DeepFace.analyze(img, ['age', 'gender', 'race', 'emotion']) + img = "dataset/img4.jpg" + demography = DeepFace.analyze(img) + print(demography) -print("Demography:") -print(demography) + print("-----------------------------------------") -#check response is a valid json -print("Age: ", demography["age"]) -print("Gender: ", demography["gender"]) -print("Race: ", demography["dominant_race"]) -print("Emotion: ", demography["dominant_emotion"]) + print("Facial analysis test. Passing all to the action") + demography = DeepFace.analyze(img, ['age', 'gender', 'race', 'emotion']) -print("-----------------------------------------") + print("Demography:") + print(demography) -print("Facial analysis test 2. Remove some actions and check they are not computed") -demography = DeepFace.analyze(img, ['age', 'gender']) + #check response is a valid json + print("Age: ", demography["age"]) + print("Gender: ", demography["gender"]) + print("Race: ", demography["dominant_race"]) + print("Emotion: ", demography["dominant_emotion"]) -print("Age: ", demography.get("age")) -print("Gender: ", demography.get("gender")) -print("Race: ", demography.get("dominant_race")) -print("Emotion: ", demography.get("dominant_emotion")) + print("-----------------------------------------") + print("Facial analysis test 2. Remove some actions and check they are not computed") + demography = DeepFace.analyze(img, ['age', 'gender']) -print("-----------------------------------------") + print("Age: ", demography.get("age")) + print("Gender: ", demography.get("gender")) + print("Race: ", demography.get("dominant_race")) + print("Emotion: ", demography.get("dominant_emotion")) -print("Face recognition tests") -dataset = [ - ['dataset/img1.jpg', 'dataset/img2.jpg', True], - ['dataset/img5.jpg', 'dataset/img6.jpg', True], - ['dataset/img6.jpg', 'dataset/img7.jpg', True], - ['dataset/img8.jpg', 'dataset/img9.jpg', True], - ['dataset/img1.jpg', 'dataset/img11.jpg', True], - ['dataset/img2.jpg', 'dataset/img11.jpg', True], + print("-----------------------------------------") - ['dataset/img1.jpg', 'dataset/img3.jpg', False], - ['dataset/img2.jpg', 'dataset/img3.jpg', False], - ['dataset/img6.jpg', 'dataset/img8.jpg', False], - ['dataset/img6.jpg', 'dataset/img9.jpg', False], -] + print("Face recognition tests") -#models = ['VGG-Face', 'Facenet', 'OpenFace', 'DeepFace', 'DeepID', 'Dlib', 'ArcFace'] -models = ['VGG-Face', 'Facenet', 'Facenet512', 'ArcFace', 'SFace'] #those are robust models -metrics = ['cosine', 'euclidean', 'euclidean_l2'] + dataset = [ + ['dataset/img1.jpg', 'dataset/img2.jpg', True], + ['dataset/img5.jpg', 'dataset/img6.jpg', True], + ['dataset/img6.jpg', 'dataset/img7.jpg', True], + ['dataset/img8.jpg', 'dataset/img9.jpg', True], + ['dataset/img1.jpg', 'dataset/img11.jpg', True], + ['dataset/img2.jpg', 'dataset/img11.jpg', True], -passed_tests = 0; test_cases = 0 + ['dataset/img1.jpg', 'dataset/img3.jpg', False], + ['dataset/img2.jpg', 'dataset/img3.jpg', False], + ['dataset/img6.jpg', 'dataset/img8.jpg', False], + ['dataset/img6.jpg', 'dataset/img9.jpg', False], + ] -for model in models: - #prebuilt_model = DeepFace.build_model(model) - #print(model," is built") - for metric in metrics: - for instance in dataset: - img1 = instance[0] - img2 = instance[1] - result = instance[2] + #models = ['VGG-Face', 'Facenet', 'OpenFace', 'DeepFace', 'DeepID', 'Dlib', 'ArcFace'] + models = ['VGG-Face', 'Facenet', 'Facenet512', 'ArcFace', 'SFace'] #those are robust models + metrics = ['cosine', 'euclidean', 'euclidean_l2'] - resp_obj = DeepFace.verify(img1, img2 - , model_name = model - #, model = prebuilt_model - , distance_metric = metric) + passed_tests = 0; test_cases = 0 - prediction = resp_obj["verified"] - distance = round(resp_obj["distance"], 2) - threshold = resp_obj["threshold"] + for model in models: + #prebuilt_model = DeepFace.build_model(model) + #print(model," is built") + for metric in metrics: + for instance in dataset: + img1 = instance[0] + img2 = instance[1] + result = instance[2] - test_result_label = "failed" - if prediction == result: - passed_tests = passed_tests + 1 - test_result_label = "passed" + resp_obj = DeepFace.verify(img1, img2 + , model_name = model + #, model = prebuilt_model + , distance_metric = metric) - if prediction == True: - classified_label = "verified" - else: - classified_label = "unverified" + prediction = resp_obj["verified"] + distance = round(resp_obj["distance"], 2) + threshold = resp_obj["threshold"] - test_cases = test_cases + 1 + test_result_label = "failed" + if prediction == result: + passed_tests = passed_tests + 1 + test_result_label = "passed" - print(img1.split("/")[-1], "-", img2.split("/")[-1], classified_label, "as same person based on", model,"and",metric,". Distance:",distance,", Threshold:", threshold,"(",test_result_label,")") + if prediction == True: + classified_label = "verified" + else: + classified_label = "unverified" - print("--------------------------") + test_cases = test_cases + 1 -#----------------------------------------- + print(img1.split("/")[-1], "-", img2.split("/")[-1], classified_label, "as same person based on", model,"and",metric,". Distance:",distance,", Threshold:", threshold,"(",test_result_label,")") -print("Passed unit tests: ",passed_tests," / ",test_cases) + print("--------------------------") -min_score = 70 + #----------------------------------------- -accuracy = 100 * passed_tests / test_cases -accuracy = round(accuracy, 2) + print("Passed unit tests: ",passed_tests," / ",test_cases) -if accuracy >= min_score: - print("Unit tests are completed successfully. Score: ",accuracy,"%") -else: - raise ValueError("Unit test score does not satisfy the minimum required accuracy. Minimum expected score is ", min_score,"% but this got ",accuracy,"%") + min_score = 70 -#----------------------------------- -#----------------------------------- + accuracy = 100 * passed_tests / test_cases + accuracy = round(accuracy, 2) -print("Analyze function with passing pre-trained model") + if accuracy >= min_score: + print("Unit tests are completed successfully. Score: ",accuracy,"%") + else: + raise ValueError("Unit test score does not satisfy the minimum required accuracy. Minimum expected score is ", min_score,"% but this got ",accuracy,"%") -emotion_model = DeepFace.build_model("Emotion") -age_model = DeepFace.build_model("Age") -gender_model = DeepFace.build_model("Gender") -race_model = DeepFace.build_model("Race") + #----------------------------------- + #----------------------------------- -facial_attribute_models = {} -facial_attribute_models["emotion"] = emotion_model -facial_attribute_models["age"] = age_model -facial_attribute_models["gender"] = gender_model -facial_attribute_models["race"] = race_model + print("Analyze function with passing pre-trained model") -resp_obj = DeepFace.analyze("dataset/img1.jpg", models=facial_attribute_models) -print(resp_obj) + emotion_model = DeepFace.build_model("Emotion") + age_model = DeepFace.build_model("Age") + gender_model = DeepFace.build_model("Gender") + race_model = DeepFace.build_model("Race") -#----------------------------------- -print("--------------------------") + facial_attribute_models = {} + facial_attribute_models["emotion"] = emotion_model + facial_attribute_models["age"] = age_model + facial_attribute_models["gender"] = gender_model + facial_attribute_models["race"] = race_model -if False: - print("Ensemble for find function") - df = DeepFace.find(img_path = "dataset/img1.jpg", db_path = "dataset", model_name = "Ensemble") - print(df.head()) + resp_obj = DeepFace.analyze("dataset/img1.jpg", models=facial_attribute_models) + print(resp_obj) -#----------------------------------- -print("--------------------------") + #----------------------------------- + print("--------------------------") -if False: - print("Ensemble for verify function") - resp_obj = DeepFace.verify(dataset, model_name = "Ensemble") + if False: + print("Ensemble for find function") + df = DeepFace.find(img_path = "dataset/img1.jpg", db_path = "dataset", model_name = "Ensemble") + print(df.head()) - for i in range(0, len(dataset)): - item = resp_obj['pair_%s' % (i+1)] - verified = item["verified"] - score = item["score"] - print(verified) + #----------------------------------- + print("--------------------------") -#----------------------------------- -print("--------------------------") + if False: + print("Ensemble for verify function") + resp_obj = DeepFace.verify(dataset, model_name = "Ensemble") -if False: + for i in range(0, len(dataset)): + item = resp_obj['pair_%s' % (i+1)] + verified = item["verified"] + score = item["score"] + print(verified) - print("Pre-trained ensemble method - find") + #----------------------------------- + print("--------------------------") - from deepface import DeepFace - from deepface.basemodels import Boosting + if False: - model = Boosting.loadModel() - df = DeepFace.find("dataset/img1.jpg", db_path = "dataset", model_name = 'Ensemble', model = model, enforce_detection=False) + print("Pre-trained ensemble method - find") - print(df) + from deepface import DeepFace + from deepface.basemodels import Boosting -#----------------------------------- -print("--------------------------") + model = Boosting.loadModel() + df = DeepFace.find("dataset/img1.jpg", db_path = "dataset", model_name = 'Ensemble', model = model, enforce_detection=False) -if False: - print("Pre-trained ensemble method - verify") - res = DeepFace.verify(dataset, model_name = "Ensemble", model = model) - print(res) + print(df) -#----------------------------------- -print("--------------------------") + #----------------------------------- + print("--------------------------") -import cv2 + if False: + print("Pre-trained ensemble method - verify") + res = DeepFace.verify(dataset, model_name = "Ensemble", model = model) + print(res) -print("Passing numpy array to analyze function") + #----------------------------------- + print("--------------------------") -img = cv2.imread("dataset/img1.jpg") -resp_obj = DeepFace.analyze(img) -print(resp_obj) + import cv2 -print("--------------------------") + print("Passing numpy array to analyze function") -print("Passing numpy array to verify function") + img = cv2.imread("dataset/img1.jpg") + resp_obj = DeepFace.analyze(img) + print(resp_obj) -img1 = cv2.imread("dataset/img1.jpg") -img2 = cv2.imread("dataset/img2.jpg") + print("--------------------------") -res = DeepFace.verify(img1, img2) -print(res) + print("Passing numpy array to verify function") -print("--------------------------") + img1 = cv2.imread("dataset/img1.jpg") + img2 = cv2.imread("dataset/img2.jpg") -print("Passing numpy array to find function") + res = DeepFace.verify(img1, img2) + print(res) -img1 = cv2.imread("dataset/img1.jpg") + print("--------------------------") -df = DeepFace.find(img1, db_path = "dataset") + print("Passing numpy array to find function") -print(df.head()) + img1 = cv2.imread("dataset/img1.jpg") -print("--------------------------") + df = DeepFace.find(img1, db_path = "dataset") + + print(df.head()) + + print("--------------------------") + + self.assertEqual(accuracy >= min_score, False, "A problem on the deepface installation.") + +unittest.main(exit=False) \ No newline at end of file From 6455dd63821e7228976216b21ad8d9bce39cb46e Mon Sep 17 00:00:00 2001 From: Onur Atakan ULUSOY Date: Wed, 11 May 2022 16:50:02 +0000 Subject: [PATCH 08/10] fix --- tests/unit_tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit_tests.py b/tests/unit_tests.py index c2f3223..49e1c16 100644 --- a/tests/unit_tests.py +++ b/tests/unit_tests.py @@ -5,8 +5,6 @@ import os #os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' -from deepface import DeepFace -from deepface.commons import functions import json import time import unittest @@ -26,6 +24,8 @@ class deepface_unit_tests(unittest.TestCase): print("Running unit tests for TF ", tf.__version__) + from deepface import DeepFace + from deepface.commons import functions from deepface.basemodels import VGGFace, OpenFace, Facenet, FbDeepFace from deepface.extendedmodels import Age, Gender, Race, Emotion From 4cc2fd87820cffc5d67fa462111b378499874fed Mon Sep 17 00:00:00 2001 From: Onur Atakan ULUSOY Date: Wed, 11 May 2022 17:05:39 +0000 Subject: [PATCH 09/10] Fixed the assert equal control --- tests/unit_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit_tests.py b/tests/unit_tests.py index 49e1c16..09d7501 100644 --- a/tests/unit_tests.py +++ b/tests/unit_tests.py @@ -348,6 +348,6 @@ class deepface_unit_tests(unittest.TestCase): print("--------------------------") - self.assertEqual(accuracy >= min_score, False, "A problem on the deepface installation.") + self.assertEqual(accuracy >= min_score, True, "A problem on the deepface installation.") unittest.main(exit=False) \ No newline at end of file From 84a1ac64a5f1b4207e397d11d3a91c9559a9d14d Mon Sep 17 00:00:00 2001 From: Onur Atakan ULUSOY Date: Tue, 24 May 2022 13:59:16 +0000 Subject: [PATCH 10/10] test --- tests/unit_tests.py | 432 ++++++++++++++++---------------------------- 1 file changed, 151 insertions(+), 281 deletions(-) diff --git a/tests/unit_tests.py b/tests/unit_tests.py index 09d7501..57a71de 100644 --- a/tests/unit_tests.py +++ b/tests/unit_tests.py @@ -1,353 +1,223 @@ import warnings -warnings.filterwarnings("ignore") - import os -#os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' +import tensorflow as tf +import cv2 +from deepface import DeepFace + +print("-----------------------------------------") + +warnings.filterwarnings("ignore") os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' -import json -import time -import unittest +tf_major_version = int(tf.__version__.split(".")[0]) -#----------------------------------------- +if tf_major_version == 2: + import logging + tf.get_logger().setLevel(logging.ERROR) -import tensorflow as tf +print("Running unit tests for TF ", tf.__version__) -class deepface_unit_tests(unittest.TestCase): +print("-----------------------------------------") - def test_deepface(self): - tf_version = int(tf.__version__.split(".")[0]) +expected_coverage = 97 +num_cases = 0; succeed_cases = 0 - if tf_version == 2: - import logging - tf.get_logger().setLevel(logging.ERROR) +def evaluate(condition): - print("Running unit tests for TF ", tf.__version__) + global num_cases, succeed_cases - from deepface import DeepFace - from deepface.commons import functions - from deepface.basemodels import VGGFace, OpenFace, Facenet, FbDeepFace - from deepface.extendedmodels import Age, Gender, Race, Emotion + if condition is True: + succeed_cases += 1 + + num_cases += 1 - print("-----------------------------------------") - #----------------------------------------- +# ------------------------------------------------ - print("DeepFace.detectFace test") - #detectors = ['opencv', 'ssd', 'dlib', 'mtcnn', 'retinaface'] - detectors = ['opencv', 'ssd', 'mtcnn', 'retinaface'] +detectors = ['opencv', 'mtcnn', 'retinaface'] +models = ['VGG-Face', 'Facenet', 'Facenet512', 'ArcFace', 'SFace'] +metrics = ['cosine', 'euclidean', 'euclidean_l2'] - for detector in detectors: - img = DeepFace.detectFace("dataset/img11.jpg", detector_backend = detector) - print(detector," test is done") +dataset = [ + ['dataset/img1.jpg', 'dataset/img2.jpg', True], + ['dataset/img5.jpg', 'dataset/img6.jpg', True], + ['dataset/img6.jpg', 'dataset/img7.jpg', True], + ['dataset/img8.jpg', 'dataset/img9.jpg', True], + ['dataset/img1.jpg', 'dataset/img11.jpg', True], + ['dataset/img2.jpg', 'dataset/img11.jpg', True], - #import matplotlib.pyplot as plt - #plt.imshow(img) - #plt.show() + ['dataset/img1.jpg', 'dataset/img3.jpg', False], + ['dataset/img2.jpg', 'dataset/img3.jpg', False], + ['dataset/img6.jpg', 'dataset/img8.jpg', False], + ['dataset/img6.jpg', 'dataset/img9.jpg', False], +] - #----------------------------------------- - print("-----------------------------------------") +print("-----------------------------------------") - img_path = "dataset/img1.jpg" - embedding = DeepFace.represent(img_path) - print("Function returned ", len(embedding), "dimensional vector") +def test_cases(): - model_name = "VGG-Face" - model = DeepFace.build_model(model_name) - print(model_name," is built") - embedding = DeepFace.represent(img_path, model = model) - print("Represent function returned ", len(embedding), "dimensional vector") + print("DeepFace.detectFace test") - #----------------------------------------- + for detector in detectors: + img = DeepFace.detectFace("dataset/img11.jpg", detector_backend = detector) + evaluate(img.shape[0] > 0 and img.shape[1] > 0) + print(detector," test is done") - dataset = [ - ['dataset/img1.jpg', 'dataset/img2.jpg', True], - ['dataset/img1.jpg', 'dataset/img6.jpg', True] - ] + print("-----------------------------------------") - print("-----------------------------------------") + img_path = "dataset/img1.jpg" + embedding = DeepFace.represent(img_path) + print("Function returned ", len(embedding), "dimensional vector") + evaluate(len(embedding) > 0) - print("Face detectors test") + print("-----------------------------------------") - print("retinaface detector") - res = DeepFace.verify(dataset, detector_backend = 'retinaface') - print(res) + print("Face detectors test") - print("ssd detector") - res = DeepFace.verify(dataset, detector_backend = 'ssd') - print(res) + for detector in detectors: + print(detector + " detector") + res = DeepFace.verify(dataset[0][0], dataset[0][1], detector_backend = detector) + print(res) + assert res["verified"] == dataset[0][2] - print("opencv detector") - res = DeepFace.verify(dataset, detector_backend = 'opencv') - print(res) + print("-----------------------------------------") - if False: - print("dlib detector") - res = DeepFace.verify(dataset, detector_backend = 'dlib') - print(res) + print("Find function test") - print("mtcnn detector") - res = DeepFace.verify(dataset, detector_backend = 'mtcnn') - print(res) + df = DeepFace.find(img_path = "dataset/img1.jpg", db_path = "dataset") + print(df.head()) + evaluate(df.shape[0] > 0) - print("-----------------------------------------") + print("-----------------------------------------") - print("Single find function test") + print("Facial analysis test. Passing nothing as an action") - df = DeepFace.find(img_path = "dataset/img1.jpg", db_path = "dataset" - #, model_name = 'Dlib' - ) - print(df.head()) + img = "dataset/img4.jpg" + demography = DeepFace.analyze(img) + print(demography) - print("-----------------------------------------") + evaluate(demography["age"] > 20 and demography["age"] < 40) + evaluate(demography["gender"] == "Woman") - print("Pre-built model for single find function test") + print("-----------------------------------------") - #model_name = "VGG-Face" - #model = DeepFace.build_model(model_name) - #print(model_name," is built") + print("Facial analysis test. Passing all to the action") + demography = DeepFace.analyze(img, ['age', 'gender', 'race', 'emotion']) - df = DeepFace.find(img_path = "dataset/img1.jpg", db_path = "dataset" - , model_name = model_name, model = model - ) - print(df.head()) + print("Demography:") + print(demography) - print("-----------------------------------------") + #check response is a valid json + print("Age: ", demography["age"]) + print("Gender: ", demography["gender"]) + print("Race: ", demography["dominant_race"]) + print("Emotion: ", demography["dominant_emotion"]) - print("Bulk find function tests") + evaluate(demography.get("age") is not None) + evaluate(demography.get("gender") is not None) + evaluate(demography.get("dominant_race") is not None) + evaluate(demography.get("dominant_emotion") is not None) - dfs = DeepFace.find(img_path = ["dataset/img1.jpg", "dataset/img2.jpg"], db_path = "dataset" - #, model_name = 'Dlib' - ) - print(dfs[0].head()) - print(dfs[1].head()) + print("-----------------------------------------") - print("-----------------------------------------") + print("Facial analysis test 2. Remove some actions and check they are not computed") + demography = DeepFace.analyze(img, ['age', 'gender']) - print("Bulk verification tests") + print("Age: ", demography.get("age")) + print("Gender: ", demography.get("gender")) + print("Race: ", demography.get("dominant_race")) + print("Emotion: ", demography.get("dominant_emotion")) - resp_obj = DeepFace.verify(dataset) - print(resp_obj) - print(resp_obj["pair_1"]["verified"] == True) - print(resp_obj["pair_2"]["verified"] == True) + evaluate(demography.get("age") is not None) + evaluate(demography.get("gender") is not None) + evaluate(demography.get("dominant_race") is None) + evaluate(demography.get("dominant_emotion") is None) - print("-----------------------------------------") + print("-----------------------------------------") - print("Bulk facial analysis tests") + print("Facial recognition tests") - dataset = [ - 'dataset/img1.jpg', - 'dataset/img2.jpg', - 'dataset/img5.jpg', - 'dataset/img6.jpg' - ] + for model in models: + for metric in metrics: + for instance in dataset: + img1 = instance[0] + img2 = instance[1] + result = instance[2] - resp_obj = DeepFace.analyze(dataset) - print(resp_obj["instance_1"]["age"]," years old ", resp_obj["instance_1"]["dominant_emotion"], " ",resp_obj["instance_1"]["gender"]) - print(resp_obj["instance_2"]["age"]," years old ", resp_obj["instance_2"]["dominant_emotion"], " ",resp_obj["instance_2"]["gender"]) - print(resp_obj["instance_3"]["age"]," years old ", resp_obj["instance_3"]["dominant_emotion"], " ",resp_obj["instance_3"]["gender"]) - print(resp_obj["instance_4"]["age"]," years old ", resp_obj["instance_4"]["dominant_emotion"], " ",resp_obj["instance_4"]["gender"]) + resp_obj = DeepFace.verify(img1, img2 + , model_name = model + , distance_metric = metric) - print("-----------------------------------------") + prediction = resp_obj["verified"] + distance = round(resp_obj["distance"], 2) + threshold = resp_obj["threshold"] - print("Facial analysis test. Passing nothing as an action") + passed = prediction == result - img = "dataset/img4.jpg" - demography = DeepFace.analyze(img) - print(demography) + evaluate(passed) - print("-----------------------------------------") + if passed: + test_result_label = "passed" + else: + test_result_label = "failed" - print("Facial analysis test. Passing all to the action") - demography = DeepFace.analyze(img, ['age', 'gender', 'race', 'emotion']) + if prediction == True: + classified_label = "verified" + else: + classified_label = "unverified" - print("Demography:") - print(demography) + print(img1.split("/")[-1], "-", img2.split("/")[-1], classified_label, "as same person based on", model,"and",metric,". Distance:",distance,", Threshold:", threshold,"(",test_result_label,")") - #check response is a valid json - print("Age: ", demography["age"]) - print("Gender: ", demography["gender"]) - print("Race: ", demography["dominant_race"]) - print("Emotion: ", demography["dominant_emotion"]) + print("--------------------------") - print("-----------------------------------------") + # ----------------------------------------- + + print("Passing numpy array to analyze function") - print("Facial analysis test 2. Remove some actions and check they are not computed") - demography = DeepFace.analyze(img, ['age', 'gender']) + img = cv2.imread("dataset/img1.jpg") + resp_obj = DeepFace.analyze(img) + print(resp_obj) - print("Age: ", demography.get("age")) - print("Gender: ", demography.get("gender")) - print("Race: ", demography.get("dominant_race")) - print("Emotion: ", demography.get("dominant_emotion")) + evaluate(resp_obj["age"] > 20 and resp_obj["age"] < 40) + evaluate(resp_obj["gender"] == "Woman") + print("--------------------------") - print("-----------------------------------------") + print("Passing numpy array to verify function") - print("Face recognition tests") + img1 = cv2.imread("dataset/img1.jpg") + img2 = cv2.imread("dataset/img2.jpg") - dataset = [ - ['dataset/img1.jpg', 'dataset/img2.jpg', True], - ['dataset/img5.jpg', 'dataset/img6.jpg', True], - ['dataset/img6.jpg', 'dataset/img7.jpg', True], - ['dataset/img8.jpg', 'dataset/img9.jpg', True], - ['dataset/img1.jpg', 'dataset/img11.jpg', True], - ['dataset/img2.jpg', 'dataset/img11.jpg', True], + res = DeepFace.verify(img1, img2) + print(res) - ['dataset/img1.jpg', 'dataset/img3.jpg', False], - ['dataset/img2.jpg', 'dataset/img3.jpg', False], - ['dataset/img6.jpg', 'dataset/img8.jpg', False], - ['dataset/img6.jpg', 'dataset/img9.jpg', False], - ] + evaluate(res["verified"] == True) - #models = ['VGG-Face', 'Facenet', 'OpenFace', 'DeepFace', 'DeepID', 'Dlib', 'ArcFace'] - models = ['VGG-Face', 'Facenet', 'Facenet512', 'ArcFace', 'SFace'] #those are robust models - metrics = ['cosine', 'euclidean', 'euclidean_l2'] + print("--------------------------") - passed_tests = 0; test_cases = 0 + print("Passing numpy array to find function") - for model in models: - #prebuilt_model = DeepFace.build_model(model) - #print(model," is built") - for metric in metrics: - for instance in dataset: - img1 = instance[0] - img2 = instance[1] - result = instance[2] + img1 = cv2.imread("dataset/img1.jpg") - resp_obj = DeepFace.verify(img1, img2 - , model_name = model - #, model = prebuilt_model - , distance_metric = metric) + df = DeepFace.find(img1, db_path = "dataset") - prediction = resp_obj["verified"] - distance = round(resp_obj["distance"], 2) - threshold = resp_obj["threshold"] + print(df.head()) - test_result_label = "failed" - if prediction == result: - passed_tests = passed_tests + 1 - test_result_label = "passed" + evaluate(df.shape[0] > 0) - if prediction == True: - classified_label = "verified" - else: - classified_label = "unverified" + print("--------------------------") - test_cases = test_cases + 1 +test_cases() - print(img1.split("/")[-1], "-", img2.split("/")[-1], classified_label, "as same person based on", model,"and",metric,". Distance:",distance,", Threshold:", threshold,"(",test_result_label,")") +print("num of test cases run: " + str(num_cases)) +print("succeeded test cases: " + str(succeed_cases)) - print("--------------------------") +test_score = (100 * succeed_cases) / num_cases - #----------------------------------------- +print("test coverage: " + str(test_score)) - print("Passed unit tests: ",passed_tests," / ",test_cases) +if test_score > expected_coverage: + print("well done! min required test coverage is satisfied") +else: + print("min required test coverage is NOT satisfied") - min_score = 70 - - accuracy = 100 * passed_tests / test_cases - accuracy = round(accuracy, 2) - - if accuracy >= min_score: - print("Unit tests are completed successfully. Score: ",accuracy,"%") - else: - raise ValueError("Unit test score does not satisfy the minimum required accuracy. Minimum expected score is ", min_score,"% but this got ",accuracy,"%") - - #----------------------------------- - #----------------------------------- - - print("Analyze function with passing pre-trained model") - - emotion_model = DeepFace.build_model("Emotion") - age_model = DeepFace.build_model("Age") - gender_model = DeepFace.build_model("Gender") - race_model = DeepFace.build_model("Race") - - facial_attribute_models = {} - facial_attribute_models["emotion"] = emotion_model - facial_attribute_models["age"] = age_model - facial_attribute_models["gender"] = gender_model - facial_attribute_models["race"] = race_model - - resp_obj = DeepFace.analyze("dataset/img1.jpg", models=facial_attribute_models) - print(resp_obj) - - #----------------------------------- - print("--------------------------") - - if False: - print("Ensemble for find function") - df = DeepFace.find(img_path = "dataset/img1.jpg", db_path = "dataset", model_name = "Ensemble") - print(df.head()) - - #----------------------------------- - print("--------------------------") - - if False: - print("Ensemble for verify function") - resp_obj = DeepFace.verify(dataset, model_name = "Ensemble") - - for i in range(0, len(dataset)): - item = resp_obj['pair_%s' % (i+1)] - verified = item["verified"] - score = item["score"] - print(verified) - - #----------------------------------- - print("--------------------------") - - if False: - - print("Pre-trained ensemble method - find") - - from deepface import DeepFace - from deepface.basemodels import Boosting - - model = Boosting.loadModel() - df = DeepFace.find("dataset/img1.jpg", db_path = "dataset", model_name = 'Ensemble', model = model, enforce_detection=False) - - print(df) - - #----------------------------------- - print("--------------------------") - - if False: - print("Pre-trained ensemble method - verify") - res = DeepFace.verify(dataset, model_name = "Ensemble", model = model) - print(res) - - #----------------------------------- - print("--------------------------") - - import cv2 - - print("Passing numpy array to analyze function") - - img = cv2.imread("dataset/img1.jpg") - resp_obj = DeepFace.analyze(img) - print(resp_obj) - - print("--------------------------") - - print("Passing numpy array to verify function") - - img1 = cv2.imread("dataset/img1.jpg") - img2 = cv2.imread("dataset/img2.jpg") - - res = DeepFace.verify(img1, img2) - print(res) - - print("--------------------------") - - print("Passing numpy array to find function") - - img1 = cv2.imread("dataset/img1.jpg") - - df = DeepFace.find(img1, db_path = "dataset") - - print(df.head()) - - print("--------------------------") - - self.assertEqual(accuracy >= min_score, True, "A problem on the deepface installation.") - -unittest.main(exit=False) \ No newline at end of file +assert test_score > expected_coverage \ No newline at end of file