{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import itertools\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from tqdm import tqdm\n",
    "tqdm.pandas()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Summary\n",
    "\n",
    "Face recognition models are regular convolutional neural networks models. They represent face photos as vectors. We find the distance between these two vectors to compare two faces. Finally, we classify two faces as same person whose distance is less than a threshold value.\n",
    "\n",
    "The question is that how to determine the threshold. In this notebook, we will find the best split point for a threshold."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Ref: https://github.com/serengil/deepface/tree/master/tests/dataset\n",
    "idendities = {\n",
    "    \"Angelina\": [\"img1.jpg\", \"img2.jpg\", \"img4.jpg\", \"img5.jpg\", \"img6.jpg\", \"img7.jpg\", \"img10.jpg\", \"img11.jpg\"],\n",
    "    \"Scarlett\": [\"img8.jpg\", \"img9.jpg\"],\n",
    "    \"Jennifer\": [\"img3.jpg\", \"img12.jpg\"],\n",
    "    \"Mark\": [\"img13.jpg\", \"img14.jpg\", \"img15.jpg\"],\n",
    "    \"Jack\": [\"img16.jpg\", \"img17.jpg\"],\n",
    "    \"Elon\": [\"img18.jpg\", \"img19.jpg\"],\n",
    "    \"Jeff\": [\"img20.jpg\", \"img21.jpg\"],\n",
    "    \"Marissa\": [\"img22.jpg\", \"img23.jpg\"],\n",
    "    \"Sundar\": [\"img24.jpg\", \"img25.jpg\"]\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Positive samples\n",
    "Find different photos of same people"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "positives = []\n",
    "\n",
    "for key, values in idendities.items():\n",
    "    \n",
    "    #print(key)\n",
    "    for i in range(0, len(values)-1):\n",
    "        for j in range(i+1, len(values)):\n",
    "            #print(values[i], \" and \", values[j])\n",
    "            positive = []\n",
    "            positive.append(values[i])\n",
    "            positive.append(values[j])\n",
    "            positives.append(positive)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "positives = pd.DataFrame(positives, columns = [\"file_x\", \"file_y\"])\n",
    "positives[\"decision\"] = \"Yes\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Negative samples\n",
    "Compare photos of different people"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "samples_list = list(idendities.values())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "negatives = []\n",
    "\n",
    "for i in range(0, len(idendities) - 1):\n",
    "    for j in range(i+1, len(idendities)):\n",
    "        #print(samples_list[i], \" vs \",samples_list[j]) \n",
    "        cross_product = itertools.product(samples_list[i], samples_list[j])\n",
    "        cross_product = list(cross_product)\n",
    "        #print(cross_product)\n",
    "        \n",
    "        for cross_sample in cross_product:\n",
    "            #print(cross_sample[0], \" vs \", cross_sample[1])\n",
    "            negative = []\n",
    "            negative.append(cross_sample[0])\n",
    "            negative.append(cross_sample[1])\n",
    "            negatives.append(negative)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "negatives = pd.DataFrame(negatives, columns = [\"file_x\", \"file_y\"])\n",
    "negatives[\"decision\"] = \"No\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Merge Positives and Negative Samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.concat([positives, negatives]).reset_index(drop = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(300, 3)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "No     262\n",
       "Yes     38\n",
       "Name: decision, dtype: int64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.decision.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.file_x = \"deepface/tests/dataset/\"+df.file_x\n",
    "df.file_y = \"deepface/tests/dataset/\"+df.file_y"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DeepFace"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from deepface import DeepFace"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "instances = df[[\"file_x\", \"file_y\"]].values.tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_name = \"VGG-Face\"\n",
    "distance_metric = \"cosine\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using VGG-Face model backend and cosine distance.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Verification: 100%|██████████| 300/300 [11:35<00:00,  2.32s/it]\n"
     ]
    }
   ],
   "source": [
    "resp_obj = DeepFace.verify(instances, model_name = model_name, distance_metric = distance_metric)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "distances = []\n",
    "for i in range(0, len(instances)):\n",
    "    distance = round(resp_obj[\"pair_%s\" % (i+1)][\"distance\"], 4)\n",
    "    distances.append(distance)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"distance\"] = distances"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Analyzing Distances"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "tp_mean = round(df[df.decision == \"Yes\"].mean().values[0], 4)\n",
    "tp_std = round(df[df.decision == \"Yes\"].std().values[0], 4)\n",
    "fp_mean = round(df[df.decision == \"No\"].mean().values[0], 4)\n",
    "fp_std = round(df[df.decision == \"No\"].std().values[0], 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean of true positives:  0.2263\n",
      "Std of true positives:  0.0744\n",
      "Mean of false positives:  0.6489\n",
      "Std of false positives:  0.12\n"
     ]
    }
   ],
   "source": [
    "print(\"Mean of true positives: \", tp_mean)\n",
    "print(\"Std of true positives: \", tp_std)\n",
    "print(\"Mean of false positives: \", fp_mean)\n",
    "print(\"Std of false positives: \", fp_std)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Distribution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x18125a54b00>"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 16724 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df[df.decision == \"Yes\"].distance.plot.kde()\n",
    "df[df.decision == \"No\"].distance.plot.kde()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Best Split Point"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "from chefboost import Chefboost as chef"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "config = {'algorithm': 'C4.5'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C4.5  tree is going to be built...\n",
      "Accuracy:  98.66666666666667 % on  300  instances\n",
      "finished in  3.5094187259674072  seconds\n"
     ]
    }
   ],
   "source": [
    "tmp_df = df[['distance', 'decision']].rename(columns = {\"decision\": \"Decision\"}).copy()\n",
    "model = chef.fit(tmp_df, config)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sigma"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "threshold:  0.3147\n"
     ]
    }
   ],
   "source": [
    "sigma = 2\n",
    "#2 sigma corresponds 95.45% confidence, and 3 sigma corresponds 99.73% confidence\n",
    "\n",
    "#threshold = round(tp_mean + sigma * tp_std, 4)\n",
    "threshold = 0.3147 #comes from c4.5 algorithm\n",
    "print(\"threshold: \", threshold)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.3637"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df.decision == 'Yes'].distance.max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.3186"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df.decision == 'No'].distance.min()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"prediction\"] = \"No\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "idx = df[df.distance <= threshold].index\n",
    "df.loc[idx, 'prediction'] = 'Yes'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file_x</th>\n",
       "      <th>file_y</th>\n",
       "      <th>decision</th>\n",
       "      <th>distance</th>\n",
       "      <th>prediction</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>150</th>\n",
       "      <td>deepface/tests/dataset/img16.jpg</td>\n",
       "      <td>deepface/tests/dataset/img4.jpg</td>\n",
       "      <td>No</td>\n",
       "      <td>0.7178</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>deepface/tests/dataset/img4.jpg</td>\n",
       "      <td>deepface/tests/dataset/img7.jpg</td>\n",
       "      <td>Yes</td>\n",
       "      <td>0.2450</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>214</th>\n",
       "      <td>deepface/tests/dataset/img24.jpg</td>\n",
       "      <td>deepface/tests/dataset/img4.jpg</td>\n",
       "      <td>No</td>\n",
       "      <td>0.7362</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>135</th>\n",
       "      <td>deepface/tests/dataset/img16.jpg</td>\n",
       "      <td>deepface/tests/dataset/img14.jpg</td>\n",
       "      <td>No</td>\n",
       "      <td>0.5281</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63</th>\n",
       "      <td>deepface/tests/dataset/img19.jpg</td>\n",
       "      <td>deepface/tests/dataset/img23.jpg</td>\n",
       "      <td>No</td>\n",
       "      <td>0.6546</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                               file_x                            file_y  \\\n",
       "150  deepface/tests/dataset/img16.jpg   deepface/tests/dataset/img4.jpg   \n",
       "25    deepface/tests/dataset/img4.jpg   deepface/tests/dataset/img7.jpg   \n",
       "214  deepface/tests/dataset/img24.jpg   deepface/tests/dataset/img4.jpg   \n",
       "135  deepface/tests/dataset/img16.jpg  deepface/tests/dataset/img14.jpg   \n",
       "63   deepface/tests/dataset/img19.jpg  deepface/tests/dataset/img23.jpg   \n",
       "\n",
       "    decision  distance prediction  \n",
       "150       No    0.7178         No  \n",
       "25       Yes    0.2450        Yes  \n",
       "214       No    0.7362         No  \n",
       "135       No    0.5281         No  \n",
       "63        No    0.6546         No  "
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.sample(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "cm = confusion_matrix(df.decision.values, df.prediction.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[262,   0],\n",
       "       [  4,  34]], dtype=int64)"
      ]
     },
     "execution_count": 121,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "tn, fp, fn, tp = cm.ravel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(262, 0, 4, 34)"
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tn, fp, fn, tp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "recall = tp / (tp + fn)\n",
    "precision = tp / (tp + fp)\n",
    "accuracy = (tp + tn)/(tn + fp +  fn + tp)\n",
    "f1 = 2 * (precision * recall) / (precision + recall)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision:  100.0 %\n",
      "Recall:  89.47368421052632 %\n",
      "F1 score  94.44444444444444 %\n",
      "Accuracy:  98.66666666666667 %\n"
     ]
    }
   ],
   "source": [
    "print(\"Precision: \", 100*precision,\"%\")\n",
    "print(\"Recall: \", 100*recall,\"%\")\n",
    "print(\"F1 score \",100*f1, \"%\")\n",
    "print(\"Accuracy: \", 100*accuracy,\"%\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv(\"threshold_pivot.csv\", index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test results\n",
    "\n",
    "### Threshold = 0.3147 (C4.5 best split point)\n",
    "\n",
    "Precision:  100.0 %\n",
    "\n",
    "Recall:  89.47368421052632 %\n",
    "\n",
    "F1 score  94.44444444444444%\n",
    "\n",
    "Accuracy:  98.66666666666667 %\n",
    "\n",
    "### Threshold = 0.3751 (2 sigma)\n",
    "\n",
    "Precision:  90.47619047619048 %\n",
    "\n",
    "Recall:  100.0 %\n",
    "\n",
    "F1 score  95.0 %\n",
    "\n",
    "Accuracy:  98.66666666666667 %"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
