Skip to content

Commit

Permalink
SE of accuracy
Browse files Browse the repository at this point in the history
  • Loading branch information
@Amandarg
Amandarg committed May 2, 2018
1 parent b9d9853 commit 6292f71
Show file tree
Hide file tree
Showing 2 changed files with 38 additions and 32 deletions.
35 changes: 19 additions & 16 deletions .ipynb_checkpoints/COMPAS_debias_experiments-checkpoint.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1381,7 +1381,7 @@
},
{
"cell_type": "code",
"execution_count": 19,
"execution_count": 42,
"metadata": {
"collapsed": false
},
Expand All @@ -1393,31 +1393,31 @@
"WITHOUT DEBIASING averaged over 30 splits of the data:\n",
" COMPAS Logistic Regression \n",
" FPR FNR FPR FPR SE FNR FNR SE\n",
"population 0.32 0.37 0.32 0.01 0.33 0.01\n",
"Caucasians 0.22 0.51 0.23 0.02 0.5 0.03\n",
"African-Americans 0.41 0.3 0.4 0.02 0.24 0.01\n",
"population 0.32 0.37 0.08 0.01 0.68 0.01\n",
"Caucasians 0.21 0.51 0.04 0.01 0.8 0.02\n",
"African-Americans 0.41 0.3 0.11 0.01 0.62 0.01\n",
"-----------\n",
"WITH DEBIASING averaged over 30 splits of the data:\n",
" COMPAS Logistic Regression \n",
" FPR FNR FPR FPR SE FNR FNR SE\n",
"population 0.32 0.37 0.34 0.01 0.34 0.01\n",
"Caucasians 0.22 0.51 0.42 0.03 0.32 0.03\n",
"African-Americans 0.41 0.3 0.28 0.02 0.35 0.02\n",
"population 0.32 0.37 0.09 0.01 0.69 0.01\n",
"Caucasians 0.21 0.51 0.09 0.01 0.7 0.03\n",
"African-Americans 0.41 0.3 0.09 0.01 0.68 0.01\n",
"-----------\n",
"Average biased logistic regression accuracy: 0.673539719626\n",
"Average debiased logistic regression accuracy: 0.656386292835\n",
"Average COMPAS accuracy: 0.606775700935\n",
"Average biased logistic regression accuracy: 0.617757009346 SE 0.0102971754518\n",
"Average debiased logistic regression accuracy: 0.610961838006 SE 0.0100114429306\n",
"Average COMPAS accuracy: 0.60632788162 SE 0.0106307163105\n",
"-----------\n",
"Average logistic regression biased threshold: 0.487343297362\n",
"Average logistic regression debiased threshold: 0.482037709456\n"
"Average logistic regression biased threshold: 0.654647902423\n",
"Average logistic regression debiased threshold: 0.641859106291\n"
]
}
],
"source": [
"num_splits = 30\n",
"compas_score_threshold = 5\n",
"compass_high_and_medium = False\n",
"logistic_quantile = 50\n",
"logistic_quantile = 80\n",
"\n",
"biased_accuracy = []\n",
"debiased_accuracy = []\n",
Expand Down Expand Up @@ -1472,6 +1472,9 @@
"rate_biased = np.mean(rates_biased, axis=0)\n",
"se_debiased = np.std(rates_debiased, axis=0)\n",
"se_biased = np.std(rates_biased, axis=0)\n",
"se_compas_acc = np.std(compas_accuracy)\n",
"se_biased_acc = np.std(biased_accuracy)\n",
"se_debiased_acc = np.std(debiased_accuracy)\n",
"\n",
"dfMatBiased = np.zeros((3,8))\n",
"dfMatBiased[:,0:7:2] = np.reshape(rate_biased,(3,4)).round(2)\n",
Expand All @@ -1491,9 +1494,9 @@
"df = pd.DataFrame(np.concatenate((np.reshape(np.array([\"FPR\", \"FNR\", \"FPR\", \"FPR SE\", \"FNR\", \"FNR SE\"]), (1,6)), dfMatDebiased)), index= [\"\", \"population\", \"Caucasians\", \"African-Americans\"], columns=[\"COMPAS\", \"\", \"\", \"Logistic\", \"Regression\", \"\"])\n",
"print(df)\n",
"print('-----------')\n",
"print('Average biased logistic regression accuracy:', np.mean(biased_accuracy))\n",
"print('Average debiased logistic regression accuracy:', np.mean(debiased_accuracy))\n",
"print('Average COMPAS accuracy:', np.mean(compas_accuracy))\n",
"print('Average biased logistic regression accuracy:', np.mean(biased_accuracy), 'SE', se_biased_acc)\n",
"print('Average debiased logistic regression accuracy:', np.mean(debiased_accuracy), 'SE',se_debiased_acc)\n",
"print('Average COMPAS accuracy:', np.mean(compas_accuracy), 'SE', se_compas_acc)\n",
"print('-----------')\n",
"print('Average logistic regression biased threshold:', np.mean(biased_logistic_regression_threshold))\n",
"print('Average logistic regression debiased threshold:', np.mean(debiased_logistic_regression_threshold))"
Expand Down
35 changes: 19 additions & 16 deletions COMPAS_debias_experiments.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1381,7 +1381,7 @@
},
{
"cell_type": "code",
"execution_count": 19,
"execution_count": 42,
"metadata": {
"collapsed": false
},
Expand All @@ -1393,31 +1393,31 @@
"WITHOUT DEBIASING averaged over 30 splits of the data:\n",
" COMPAS Logistic Regression \n",
" FPR FNR FPR FPR SE FNR FNR SE\n",
"population 0.32 0.37 0.32 0.01 0.33 0.01\n",
"Caucasians 0.22 0.51 0.23 0.02 0.5 0.03\n",
"African-Americans 0.41 0.3 0.4 0.02 0.24 0.01\n",
"population 0.32 0.37 0.08 0.01 0.68 0.01\n",
"Caucasians 0.21 0.51 0.04 0.01 0.8 0.02\n",
"African-Americans 0.41 0.3 0.11 0.01 0.62 0.01\n",
"-----------\n",
"WITH DEBIASING averaged over 30 splits of the data:\n",
" COMPAS Logistic Regression \n",
" FPR FNR FPR FPR SE FNR FNR SE\n",
"population 0.32 0.37 0.34 0.01 0.34 0.01\n",
"Caucasians 0.22 0.51 0.42 0.03 0.32 0.03\n",
"African-Americans 0.41 0.3 0.28 0.02 0.35 0.02\n",
"population 0.32 0.37 0.09 0.01 0.69 0.01\n",
"Caucasians 0.21 0.51 0.09 0.01 0.7 0.03\n",
"African-Americans 0.41 0.3 0.09 0.01 0.68 0.01\n",
"-----------\n",
"Average biased logistic regression accuracy: 0.673539719626\n",
"Average debiased logistic regression accuracy: 0.656386292835\n",
"Average COMPAS accuracy: 0.606775700935\n",
"Average biased logistic regression accuracy: 0.617757009346 SE 0.0102971754518\n",
"Average debiased logistic regression accuracy: 0.610961838006 SE 0.0100114429306\n",
"Average COMPAS accuracy: 0.60632788162 SE 0.0106307163105\n",
"-----------\n",
"Average logistic regression biased threshold: 0.487343297362\n",
"Average logistic regression debiased threshold: 0.482037709456\n"
"Average logistic regression biased threshold: 0.654647902423\n",
"Average logistic regression debiased threshold: 0.641859106291\n"
]
}
],
"source": [
"num_splits = 30\n",
"compas_score_threshold = 5\n",
"compass_high_and_medium = False\n",
"logistic_quantile = 50\n",
"logistic_quantile = 80\n",
"\n",
"biased_accuracy = []\n",
"debiased_accuracy = []\n",
Expand Down Expand Up @@ -1472,6 +1472,9 @@
"rate_biased = np.mean(rates_biased, axis=0)\n",
"se_debiased = np.std(rates_debiased, axis=0)\n",
"se_biased = np.std(rates_biased, axis=0)\n",
"se_compas_acc = np.std(compas_accuracy)\n",
"se_biased_acc = np.std(biased_accuracy)\n",
"se_debiased_acc = np.std(debiased_accuracy)\n",
"\n",
"dfMatBiased = np.zeros((3,8))\n",
"dfMatBiased[:,0:7:2] = np.reshape(rate_biased,(3,4)).round(2)\n",
Expand All @@ -1491,9 +1494,9 @@
"df = pd.DataFrame(np.concatenate((np.reshape(np.array([\"FPR\", \"FNR\", \"FPR\", \"FPR SE\", \"FNR\", \"FNR SE\"]), (1,6)), dfMatDebiased)), index= [\"\", \"population\", \"Caucasians\", \"African-Americans\"], columns=[\"COMPAS\", \"\", \"\", \"Logistic\", \"Regression\", \"\"])\n",
"print(df)\n",
"print('-----------')\n",
"print('Average biased logistic regression accuracy:', np.mean(biased_accuracy))\n",
"print('Average debiased logistic regression accuracy:', np.mean(debiased_accuracy))\n",
"print('Average COMPAS accuracy:', np.mean(compas_accuracy))\n",
"print('Average biased logistic regression accuracy:', np.mean(biased_accuracy), 'SE', se_biased_acc)\n",
"print('Average debiased logistic regression accuracy:', np.mean(debiased_accuracy), 'SE',se_debiased_acc)\n",
"print('Average COMPAS accuracy:', np.mean(compas_accuracy), 'SE', se_compas_acc)\n",
"print('-----------')\n",
"print('Average logistic regression biased threshold:', np.mean(biased_logistic_regression_threshold))\n",
"print('Average logistic regression debiased threshold:', np.mean(debiased_logistic_regression_threshold))"
Expand Down

0 comments on commit 6292f71

Please sign in to comment.