accuracy for high vs high and medium for compas scores

Amandarg · May 1, 2018 · 62cf6c1 · 62cf6c1
1 parent 3cfdeb9
commit 62cf6c1
Show file tree

Hide file tree

Showing 2 changed files with 36 additions and 28 deletions.
diff --git a/.ipynb_checkpoints/COMPAS_debias_experiments-checkpoint.ipynb b/.ipynb_checkpoints/COMPAS_debias_experiments-checkpoint.ipynb
@@ -1275,7 +1275,7 @@
  },
  {
  "cell_type": "code",
- "execution_count": 43,
+ "execution_count": 45,
  "metadata": {
  "collapsed": false
  },
@@ -1287,30 +1287,31 @@
  "WITHOUT DEBIASING averaged over 30 splits of the data:\n",
  " COMPAS Logistic Regression \n",
  " FPR FNR FPR FPR SE FNR FNR SE\n",
- "population 0.32 0.36 0.32 0.01 0.32 0.01\n",
- "Caucasians 0.22 0.51 0.23 0.02 0.49 0.02\n",
- "African-Americans 0.41 0.29 0.4 0.02 0.24 0.01\n",
+ "population 0.32 0.37 0.08 0.01 0.68 0.01\n",
+ "Caucasians 0.22 0.51 0.05 0.01 0.81 0.02\n",
+ "African-Americans 0.41  0.3 0.11 0.01 0.62 0.01\n",
  "-----------\n",
  "WITH DEBIASING averaged over 30 splits of the data:\n",
  " COMPAS Logistic Regression \n",
  " FPR FNR FPR FPR SE FNR FNR SE\n",
- "population 0.32 0.36 0.34 0.01 0.34 0.01\n",
- "Caucasians 0.22 0.51 0.42 0.03 0.31 0.03\n",
- "African-Americans 0.41 0.29 0.27 0.02 0.36 0.02\n",
+ "population 0.32 0.37 0.09 0.01 0.69 0.01\n",
+ "Caucasians 0.22 0.51 0.09 0.02 0.72 0.03\n",
+ "African-Americans 0.41  0.3 0.09 0.01 0.68 0.02\n",
  "-----------\n",
- "Average biased logistic regression accuracy: 0.678154205607\n",
- "Average debiased logistic regression accuracy: 0.658625389408\n",
- "Average COMPAS accuracy: 0.656191588785\n",
+ "Average biased logistic regression accuracy: 0.613590342679\n",
+ "Average debiased logistic regression accuracy: 0.605802180685\n",
+ "Average COMPAS accuracy: 0.653348909657\n",
  "-----------\n",
- "Average logistic regression biased threshold: 0.490145795433\n",
- "Average logistic regression debiased threshold: 0.486587390083\n"
+ "Average logistic regression biased threshold: 0.653703952872\n",
+ "Average logistic regression debiased threshold: 0.643192162419\n"
  ]
  }
  ],
  "source": [
  "num_splits = 30\n",
  "compas_score_threshold = 5\n",
- "logistic_quantile = 50\n",
+ "compass_high_and_medium = True\n",
+ "logistic_quantile = 80\n",
  "\n",
  "biased_accuracy = []\n",
  "debiased_accuracy = []\n",
@@ -1353,7 +1354,10 @@
  "\n",
  " biased_classification = [1 if x > high_risk_cutoff_biased else 0 for x in probabilities_biased]\n",
  " debiased_classification = [1 if x > high_risk_cutoff_debiased else 0 for x in probabilities_debiased]\n",
- " compas_classification = [1 if x == \"High\" or x == \"Medium\" else 0 for x in score_test]\n",
+ " if compass_high_and_medium == True:\n",
+ " compas_classification = [1 if x == \"High\" or x == \"Medium\" else 0 for x in score_test]\n",
+ " else:\n",
+ " compas_classification = [1 if x == \"High\" else 0 for x in score_test]\n",
  "\n",
  " biased_accuracy.append(sum(biased_classification == Y_test[:,0])/len(Y_test[:,0])) \n",
  " debiased_accuracy.append(sum(debiased_classification == Y_test[:,0]) / len(Y_test[:,0]))\n",

diff --git a/COMPAS_debias_experiments.ipynb b/COMPAS_debias_experiments.ipynb
@@ -1275,7 +1275,7 @@
  },
  {
  "cell_type": "code",
- "execution_count": 43,
+ "execution_count": 45,
  "metadata": {
  "collapsed": false
  },
@@ -1287,30 +1287,31 @@
  "WITHOUT DEBIASING averaged over 30 splits of the data:\n",
  " COMPAS Logistic Regression \n",
  " FPR FNR FPR FPR SE FNR FNR SE\n",
- "population 0.32 0.36 0.32 0.01 0.32 0.01\n",
- "Caucasians 0.22 0.51 0.23 0.02 0.49 0.02\n",
- "African-Americans 0.41 0.29 0.4 0.02 0.24 0.01\n",
+ "population 0.32 0.37 0.08 0.01 0.68 0.01\n",
+ "Caucasians 0.22 0.51 0.05 0.01 0.81 0.02\n",
+ "African-Americans 0.41  0.3 0.11 0.01 0.62 0.01\n",
  "-----------\n",
  "WITH DEBIASING averaged over 30 splits of the data:\n",
  " COMPAS Logistic Regression \n",
  " FPR FNR FPR FPR SE FNR FNR SE\n",
- "population 0.32 0.36 0.34 0.01 0.34 0.01\n",
- "Caucasians 0.22 0.51 0.42 0.03 0.31 0.03\n",
- "African-Americans 0.41 0.29 0.27 0.02 0.36 0.02\n",
+ "population 0.32 0.37 0.09 0.01 0.69 0.01\n",
+ "Caucasians 0.22 0.51 0.09 0.02 0.72 0.03\n",
+ "African-Americans 0.41  0.3 0.09 0.01 0.68 0.02\n",
  "-----------\n",
- "Average biased logistic regression accuracy: 0.678154205607\n",
- "Average debiased logistic regression accuracy: 0.658625389408\n",
- "Average COMPAS accuracy: 0.656191588785\n",
+ "Average biased logistic regression accuracy: 0.613590342679\n",
+ "Average debiased logistic regression accuracy: 0.605802180685\n",
+ "Average COMPAS accuracy: 0.653348909657\n",
  "-----------\n",
- "Average logistic regression biased threshold: 0.490145795433\n",
- "Average logistic regression debiased threshold: 0.486587390083\n"
+ "Average logistic regression biased threshold: 0.653703952872\n",
+ "Average logistic regression debiased threshold: 0.643192162419\n"
  ]
  }
  ],
  "source": [
  "num_splits = 30\n",
  "compas_score_threshold = 5\n",
- "logistic_quantile = 50\n",
+ "compass_high_and_medium = True\n",
+ "logistic_quantile = 80\n",
  "\n",
  "biased_accuracy = []\n",
  "debiased_accuracy = []\n",
@@ -1353,7 +1354,10 @@
  "\n",
  " biased_classification = [1 if x > high_risk_cutoff_biased else 0 for x in probabilities_biased]\n",
  " debiased_classification = [1 if x > high_risk_cutoff_debiased else 0 for x in probabilities_debiased]\n",
- " compas_classification = [1 if x == \"High\" or x == \"Medium\" else 0 for x in score_test]\n",
+ " if compass_high_and_medium == True:\n",
+ " compas_classification = [1 if x == \"High\" or x == \"Medium\" else 0 for x in score_test]\n",
+ " else:\n",
+ " compas_classification = [1 if x == \"High\" else 0 for x in score_test]\n",
  "\n",
  " biased_accuracy.append(sum(biased_classification == Y_test[:,0])/len(Y_test[:,0])) \n",
  " debiased_accuracy.append(sum(debiased_classification == Y_test[:,0]) / len(Y_test[:,0]))\n",