DP_Algorithm_Grapher.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from generators import generate_databases\n",
    "from mechanisms import *\n",
    "from estimator import *\n",
    "import collections\n",
    "import matplotlib.pyplot as plt\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_results(algorithm, input_list, epsilon_list, n_mean=10 ):\n",
    "    \n",
    "    delta_list12 = np.zeros([len(input_list), len(epsilon_list)])\n",
    "    delta_list21 = np.zeros([len(input_list), len(epsilon_list)])\n",
    "\n",
    "    for i in tqdm(range(len(input_list))):\n",
    "        (d1,d2, kwargs) = input_list[i]\n",
    "        np.random.seed()\n",
    "\n",
    "        n1 = np.random.poisson(n_mean)\n",
    "        n2 = np.random.poisson(n_mean)\n",
    "        n12 = np.random.poisson(n_mean)\n",
    "        n22 = np.random.poisson(n_mean)\n",
    "        if algorithm in (noisy_max_v1a, dp_max, simple_max, noisy_max_v2a,laplace_mechanism,truncated_geometric):\n",
    "            result_d1 = []\n",
    "            result_d2 = []\n",
    "            result_d12 = []\n",
    "            result_d22 = []\n",
    "            for _ in range(n1):\n",
    "                result_d1.append(algorithm(d1, **kwargs))\n",
    "            for _ in range(n2):\n",
    "                result_d2.append(algorithm(d2, **kwargs))\n",
    "            for _ in range(n12):\n",
    "                result_d12.append(algorithm(d1, **kwargs))\n",
    "            for _ in range(n22):\n",
    "                result_d22.append(algorithm(d2, **kwargs))\n",
    "\n",
    "\n",
    "            count_d1 = dict(zip(*np.unique(result_d1, return_counts=True)))\n",
    "            count_d2 = dict(zip(*np.unique(result_d2, return_counts=True)))\n",
    "            count_d12 = dict(zip(*np.unique(result_d12, return_counts=True)))\n",
    "            count_d22 = dict(zip(*np.unique(result_d22, return_counts=True)))\n",
    "\n",
    "\n",
    "        elif algorithm in (noisy_max_v1b, noisy_max_v2b,histogram_eps,histogram):\n",
    "            result_d1 = []\n",
    "            result_d2 = []\n",
    "            result_d12 = []\n",
    "            result_d22 = []\n",
    "            for _ in range(n1):\n",
    "                result_d1.append(algorithm(d1, **kwargs))\n",
    "            for _ in range(n2):\n",
    "                result_d2.append(algorithm(d2, **kwargs))\n",
    "            for _ in range(n12):\n",
    "                result_d12.append(algorithm(d1, **kwargs))\n",
    "            for _ in range(n22):\n",
    "                result_d22.append(algorithm(d2, **kwargs))\n",
    "            range_max = max(np.max(result_d1), np.max(result_d2), np.max(result_d12), np.max(result_d22))\n",
    "            range_min = min(np.min(result_d1), np.min(result_d2), np.min(result_d12), np.min(result_d22))\n",
    "            hist1, bin_edges = np.histogram(result_d1, bins = 100, range = (range_min,range_max))\n",
    "            hist2, bin_edges = np.histogram(result_d2, bins = 100, range = (range_min,range_max))\n",
    "            hist12, bin_edges = np.histogram(result_d12, bins=100, range=(range_min, range_max))\n",
    "            hist22, bin_edges = np.histogram(result_d22, bins=100, range=(range_min, range_max))\n",
    "\n",
    "            intervals = [(bin_edges[i],bin_edges[i+1]) for i in range(len(bin_edges)-1)]\n",
    "            count_d1 = dict(zip(intervals,hist1))\n",
    "            count_d2 = dict(zip(intervals,hist2))\n",
    "            count_d12 = dict(zip(intervals, hist12))\n",
    "            count_d22 = dict(zip(intervals, hist22))\n",
    "\n",
    "        elif algorithm in ( iSVT1, iSVT2):\n",
    "            result_d1 = []\n",
    "            result_d2 = []\n",
    "            result_d12 = []\n",
    "            result_d22 = []\n",
    "            for _ in range(n1):\n",
    "                result_d1.append(tuple(algorithm(d1, **kwargs)))\n",
    "            for _ in range(n2):\n",
    "                result_d2.append(tuple(algorithm(d2, **kwargs)))\n",
    "            for _ in range(n12):\n",
    "                result_d12.append(tuple(algorithm(d1, **kwargs)))\n",
    "            for _ in range(n22):\n",
    "                result_d22.append(tuple(algorithm(d2, **kwargs)))\n",
    "\n",
    "            value_1, count_1 = np.unique(result_d1, return_counts=True,axis=0)\n",
    "            count_d1 = dict(zip([tuple(x) for x in value_1], count_1))\n",
    "            value_2, count_2 = np.unique(result_d2, return_counts=True, axis=0)\n",
    "            count_d2 = dict(zip([tuple(x) for x in value_2], count_2))\n",
    "            value_12, count_12 = np.unique(result_d12, return_counts=True, axis=0)\n",
    "            count_d12 = dict(zip([tuple(x) for x in value_12], count_12))\n",
    "            value_22, count_22 = np.unique(result_d22, return_counts=True, axis=0)\n",
    "            count_d22 = dict(zip([tuple(x) for x in value_22], count_22))\n",
    "\n",
    "        elif algorithm in (SVT,iSVT3):\n",
    "            result_d1 = []\n",
    "            result_d2 = []\n",
    "            result_d12 = []\n",
    "            result_d22 = []\n",
    "            for _ in range(n1):\n",
    "                result_d1.append(tuple(algorithm(d1, **kwargs)))\n",
    "            for _ in range(n2):\n",
    "                result_d2.append(tuple(algorithm(d2, **kwargs)))\n",
    "            for _ in range(n12):\n",
    "                result_d12.append(tuple(algorithm(d1, **kwargs)))\n",
    "            for _ in range(n22):\n",
    "                result_d22.append(tuple(algorithm(d2, **kwargs)))\n",
    "\n",
    "            count_d1 = dict(zip(*np.unique(result_d1, return_counts=True)))\n",
    "            count_d2 = dict(zip(*np.unique(result_d2, return_counts=True)))\n",
    "            count_d12 = dict(zip(*np.unique(result_d12, return_counts=True)))\n",
    "            count_d22 = dict(zip(*np.unique(result_d22, return_counts=True)))\n",
    "        else:\n",
    "            raise ValueError('Unsupported algorithm')\n",
    "\n",
    "\n",
    "\n",
    "        support = list(set(list(count_d1.keys()) + list(count_d2.keys())+list(count_d12.keys())+list(count_d22.keys()) ))\n",
    "\n",
    "        for event in support:\n",
    "            if event not in count_d1.keys():\n",
    "                count_d1[event] = 0\n",
    "            if event not in count_d2.keys():\n",
    "                count_d2[event] = 0\n",
    "            if event not in count_d12.keys():\n",
    "                count_d12[event] = 0\n",
    "            if event not in count_d22.keys():\n",
    "                count_d22[event] = 0\n",
    "                \n",
    "        \n",
    "\n",
    "        for j in range(len(epsilon_list)):\n",
    "            epsilon = epsilon_list[j]\n",
    "            delta12 = opt_estimator(count_d1, count_d2, count_d12, count_d22,n1,n2,n12,n22, n_mean, epsilon,c_1 = 4, c_2 = 0.1, c_3 = 0.9)\n",
    "            delta21 = opt_estimator(count_d2, count_d1, count_d22, count_d12,n2,n1,n22,n12, n_mean, epsilon,c_1 = 4, c_2 = 0.1, c_3 = 0.9)\n",
    "            delta_list12[i][j] = delta12\n",
    "            delta_list21[i][j] = delta21\n",
    "            \n",
    "    return delta_list12,delta_list21\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For a properly functioning algorithm with (ε0,δ)-DP, $\\hat{\\delta}$ should approach δ as ε approaches ε0."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [21:37<00:00, 81.12s/it]\n",
      "100%|██████████| 16/16 [20:36<00:00, 77.30s/it]\n",
      " 38%|███▊      | 6/16 [07:30<12:31, 75.10s/it]"
     ]
    }
   ],
   "source": [
    "#the arguments needed for each supported algorithm\n",
    "algorithm_args = {noisy_max_v1a: {},\n",
    "         noisy_max_v1b: {},\n",
    "         noisy_max_v2a: {},\n",
    "         noisy_max_v2b: {},\n",
    "         histogram: {},\n",
    "         dp_max:{},\n",
    "         simple_max:{},\n",
    "         histogram_eps: {},\n",
    "         SVT: {'T': 0.5, 'N': 1},\n",
    "         iSVT1: {'T': 1},\n",
    "         iSVT2: {'T': 1},\n",
    "         iSVT3: {'T': 1, 'N': 1},\n",
    "         truncated_geometric: {'delta':0.7}}\n",
    "\n",
    "algorithm = dp_max #the algorithm used in the current run\n",
    "\n",
    "\n",
    "\n",
    "for epsilon_nought in np.linspace(.1,1,4):\n",
    "    kwargs = algorithm_args[algorithm]\n",
    "    #epsilon_nought = 0.3\n",
    "    kwargs['epsilon'] = epsilon_nought\n",
    "\n",
    "    \n",
    "    epsilon_list =  np.linspace(0,epsilon_nought + epsilon_nought/10,11)\n",
    "\n",
    "    num_input = (5,10)\n",
    "\n",
    "\n",
    "    input_list = []\n",
    "    for num in num_input:\n",
    "        input_list.extend(generate_databases(algorithm, num, default_kwargs=kwargs))\n",
    "\n",
    "        delta_list_list = []\n",
    "    num_runs = 2\n",
    "    for i in range(num_runs):\n",
    "        delta_list12, delta_list21 = test_results(algorithm = algorithm, input_list = input_list, epsilon_list=epsilon_list, n_mean=100000 )\n",
    "        delta_list = np.maximum(np.amax(delta_list12, axis = 0), np.amax(delta_list21, axis = 0))\n",
    "        delta_list_list.append(delta_list)\n",
    "\n",
    "    mean_delta = np.mean(np.array(delta_list_list),axis = 0)\n",
    "    ste_delta = np.std(np.array(delta_list_list),axis = 0)/np.sqrt(num_runs)\n",
    "\n",
    "    plt.rc('text', usetex=False)\n",
    "    plt.title( algorithm.__name__)\n",
    "    plt.errorbar(epsilon_list, delta_list,ste_delta,marker = \".\",label = \"$\\epsilon$0: \" + str(epsilon_nought))\n",
    "    plt.xlabel(\"$\\epsilon$\")\n",
    "    plt.ylabel(\"$\\hat{\\delta}$\")\n",
    "    plt.grid()\n",
    "    plt.legend(loc=\"upper left\", bbox_to_anchor=(1,1))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  },
  "latex_envs": {
   "LaTeX_envs_menu_present": true,
   "autoclose": false,
   "autocomplete": false,
   "bibliofile": "ref.bib",
   "cite_by": "apalike",
   "current_citInitial": 1,
   "eqLabelWithNumbers": true,
   "eqNumInitial": 1,
   "hotkeys": {
    "equation": "Ctrl-E",
    "itemize": "Ctrl-I"
   },
   "labels_anchors": false,
   "latex_user_defs": false,
   "report_style_numbering": true,
   "user_envs_cfg": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}