dgergel/HW4.ipynb

{
 "metadata": {
  "name": "",
  "signature": "sha256:29a09a39797515610e0dea130f8bdcecaf3dd7074554a55f5333fa1a5267173e"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import pandas as pd\n",
      "import numpy as np\n",
      "url = 'https://ghdx.healthdata.org/sites/default/files/'\\\n",
      "      'record-attached-files/IHME_PHMRC_VA_DATA_ADULT_Y2013M09D11_0.csv'\n",
      "df = pd.read_csv(url, low_memory=False)\n",
      "## url = 'https://ghdx.healthdata.org/sites/default/files/'\\\n",
      "      ## 'record-attached-files/IHME_PHMRC_VA_DATA_CODEBOOK_Y2013M09D11_0.xlsx'\n",
      "cb = pd.read_excel('/Users/diana/Downloads/IHME_PHMRC_VA_DATA_CODEBOOK_Y2013M09D11_0.xlsx')\n",
      "cb.head()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>variable</th>\n",
        "      <th>question</th>\n",
        "      <th>module</th>\n",
        "      <th>health_care_experience</th>\n",
        "      <th>coding</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>0</th>\n",
        "      <td>         site</td>\n",
        "      <td>                                Site</td>\n",
        "      <td> General</td>\n",
        "      <td> 0</td>\n",
        "      <td> NaN</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1</th>\n",
        "      <td>        newid</td>\n",
        "      <td>                            Study ID</td>\n",
        "      <td> General</td>\n",
        "      <td> 0</td>\n",
        "      <td> NaN</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
        "      <td> gs_diagnosis</td>\n",
        "      <td>        Gold Standard Diagnosis Code</td>\n",
        "      <td> General</td>\n",
        "      <td> 0</td>\n",
        "      <td> NaN</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>3</th>\n",
        "      <td> gs_comorbid1</td>\n",
        "      <td> Gold Standard Comorbid Conditions 1</td>\n",
        "      <td> General</td>\n",
        "      <td> 0</td>\n",
        "      <td> NaN</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>4</th>\n",
        "      <td> gs_comorbid2</td>\n",
        "      <td> Gold Standard Comorbid Conditions 2</td>\n",
        "      <td> General</td>\n",
        "      <td> 0</td>\n",
        "      <td> NaN</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 2,
       "text": [
        "       variable                             question   module  \\\n",
        "0          site                                 Site  General   \n",
        "1         newid                             Study ID  General   \n",
        "2  gs_diagnosis         Gold Standard Diagnosis Code  General   \n",
        "3  gs_comorbid1  Gold Standard Comorbid Conditions 1  General   \n",
        "4  gs_comorbid2  Gold Standard Comorbid Conditions 2  General   \n",
        "\n",
        "   health_care_experience coding  \n",
        "0                       0    NaN  \n",
        "1                       0    NaN  \n",
        "2                       0    NaN  \n",
        "3                       0    NaN  \n",
        "4                       0    NaN  "
       ]
      }
     ],
     "prompt_number": 2
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Use the codebook to find the column which corresponds to the question \"Did __ have a fever?\"\n",
      "print ('the column which corresponds to the question of does [name] have a fever? is %2.f' %79)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "the column which corresponds to the question of does [name] have a fever? is 79\n"
       ]
      }
     ],
     "prompt_number": 3
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# df.groupby('gs_text34').word_fever.max()\n",
      "df.groupby('gs_text55').word_fever.max()\n",
      "df.groupby('gs_text46').word_fever.max()\n",
      "print ('the causes with the highest endorsement rate for the word fever are pneumonia, renal failure and AIDS')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "the causes with the highest endorsement rate for the word fever are pneumonia, renal failure and AIDS\n"
       ]
      }
     ],
     "prompt_number": 4
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Display the cause-specific endorsement rates visually\n",
      "import matplotlib.pyplot as plt\n",
      "# do some plotting like we did in class\n",
      "(100 * df.gs_text34.value_counts(normalize=True).round(3)).plot(kind='barh',figsize=(12,8))\n",
      "plt.xlabel('Endorsement Rate (%)')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 5,
       "text": [
        "<matplotlib.text.Text at 0x10745ff90>"
       ]
      }
     ],
     "prompt_number": 5
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "(100 * df.word_fever.value_counts(normalize=True).round(3)).plot(kind='barh',figsize=(12,8))\n",
      "plt.xlabel('Endorsement Rate (%)')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 6,
       "text": [
        "<matplotlib.text.Text at 0x10745ff90>"
       ]
      }
     ],
     "prompt_number": 6
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Find the question that asks if the deceased had AIDS\n",
      "cb.question[50:100]\n",
      "print('the question that asks if the decedant had AIDS is in the %2.f th row' %77)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "the question that asks if the decedant had AIDS is in the 77 th row\n"
       ]
      }
     ],
     "prompt_number": 7
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Make a 2x2 table showing the number of adult deceased with and without this question endorsed for decedents with \n",
      "# underlying cause AIDS and not AIDS\n",
      "df['aids_question'] = df[cb.variable[76]]=='Yes'\n",
      "df['cause_aids'] = (df['gs_text34']=='AIDS').astype(int)\n",
      "df['cause_not_aids'] = 1 - df['cause_aids']\n",
      "df.pivot_table(index='aids_question',values=['cause_aids','cause_not_aids'],aggfunc=sum)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>cause_aids</th>\n",
        "      <th>cause_not_aids</th>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>aids_question</th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>False</th>\n",
        "      <td> 194</td>\n",
        "      <td> 7204</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>True </th>\n",
        "      <td> 308</td>\n",
        "      <td>  135</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 8,
       "text": [
        "               cause_aids  cause_not_aids\n",
        "aids_question                            \n",
        "False                 194            7204\n",
        "True                  308             135"
       ]
      }
     ],
     "prompt_number": 8
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Compare this 2x2 table across study sites\n",
      "for site in np.unique(df['site']):\n",
      "    tmpdf = df[df['site']==site]\n",
      "    print('\\n'+site)\n",
      "    print(tmpdf.pivot_table(index='aids_question',values=['cause_aids','cause_not_aids'],aggfunc=sum))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "AP\n",
        "               cause_aids  cause_not_aids\n",
        "aids_question                            \n",
        "False                  75            1404\n",
        "True                   61              14\n",
        "\n",
        "Bohol\n",
        "               cause_aids  cause_not_aids\n",
        "aids_question                            \n",
        "False                   0            1259"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "\n",
        "Dar"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "               cause_aids  cause_not_aids\n",
        "aids_question                            \n",
        "False                  88            1414\n",
        "True                  115             109\n",
        "\n",
        "Mexico"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "               cause_aids  cause_not_aids\n",
        "aids_question                            \n",
        "False                  29            1460\n",
        "True                   91               6\n",
        "\n",
        "Pemba\n",
        "               cause_aids  cause_not_aids\n",
        "aids_question                            \n",
        "False                   0             294\n",
        "True                    0               3\n",
        "\n",
        "UP"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "               cause_aids  cause_not_aids\n",
        "aids_question                            \n",
        "False                   2            1373\n",
        "True                   41               3\n"
       ]
      }
     ],
     "prompt_number": 9
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Now do a series of 2x2 tables comparing the percent of deaths truely due to AIDS for which \"Had AIDS\" question was endorsed\n",
      "ct = pd.crosstab(df['site'], [df['cause_aids'], df['cause_not_aids']])\n",
      "pd.options.display.precision = 3\n",
      "percentify = lambda r: r / r.sum() * 100\n",
      "print(ct.apply(percentify, axis=1))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "cause_aids           0      1\n",
        "cause_not_aids       1      0\n",
        "site                         \n",
        "AP               91.25   8.75\n",
        "Bohol           100.00   0.00\n",
        "Dar              88.24  11.76\n",
        "Mexico           92.43   7.57\n",
        "Pemba           100.00   0.00\n",
        "UP               96.97   3.03\n"
       ]
      },
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 14,
       "text": [
        "\"\\nfor site in np.unique(df['site']):\\n    tmpdf = df[df['site']==site]\\n    print('\\n'+site)\\n    print(tmpdf.pivot_table(index='aids_question',values=['cause_aids','cause_not_aids'],aggfunc=sum))\\n\""
       ]
      }
     ],
     "prompt_number": 14
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": []
    }
   ],
   "metadata": {}
  }
 ]
}