motion_sensor_predictions.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#imports\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import datetime as dt\n",
    "from sqlalchemy import create_engine\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.cross_validation import train_test_split\n",
    "from sklearn import metrics\n",
    "from sklearn.cross_validation import cross_val_score\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from collections import Counter\n",
    "from operator import itemgetter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>sequential_number</th>\n",
       "      <th>x_acceleration</th>\n",
       "      <th>y_acceleration</th>\n",
       "      <th>z_acceleration</th>\n",
       "      <th>label</th>\n",
       "      <th>participant_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 1502</td>\n",
       "      <td> 2215</td>\n",
       "      <td> 2153</td>\n",
       "      <td> 1</td>\n",
       "      <td> 1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td> 1</td>\n",
       "      <td> 1</td>\n",
       "      <td> 1667</td>\n",
       "      <td> 2072</td>\n",
       "      <td> 2047</td>\n",
       "      <td> 1</td>\n",
       "      <td> 1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td> 2</td>\n",
       "      <td> 2</td>\n",
       "      <td> 1611</td>\n",
       "      <td> 1957</td>\n",
       "      <td> 1906</td>\n",
       "      <td> 1</td>\n",
       "      <td> 1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td> 3</td>\n",
       "      <td> 3</td>\n",
       "      <td> 1601</td>\n",
       "      <td> 1939</td>\n",
       "      <td> 1831</td>\n",
       "      <td> 1</td>\n",
       "      <td> 1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td> 4</td>\n",
       "      <td> 4</td>\n",
       "      <td> 1643</td>\n",
       "      <td> 1965</td>\n",
       "      <td> 1879</td>\n",
       "      <td> 1</td>\n",
       "      <td> 1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td> 5</td>\n",
       "      <td> 5</td>\n",
       "      <td> 1604</td>\n",
       "      <td> 1959</td>\n",
       "      <td> 1921</td>\n",
       "      <td> 1</td>\n",
       "      <td> 1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td> 6</td>\n",
       "      <td> 6</td>\n",
       "      <td> 1640</td>\n",
       "      <td> 1829</td>\n",
       "      <td> 1940</td>\n",
       "      <td> 1</td>\n",
       "      <td> 1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td> 7</td>\n",
       "      <td> 7</td>\n",
       "      <td> 1607</td>\n",
       "      <td> 1910</td>\n",
       "      <td> 1910</td>\n",
       "      <td> 1</td>\n",
       "      <td> 1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td> 8</td>\n",
       "      <td> 8</td>\n",
       "      <td> 1546</td>\n",
       "      <td> 2045</td>\n",
       "      <td> 1910</td>\n",
       "      <td> 1</td>\n",
       "      <td> 1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td> 9</td>\n",
       "      <td> 9</td>\n",
       "      <td> 1529</td>\n",
       "      <td> 2049</td>\n",
       "      <td> 1972</td>\n",
       "      <td> 1</td>\n",
       "      <td> 1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   index  sequential_number  x_acceleration  y_acceleration  z_acceleration  \\\n",
       "0      0                  0            1502            2215            2153   \n",
       "1      1                  1            1667            2072            2047   \n",
       "2      2                  2            1611            1957            1906   \n",
       "3      3                  3            1601            1939            1831   \n",
       "4      4                  4            1643            1965            1879   \n",
       "5      5                  5            1604            1959            1921   \n",
       "6      6                  6            1640            1829            1940   \n",
       "7      7                  7            1607            1910            1910   \n",
       "8      8                  8            1546            2045            1910   \n",
       "9      9                  9            1529            2049            1972   \n",
       "\n",
       "   label  participant_id  \n",
       "0      1               1  \n",
       "1      1               1  \n",
       "2      1               1  \n",
       "3      1               1  \n",
       "4      1               1  \n",
       "5      1               1  \n",
       "6      1               1  \n",
       "7      1               1  \n",
       "8      1               1  \n",
       "9      1               1  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#create engine and load the full database into a pandas data frame\n",
    "#This database is created from the csv files by running makeSensorDatabase.py\n",
    "disk_engine = create_engine('sqlite:https:///sensorData.db')\n",
    "df = pd.read_sql_query('SELECT * FROM full_data',disk_engine)\n",
    "df.head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Explanation of columns:\n",
    "\n",
    "participant_id: a number 1-15 labeling the person wearing the gear.\n",
    "\n",
    "sequential_number labels the data points in sequential order for each participant.\n",
    "\n",
    "x_acceleration, y_acceleration, and z_acceleration: the x,y and z direction accelerometer readings respectively\n",
    "\n",
    "label: a number 1-7 labeling the type of activity of the accelerometer wearer. \n",
    "       \n",
    "       --- 1: Working at Computer\n",
    "       \n",
    "       --- 2: Standing Up, Walking and Going up\\down stairs\n",
    "       \n",
    "       --- 3: Standing\n",
    "       \n",
    "       --- 4: Walking\n",
    "       \n",
    "       --- 5: Going Up\\Down Stairs\n",
    "       \n",
    "       --- 6: Walking and Talking with Someone\n",
    "       \n",
    "       --- 7: Talking while Standing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index([u'index', u'sequential_number', u'x_acceleration', u'y_acceleration', u'z_acceleration', u'label', u'participant_id'], dtype='object')"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Our goal is to predict the type of activity (label) of the wearer from the accelerometer data.  Before we predict with any model, we will first establish some \"naive\" null baseline predictions to get a better idea of the predictive power of our models."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    608667\n",
       "7    593563\n",
       "4    357064\n",
       "3    216737\n",
       "5     51498\n",
       "2     47878\n",
       "6     47770\n",
       "0      3719\n",
       "dtype: int64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Find a count of each label\n",
    "df['label'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    608667\n",
       "7    593563\n",
       "4    357064\n",
       "3    216737\n",
       "5     51498\n",
       "2     47878\n",
       "6     47770\n",
       "dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#drop values where label is 0 since this corresponds to an unclassified activity\n",
    "df = df[df['label'] != 0]\n",
    "df['label'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.31649036984115347"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#compute % of total labels that are 1, the most common label\n",
    "df['label'].value_counts()[1]/float(len(df.index))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "From the above observation, we se that 31% of the data points correspond to wearers participating in activity 1 (working at a computer).  So simply by always guessing label 1, we can get 31.65% accuracy, higher than the 14.29% we would get by always guessing one of the 7 labels if there were equal amounts of data points for each.  Any effective model should be able to predict with accuracy significantly higher than the 31.65% bench mark on this data set."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As a second naive baseline, we will train a logistic regression on a random subset of 75% of the data and testing on the rest.  We will simply use x_acceleration, y_acceleration, z_acceleration as the features.  So we're simply and naively predicting the activity of the participant from a single accelerometer reading."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#make features matrix X and label vector y\n",
    "X = df[['x_acceleration','y_acceleration','z_acceleration']]\n",
    "#flatten the label vector.\n",
    "y = np.ravel(df['label'])\n",
    "#split into training and testing data\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, penalty='l2', random_state=None, tol=0.0001)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Initialize and train our logistic regression model\n",
    "toy_model = LogisticRegression()\n",
    "toy_model.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.347684564107\n"
     ]
    }
   ],
   "source": [
    "#predict test labels on test set and compute the accuracy\n",
    "predictions = toy_model.predict(X_test)\n",
    "print metrics.accuracy_score(y_test,predictions)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As we might expect with such a basic strategy, our results are not much better than the strategy of simply predicting 1 for every label.  The confusion matrix computed below also confirms this notion. By definition a confusion matrix C is such that C_{i, j} is equal to the number of observations known to be in group i but predicted to be in group j."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[106485      7      0     10      6      7  45777]\n",
      " [  7613      0      0      2      0      0   4383]\n",
      " [ 28899      0      0      0      0      0  25473]\n",
      " [ 58066      0      0      0      0      0  30961]\n",
      " [  7595      0      0      0      0      0   5102]\n",
      " [  5310      0      0      0      0      0   6772]\n",
      " [ 87647      0      0      0      0      0  60680]]\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          1       0.35      0.70      0.47    152292\n",
      "          2       0.00      0.00      0.00     11998\n",
      "          3       0.00      0.00      0.00     54372\n",
      "          4       0.00      0.00      0.00     89027\n",
      "          5       0.00      0.00      0.00     12697\n",
      "          6       0.00      0.00      0.00     12082\n",
      "          7       0.34      0.41      0.37    148327\n",
      "\n",
      "avg / total       0.22      0.35      0.26    480795\n",
      "\n"
     ]
    }
   ],
   "source": [
    "C=  metrics.confusion_matrix(y_test,predictions)\n",
    "print C\n",
    "print metrics.classification_report(y_test, predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([301615,      7,      0,     12,      6,      7, 179148])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.sum(C, axis=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "From the confusion matrix we can see our toy linear regression model classifies (seemingly randomly) almost every activity as either 1 or 7.  Note above that 7 (walking in a straight line) is the second most common activity in the data set, having nearly as many data points as 1 label one, and together they constitute around 60% of the total data points.  \n",
    "\n",
    "  So as one might expect, we cannot meaningfully classify the activity from a single accelerometer reading.  For a more effective strategy, we need to reorgnize our data frame.  instead of predicting activity from an instaneous window, we will group our data points by participant and by activity label.  Then for a fixed participant performing a fixed activity, we will cluster consecutive data points into windows of 2 seconds.  So we will the predict the activity of the wearer over features derived from this 2 second window of data points"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/waymaniac/anaconda/lib/python2.7/site-packages/IPython/kernel/__main__.py:8: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the the caveats in the documentation: http:https://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>sequential_number</th>\n",
       "      <th>x_acceleration</th>\n",
       "      <th>y_acceleration</th>\n",
       "      <th>z_acceleration</th>\n",
       "      <th>label</th>\n",
       "      <th>participant_id</th>\n",
       "      <th>record_counts</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 1502</td>\n",
       "      <td> 2215</td>\n",
       "      <td> 2153</td>\n",
       "      <td> 1</td>\n",
       "      <td> 1</td>\n",
       "      <td> 1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td> 1</td>\n",
       "      <td> 1</td>\n",
       "      <td> 1667</td>\n",
       "      <td> 2072</td>\n",
       "      <td> 2047</td>\n",
       "      <td> 1</td>\n",
       "      <td> 1</td>\n",
       "      <td> 2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td> 2</td>\n",
       "      <td> 2</td>\n",
       "      <td> 1611</td>\n",
       "      <td> 1957</td>\n",
       "      <td> 1906</td>\n",
       "      <td> 1</td>\n",
       "      <td> 1</td>\n",
       "      <td> 3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td> 3</td>\n",
       "      <td> 3</td>\n",
       "      <td> 1601</td>\n",
       "      <td> 1939</td>\n",
       "      <td> 1831</td>\n",
       "      <td> 1</td>\n",
       "      <td> 1</td>\n",
       "      <td> 4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td> 4</td>\n",
       "      <td> 4</td>\n",
       "      <td> 1643</td>\n",
       "      <td> 1965</td>\n",
       "      <td> 1879</td>\n",
       "      <td> 1</td>\n",
       "      <td> 1</td>\n",
       "      <td> 5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   index  sequential_number  x_acceleration  y_acceleration  z_acceleration  \\\n",
       "0      0                  0            1502            2215            2153   \n",
       "1      1                  1            1667            2072            2047   \n",
       "2      2                  2            1611            1957            1906   \n",
       "3      3                  3            1601            1939            1831   \n",
       "4      4                  4            1643            1965            1879   \n",
       "\n",
       "   label  participant_id  record_counts  \n",
       "0      1               1              1  \n",
       "1      1               1              2  \n",
       "2      1               1              3  \n",
       "3      1               1              4  \n",
       "4      1               1              5  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''Reorganize our data frame to group data points into 2 second windows and derive features from these\n",
    "\n",
    "\n",
    "First add column to record ordered counts of the data point grouped for each participant and each label\n",
    "This will be used to group consecutive data points into the windows.\n",
    "'''\n",
    "temp = df[['participant_id','label']]\n",
    "temp['count']=1\n",
    "counts = temp.groupby(['participant_id','label'])['count'].cumsum()\n",
    "df['record_counts']=counts\n",
    "df.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "From the data descrition, the sampling frequency of the accelerometer is 52 Hz.  We'll initialize variables to represent the sampling frequency and our desired window length (2 seconds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "  \n",
    "#one sample taken every 1/52 seconds (52 observations per second)\n",
    "frequency = 52\n",
    "#length of observation window in seconds\n",
    "window_length = 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "'''for each participant add window column which records which window the activity took place in\n",
    "this will allow for features based on values computed over observations in the window\n",
    "\n",
    "'''\n",
    "\n",
    "windows = df['record_counts'].apply(lambda x: int(x/(frequency*window_length)))\n",
    "df['window'] = windows"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "participant_id  label     \n",
       "1               1      319    104\n",
       "                       231    104\n",
       "                       297    104\n",
       "                       10     104\n",
       "                       42     104\n",
       "                       74     104\n",
       "                       106    104\n",
       "                       138    104\n",
       "                       170    104\n",
       "                       202    104\n",
       "                       234    104\n",
       "                       266    104\n",
       "                       298    104\n",
       "                       11     104\n",
       "                       43     104\n",
       "...\n",
       "15              7      27     104\n",
       "                       11     104\n",
       "                       154    104\n",
       "                       138    104\n",
       "                       122    104\n",
       "                       106    104\n",
       "                       90     104\n",
       "                       74     104\n",
       "                       58     104\n",
       "                       42     104\n",
       "                       26     104\n",
       "                       10     104\n",
       "                       153    104\n",
       "                       0      103\n",
       "                       168     29\n",
       "Length: 18543, dtype: int64"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.groupby(['participant_id','label'])['window'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "So from above we see that for a fixed participant id and label the data points are partition according to their window number, and each window has 104 points (52 observations per second (frequency) x 2 seconds (window length) except for windows at the end of the observation period (the total activity observation period may not be a multiple of 2 seconds)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For a derived features, we will also add columns to record the jerk in each direction (the derivative of acceleration)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>sequential_number</th>\n",
       "      <th>x_acceleration</th>\n",
       "      <th>y_acceleration</th>\n",
       "      <th>z_acceleration</th>\n",
       "      <th>label</th>\n",
       "      <th>participant_id</th>\n",
       "      <th>record_counts</th>\n",
       "      <th>window</th>\n",
       "      <th>x_jerk</th>\n",
       "      <th>y_jerk</th>\n",
       "      <th>z_jerk</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 1502</td>\n",
       "      <td> 2215</td>\n",
       "      <td> 2153</td>\n",
       "      <td> 1</td>\n",
       "      <td> 1</td>\n",
       "      <td>  1</td>\n",
       "      <td> 0</td>\n",
       "      <td>  NaN</td>\n",
       "      <td>  NaN</td>\n",
       "      <td>  NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td> 1</td>\n",
       "      <td> 1</td>\n",
       "      <td> 1667</td>\n",
       "      <td> 2072</td>\n",
       "      <td> 2047</td>\n",
       "      <td> 1</td>\n",
       "      <td> 1</td>\n",
       "      <td>  2</td>\n",
       "      <td> 0</td>\n",
       "      <td> 8580</td>\n",
       "      <td>-7436</td>\n",
       "      <td>-5512</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td> 2</td>\n",
       "      <td> 2</td>\n",
       "      <td> 1611</td>\n",
       "      <td> 1957</td>\n",
       "      <td> 1906</td>\n",
       "      <td> 1</td>\n",
       "      <td> 1</td>\n",
       "      <td>  3</td>\n",
       "      <td> 0</td>\n",
       "      <td>-2912</td>\n",
       "      <td>-5980</td>\n",
       "      <td>-7332</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td> 3</td>\n",
       "      <td> 3</td>\n",
       "      <td> 1601</td>\n",
       "      <td> 1939</td>\n",
       "      <td> 1831</td>\n",
       "      <td> 1</td>\n",
       "      <td> 1</td>\n",
       "      <td>  4</td>\n",
       "      <td> 0</td>\n",
       "      <td> -520</td>\n",
       "      <td> -936</td>\n",
       "      <td>-3900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td> 4</td>\n",
       "      <td> 4</td>\n",
       "      <td> 1643</td>\n",
       "      <td> 1965</td>\n",
       "      <td> 1879</td>\n",
       "      <td> 1</td>\n",
       "      <td> 1</td>\n",
       "      <td>  5</td>\n",
       "      <td> 0</td>\n",
       "      <td> 2184</td>\n",
       "      <td> 1352</td>\n",
       "      <td> 2496</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td> 5</td>\n",
       "      <td> 5</td>\n",
       "      <td> 1604</td>\n",
       "      <td> 1959</td>\n",
       "      <td> 1921</td>\n",
       "      <td> 1</td>\n",
       "      <td> 1</td>\n",
       "      <td>  6</td>\n",
       "      <td> 0</td>\n",
       "      <td>-2028</td>\n",
       "      <td> -312</td>\n",
       "      <td> 2184</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td> 6</td>\n",
       "      <td> 6</td>\n",
       "      <td> 1640</td>\n",
       "      <td> 1829</td>\n",
       "      <td> 1940</td>\n",
       "      <td> 1</td>\n",
       "      <td> 1</td>\n",
       "      <td>  7</td>\n",
       "      <td> 0</td>\n",
       "      <td> 1872</td>\n",
       "      <td>-6760</td>\n",
       "      <td>  988</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td> 7</td>\n",
       "      <td> 7</td>\n",
       "      <td> 1607</td>\n",
       "      <td> 1910</td>\n",
       "      <td> 1910</td>\n",
       "      <td> 1</td>\n",
       "      <td> 1</td>\n",
       "      <td>  8</td>\n",
       "      <td> 0</td>\n",
       "      <td>-1716</td>\n",
       "      <td> 4212</td>\n",
       "      <td>-1560</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td> 8</td>\n",
       "      <td> 8</td>\n",
       "      <td> 1546</td>\n",
       "      <td> 2045</td>\n",
       "      <td> 1910</td>\n",
       "      <td> 1</td>\n",
       "      <td> 1</td>\n",
       "      <td>  9</td>\n",
       "      <td> 0</td>\n",
       "      <td>-3172</td>\n",
       "      <td> 7020</td>\n",
       "      <td>    0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td> 9</td>\n",
       "      <td> 9</td>\n",
       "      <td> 1529</td>\n",
       "      <td> 2049</td>\n",
       "      <td> 1972</td>\n",
       "      <td> 1</td>\n",
       "      <td> 1</td>\n",
       "      <td> 10</td>\n",
       "      <td> 0</td>\n",
       "      <td> -884</td>\n",
       "      <td>  208</td>\n",
       "      <td> 3224</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   index  sequential_number  x_acceleration  y_acceleration  z_acceleration  \\\n",
       "0      0                  0            1502            2215            2153   \n",
       "1      1                  1            1667            2072            2047   \n",
       "2      2                  2            1611            1957            1906   \n",
       "3      3                  3            1601            1939            1831   \n",
       "4      4                  4            1643            1965            1879   \n",
       "5      5                  5            1604            1959            1921   \n",
       "6      6                  6            1640            1829            1940   \n",
       "7      7                  7            1607            1910            1910   \n",
       "8      8                  8            1546            2045            1910   \n",
       "9      9                  9            1529            2049            1972   \n",
       "\n",
       "   label  participant_id  record_counts  window  x_jerk  y_jerk  z_jerk  \n",
       "0      1               1              1       0     NaN     NaN     NaN  \n",
       "1      1               1              2       0    8580   -7436   -5512  \n",
       "2      1               1              3       0   -2912   -5980   -7332  \n",
       "3      1               1              4       0    -520    -936   -3900  \n",
       "4      1               1              5       0    2184    1352    2496  \n",
       "5      1               1              6       0   -2028    -312    2184  \n",
       "6      1               1              7       0    1872   -6760     988  \n",
       "7      1               1              8       0   -1716    4212   -1560  \n",
       "8      1               1              9       0   -3172    7020       0  \n",
       "9      1               1             10       0    -884     208    3224  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grouped = df.groupby(['participant_id','label'])\n",
    "#add features for jerk\n",
    "x_jerk = grouped['x_acceleration'].apply(lambda x: (x - x.shift(1))*frequency)\n",
    "y_jerk = grouped['y_acceleration'].apply(lambda x: (x - x.shift(1))*frequency)\n",
    "z_jerk = grouped['z_acceleration'].apply(lambda x: (x - x.shift(1))*frequency)\n",
    "df['x_jerk'] = x_jerk\n",
    "df['y_jerk'] = y_jerk\n",
    "df['z_jerk'] = z_jerk\n",
    "df.head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Using these windows we can compute some summary statistics on the observations to use as features for our model.  For each of the 6 variables (jerk, acceleration in the 3 directions: x,y,z) we will compute the variance of the variable, the mean and the range (max - min) over each window sample.  These will serve as the 18 features for our model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th colspan=\"3\" halign=\"left\">x_acceleration</th>\n",
       "      <th colspan=\"3\" halign=\"left\">y_acceleration</th>\n",
       "      <th colspan=\"3\" halign=\"left\">z_acceleration</th>\n",
       "      <th colspan=\"3\" halign=\"left\">x_jerk</th>\n",
       "      <th colspan=\"3\" halign=\"left\">y_jerk</th>\n",
       "      <th colspan=\"3\" halign=\"left\">z_jerk</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>&lt;lambda&gt;</th>\n",
       "      <th>mean</th>\n",
       "      <th>var</th>\n",
       "      <th>&lt;lambda&gt;</th>\n",
       "      <th>mean</th>\n",
       "      <th>var</th>\n",
       "      <th>&lt;lambda&gt;</th>\n",
       "      <th>mean</th>\n",
       "      <th>var</th>\n",
       "      <th>&lt;lambda&gt;</th>\n",
       "      <th>mean</th>\n",
       "      <th>var</th>\n",
       "      <th>&lt;lambda&gt;</th>\n",
       "      <th>mean</th>\n",
       "      <th>var</th>\n",
       "      <th>&lt;lambda&gt;</th>\n",
       "      <th>mean</th>\n",
       "      <th>var</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>participant_id</th>\n",
       "      <th>label</th>\n",
       "      <th>window</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">1</th>\n",
       "      <th rowspan=\"5\" valign=\"top\">1</th>\n",
       "      <th>0</th>\n",
       "      <td> 323</td>\n",
       "      <td> 1612.349515</td>\n",
       "      <td>  2899.190367</td>\n",
       "      <td> 771</td>\n",
       "      <td> 2048.466019</td>\n",
       "      <td> 13979.859128</td>\n",
       "      <td>  478</td>\n",
       "      <td> 2035.932039</td>\n",
       "      <td>  6592.495336</td>\n",
       "      <td>   NaN</td>\n",
       "      <td>  70.862745</td>\n",
       "      <td>  4967798.931470</td>\n",
       "      <td>   NaN</td>\n",
       "      <td> 19.882353</td>\n",
       "      <td> 42716188.025626</td>\n",
       "      <td>   NaN</td>\n",
       "      <td> -81.568627</td>\n",
       "      <td> 16198486.544749</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td> 797</td>\n",
       "      <td> 1991.836538</td>\n",
       "      <td> 36940.875934</td>\n",
       "      <td> 498</td>\n",
       "      <td> 2292.211538</td>\n",
       "      <td>  6039.488798</td>\n",
       "      <td> 1095</td>\n",
       "      <td> 1924.913462</td>\n",
       "      <td> 26570.778846</td>\n",
       "      <td> 20124</td>\n",
       "      <td>  51.500000</td>\n",
       "      <td> 12710243.631068</td>\n",
       "      <td> 44928</td>\n",
       "      <td> 75.500000</td>\n",
       "      <td> 28758110.038835</td>\n",
       "      <td> 91000</td>\n",
       "      <td> 112.500000</td>\n",
       "      <td> 62299010.699029</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td> 437</td>\n",
       "      <td> 1960.144231</td>\n",
       "      <td>  5258.978996</td>\n",
       "      <td> 355</td>\n",
       "      <td> 2358.528846</td>\n",
       "      <td>  3092.154500</td>\n",
       "      <td>  428</td>\n",
       "      <td> 2156.576923</td>\n",
       "      <td> 10580.654220</td>\n",
       "      <td> 24024</td>\n",
       "      <td> 119.000000</td>\n",
       "      <td> 15179503.766990</td>\n",
       "      <td> 22360</td>\n",
       "      <td>  4.000000</td>\n",
       "      <td>  8804522.873786</td>\n",
       "      <td> 21736</td>\n",
       "      <td>-102.500000</td>\n",
       "      <td> 14369184.990291</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td> 175</td>\n",
       "      <td> 1952.740385</td>\n",
       "      <td>   768.796023</td>\n",
       "      <td> 172</td>\n",
       "      <td> 2385.548077</td>\n",
       "      <td>   643.764656</td>\n",
       "      <td>  153</td>\n",
       "      <td> 2078.317308</td>\n",
       "      <td>  2285.403193</td>\n",
       "      <td> 11960</td>\n",
       "      <td> -12.500000</td>\n",
       "      <td>  2132510.660194</td>\n",
       "      <td> 10660</td>\n",
       "      <td>-17.000000</td>\n",
       "      <td>  2330503.689320</td>\n",
       "      <td>  4680</td>\n",
       "      <td>  63.000000</td>\n",
       "      <td>   523771.262136</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>  52</td>\n",
       "      <td> 1960.278846</td>\n",
       "      <td>    67.950616</td>\n",
       "      <td>  50</td>\n",
       "      <td> 2373.836538</td>\n",
       "      <td>    52.138069</td>\n",
       "      <td>   43</td>\n",
       "      <td> 2126.557692</td>\n",
       "      <td>    77.064600</td>\n",
       "      <td>  2236</td>\n",
       "      <td>   2.000000</td>\n",
       "      <td>   108050.951456</td>\n",
       "      <td>  2028</td>\n",
       "      <td> -1.000000</td>\n",
       "      <td>   100808.310680</td>\n",
       "      <td>  1976</td>\n",
       "      <td>  -3.000000</td>\n",
       "      <td>    98700.038835</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                            x_acceleration                             \\\n",
       "                                  <lambda>         mean           var   \n",
       "participant_id label window                                             \n",
       "1              1     0                 323  1612.349515   2899.190367   \n",
       "                     1                 797  1991.836538  36940.875934   \n",
       "                     2                 437  1960.144231   5258.978996   \n",
       "                     3                 175  1952.740385    768.796023   \n",
       "                     4                  52  1960.278846     67.950616   \n",
       "\n",
       "                            y_acceleration                             \\\n",
       "                                  <lambda>         mean           var   \n",
       "participant_id label window                                             \n",
       "1              1     0                 771  2048.466019  13979.859128   \n",
       "                     1                 498  2292.211538   6039.488798   \n",
       "                     2                 355  2358.528846   3092.154500   \n",
       "                     3                 172  2385.548077    643.764656   \n",
       "                     4                  50  2373.836538     52.138069   \n",
       "\n",
       "                            z_acceleration                             \\\n",
       "                                  <lambda>         mean           var   \n",
       "participant_id label window                                             \n",
       "1              1     0                 478  2035.932039   6592.495336   \n",
       "                     1                1095  1924.913462  26570.778846   \n",
       "                     2                 428  2156.576923  10580.654220   \n",
       "                     3                 153  2078.317308   2285.403193   \n",
       "                     4                  43  2126.557692     77.064600   \n",
       "\n",
       "                              x_jerk                                y_jerk  \\\n",
       "                            <lambda>        mean              var <lambda>   \n",
       "participant_id label window                                                  \n",
       "1              1     0           NaN   70.862745   4967798.931470      NaN   \n",
       "                     1         20124   51.500000  12710243.631068    44928   \n",
       "                     2         24024  119.000000  15179503.766990    22360   \n",
       "                     3         11960  -12.500000   2132510.660194    10660   \n",
       "                     4          2236    2.000000    108050.951456     2028   \n",
       "\n",
       "                                                          z_jerk              \\\n",
       "                                  mean              var <lambda>        mean   \n",
       "participant_id label window                                                    \n",
       "1              1     0       19.882353  42716188.025626      NaN  -81.568627   \n",
       "                     1       75.500000  28758110.038835    91000  112.500000   \n",
       "                     2        4.000000   8804522.873786    21736 -102.500000   \n",
       "                     3      -17.000000   2330503.689320     4680   63.000000   \n",
       "                     4       -1.000000    100808.310680     1976   -3.000000   \n",
       "\n",
       "                                              \n",
       "                                         var  \n",
       "participant_id label window                   \n",
       "1              1     0       16198486.544749  \n",
       "                     1       62299010.699029  \n",
       "                     2       14369184.990291  \n",
       "                     3         523771.262136  \n",
       "                     4          98700.038835  "
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "variables = ['x_acceleration','y_acceleration','z_acceleration','x_jerk','y_jerk','z_jerk']\n",
    "data = df.groupby(['participant_id','label','window'])[variables].agg([(lambda x: max(x)-min(x)),np.mean,np.var])\n",
    "data.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>participant_id</th>\n",
       "      <th>label</th>\n",
       "      <th>window</th>\n",
       "      <th colspan=\"3\" halign=\"left\">x_acceleration</th>\n",
       "      <th colspan=\"3\" halign=\"left\">y_acceleration</th>\n",
       "      <th colspan=\"3\" halign=\"left\">z_acceleration</th>\n",
       "      <th colspan=\"3\" halign=\"left\">x_jerk</th>\n",
       "      <th colspan=\"3\" halign=\"left\">y_jerk</th>\n",
       "      <th colspan=\"3\" halign=\"left\">z_jerk</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>&lt;lambda&gt;</th>\n",
       "      <th>mean</th>\n",
       "      <th>var</th>\n",
       "      <th>&lt;lambda&gt;</th>\n",
       "      <th>mean</th>\n",
       "      <th>var</th>\n",
       "      <th>&lt;lambda&gt;</th>\n",
       "      <th>...</th>\n",
       "      <th>var</th>\n",
       "      <th>&lt;lambda&gt;</th>\n",
       "      <th>mean</th>\n",
       "      <th>var</th>\n",
       "      <th>&lt;lambda&gt;</th>\n",
       "      <th>mean</th>\n",
       "      <th>var</th>\n",
       "      <th>&lt;lambda&gt;</th>\n",
       "      <th>mean</th>\n",
       "      <th>var</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td> 1</td>\n",
       "      <td> 1</td>\n",
       "      <td> 1</td>\n",
       "      <td> 797</td>\n",
       "      <td> 1991.836538</td>\n",
       "      <td> 36940.875934</td>\n",
       "      <td> 498</td>\n",
       "      <td> 2292.211538</td>\n",
       "      <td> 6039.488798</td>\n",
       "      <td> 1095</td>\n",
       "      <td>...</td>\n",
       "      <td> 26570.778846</td>\n",
       "      <td> 20124</td>\n",
       "      <td>  51.5</td>\n",
       "      <td> 12710243.631068</td>\n",
       "      <td> 44928</td>\n",
       "      <td> 75.5</td>\n",
       "      <td> 28758110.038835</td>\n",
       "      <td> 91000</td>\n",
       "      <td> 112.5</td>\n",
       "      <td> 62299010.699029</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td> 1</td>\n",
       "      <td> 1</td>\n",
       "      <td> 2</td>\n",
       "      <td> 437</td>\n",
       "      <td> 1960.144231</td>\n",
       "      <td>  5258.978996</td>\n",
       "      <td> 355</td>\n",
       "      <td> 2358.528846</td>\n",
       "      <td> 3092.154500</td>\n",
       "      <td>  428</td>\n",
       "      <td>...</td>\n",
       "      <td> 10580.654220</td>\n",
       "      <td> 24024</td>\n",
       "      <td> 119.0</td>\n",
       "      <td> 15179503.766990</td>\n",
       "      <td> 22360</td>\n",
       "      <td>  4.0</td>\n",
       "      <td>  8804522.873786</td>\n",
       "      <td> 21736</td>\n",
       "      <td>-102.5</td>\n",
       "      <td> 14369184.990291</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td> 1</td>\n",
       "      <td> 1</td>\n",
       "      <td> 3</td>\n",
       "      <td> 175</td>\n",
       "      <td> 1952.740385</td>\n",
       "      <td>   768.796023</td>\n",
       "      <td> 172</td>\n",
       "      <td> 2385.548077</td>\n",
       "      <td>  643.764656</td>\n",
       "      <td>  153</td>\n",
       "      <td>...</td>\n",
       "      <td>  2285.403193</td>\n",
       "      <td> 11960</td>\n",
       "      <td> -12.5</td>\n",
       "      <td>  2132510.660194</td>\n",
       "      <td> 10660</td>\n",
       "      <td>-17.0</td>\n",
       "      <td>  2330503.689320</td>\n",
       "      <td>  4680</td>\n",
       "      <td>  63.0</td>\n",
       "      <td>   523771.262136</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td> 1</td>\n",
       "      <td> 1</td>\n",
       "      <td> 4</td>\n",
       "      <td>  52</td>\n",
       "      <td> 1960.278846</td>\n",
       "      <td>    67.950616</td>\n",
       "      <td>  50</td>\n",
       "      <td> 2373.836538</td>\n",
       "      <td>   52.138069</td>\n",
       "      <td>   43</td>\n",
       "      <td>...</td>\n",
       "      <td>    77.064600</td>\n",
       "      <td>  2236</td>\n",
       "      <td>   2.0</td>\n",
       "      <td>   108050.951456</td>\n",
       "      <td>  2028</td>\n",
       "      <td> -1.0</td>\n",
       "      <td>   100808.310680</td>\n",
       "      <td>  1976</td>\n",
       "      <td>  -3.0</td>\n",
       "      <td>    98700.038835</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td> 1</td>\n",
       "      <td> 1</td>\n",
       "      <td> 5</td>\n",
       "      <td>  40</td>\n",
       "      <td> 1963.625000</td>\n",
       "      <td>    39.401699</td>\n",
       "      <td>  29</td>\n",
       "      <td> 2376.442308</td>\n",
       "      <td>   31.763630</td>\n",
       "      <td>   44</td>\n",
       "      <td>...</td>\n",
       "      <td>   103.984223</td>\n",
       "      <td>  1820</td>\n",
       "      <td>  -1.0</td>\n",
       "      <td>    95452.815534</td>\n",
       "      <td>  1352</td>\n",
       "      <td>  0.0</td>\n",
       "      <td>    66891.184466</td>\n",
       "      <td>  1872</td>\n",
       "      <td> -11.5</td>\n",
       "      <td>    89886.038835</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  participant_id label window x_acceleration                             \\\n",
       "                                    <lambda>         mean           var   \n",
       "0              1     1      1            797  1991.836538  36940.875934   \n",
       "1              1     1      2            437  1960.144231   5258.978996   \n",
       "2              1     1      3            175  1952.740385    768.796023   \n",
       "3              1     1      4             52  1960.278846     67.950616   \n",
       "4              1     1      5             40  1963.625000     39.401699   \n",
       "\n",
       "  y_acceleration                           z_acceleration       ...         \\\n",
       "        <lambda>         mean          var       <lambda>       ...          \n",
       "0            498  2292.211538  6039.488798           1095       ...          \n",
       "1            355  2358.528846  3092.154500            428       ...          \n",
       "2            172  2385.548077   643.764656            153       ...          \n",
       "3             50  2373.836538    52.138069             43       ...          \n",
       "4             29  2376.442308    31.763630             44       ...          \n",
       "\n",
       "                  x_jerk                           y_jerk        \\\n",
       "            var <lambda>   mean              var <lambda>  mean   \n",
       "0  26570.778846    20124   51.5  12710243.631068    44928  75.5   \n",
       "1  10580.654220    24024  119.0  15179503.766990    22360   4.0   \n",
       "2   2285.403193    11960  -12.5   2132510.660194    10660 -17.0   \n",
       "3     77.064600     2236    2.0    108050.951456     2028  -1.0   \n",
       "4    103.984223     1820   -1.0     95452.815534     1352   0.0   \n",
       "\n",
       "                     z_jerk                          \n",
       "               var <lambda>   mean              var  \n",
       "0  28758110.038835    91000  112.5  62299010.699029  \n",
       "1   8804522.873786    21736 -102.5  14369184.990291  \n",
       "2   2330503.689320     4680   63.0    523771.262136  \n",
       "3    100808.310680     1976   -3.0     98700.038835  \n",
       "4     66891.184466     1872  -11.5     89886.038835  \n",
       "\n",
       "[5 rows x 21 columns]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#clean up data frame by reseting to a simple index\n",
    "cleaned_data = data.dropna()\n",
    "cleaned_data = cleaned_data.reset_index()\n",
    "cleaned_data.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>participant_id</th>\n",
       "      <th>label</th>\n",
       "      <th>window</th>\n",
       "      <th>x_accel_peaks</th>\n",
       "      <th>x_accel_mean</th>\n",
       "      <th>x_accel_var</th>\n",
       "      <th>y_accel_peaks</th>\n",
       "      <th>y_accel_mean</th>\n",
       "      <th>y_accel_var</th>\n",
       "      <th>z_accel_peaks</th>\n",
       "      <th>...</th>\n",
       "      <th>z_accel_var</th>\n",
       "      <th>x_jerk_peaks</th>\n",
       "      <th>x_jerk_mean</th>\n",
       "      <th>x_jerk_var</th>\n",
       "      <th>y_jerk_peaks</th>\n",
       "      <th>y_jerk_mean</th>\n",
       "      <th>y_jerk_var</th>\n",
       "      <th>z_jerk_peaks</th>\n",
       "      <th>z_jerk_mean</th>\n",
       "      <th>z_jerk_var</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td> 1</td>\n",
       "      <td> 1</td>\n",
       "      <td> 1</td>\n",
       "      <td> 797</td>\n",
       "      <td> 1991.836538</td>\n",
       "      <td> 36940.875934</td>\n",
       "      <td> 498</td>\n",
       "      <td> 2292.211538</td>\n",
       "      <td> 6039.488798</td>\n",
       "      <td> 1095</td>\n",
       "      <td>...</td>\n",
       "      <td> 26570.778846</td>\n",
       "      <td> 20124</td>\n",
       "      <td>  51.5</td>\n",
       "      <td> 12710243.631068</td>\n",
       "      <td> 44928</td>\n",
       "      <td> 75.5</td>\n",
       "      <td> 28758110.038835</td>\n",
       "      <td> 91000</td>\n",
       "      <td> 112.5</td>\n",
       "      <td> 62299010.699029</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td> 1</td>\n",
       "      <td> 1</td>\n",
       "      <td> 2</td>\n",
       "      <td> 437</td>\n",
       "      <td> 1960.144231</td>\n",
       "      <td>  5258.978996</td>\n",
       "      <td> 355</td>\n",
       "      <td> 2358.528846</td>\n",
       "      <td> 3092.154500</td>\n",
       "      <td>  428</td>\n",
       "      <td>...</td>\n",
       "      <td> 10580.654220</td>\n",
       "      <td> 24024</td>\n",
       "      <td> 119.0</td>\n",
       "      <td> 15179503.766990</td>\n",
       "      <td> 22360</td>\n",
       "      <td>  4.0</td>\n",
       "      <td>  8804522.873786</td>\n",
       "      <td> 21736</td>\n",
       "      <td>-102.5</td>\n",
       "      <td> 14369184.990291</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td> 1</td>\n",
       "      <td> 1</td>\n",
       "      <td> 3</td>\n",
       "      <td> 175</td>\n",
       "      <td> 1952.740385</td>\n",
       "      <td>   768.796023</td>\n",
       "      <td> 172</td>\n",
       "      <td> 2385.548077</td>\n",
       "      <td>  643.764656</td>\n",
       "      <td>  153</td>\n",
       "      <td>...</td>\n",
       "      <td>  2285.403193</td>\n",
       "      <td> 11960</td>\n",
       "      <td> -12.5</td>\n",
       "      <td>  2132510.660194</td>\n",
       "      <td> 10660</td>\n",
       "      <td>-17.0</td>\n",
       "      <td>  2330503.689320</td>\n",
       "      <td>  4680</td>\n",
       "      <td>  63.0</td>\n",
       "      <td>   523771.262136</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td> 1</td>\n",
       "      <td> 1</td>\n",
       "      <td> 4</td>\n",
       "      <td>  52</td>\n",
       "      <td> 1960.278846</td>\n",
       "      <td>    67.950616</td>\n",
       "      <td>  50</td>\n",
       "      <td> 2373.836538</td>\n",
       "      <td>   52.138069</td>\n",
       "      <td>   43</td>\n",
       "      <td>...</td>\n",
       "      <td>    77.064600</td>\n",
       "      <td>  2236</td>\n",
       "      <td>   2.0</td>\n",
       "      <td>   108050.951456</td>\n",
       "      <td>  2028</td>\n",
       "      <td> -1.0</td>\n",
       "      <td>   100808.310680</td>\n",
       "      <td>  1976</td>\n",
       "      <td>  -3.0</td>\n",
       "      <td>    98700.038835</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td> 1</td>\n",
       "      <td> 1</td>\n",
       "      <td> 5</td>\n",
       "      <td>  40</td>\n",
       "      <td> 1963.625000</td>\n",
       "      <td>    39.401699</td>\n",
       "      <td>  29</td>\n",
       "      <td> 2376.442308</td>\n",
       "      <td>   31.763630</td>\n",
       "      <td>   44</td>\n",
       "      <td>...</td>\n",
       "      <td>   103.984223</td>\n",
       "      <td>  1820</td>\n",
       "      <td>  -1.0</td>\n",
       "      <td>    95452.815534</td>\n",
       "      <td>  1352</td>\n",
       "      <td>  0.0</td>\n",
       "      <td>    66891.184466</td>\n",
       "      <td>  1872</td>\n",
       "      <td> -11.5</td>\n",
       "      <td>    89886.038835</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   participant_id  label  window  x_accel_peaks  x_accel_mean   x_accel_var  \\\n",
       "0               1      1       1            797   1991.836538  36940.875934   \n",
       "1               1      1       2            437   1960.144231   5258.978996   \n",
       "2               1      1       3            175   1952.740385    768.796023   \n",
       "3               1      1       4             52   1960.278846     67.950616   \n",
       "4               1      1       5             40   1963.625000     39.401699   \n",
       "\n",
       "   y_accel_peaks  y_accel_mean  y_accel_var  z_accel_peaks       ...         \\\n",
       "0            498   2292.211538  6039.488798           1095       ...          \n",
       "1            355   2358.528846  3092.154500            428       ...          \n",
       "2            172   2385.548077   643.764656            153       ...          \n",
       "3             50   2373.836538    52.138069             43       ...          \n",
       "4             29   2376.442308    31.763630             44       ...          \n",
       "\n",
       "    z_accel_var  x_jerk_peaks  x_jerk_mean       x_jerk_var  y_jerk_peaks  \\\n",
       "0  26570.778846         20124         51.5  12710243.631068         44928   \n",
       "1  10580.654220         24024        119.0  15179503.766990         22360   \n",
       "2   2285.403193         11960        -12.5   2132510.660194         10660   \n",
       "3     77.064600          2236          2.0    108050.951456          2028   \n",
       "4    103.984223          1820         -1.0     95452.815534          1352   \n",
       "\n",
       "   y_jerk_mean       y_jerk_var  z_jerk_peaks  z_jerk_mean       z_jerk_var  \n",
       "0         75.5  28758110.038835         91000        112.5  62299010.699029  \n",
       "1          4.0   8804522.873786         21736       -102.5  14369184.990291  \n",
       "2        -17.0   2330503.689320          4680         63.0    523771.262136  \n",
       "3         -1.0    100808.310680          1976         -3.0     98700.038835  \n",
       "4          0.0     66891.184466          1872        -11.5     89886.038835  \n",
       "\n",
       "[5 rows x 21 columns]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#rename columns and use simple index.\n",
    "features =  ['x_accel_peaks','x_accel_mean','x_accel_var',\n",
    "             'y_accel_peaks','y_accel_mean','y_accel_var',\n",
    "             'z_accel_peaks','z_accel_mean','z_accel_var',\n",
    "             'x_jerk_peaks','x_jerk_mean','x_jerk_var',\n",
    "             'y_jerk_peaks','y_jerk_mean','y_jerk_var',\n",
    "             'z_jerk_peaks','z_jerk_mean','z_jerk_var'\n",
    "                     ]\n",
    "columns = ['participant_id','label','window'] + features\n",
    "cleaned_data.columns = columns\n",
    "cleaned_data.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Next we will train the data and check the accuracy with cross validation using a leave one participant out.  That is, we'll loop through the 15 different participant.  At each step we will save the data for one participant and train on the data for the remaining 14.  We then test out of sample on the 15th participant and record the accuracy.  If our features are meaningful, we would expect to get a higher accuracy than guessing.  However, we might also expect to get lower accuracy than if we did a random 15-fold cross validation because in this model, we do not train on the left out participant, so when we test it's the first time data for that particpant has been observed.  If the activity patterns are significantly different for each participant, than the model could have a harder time predicting the labels for the new participant.  On the other hand, for the random cross-validation, the model is trained on all 15 participants, so the model has already learned the activity patterns of each individual.\n",
    "   The logistic regression provides a relatively simple and flexible model that is not too complicated to fit.  It is useful for preliminary data analysis and for examining which features are useful to select before possibly choosing a more advanced model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X = cleaned_data[features]\n",
    "y = cleaned_data['label']\n",
    "model = LogisticRegression()\n",
    "accuracy_scores = []\n",
    "label_counts = []\n",
    "for test_id in range(1,16):\n",
    "    train_indices = cleaned_data.index[cleaned_data['participant_id'] !=test_id]\n",
    "    test_indices =  cleaned_data.index[cleaned_data['participant_id'] ==test_id]\n",
    "    X_train = X.loc[train_indices,:]\n",
    "    X_test = X.loc[test_indices,:]\n",
    "    y_train = y.loc[train_indices]\n",
    "    y_test = y.loc[test_indices]\n",
    "    model.fit(X_train,y_train)\n",
    "    predictions = model.predict(X_test)\n",
    "    accuracy = metrics.accuracy_score(y_test,predictions)\n",
    "    accuracy_scores.append(accuracy)\n",
    "    label_counts.append(Counter(y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.45221295702373315, 0.59727479182437548, 0.69622833843017329, 0.4578005115089514, 0.41107491856677525, 0.59198813056379818, 0.38835572616762637, 0.56245268735806209, 0.37555697008274985, 0.562962962962963, 0.58541458541458546, 0.5868971792538672, 0.54489164086687303, 0.65827338129496404, 0.57661290322580649]\n"
     ]
    }
   ],
   "source": [
    "print accuracy_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean accuracy: 0.53653317897\n",
      "variance of the accuracy: 0.0087985965473\n"
     ]
    }
   ],
   "source": [
    "accuracy_scores = np.array(accuracy_scores)\n",
    "print \"mean accuracy: \"+ str(accuracy_scores.mean())\n",
    "print \"variance of the accuracy: \"+str(accuracy_scores.var())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[354   0   0  15   0   0 127]\n",
      " [ 17   0   0   3   0   0  24]\n",
      " [  5   0   0  37   0   0  34]\n",
      " [ 19   0   1 129   0   0  19]\n",
      " [  2   0   0  17   0   0  12]\n",
      " [  0   0   0   9   0   0   0]\n",
      " [ 45   0   0  34   0   0  89]]\n",
      "Number of predictions for each label: \n",
      "[442   0   1 244   0   0 305]\n",
      "Number of data points for each label: \n",
      "Counter({1: 496, 4: 168, 7: 168, 3: 76, 2: 44, 5: 31, 6: 9})\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          1       0.80      0.71      0.75       496\n",
      "          2       0.00      0.00      0.00        44\n",
      "          3       0.00      0.00      0.00        76\n",
      "          4       0.53      0.77      0.63       168\n",
      "          5       0.00      0.00      0.00        31\n",
      "          6       0.00      0.00      0.00         9\n",
      "          7       0.29      0.53      0.38       168\n",
      "\n",
      "avg / total       0.54      0.58      0.55       992\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#check the confusion matrix when testing on participant 15 for the model trained on 1-14\n",
    "C=  metrics.confusion_matrix(y_test,predictions)\n",
    "print C\n",
    "print \"Number of predictions for each label: \\n\" + str(C.sum(axis= 0))\n",
    "print \"Number of data points for each label: \\n\" + str(label_counts[14])\n",
    "print metrics.classification_report(y_test, predictions)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "From the above confusion matrix (  C_{i, j} is equal to the number of observations known to be in group i but predicted to be in group j), we observe that again most of the observations are classified either as label 1 (working at a computer) or label 7 (talking while standing), but now we have also a good portion classified as state 4 ( walking).  However, no data points (save 1) are predicted to belong in classes 2,3,5 and 6.  It is also, interesting to note that the precision for label 1 (correct label 1 predicitions/ total # of label 1 predictions) is very high at 80%, but only 29% for label 7."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>x_accel_peaks</th>\n",
       "      <td>-0.000033</td>\n",
       "      <td> 1.004028e-05</td>\n",
       "      <td>-8.685992e-07</td>\n",
       "      <td> 7.469256e-03</td>\n",
       "      <td>-1.600595e-04</td>\n",
       "      <td> 7.333645e-05</td>\n",
       "      <td> 4.771030e-05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>x_accel_mean</th>\n",
       "      <td>-0.000020</td>\n",
       "      <td>-6.084537e-04</td>\n",
       "      <td>-3.095777e-04</td>\n",
       "      <td>-4.389287e-03</td>\n",
       "      <td>-1.826498e-04</td>\n",
       "      <td>-2.669642e-04</td>\n",
       "      <td>-4.811946e-05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>x_accel_var</th>\n",
       "      <td>-0.000211</td>\n",
       "      <td> 1.379450e-04</td>\n",
       "      <td> 1.214333e-05</td>\n",
       "      <td> 1.457178e-05</td>\n",
       "      <td>-3.503694e-05</td>\n",
       "      <td> 1.472086e-04</td>\n",
       "      <td> 4.480283e-05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>y_accel_peaks</th>\n",
       "      <td>-0.000035</td>\n",
       "      <td> 2.638802e-06</td>\n",
       "      <td> 1.173925e-06</td>\n",
       "      <td> 8.597318e-03</td>\n",
       "      <td> 3.367239e-04</td>\n",
       "      <td> 5.104320e-05</td>\n",
       "      <td> 3.477941e-05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>y_accel_mean</th>\n",
       "      <td> 0.000005</td>\n",
       "      <td>-6.853135e-04</td>\n",
       "      <td>-3.909130e-04</td>\n",
       "      <td> 3.782417e-03</td>\n",
       "      <td>-1.407340e-03</td>\n",
       "      <td>-7.539034e-04</td>\n",
       "      <td>-1.153178e-04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>y_accel_var</th>\n",
       "      <td>-0.000393</td>\n",
       "      <td>-1.585969e-04</td>\n",
       "      <td>-6.827102e-05</td>\n",
       "      <td> 1.297311e-04</td>\n",
       "      <td> 3.266384e-04</td>\n",
       "      <td> 2.775060e-05</td>\n",
       "      <td>-5.432287e-04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>z_accel_peaks</th>\n",
       "      <td>-0.000027</td>\n",
       "      <td> 1.474185e-06</td>\n",
       "      <td> 2.808337e-06</td>\n",
       "      <td> 4.402738e-03</td>\n",
       "      <td>-1.446679e-04</td>\n",
       "      <td> 6.343948e-05</td>\n",
       "      <td> 5.034281e-05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>z_accel_mean</th>\n",
       "      <td> 0.000037</td>\n",
       "      <td>-6.009011e-04</td>\n",
       "      <td>-3.274423e-04</td>\n",
       "      <td>-2.415806e-03</td>\n",
       "      <td>-2.976586e-04</td>\n",
       "      <td>-7.430820e-04</td>\n",
       "      <td>-1.222782e-04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>z_accel_var</th>\n",
       "      <td>-0.000108</td>\n",
       "      <td>-5.608652e-05</td>\n",
       "      <td> 7.434234e-05</td>\n",
       "      <td>-1.295255e-04</td>\n",
       "      <td> 1.677517e-04</td>\n",
       "      <td>-5.762908e-06</td>\n",
       "      <td> 2.518509e-04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>x_jerk_peaks</th>\n",
       "      <td>-0.000040</td>\n",
       "      <td> 8.613060e-05</td>\n",
       "      <td>-1.042191e-05</td>\n",
       "      <td>-5.707369e-05</td>\n",
       "      <td> 6.691327e-05</td>\n",
       "      <td> 6.956292e-05</td>\n",
       "      <td>-1.726091e-05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>x_jerk_mean</th>\n",
       "      <td> 0.000005</td>\n",
       "      <td>-1.320900e-06</td>\n",
       "      <td> 1.102946e-07</td>\n",
       "      <td>-1.835390e-03</td>\n",
       "      <td>-2.119019e-04</td>\n",
       "      <td> 9.296737e-07</td>\n",
       "      <td>-7.944720e-08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>x_jerk_var</th>\n",
       "      <td> 0.000000</td>\n",
       "      <td>-1.931424e-07</td>\n",
       "      <td> 9.588112e-09</td>\n",
       "      <td>-3.864821e-08</td>\n",
       "      <td>-5.122850e-07</td>\n",
       "      <td>-8.666461e-07</td>\n",
       "      <td>-2.469737e-08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>y_jerk_peaks</th>\n",
       "      <td>-0.000140</td>\n",
       "      <td> 1.719934e-05</td>\n",
       "      <td> 4.702065e-05</td>\n",
       "      <td> 1.730341e-05</td>\n",
       "      <td> 1.967943e-05</td>\n",
       "      <td> 6.942743e-05</td>\n",
       "      <td> 4.061812e-05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>y_jerk_mean</th>\n",
       "      <td>-0.000001</td>\n",
       "      <td> 1.195911e-06</td>\n",
       "      <td>-1.967043e-07</td>\n",
       "      <td>-8.004333e-04</td>\n",
       "      <td> 2.647920e-04</td>\n",
       "      <td>-7.564928e-06</td>\n",
       "      <td>-1.612919e-06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>y_jerk_var</th>\n",
       "      <td> 0.000000</td>\n",
       "      <td> 9.342784e-08</td>\n",
       "      <td>-2.610296e-10</td>\n",
       "      <td>-3.155099e-07</td>\n",
       "      <td>-4.101702e-07</td>\n",
       "      <td>-8.783491e-08</td>\n",
       "      <td>-2.663964e-09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>z_jerk_peaks</th>\n",
       "      <td>-0.000100</td>\n",
       "      <td> 3.252189e-05</td>\n",
       "      <td> 5.152277e-05</td>\n",
       "      <td>-4.132963e-06</td>\n",
       "      <td> 3.019085e-05</td>\n",
       "      <td> 2.403237e-05</td>\n",
       "      <td> 1.154993e-04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>z_jerk_mean</th>\n",
       "      <td> 0.000001</td>\n",
       "      <td> 2.492562e-06</td>\n",
       "      <td> 2.286503e-07</td>\n",
       "      <td> 2.533392e-04</td>\n",
       "      <td>-3.169205e-04</td>\n",
       "      <td> 3.768691e-06</td>\n",
       "      <td> 1.399917e-06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>z_jerk_var</th>\n",
       "      <td> 0.000000</td>\n",
       "      <td>-5.051080e-08</td>\n",
       "      <td>-2.229178e-07</td>\n",
       "      <td>-6.372811e-09</td>\n",
       "      <td>-3.692003e-08</td>\n",
       "      <td>-5.677005e-08</td>\n",
       "      <td>-4.204116e-07</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      0             1             2             3  \\\n",
       "x_accel_peaks -0.000033  1.004028e-05 -8.685992e-07  7.469256e-03   \n",
       "x_accel_mean  -0.000020 -6.084537e-04 -3.095777e-04 -4.389287e-03   \n",
       "x_accel_var   -0.000211  1.379450e-04  1.214333e-05  1.457178e-05   \n",
       "y_accel_peaks -0.000035  2.638802e-06  1.173925e-06  8.597318e-03   \n",
       "y_accel_mean   0.000005 -6.853135e-04 -3.909130e-04  3.782417e-03   \n",
       "y_accel_var   -0.000393 -1.585969e-04 -6.827102e-05  1.297311e-04   \n",
       "z_accel_peaks -0.000027  1.474185e-06  2.808337e-06  4.402738e-03   \n",
       "z_accel_mean   0.000037 -6.009011e-04 -3.274423e-04 -2.415806e-03   \n",
       "z_accel_var   -0.000108 -5.608652e-05  7.434234e-05 -1.295255e-04   \n",
       "x_jerk_peaks  -0.000040  8.613060e-05 -1.042191e-05 -5.707369e-05   \n",
       "x_jerk_mean    0.000005 -1.320900e-06  1.102946e-07 -1.835390e-03   \n",
       "x_jerk_var     0.000000 -1.931424e-07  9.588112e-09 -3.864821e-08   \n",
       "y_jerk_peaks  -0.000140  1.719934e-05  4.702065e-05  1.730341e-05   \n",
       "y_jerk_mean   -0.000001  1.195911e-06 -1.967043e-07 -8.004333e-04   \n",
       "y_jerk_var     0.000000  9.342784e-08 -2.610296e-10 -3.155099e-07   \n",
       "z_jerk_peaks  -0.000100  3.252189e-05  5.152277e-05 -4.132963e-06   \n",
       "z_jerk_mean    0.000001  2.492562e-06  2.286503e-07  2.533392e-04   \n",
       "z_jerk_var     0.000000 -5.051080e-08 -2.229178e-07 -6.372811e-09   \n",
       "\n",
       "                          4             5             6  \n",
       "x_accel_peaks -1.600595e-04  7.333645e-05  4.771030e-05  \n",
       "x_accel_mean  -1.826498e-04 -2.669642e-04 -4.811946e-05  \n",
       "x_accel_var   -3.503694e-05  1.472086e-04  4.480283e-05  \n",
       "y_accel_peaks  3.367239e-04  5.104320e-05  3.477941e-05  \n",
       "y_accel_mean  -1.407340e-03 -7.539034e-04 -1.153178e-04  \n",
       "y_accel_var    3.266384e-04  2.775060e-05 -5.432287e-04  \n",
       "z_accel_peaks -1.446679e-04  6.343948e-05  5.034281e-05  \n",
       "z_accel_mean  -2.976586e-04 -7.430820e-04 -1.222782e-04  \n",
       "z_accel_var    1.677517e-04 -5.762908e-06  2.518509e-04  \n",
       "x_jerk_peaks   6.691327e-05  6.956292e-05 -1.726091e-05  \n",
       "x_jerk_mean   -2.119019e-04  9.296737e-07 -7.944720e-08  \n",
       "x_jerk_var    -5.122850e-07 -8.666461e-07 -2.469737e-08  \n",
       "y_jerk_peaks   1.967943e-05  6.942743e-05  4.061812e-05  \n",
       "y_jerk_mean    2.647920e-04 -7.564928e-06 -1.612919e-06  \n",
       "y_jerk_var    -4.101702e-07 -8.783491e-08 -2.663964e-09  \n",
       "z_jerk_peaks   3.019085e-05  2.403237e-05  1.154993e-04  \n",
       "z_jerk_mean   -3.169205e-04  3.768691e-06  1.399917e-06  \n",
       "z_jerk_var    -3.692003e-08 -5.677005e-08 -4.204116e-07  "
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "coefficientDF = pd.DataFrame(np.transpose(model.coef_),index =X.columns )\n",
    "coefficientDF"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now again, we will fit our data to a logistic regression model, but this time we will use a random 15-fold cross validation so that the model is trained and tested on data involving all 15 participants."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#try random 15-fold cross validation.  See if the results are improved\n",
    "scores = cross_val_score(LogisticRegression(), X, y, scoring='accuracy', cv=15)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0.54987835  0.52311436  0.62956946  0.5264013   0.5264013   0.47684809\n",
      "  0.59886086  0.58469055  0.52198697  0.57131214  0.47514262  0.37326813\n",
      "  0.57294214  0.58516707  0.51507742]\n",
      "Mean of the scores: 0.535377384622\n",
      "Variance of the scores: 0.00366413485135\n"
     ]
    }
   ],
   "source": [
    "print scores\n",
    "print \"Mean of the scores: \" + str(scores.mean())\n",
    "print \"Variance of the scores: \" + str(scores.var())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Interestingly, the accuracy in this cross-validation is similar to the leave one participant out validation done above, indicating that the model likely is not detecting significant variation in the activities of the different participants."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For a second model, we will train a random forest a validate it using one participant left out cross-validation.  Random forests are an effective prediction tool that benefits from the ability of random trees to capture the effects of complex structures in the data.  Trees tend to be low bias but have high variance.  Random forests are effective at reducing this variance by predicting using a majority vote from an ensemble of random trees.  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "X = cleaned_data[features]\n",
    "y = cleaned_data['label']\n",
    "forest = RandomForestClassifier()\n",
    "forest_accuracy_scores = []\n",
    "forest_label_counts = []\n",
    "y_test = None\n",
    "predictions = None\n",
    "for test_id in range(1,16):\n",
    "    train_indices = cleaned_data.index[cleaned_data['participant_id'] !=test_id]\n",
    "    test_indices =  cleaned_data.index[cleaned_data['participant_id'] ==test_id]\n",
    "    X_train = X.loc[train_indices,:]\n",
    "    X_test = X.loc[test_indices,:]\n",
    "    y_train = y.loc[train_indices]\n",
    "    y_test = y.loc[test_indices]\n",
    "    forest.fit(X_train,y_train)\n",
    "    predictions = forest.predict(X_test)\n",
    "    accuracy = metrics.accuracy_score(y_test,predictions)\n",
    "    forest_accuracy_scores.append(accuracy)\n",
    "    forest_label_counts.append(Counter(y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0.38037203  0.666919    0.66258919  0.62489344  0.36286645  0.68026706\n",
      "  0.56174024  0.43830431  0.32399745  0.47572016  0.4995005   0.64422202\n",
      "  0.42414861  0.6807554   0.48487903]\n",
      "mean accuracy: 0.527411660469\n",
      "variance of the accuracy: 0.0148584473449\n"
     ]
    }
   ],
   "source": [
    "forest_accuracy_scores = np.array(forest_accuracy_scores)\n",
    "print forest_accuracy_scores\n",
    "print \"mean accuracy: \"+ str(forest_accuracy_scores.mean())\n",
    "print \"variance of the accuracy: \"+str(forest_accuracy_scores.var())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[310   2  37   8   0   1 138]\n",
      " [  9   0   7   2   3   2  21]\n",
      " [ 13   0  19  29   0   0  15]\n",
      " [ 15   1  25  95   3   0  29]\n",
      " [  6   0   5  13   0   0   7]\n",
      " [  0   0   5   4   0   0   0]\n",
      " [ 61   1  38  10   0   1  57]]\n",
      "Number of predictions for each label: \n",
      "[414   4 136 161   6   4 267]\n",
      "Number of data points for each label: \n",
      "Counter({1: 496, 4: 168, 7: 168, 3: 76, 2: 44, 5: 31, 6: 9})\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          1       0.75      0.62      0.68       496\n",
      "          2       0.00      0.00      0.00        44\n",
      "          3       0.14      0.25      0.18        76\n",
      "          4       0.59      0.57      0.58       168\n",
      "          5       0.00      0.00      0.00        31\n",
      "          6       0.00      0.00      0.00         9\n",
      "          7       0.21      0.34      0.26       168\n",
      "\n",
      "avg / total       0.52      0.48      0.50       992\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#check the confusion matrix for the random forest when testing on participant 15 for the model trained on 1-14\n",
    "C=  metrics.confusion_matrix(y_test,predictions)\n",
    "print C\n",
    "print \"Number of predictions for each label: \\n\" + str(C.sum(axis= 0))\n",
    "print \"Number of data points for each label: \\n\" + str(label_counts[14])\n",
    "print metrics.classification_report(y_test, predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x_accel_peaks: 0.0436406648664\n",
      "x_accel_mean: 0.130897482573\n",
      "x_accel_var: 0.0424317282689\n",
      "y_accel_peaks: 0.0808949259656\n",
      "y_accel_mean: 0.119754465254\n",
      "y_accel_var: 0.0997967662228\n",
      "z_accel_peaks: 0.0229688216811\n",
      "z_accel_mean: 0.162821791487\n",
      "z_accel_var: 0.0393893480595\n",
      "x_jerk_peaks: 0.033697340586\n",
      "x_jerk_mean: 0.0194047738919\n",
      "x_jerk_var: 0.0475627005238\n",
      "y_jerk_peaks: 0.0218903718842\n",
      "y_jerk_mean: 0.0173413394158\n",
      "y_jerk_var: 0.0316018321705\n",
      "z_jerk_peaks: 0.0195097200995\n",
      "z_jerk_mean: 0.0190427823123\n",
      "z_jerk_var: 0.0473531447383\n"
     ]
    }
   ],
   "source": [
    "#check feature importances (higher the value, the more important the feature)\n",
    "feature_scores = zip(features,forest.feature_importances_)\n",
    "for x in feature_scores:\n",
    "    print x[0] + \": \"+ str(x[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "So the mean acceleration in the z direction, x direction and y direction respecively rank as the three most importand features."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The accuracy scores on the forest tested on participant 15 are similar to those for the logistic regression model.  For improved performance, we likely need to use more advanced features to describe the jerk and acceleration time series for each window.  It might be worthwhile to plot a few of the time series to see if any distinct differences show up that could be incorporated as features.  Perhaps some features involving the shape of the acceleration trajectories could be informative.  It also might be worthwhile to use more advanced model, such as one using neural networks."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}