Lec 31 - Concatenate.ipynb

{

 "metadata": {

  "name": "",

  "signature": "sha256:b8254d0b5e2d0fe0307b2a54dea2f81ade079dec20d3c5a85027f9a405b6b612"

 },

 "nbformat": 3,

 "nbformat_minor": 0,

 "worksheets": [

  {

   "cells": [

    {

     "cell_type": "code",

     "collapsed": false,

     "input": [

      "# Now we'll learn about concatenating along an axis\n",

      "import numpy as np\n",

      "import pandas as pd\n",

      "from pandas import Series, DataFrame"

     ],

     "language": "python",

     "metadata": {},

     "outputs": [],

     "prompt_number": 2

    },

    {

     "cell_type": "code",

     "collapsed": false,

     "input": [

      "# First in just Numpy"

     ],

     "language": "python",

     "metadata": {},

     "outputs": [],

     "prompt_number": 3

    },

    {

     "cell_type": "code",

     "collapsed": false,

     "input": [

      "# Create a matrix \n",

      "arr1 = np.arange(9).reshape((3,3))"

     ],

     "language": "python",

     "metadata": {},

     "outputs": [],

     "prompt_number": 4

    },

    {

     "cell_type": "code",

     "collapsed": false,

     "input": [

      "# Show\n",

      "arr1"

     ],

     "language": "python",

     "metadata": {},

     "outputs": [

      {

       "metadata": {},

       "output_type": "pyout",

       "prompt_number": 5,

       "text": [

        "array([[0, 1, 2],\n",

        "       [3, 4, 5],\n",

        "       [6, 7, 8]])"

       ]

      }

     ],

     "prompt_number": 5

    },

    {

     "cell_type": "code",

     "collapsed": false,

     "input": [

      "# Concatenate along axis 1\n",

      "np.concatenate([arr1,arr1],axis=1)"

     ],

     "language": "python",

     "metadata": {},

     "outputs": [

      {

       "metadata": {},

       "output_type": "pyout",

       "prompt_number": 6,

       "text": [

        "array([[0, 1, 2, 0, 1, 2],\n",

        "       [3, 4, 5, 3, 4, 5],\n",

        "       [6, 7, 8, 6, 7, 8]])"

       ]

      }

     ],

     "prompt_number": 6

    },

    {

     "cell_type": "code",

     "collapsed": false,

     "input": [

      "# Let's see other axis options\n",

      "np.concatenate([arr1,arr1],axis=0)"

     ],

     "language": "python",

     "metadata": {},

     "outputs": [

      {

       "metadata": {},

       "output_type": "pyout",

       "prompt_number": 7,

       "text": [

        "array([[0, 1, 2],\n",

        "       [3, 4, 5],\n",

        "       [6, 7, 8],\n",

        "       [0, 1, 2],\n",

        "       [3, 4, 5],\n",

        "       [6, 7, 8]])"

       ]

      }

     ],

     "prompt_number": 7

    },

    {

     "cell_type": "code",

     "collapsed": false,

     "input": [

      "# Now let's see how this works in pandas"

     ],

     "language": "python",

     "metadata": {},

     "outputs": [],

     "prompt_number": 8

    },

    {

     "cell_type": "code",

     "collapsed": false,

     "input": [

      "# Lets create two Series with no overlap\n",

      "ser1 =  Series([0,1,2],index=['T','U','V'])\n",

      "\n",

      "ser2 = Series([3,4],index=['X','Y'])\n",

      "\n",

      "#Now let use concat (default is axis=0)\n",

      "pd.concat([ser1,ser2])"

     ],

     "language": "python",

     "metadata": {},

     "outputs": [

      {

       "metadata": {},

       "output_type": "pyout",

       "prompt_number": 9,

       "text": [

        "T    0\n",

        "U    1\n",

        "V    2\n",

        "X    3\n",

        "Y    4\n",

        "dtype: int64"

       ]

      }

     ],

     "prompt_number": 9

    },

    {

     "cell_type": "code",

     "collapsed": false,

     "input": [

      "# Now passing along another axis will produce a DataFrame\n",

      "pd.concat([ser1,ser2],axis=1)"

     ],

     "language": "python",

     "metadata": {},

     "outputs": [

      {

       "html": [

        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",

        "<table border=\"1\" class=\"dataframe\">\n",

        "  <thead>\n",

        "    <tr style=\"text-align: right;\">\n",

        "      <th></th>\n",

        "      <th>0</th>\n",

        "      <th>1</th>\n",

        "    </tr>\n",

        "  </thead>\n",

        "  <tbody>\n",

        "    <tr>\n",

        "      <th>T</th>\n",

        "      <td>  0</td>\n",

        "      <td>NaN</td>\n",

        "    </tr>\n",

        "    <tr>\n",

        "      <th>U</th>\n",

        "      <td>  1</td>\n",

        "      <td>NaN</td>\n",

        "    </tr>\n",

        "    <tr>\n",

        "      <th>V</th>\n",

        "      <td>  2</td>\n",

        "      <td>NaN</td>\n",

        "    </tr>\n",

        "    <tr>\n",

        "      <th>X</th>\n",

        "      <td>NaN</td>\n",

        "      <td>  3</td>\n",

        "    </tr>\n",

        "    <tr>\n",

        "      <th>Y</th>\n",

        "      <td>NaN</td>\n",

        "      <td>  4</td>\n",

        "    </tr>\n",

        "  </tbody>\n",

        "</table>\n",

        "</div>"

       ],

       "metadata": {},

       "output_type": "pyout",

       "prompt_number": 10,

       "text": [

        "    0   1\n",

        "T   0 NaN\n",

        "U   1 NaN\n",

        "V   2 NaN\n",

        "X NaN   3\n",

        "Y NaN   4"

       ]

      }

     ],

     "prompt_number": 10

    },

    {

     "cell_type": "code",

     "collapsed": false,

     "input": [

      "# We can specify which specific axes to be used\n",

      "pd.concat([ser1,ser2],axis=1,join_axes=[['U','V','Y']])"

     ],

     "language": "python",

     "metadata": {},

     "outputs": [

      {

       "html": [

        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",

        "<table border=\"1\" class=\"dataframe\">\n",

        "  <thead>\n",

        "    <tr style=\"text-align: right;\">\n",

        "      <th></th>\n",

        "      <th>0</th>\n",

        "      <th>1</th>\n",

        "    </tr>\n",

        "  </thead>\n",

        "  <tbody>\n",

        "    <tr>\n",

        "      <th>U</th>\n",

        "      <td>  1</td>\n",

        "      <td>NaN</td>\n",

        "    </tr>\n",

        "    <tr>\n",

        "      <th>V</th>\n",

        "      <td>  2</td>\n",

        "      <td>NaN</td>\n",

        "    </tr>\n",

        "    <tr>\n",

        "      <th>Y</th>\n",

        "      <td>NaN</td>\n",

        "      <td>  4</td>\n",

        "    </tr>\n",

        "  </tbody>\n",

        "</table>\n",

        "</div>"

       ],

       "metadata": {},

       "output_type": "pyout",

       "prompt_number": 17,

       "text": [

        "    0   1\n",

        "U   1 NaN\n",

        "V   2 NaN\n",

        "Y NaN   4"

       ]

      }

     ],

     "prompt_number": 17

    },

    {

     "cell_type": "code",

     "collapsed": false,

     "input": [

      "# Lets say we wanted to add markers.keys to the concatenation result\n",

      "\n",

      "# WE can do this with a hierarchical index\n",

      "pd.concat([ser1,ser2],keys=['cat1','cat2'])"

     ],

     "language": "python",

     "metadata": {},

     "outputs": [

      {

       "metadata": {},

       "output_type": "pyout",

       "prompt_number": 11,

       "text": [

        "cat1  T    0\n",

        "      U    1\n",

        "      V    2\n",

        "cat2  X    3\n",

        "      Y    4\n",

        "dtype: int64"

       ]

      }

     ],

     "prompt_number": 11

    },

    {

     "cell_type": "code",

     "collapsed": false,

     "input": [

      "# Along the axis=1 then these Keys become column headers\n",

      "pd.concat([ser1,ser2],axis=1,keys=['cat1','cat2'])"

     ],

     "language": "python",

     "metadata": {},

     "outputs": [

      {

       "html": [

        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",

        "<table border=\"1\" class=\"dataframe\">\n",

        "  <thead>\n",

        "    <tr style=\"text-align: right;\">\n",

        "      <th></th>\n",

        "      <th>cat1</th>\n",

        "      <th>cat2</th>\n",

        "    </tr>\n",

        "  </thead>\n",

        "  <tbody>\n",

        "    <tr>\n",

        "      <th>T</th>\n",

        "      <td>  0</td>\n",

        "      <td>NaN</td>\n",

        "    </tr>\n",

        "    <tr>\n",

        "      <th>U</th>\n",

        "      <td>  1</td>\n",

        "      <td>NaN</td>\n",

        "    </tr>\n",

        "    <tr>\n",

        "      <th>V</th>\n",

        "      <td>  2</td>\n",

        "      <td>NaN</td>\n",

        "    </tr>\n",

        "    <tr>\n",

        "      <th>X</th>\n",

        "      <td>NaN</td>\n",

        "      <td>  3</td>\n",

        "    </tr>\n",

        "    <tr>\n",

        "      <th>Y</th>\n",

        "      <td>NaN</td>\n",

        "      <td>  4</td>\n",

        "    </tr>\n",

        "  </tbody>\n",

        "</table>\n",

        "</div>"

       ],

       "metadata": {},

       "output_type": "pyout",

       "prompt_number": 12,

       "text": [

        "   cat1  cat2\n",

        "T     0   NaN\n",

        "U     1   NaN\n",

        "V     2   NaN\n",

        "X   NaN     3\n",

        "Y   NaN     4"

       ]

      }

     ],

     "prompt_number": 12

    },

    {

     "cell_type": "code",

     "collapsed": false,

     "input": [

      "#Lastly, everything works similarly in DataFrames\n",

      "\n",

      "dframe1 = DataFrame(np.random.randn(4,3), columns=['X', 'Y', 'Z'])\n",

      "dframe2 = DataFrame(np.random.randn(3, 3), columns=['Y', 'Q', 'X'])"

     ],

     "language": "python",

     "metadata": {},

     "outputs": [],

     "prompt_number": 14

    },

    {

     "cell_type": "code",

     "collapsed": false,

     "input": [

      "#Concat on DataFrame\n",

      "pd.concat([dframe1,dframe2])"

     ],

     "language": "python",

     "metadata": {},

     "outputs": [

      {

       "html": [

        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",

        "<table border=\"1\" class=\"dataframe\">\n",

        "  <thead>\n",

        "    <tr style=\"text-align: right;\">\n",

        "      <th></th>\n",

        "      <th>Q</th>\n",

        "      <th>X</th>\n",

        "      <th>Y</th>\n",

        "      <th>Z</th>\n",

        "    </tr>\n",

        "  </thead>\n",

        "  <tbody>\n",

        "    <tr>\n",

        "      <th>0</th>\n",

        "      <td>      NaN</td>\n",

        "      <td> 1.096040</td>\n",

        "      <td>-1.366980</td>\n",

        "      <td> 0.546707</td>\n",

        "    </tr>\n",

        "    <tr>\n",

        "      <th>1</th>\n",

        "      <td>      NaN</td>\n",

        "      <td>-1.406425</td>\n",

        "      <td> 0.484748</td>\n",

        "      <td>-1.156143</td>\n",

        "    </tr>\n",

        "    <tr>\n",

        "      <th>2</th>\n",

        "      <td>      NaN</td>\n",

        "      <td> 1.155464</td>\n",

        "      <td> 1.166407</td>\n",

        "      <td>-0.245477</td>\n",

        "    </tr>\n",

        "    <tr>\n",

        "      <th>3</th>\n",

        "      <td>      NaN</td>\n",

        "      <td>-0.153330</td>\n",

        "      <td> 2.185743</td>\n",

        "      <td> 0.307704</td>\n",

        "    </tr>\n",

        "    <tr>\n",

        "      <th>0</th>\n",

        "      <td> 0.789881</td>\n",

        "      <td> 1.616933</td>\n",

        "      <td>-0.961830</td>\n",

        "      <td>      NaN</td>\n",

        "    </tr>\n",

        "    <tr>\n",

        "      <th>1</th>\n",

        "      <td> 0.201265</td>\n",

        "      <td> 0.293210</td>\n",

        "      <td>-0.277847</td>\n",

        "      <td>      NaN</td>\n",

        "    </tr>\n",

        "    <tr>\n",

        "      <th>2</th>\n",

        "      <td>-0.121395</td>\n",

        "      <td> 0.959849</td>\n",

        "      <td>-1.360611</td>\n",

        "      <td>      NaN</td>\n",

        "    </tr>\n",

        "  </tbody>\n",

        "</table>\n",

        "</div>"

       ],

       "metadata": {},

       "output_type": "pyout",

       "prompt_number": 16,

       "text": [

        "          Q         X         Y         Z\n",

        "0       NaN  1.096040 -1.366980  0.546707\n",

        "1       NaN -1.406425  0.484748 -1.156143\n",

        "2       NaN  1.155464  1.166407 -0.245477\n",

        "3       NaN -0.153330  2.185743  0.307704\n",

        "0  0.789881  1.616933 -0.961830       NaN\n",

        "1  0.201265  0.293210 -0.277847       NaN\n",

        "2 -0.121395  0.959849 -1.360611       NaN"

       ]

      }

     ],

     "prompt_number": 16

    },

    {

     "cell_type": "code",

     "collapsed": false,

     "input": [

      "#If we dont care about the index info and just awnt to make a complete DataFrame, just use ignore_index\n",

      "pd.concat([dframe1,dframe2],ignore_index=True)"

     ],

     "language": "python",

     "metadata": {},

     "outputs": [

      {

       "html": [

        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",

        "<table border=\"1\" class=\"dataframe\">\n",

        "  <thead>\n",

        "    <tr style=\"text-align: right;\">\n",

        "      <th></th>\n",

        "      <th>Q</th>\n",

        "      <th>X</th>\n",

        "      <th>Y</th>\n",

        "      <th>Z</th>\n",

        "    </tr>\n",

        "  </thead>\n",

        "  <tbody>\n",

        "    <tr>\n",

        "      <th>0</th>\n",

        "      <td>      NaN</td>\n",

        "      <td> 1.096040</td>\n",

        "      <td>-1.366980</td>\n",

        "      <td> 0.546707</td>\n",

        "    </tr>\n",

        "    <tr>\n",

        "      <th>1</th>\n",

        "      <td>      NaN</td>\n",

        "      <td>-1.406425</td>\n",

        "      <td> 0.484748</td>\n",

        "      <td>-1.156143</td>\n",

        "    </tr>\n",

        "    <tr>\n",

        "      <th>2</th>\n",

        "      <td>      NaN</td>\n",

        "      <td> 1.155464</td>\n",

        "      <td> 1.166407</td>\n",

        "      <td>-0.245477</td>\n",

        "    </tr>\n",

        "    <tr>\n",

        "      <th>3</th>\n",

        "      <td>      NaN</td>\n",

        "      <td>-0.153330</td>\n",

        "      <td> 2.185743</td>\n",

        "      <td> 0.307704</td>\n",

        "    </tr>\n",

        "    <tr>\n",

        "      <th>4</th>\n",

        "      <td> 0.789881</td>\n",

        "      <td> 1.616933</td>\n",

        "      <td>-0.961830</td>\n",

        "      <td>      NaN</td>\n",

        "    </tr>\n",

        "    <tr>\n",

        "      <th>5</th>\n",

        "      <td> 0.201265</td>\n",

        "      <td> 0.293210</td>\n",

        "      <td>-0.277847</td>\n",

        "      <td>      NaN</td>\n",

        "    </tr>\n",

        "    <tr>\n",

        "      <th>6</th>\n",

        "      <td>-0.121395</td>\n",

        "      <td> 0.959849</td>\n",

        "      <td>-1.360611</td>\n",

        "      <td>      NaN</td>\n",

        "    </tr>\n",

        "  </tbody>\n",

        "</table>\n",

        "</div>"

       ],

       "metadata": {},

       "output_type": "pyout",

       "prompt_number": 17,

       "text": [

        "          Q         X         Y         Z\n",

        "0       NaN  1.096040 -1.366980  0.546707\n",

        "1       NaN -1.406425  0.484748 -1.156143\n",

        "2       NaN  1.155464  1.166407 -0.245477\n",

        "3       NaN -0.153330  2.185743  0.307704\n",

        "4  0.789881  1.616933 -0.961830       NaN\n",

        "5  0.201265  0.293210 -0.277847       NaN\n",

        "6 -0.121395  0.959849 -1.360611       NaN"

       ]

      }

     ],

     "prompt_number": 17

    },

    {

     "cell_type": "code",

     "collapsed": false,

     "input": [

      "#For more info in documentation:\n",

      "url='https://pandas.pydata.org/pandas-docs/stable/generated/pandas.concat.html'"

     ],

     "language": "python",

     "metadata": {},

     "outputs": [],

     "prompt_number": 18

    },

    {

     "cell_type": "code",

     "collapsed": false,

     "input": [

      "#Next up: More on Combining DataFrames with Overlapping Indexes!"

     ],

     "language": "python",

     "metadata": {},

     "outputs": []

    }

   ],

   "metadata": {}

  }

 ]

}