Uber Data Pipeline (Fixed Version).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "368d2580",
   "metadata": {},
   "outputs": [],
   "source": [
    "import io\n",
    "import pandas as pd\n",
    "import requests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "500446cc",
   "metadata": {},
   "outputs": [],
   "source": [
    "url = 'https://storage.googleapis.com/uber-data-engineering-project/uber_data.csv'\n",
    "response = requests.get(url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "b37a1ace",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(io.StringIO(response.text), sep=',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "edc6d495",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['tpep_pickup_datetime'] = pd.to_datetime(df['tpep_pickup_datetime'])\n",
    "df['tpep_dropoff_datetime'] = pd.to_datetime(df['tpep_dropoff_datetime'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "4d0c8281",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.drop_duplicates().reset_index(drop=True)\n",
    "df['trip_id'] = df.index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "a35f32dd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>VendorID</th>\n",
       "      <th>tpep_pickup_datetime</th>\n",
       "      <th>tpep_dropoff_datetime</th>\n",
       "      <th>passenger_count</th>\n",
       "      <th>trip_distance</th>\n",
       "      <th>pickup_longitude</th>\n",
       "      <th>pickup_latitude</th>\n",
       "      <th>RatecodeID</th>\n",
       "      <th>store_and_fwd_flag</th>\n",
       "      <th>dropoff_longitude</th>\n",
       "      <th>dropoff_latitude</th>\n",
       "      <th>payment_type</th>\n",
       "      <th>fare_amount</th>\n",
       "      <th>extra</th>\n",
       "      <th>mta_tax</th>\n",
       "      <th>tip_amount</th>\n",
       "      <th>tolls_amount</th>\n",
       "      <th>improvement_surcharge</th>\n",
       "      <th>total_amount</th>\n",
       "      <th>trip_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>2016-03-01</td>\n",
       "      <td>2016-03-01 00:07:55</td>\n",
       "      <td>1</td>\n",
       "      <td>2.50</td>\n",
       "      <td>-73.976746</td>\n",
       "      <td>40.765152</td>\n",
       "      <td>1</td>\n",
       "      <td>N</td>\n",
       "      <td>-74.004265</td>\n",
       "      <td>40.746128</td>\n",
       "      <td>1</td>\n",
       "      <td>9.0</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>2.05</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.3</td>\n",
       "      <td>12.35</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>2016-03-01</td>\n",
       "      <td>2016-03-01 00:11:06</td>\n",
       "      <td>1</td>\n",
       "      <td>2.90</td>\n",
       "      <td>-73.983482</td>\n",
       "      <td>40.767925</td>\n",
       "      <td>1</td>\n",
       "      <td>N</td>\n",
       "      <td>-74.005943</td>\n",
       "      <td>40.733166</td>\n",
       "      <td>1</td>\n",
       "      <td>11.0</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>3.05</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.3</td>\n",
       "      <td>15.35</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>2016-03-01</td>\n",
       "      <td>2016-03-01 00:31:06</td>\n",
       "      <td>2</td>\n",
       "      <td>19.98</td>\n",
       "      <td>-73.782021</td>\n",
       "      <td>40.644810</td>\n",
       "      <td>1</td>\n",
       "      <td>N</td>\n",
       "      <td>-73.974541</td>\n",
       "      <td>40.675770</td>\n",
       "      <td>1</td>\n",
       "      <td>54.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>8.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.3</td>\n",
       "      <td>63.80</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2</td>\n",
       "      <td>2016-03-01</td>\n",
       "      <td>2016-03-01 00:00:00</td>\n",
       "      <td>3</td>\n",
       "      <td>10.78</td>\n",
       "      <td>-73.863419</td>\n",
       "      <td>40.769814</td>\n",
       "      <td>1</td>\n",
       "      <td>N</td>\n",
       "      <td>-73.969650</td>\n",
       "      <td>40.757767</td>\n",
       "      <td>1</td>\n",
       "      <td>31.5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.5</td>\n",
       "      <td>3.78</td>\n",
       "      <td>5.54</td>\n",
       "      <td>0.3</td>\n",
       "      <td>41.62</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2</td>\n",
       "      <td>2016-03-01</td>\n",
       "      <td>2016-03-01 00:00:00</td>\n",
       "      <td>5</td>\n",
       "      <td>30.43</td>\n",
       "      <td>-73.971741</td>\n",
       "      <td>40.792183</td>\n",
       "      <td>3</td>\n",
       "      <td>N</td>\n",
       "      <td>-74.177170</td>\n",
       "      <td>40.695053</td>\n",
       "      <td>1</td>\n",
       "      <td>98.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>15.50</td>\n",
       "      <td>0.3</td>\n",
       "      <td>113.80</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   VendorID tpep_pickup_datetime tpep_dropoff_datetime  passenger_count  \\\n",
       "0         1           2016-03-01   2016-03-01 00:07:55                1   \n",
       "1         1           2016-03-01   2016-03-01 00:11:06                1   \n",
       "2         2           2016-03-01   2016-03-01 00:31:06                2   \n",
       "3         2           2016-03-01   2016-03-01 00:00:00                3   \n",
       "4         2           2016-03-01   2016-03-01 00:00:00                5   \n",
       "\n",
       "   trip_distance  pickup_longitude  pickup_latitude  RatecodeID  \\\n",
       "0           2.50        -73.976746        40.765152           1   \n",
       "1           2.90        -73.983482        40.767925           1   \n",
       "2          19.98        -73.782021        40.644810           1   \n",
       "3          10.78        -73.863419        40.769814           1   \n",
       "4          30.43        -73.971741        40.792183           3   \n",
       "\n",
       "  store_and_fwd_flag  dropoff_longitude  dropoff_latitude  payment_type  \\\n",
       "0                  N         -74.004265         40.746128             1   \n",
       "1                  N         -74.005943         40.733166             1   \n",
       "2                  N         -73.974541         40.675770             1   \n",
       "3                  N         -73.969650         40.757767             1   \n",
       "4                  N         -74.177170         40.695053             1   \n",
       "\n",
       "   fare_amount  extra  mta_tax  tip_amount  tolls_amount  \\\n",
       "0          9.0    0.5      0.5        2.05          0.00   \n",
       "1         11.0    0.5      0.5        3.05          0.00   \n",
       "2         54.5    0.5      0.5        8.00          0.00   \n",
       "3         31.5    0.0      0.5        3.78          5.54   \n",
       "4         98.0    0.0      0.0        0.00         15.50   \n",
       "\n",
       "   improvement_surcharge  total_amount  trip_id  \n",
       "0                    0.3         12.35        0  \n",
       "1                    0.3         15.35        1  \n",
       "2                    0.3         63.80        2  \n",
       "3                    0.3         41.62        3  \n",
       "4                    0.3        113.80        4  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "4a148bc6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>datetime_id</th>\n",
       "      <th>tpep_pickup_datetime</th>\n",
       "      <th>pick_hour</th>\n",
       "      <th>pick_day</th>\n",
       "      <th>pick_month</th>\n",
       "      <th>pick_year</th>\n",
       "      <th>pick_weekday</th>\n",
       "      <th>tpep_dropoff_datetime</th>\n",
       "      <th>drop_hour</th>\n",
       "      <th>drop_day</th>\n",
       "      <th>drop_month</th>\n",
       "      <th>drop_year</th>\n",
       "      <th>drop_weekday</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>2016-03-01</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2016</td>\n",
       "      <td>1</td>\n",
       "      <td>2016-03-01 00:07:55</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2016</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>2016-03-01</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2016</td>\n",
       "      <td>1</td>\n",
       "      <td>2016-03-01 00:11:06</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2016</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>2016-03-01</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2016</td>\n",
       "      <td>1</td>\n",
       "      <td>2016-03-01 00:31:06</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2016</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>2016-03-01</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2016</td>\n",
       "      <td>1</td>\n",
       "      <td>2016-03-01 00:00:00</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2016</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>2016-03-01</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2016</td>\n",
       "      <td>1</td>\n",
       "      <td>2016-03-01 00:00:00</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2016</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   datetime_id tpep_pickup_datetime  pick_hour  pick_day  pick_month  \\\n",
       "0            0           2016-03-01          0         1           3   \n",
       "1            1           2016-03-01          0         1           3   \n",
       "2            2           2016-03-01          0         1           3   \n",
       "3            3           2016-03-01          0         1           3   \n",
       "4            4           2016-03-01          0         1           3   \n",
       "\n",
       "   pick_year  pick_weekday tpep_dropoff_datetime  drop_hour  drop_day  \\\n",
       "0       2016             1   2016-03-01 00:07:55          0         1   \n",
       "1       2016             1   2016-03-01 00:11:06          0         1   \n",
       "2       2016             1   2016-03-01 00:31:06          0         1   \n",
       "3       2016             1   2016-03-01 00:00:00          0         1   \n",
       "4       2016             1   2016-03-01 00:00:00          0         1   \n",
       "\n",
       "   drop_month  drop_year  drop_weekday  \n",
       "0           3       2016             1  \n",
       "1           3       2016             1  \n",
       "2           3       2016             1  \n",
       "3           3       2016             1  \n",
       "4           3       2016             1  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "datetime_dim = df[['tpep_pickup_datetime','tpep_dropoff_datetime']].reset_index(drop=True)\n",
    "datetime_dim['tpep_pickup_datetime'] = datetime_dim['tpep_pickup_datetime']\n",
    "datetime_dim['pick_hour'] = datetime_dim['tpep_pickup_datetime'].dt.hour\n",
    "datetime_dim['pick_day'] = datetime_dim['tpep_pickup_datetime'].dt.day\n",
    "datetime_dim['pick_month'] = datetime_dim['tpep_pickup_datetime'].dt.month\n",
    "datetime_dim['pick_year'] = datetime_dim['tpep_pickup_datetime'].dt.year\n",
    "datetime_dim['pick_weekday'] = datetime_dim['tpep_pickup_datetime'].dt.weekday\n",
    "\n",
    "datetime_dim['tpep_dropoff_datetime'] = datetime_dim['tpep_dropoff_datetime']\n",
    "datetime_dim['drop_hour'] = datetime_dim['tpep_dropoff_datetime'].dt.hour\n",
    "datetime_dim['drop_day'] = datetime_dim['tpep_dropoff_datetime'].dt.day\n",
    "datetime_dim['drop_month'] = datetime_dim['tpep_dropoff_datetime'].dt.month\n",
    "datetime_dim['drop_year'] = datetime_dim['tpep_dropoff_datetime'].dt.year\n",
    "datetime_dim['drop_weekday'] = datetime_dim['tpep_dropoff_datetime'].dt.weekday\n",
    "\n",
    "\n",
    "datetime_dim['datetime_id'] = datetime_dim.index\n",
    "\n",
    "# datetime_dim = datetime_dim.rename(columns={'tpep_pickup_datetime': 'datetime_id'}).reset_index(drop=True)\n",
    "datetime_dim = datetime_dim[['datetime_id', 'tpep_pickup_datetime', 'pick_hour', 'pick_day', 'pick_month', 'pick_year', 'pick_weekday',\n",
    "                             'tpep_dropoff_datetime', 'drop_hour', 'drop_day', 'drop_month', 'drop_year', 'drop_weekday']]\n",
    "#\n",
    "datetime_dim.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "ba67912f",
   "metadata": {},
   "outputs": [],
   "source": [
    "passenger_count_dim = df[['passenger_count']].reset_index(drop=True)\n",
    "passenger_count_dim['passenger_count_id'] = passenger_count_dim.index\n",
    "passenger_count_dim = passenger_count_dim[['passenger_count_id','passenger_count']]\n",
    "\n",
    "trip_distance_dim = df[['trip_distance']].reset_index(drop=True)\n",
    "trip_distance_dim['trip_distance_id'] = trip_distance_dim.index\n",
    "trip_distance_dim = trip_distance_dim[['trip_distance_id','trip_distance']]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "fb7c9704",
   "metadata": {},
   "outputs": [],
   "source": [
    "rate_code_type = {\n",
    "    1:\"Standard rate\",\n",
    "    2:\"JFK\",\n",
    "    3:\"Newark\",\n",
    "    4:\"Nassau or Westchester\",\n",
    "    5:\"Negotiated fare\",\n",
    "    6:\"Group ride\"\n",
    "}\n",
    "\n",
    "rate_code_dim = df[['RatecodeID']].reset_index(drop=True)\n",
    "rate_code_dim['rate_code_id'] = rate_code_dim.index\n",
    "rate_code_dim['rate_code_name'] = rate_code_dim['RatecodeID'].map(rate_code_type)\n",
    "rate_code_dim = rate_code_dim[['rate_code_id','RatecodeID','rate_code_name']]\n",
    "\n",
    "# rate_code_dim.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "4db826a1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>rate_code_id</th>\n",
       "      <th>RatecodeID</th>\n",
       "      <th>rate_code_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>Standard rate</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>Newark</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>JFK</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>Negotiated fare</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>Nassau or Westchester</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   rate_code_id  RatecodeID         rate_code_name\n",
       "0             0           1          Standard rate\n",
       "1             1           3                 Newark\n",
       "2             2           2                    JFK\n",
       "3             3           5        Negotiated fare\n",
       "4             4           4  Nassau or Westchester"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rate_code_dim.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "8048bdef",
   "metadata": {},
   "outputs": [],
   "source": [
    "pickup_location_dim = df[['pickup_longitude', 'pickup_latitude']].reset_index(drop=True)\n",
    "pickup_location_dim['pickup_location_id'] = pickup_location_dim.index\n",
    "pickup_location_dim = pickup_location_dim[['pickup_location_id','pickup_latitude','pickup_longitude']] \n",
    "\n",
    "\n",
    "dropoff_location_dim = df[['dropoff_longitude', 'dropoff_latitude']].reset_index(drop=True)\n",
    "dropoff_location_dim['dropoff_location_id'] = dropoff_location_dim.index\n",
    "dropoff_location_dim = dropoff_location_dim[['dropoff_location_id','dropoff_latitude','dropoff_longitude']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "bfb04993",
   "metadata": {},
   "outputs": [],
   "source": [
    "payment_type_name = {\n",
    "    1:\"Credit card\",\n",
    "    2:\"Cash\",\n",
    "    3:\"No charge\",\n",
    "    4:\"Dispute\",\n",
    "    5:\"Unknown\",\n",
    "    6:\"Voided trip\"\n",
    "}\n",
    "payment_type_dim = df[['payment_type']].reset_index(drop=True)\n",
    "payment_type_dim['payment_type_id'] = payment_type_dim.index\n",
    "payment_type_dim['payment_type_name'] = payment_type_dim['payment_type'].map(payment_type_name)\n",
    "payment_type_dim = payment_type_dim[['payment_type_id','payment_type','payment_type_name']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "e747865b",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "fact_table = df.merge(passenger_count_dim, left_on='trip_id', right_on='passenger_count_id') \\\n",
    "             .merge(trip_distance_dim, left_on='trip_id', right_on='trip_distance_id') \\\n",
    "             .merge(rate_code_dim, left_on='trip_id', right_on='rate_code_id') \\\n",
    "             .merge(pickup_location_dim, left_on='trip_id', right_on='pickup_location_id') \\\n",
    "             .merge(dropoff_location_dim, left_on='trip_id', right_on='dropoff_location_id')\\\n",
    "             .merge(datetime_dim, left_on='trip_id', right_on='datetime_id') \\\n",
    "             .merge(payment_type_dim, left_on='trip_id', right_on='payment_type_id') \\\n",
    "             [['trip_id','VendorID', 'datetime_id', 'passenger_count_id',\n",
    "               'trip_distance_id', 'rate_code_id', 'store_and_fwd_flag', 'pickup_location_id', 'dropoff_location_id',\n",
    "               'payment_type_id', 'fare_amount', 'extra', 'mta_tax', 'tip_amount', 'tolls_amount',\n",
    "               'improvement_surcharge', 'total_amount']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "9acdb41a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['payment_type_id', 'payment_type', 'payment_type_name'], dtype='object')"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "payment_type_dim.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "62e05477",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['trip_id', 'VendorID', 'datetime_id', 'passenger_count_id',\n",
       "       'trip_distance_id', 'rate_code_id', 'store_and_fwd_flag',\n",
       "       'pickup_location_id', 'dropoff_location_id', 'payment_type_id',\n",
       "       'fare_amount', 'extra', 'mta_tax', 'tip_amount', 'tolls_amount',\n",
       "       'improvement_surcharge', 'total_amount'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fact_table.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6162ee05",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "254ff9ad",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>trip_id</th>\n",
       "      <th>VendorID</th>\n",
       "      <th>datetime_id</th>\n",
       "      <th>passenger_count_id</th>\n",
       "      <th>trip_distance_id</th>\n",
       "      <th>rate_code_id</th>\n",
       "      <th>store_and_fwd_flag</th>\n",
       "      <th>pickup_location_id</th>\n",
       "      <th>dropoff_location_id</th>\n",
       "      <th>payment_type_id</th>\n",
       "      <th>fare_amount</th>\n",
       "      <th>extra</th>\n",
       "      <th>mta_tax</th>\n",
       "      <th>tip_amount</th>\n",
       "      <th>tolls_amount</th>\n",
       "      <th>improvement_surcharge</th>\n",
       "      <th>total_amount</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>N</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>2.05</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.3</td>\n",
       "      <td>12.35</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>N</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>11.0</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>3.05</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.3</td>\n",
       "      <td>15.35</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>N</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>54.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>8.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.3</td>\n",
       "      <td>63.80</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>N</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>31.5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.5</td>\n",
       "      <td>3.78</td>\n",
       "      <td>5.54</td>\n",
       "      <td>0.3</td>\n",
       "      <td>41.62</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>N</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>98.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>15.50</td>\n",
       "      <td>0.3</td>\n",
       "      <td>113.80</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99995</th>\n",
       "      <td>99995</td>\n",
       "      <td>1</td>\n",
       "      <td>99995</td>\n",
       "      <td>99995</td>\n",
       "      <td>99995</td>\n",
       "      <td>99995</td>\n",
       "      <td>N</td>\n",
       "      <td>99995</td>\n",
       "      <td>99995</td>\n",
       "      <td>99995</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.3</td>\n",
       "      <td>5.80</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99996</th>\n",
       "      <td>99996</td>\n",
       "      <td>1</td>\n",
       "      <td>99996</td>\n",
       "      <td>99996</td>\n",
       "      <td>99996</td>\n",
       "      <td>99996</td>\n",
       "      <td>N</td>\n",
       "      <td>99996</td>\n",
       "      <td>99996</td>\n",
       "      <td>99996</td>\n",
       "      <td>14.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.5</td>\n",
       "      <td>2.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.3</td>\n",
       "      <td>16.80</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99997</th>\n",
       "      <td>99997</td>\n",
       "      <td>1</td>\n",
       "      <td>99997</td>\n",
       "      <td>99997</td>\n",
       "      <td>99997</td>\n",
       "      <td>99997</td>\n",
       "      <td>N</td>\n",
       "      <td>99997</td>\n",
       "      <td>99997</td>\n",
       "      <td>99997</td>\n",
       "      <td>29.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.5</td>\n",
       "      <td>8.80</td>\n",
       "      <td>5.54</td>\n",
       "      <td>0.3</td>\n",
       "      <td>44.14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99998</th>\n",
       "      <td>99998</td>\n",
       "      <td>2</td>\n",
       "      <td>99998</td>\n",
       "      <td>99998</td>\n",
       "      <td>99998</td>\n",
       "      <td>99998</td>\n",
       "      <td>N</td>\n",
       "      <td>99998</td>\n",
       "      <td>99998</td>\n",
       "      <td>99998</td>\n",
       "      <td>5.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>1.36</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.3</td>\n",
       "      <td>8.16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99999</th>\n",
       "      <td>99999</td>\n",
       "      <td>1</td>\n",
       "      <td>99999</td>\n",
       "      <td>99999</td>\n",
       "      <td>99999</td>\n",
       "      <td>99999</td>\n",
       "      <td>N</td>\n",
       "      <td>99999</td>\n",
       "      <td>99999</td>\n",
       "      <td>99999</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.3</td>\n",
       "      <td>6.80</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100000 rows × 17 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       trip_id  VendorID  datetime_id  passenger_count_id  trip_distance_id  \\\n",
       "0            0         1            0                   0                 0   \n",
       "1            1         1            1                   1                 1   \n",
       "2            2         2            2                   2                 2   \n",
       "3            3         2            3                   3                 3   \n",
       "4            4         2            4                   4                 4   \n",
       "...        ...       ...          ...                 ...               ...   \n",
       "99995    99995         1        99995               99995             99995   \n",
       "99996    99996         1        99996               99996             99996   \n",
       "99997    99997         1        99997               99997             99997   \n",
       "99998    99998         2        99998               99998             99998   \n",
       "99999    99999         1        99999               99999             99999   \n",
       "\n",
       "       rate_code_id store_and_fwd_flag  pickup_location_id  \\\n",
       "0                 0                  N                   0   \n",
       "1                 1                  N                   1   \n",
       "2                 2                  N                   2   \n",
       "3                 3                  N                   3   \n",
       "4                 4                  N                   4   \n",
       "...             ...                ...                 ...   \n",
       "99995         99995                  N               99995   \n",
       "99996         99996                  N               99996   \n",
       "99997         99997                  N               99997   \n",
       "99998         99998                  N               99998   \n",
       "99999         99999                  N               99999   \n",
       "\n",
       "       dropoff_location_id  payment_type_id  fare_amount  extra  mta_tax  \\\n",
       "0                        0                0          9.0    0.5      0.5   \n",
       "1                        1                1         11.0    0.5      0.5   \n",
       "2                        2                2         54.5    0.5      0.5   \n",
       "3                        3                3         31.5    0.0      0.5   \n",
       "4                        4                4         98.0    0.0      0.0   \n",
       "...                    ...              ...          ...    ...      ...   \n",
       "99995                99995            99995          5.0    0.0      0.5   \n",
       "99996                99996            99996         14.0    0.0      0.5   \n",
       "99997                99997            99997         29.0    0.0      0.5   \n",
       "99998                99998            99998          5.5    0.5      0.5   \n",
       "99999                99999            99999          6.0    0.0      0.5   \n",
       "\n",
       "       tip_amount  tolls_amount  improvement_surcharge  total_amount  \n",
       "0            2.05          0.00                    0.3         12.35  \n",
       "1            3.05          0.00                    0.3         15.35  \n",
       "2            8.00          0.00                    0.3         63.80  \n",
       "3            3.78          5.54                    0.3         41.62  \n",
       "4            0.00         15.50                    0.3        113.80  \n",
       "...           ...           ...                    ...           ...  \n",
       "99995        0.00          0.00                    0.3          5.80  \n",
       "99996        2.00          0.00                    0.3         16.80  \n",
       "99997        8.80          5.54                    0.3         44.14  \n",
       "99998        1.36          0.00                    0.3          8.16  \n",
       "99999        0.00          0.00                    0.3          6.80  \n",
       "\n",
       "[100000 rows x 17 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fact_table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a45c58d1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# CREATE OR REPLACE TABLE `data-with-darshil.uber_dataset.tbl_analysis_report` AS (\n",
    "# SELECT\n",
    "#   f.VendorID,\n",
    "#   f.tpep_pickup_datetime,\n",
    "#   f.tpep_dropoff_datetime,\n",
    "#   p.passenger_count,\n",
    "#   td.trip_distance,\n",
    "#   rc.RatecodeID,\n",
    "#   f.store_and_fwd_flag,\n",
    "#   pl.pickup_latitude,\n",
    "#   pl.pickup_longitude,\n",
    "#   dl.dropoff_latitude,\n",
    "#   dl.dropoff_longitude,\n",
    "#   pt.payment_type,\n",
    "#   f.fare_amount,\n",
    "#   f.extra,\n",
    "#   f.mta_tax,\n",
    "#   f.tip_amount,\n",
    "#   f.tolls_amount,\n",
    "#   f.improvement_surcharge,\n",
    "#   f.total_amount\n",
    "# FROM\n",
    "#   `data-with-darshil.uber_dataset.fact_table` f\n",
    "#   JOIN `data-with-darshil.uber_dataset.passenger_count_dim` p ON f.passenger_count_id = p.passenger_count_id\n",
    "#   JOIN `data-with-darshil.uber_dataset.trip_distance_dim` td ON f.trip_distance_id = td.trip_distance_id\n",
    "#   JOIN `data-with-darshil.uber_dataset.rate_code_dim` rc ON f.rate_code_id = rc.rate_code_id\n",
    "#   JOIN `data-with-darshil.uber_dataset.pickup_location_dim` pl ON f.pickup_location_id = pl.pickup_location_id\n",
    "#   JOIN `data-with-darshil.uber_dataset.dropoff_location_dim` dl ON f.dropoff_location_id = dl.dropoff_location_id\n",
    "#   JOIN `data-with-darshil.uber_dataset.payment_type_dim` pt ON f.payment_type_id = pt.payment_type_id);"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}