{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from datetime import datetime\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Read the Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parser(s):\n",
    "    return datetime.strptime(s, '%Y-%m')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "ice_cream_heater_df = pd.read_csv('ice_cream_vs_heater.csv', parse_dates=[0], index_col=0, squeeze=True, date_parser=parser)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "ice_cream_heater_df = ice_cream_heater_df.asfreq(pd.infer_freq(ice_cream_heater_df.index))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "heater_series = ice_cream_heater_df.heater"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Month\n",
       "2004-01-01    27\n",
       "2004-02-01    18\n",
       "2004-03-01    14\n",
       "2004-04-01    13\n",
       "2004-05-01    13\n",
       "              ..\n",
       "2020-02-01    34\n",
       "2020-03-01    25\n",
       "2020-04-01    25\n",
       "2020-05-01    27\n",
       "2020-06-01    24\n",
       "Freq: MS, Name: heater, Length: 198, dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "heater_series"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_series(series):\n",
    "    plt.figure(figsize=(12,6))\n",
    "    plt.plot(heater_series, color='red')\n",
    "    plt.ylabel('Search Frequency for \"Heater\"', fontsize=16)\n",
    "\n",
    "    for year in range(2004, 2021):\n",
    "        plt.axvline(datetime(year,1,1), linestyle='--', color='k', alpha=0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 55780 characters]\n",
      "text/plain": [
       "<Figure size 864x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_series(heater_series)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Normalize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "avg, dev = heater_series.mean(), heater_series.std()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "heater_series = (heater_series - avg) / dev"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.lines.Line2D at 0x17d2fb23888>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 56036 characters]\n",
      "text/plain": [
       "<Figure size 864x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_series(heater_series)\n",
    "plt.axhline(0, linestyle='--', color='k', alpha=0.3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Take First Difference to Remove Trend"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "heater_series = heater_series.diff().dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.lines.Line2D at 0x17d2fbd7c88>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 57084 characters]\n",
      "text/plain": [
       "<Figure size 864x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_series(heater_series)\n",
    "plt.axhline(0, linestyle='--', color='k', alpha=0.3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Remove Increasing Volatility"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "annual_volatility = heater_series.groupby(heater_series.index.year).std()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Month\n",
       "2004    0.415481\n",
       "2005    0.474527\n",
       "2006    0.400148\n",
       "2007    0.359839\n",
       "2008    0.396182\n",
       "2009    0.499810\n",
       "2010    0.459566\n",
       "2011    0.443924\n",
       "2012    0.471104\n",
       "2013    0.503587\n",
       "2014    0.855743\n",
       "2015    0.569441\n",
       "2016    0.719843\n",
       "2017    0.830886\n",
       "2018    0.987221\n",
       "2019    0.892991\n",
       "2020    0.426657\n",
       "Name: heater, dtype: float64"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "annual_volatility"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "heater_annual_vol = heater_series.index.map(lambda d: annual_volatility.loc[d.year])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Float64Index([0.41548104705328814, 0.41548104705328814, 0.41548104705328814,\n",
       "              0.41548104705328814, 0.41548104705328814, 0.41548104705328814,\n",
       "              0.41548104705328814, 0.41548104705328814, 0.41548104705328814,\n",
       "              0.41548104705328814,\n",
       "              ...\n",
       "               0.8929910269067826,  0.8929910269067826,  0.8929910269067826,\n",
       "               0.8929910269067826, 0.42665652301411977, 0.42665652301411977,\n",
       "              0.42665652301411977, 0.42665652301411977, 0.42665652301411977,\n",
       "              0.42665652301411977],\n",
       "             dtype='float64', name='Month', length=197)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "heater_annual_vol"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "heater_series = heater_series / heater_annual_vol"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.lines.Line2D at 0x17d2feb0b48>"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 67352 characters]\n",
      "text/plain": [
       "<Figure size 864x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_series(heater_series)\n",
    "plt.axhline(0, linestyle='--', color='k', alpha=0.3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Remove Seasonality"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "month_avgs = heater_series.groupby(heater_series.index.month).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Month\n",
       "1    -0.428360\n",
       "2    -1.483129\n",
       "3    -1.063595\n",
       "4    -0.442926\n",
       "5    -0.121663\n",
       "6    -0.128425\n",
       "7    -0.107093\n",
       "8     0.075720\n",
       "9     0.544638\n",
       "10    1.613170\n",
       "11    1.183118\n",
       "12    0.491641\n",
       "dtype: float64"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "month_avgs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "heater_month_avg = heater_series.index.map(lambda d: month_avgs.loc[d.month])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Float64Index([  -1.483129277176665,  -1.0635953355687688, -0.44292637101368243,\n",
       "               -0.1216626157216479, -0.12842542811874375, -0.10709260508002617,\n",
       "               0.07572031336904017,   0.5446384049209764,   1.6131699911888606,\n",
       "                1.1831180641276993,\n",
       "              ...\n",
       "                0.5446384049209764,   1.6131699911888606,   1.1831180641276993,\n",
       "                0.4916411242912191,  -0.4283597874466263,   -1.483129277176665,\n",
       "               -1.0635953355687688, -0.44292637101368243,  -0.1216626157216479,\n",
       "              -0.12842542811874375],\n",
       "             dtype='float64', name='Month', length=197)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "heater_month_avg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "heater_series = heater_series - heater_month_avg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.lines.Line2D at 0x17d2fd39788>"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 64444 characters]\n",
      "text/plain": [
       "<Figure size 864x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_series(heater_series)\n",
    "plt.axhline(0, linestyle='--', color='k', alpha=0.3)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
