{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\ritvik\\Anaconda3\\lib\\site-packages\\statsmodels\\tools\\_testing.py:19: FutureWarning: pandas.util.testing is deprecated. Use the functions in the public API at pandas.testing instead.\n",
      "  import pandas.util.testing as tm\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from datetime import datetime\n",
    "from datetime import timedelta\n",
    "from pandas.plotting import register_matplotlib_converters\n",
    "from statsmodels.tsa.stattools import acf, pacf\n",
    "from statsmodels.tsa.statespace.sarimax import SARIMAX\n",
    "register_matplotlib_converters()\n",
    "from time import time"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Catfish Sales Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parser(s):\n",
    "    return datetime.strptime(s, '%Y-%m-%d')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#read data\n",
    "catfish_sales = pd.read_csv('catfish.csv', parse_dates=[0], index_col=0, squeeze=True, date_parser=parser)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#infer the frequency of the data\n",
    "catfish_sales = catfish_sales.asfreq(pd.infer_freq(catfish_sales.index))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "start_date = datetime(1996,1,1)\n",
    "end_date = datetime(2000,1,1)\n",
    "lim_catfish_sales = catfish_sales[start_date:end_date]"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "Introduce an Anomaly"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "#At December 1 1998\n",
    "lim_catfish_sales[datetime(1998,12,1)] = 10000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 37372 characters]\n",
      "text/plain": [
       "<Figure size 720x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(10,4))\n",
    "plt.plot(lim_catfish_sales)\n",
    "plt.title('Catfish Sales in 1000s of Pounds', fontsize=20)\n",
    "plt.ylabel('Sales', fontsize=16)\n",
    "for year in range(start_date.year,end_date.year):\n",
    "    plt.axvline(pd.to_datetime(str(year)+'-01-01'), color='k', linestyle='--', alpha=0.2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Remove the trend"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "first_diff = lim_catfish_sales.diff()[1:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.lines.Line2D at 0x1f29c731908>"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 37028 characters]\n",
      "text/plain": [
       "<Figure size 720x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(10,4))\n",
    "plt.plot(first_diff)\n",
    "plt.title('Catfish Sales in 1000s of Pounds', fontsize=20)\n",
    "plt.ylabel('Sales', fontsize=16)\n",
    "for year in range(start_date.year,end_date.year):\n",
    "    plt.axvline(pd.to_datetime(str(year)+'-01-01'), color='k', linestyle='--', alpha=0.2)\n",
    "plt.axhline(0, color='k', linestyle='--', alpha=0.2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Get training and testing sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_end = datetime(1999,7,1)\n",
    "test_end = datetime(2000,1,1)\n",
    "\n",
    "test_data = lim_catfish_sales[train_end + timedelta(days=1):test_end]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Make Predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "my_order = (0,1,0)\n",
    "my_seasonal_order = (1, 0, 1, 12)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\ritvik\\Anaconda3\\lib\\site-packages\\statsmodels\\base\\model.py:512: ConvergenceWarning: Maximum Likelihood optimization failed to converge. Check mle_retvals\n",
      "  \"Check mle_retvals\", ConvergenceWarning)\n"
     ]
    }
   ],
   "source": [
    "rolling_predictions = test_data.copy()\n",
    "for train_end in test_data.index:\n",
    "    train_data = lim_catfish_sales[:train_end-timedelta(days=1)]\n",
    "    model = SARIMAX(train_data, order=my_order, seasonal_order=my_seasonal_order)\n",
    "    model_fit = model.fit()\n",
    "\n",
    "    pred = model_fit.forecast()\n",
    "    rolling_predictions[train_end] = pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "rolling_residuals = test_data - rolling_predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0, 0.5, 'Error')"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 27232 characters]\n",
      "text/plain": [
       "<Figure size 720x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(10,4))\n",
    "plt.plot(rolling_residuals)\n",
    "plt.axhline(0, linestyle='--', color='k')\n",
    "plt.title('Rolling Forecast Residuals from SARIMA Model', fontsize=20)\n",
    "plt.ylabel('Error', fontsize=16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 43524 characters]\n",
      "text/plain": [
       "<Figure size 720x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(10,4))\n",
    "\n",
    "plt.plot(lim_catfish_sales)\n",
    "plt.plot(rolling_predictions)\n",
    "\n",
    "plt.legend(('Data', 'Predictions'), fontsize=16)\n",
    "\n",
    "plt.title('Catfish Sales in 1000s of Pounds', fontsize=20)\n",
    "plt.ylabel('Production', fontsize=16)\n",
    "for year in range(start_date.year,end_date.year):\n",
    "    plt.axvline(pd.to_datetime(str(year)+'-01-01'), color='k', linestyle='--', alpha=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean Absolute Percent Error: 0.072\n"
     ]
    }
   ],
   "source": [
    "print('Mean Absolute Percent Error:', round(np.mean(abs(rolling_residuals/test_data)),4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Root Mean Squared Error: 2382.2706255447415\n"
     ]
    }
   ],
   "source": [
    "print('Root Mean Squared Error:', np.sqrt(np.mean(rolling_residuals**2)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Detecting the Anomaly"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Attempt 1: Deviation Method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 37372 characters]\n",
      "text/plain": [
       "<Figure size 720x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(10,4))\n",
    "plt.plot(lim_catfish_sales)\n",
    "plt.title('Catfish Sales in 1000s of Pounds', fontsize=20)\n",
    "plt.ylabel('Sales', fontsize=16)\n",
    "for year in range(start_date.year,end_date.year):\n",
    "    plt.axvline(pd.to_datetime(str(year)+'-01-01'), color='k', linestyle='--', alpha=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "rolling_deviations = pd.Series(dtype=float, index = lim_catfish_sales.index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "for date in rolling_deviations.index:\n",
    "    #get the window ending at this data point\n",
    "    window = lim_catfish_sales.loc[:date]\n",
    "    \n",
    "    #get the deviation within this window\n",
    "    rolling_deviations.loc[date] = window.std()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "#get the difference in deviation between one time point and the next\n",
    "diff_rolling_deviations = rolling_deviations.diff()\n",
    "diff_rolling_deviations = diff_rolling_deviations.dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 31848 characters]\n",
      "text/plain": [
       "<Figure size 720x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(10,4))\n",
    "plt.plot(diff_rolling_deviations)\n",
    "plt.title('Deviation Differences', fontsize=20)\n",
    "plt.ylabel('Sales', fontsize=16)\n",
    "for year in range(start_date.year,end_date.year):\n",
    "    plt.axvline(pd.to_datetime(str(year)+'-01-01'), color='k', linestyle='--', alpha=0.2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Attempt 2: Seasonal Method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "month_deviations = lim_catfish_sales.groupby(lambda d: d.month).std()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0, 0.5, 'Sales')"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 22980 characters]\n",
      "text/plain": [
       "<Figure size 720x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(10,4))\n",
    "plt.plot(month_deviations)\n",
    "plt.title('Deviation by Month', fontsize=20)\n",
    "plt.ylabel('Sales', fontsize=16)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## So, the anomaly occurs in a December"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "december_data = lim_catfish_sales[lim_catfish_sales.index.month == 12]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Date\n",
       "1996-12-01    16898\n",
       "1997-12-01    18278\n",
       "1998-12-01    10000\n",
       "1999-12-01    22372\n",
       "Name: Total, dtype: int64"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "december_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "min_dev = 9999999\n",
    "curr_anomaly = None\n",
    "for date in december_data.index:\n",
    "    other_data = december_data[december_data.index != date]\n",
    "    curr_dev = other_data.std()\n",
    "    if curr_dev < min_dev:\n",
    "        min_dev = curr_dev\n",
    "        curr_anomaly = date"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Timestamp('1998-12-01 00:00:00')"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "curr_anomaly"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# What to do about the anomaly?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Simple Idea: use mean of other months"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "adjusted_data = lim_catfish_sales.copy()\n",
    "adjusted_data.loc[curr_anomaly] = december_data[(december_data.index != curr_anomaly) & (december_data.index < test_data.index[0])].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.lines.Line2D at 0x26decdae708>"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 38060 characters]\n",
      "text/plain": [
       "<Figure size 720x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(10,4))\n",
    "plt.plot(lim_catfish_sales, color='firebrick', alpha=0.4)\n",
    "plt.plot(adjusted_data)\n",
    "plt.title('Catfish Sales in 1000s of Pounds', fontsize=20)\n",
    "plt.ylabel('Sales', fontsize=16)\n",
    "for year in range(start_date.year,end_date.year):\n",
    "    plt.axvline(pd.to_datetime(str(year)+'-01-01'), color='k', linestyle='--', alpha=0.2)\n",
    "plt.axvline(curr_anomaly, color='k', alpha=0.7)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Resulting Predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_end = datetime(1999,7,1)\n",
    "test_end = datetime(2000,1,1)\n",
    "\n",
    "test_data = adjusted_data[train_end + timedelta(days=1):test_end]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\ritvik\\Anaconda3\\lib\\site-packages\\statsmodels\\base\\model.py:512: ConvergenceWarning: Maximum Likelihood optimization failed to converge. Check mle_retvals\n",
      "  \"Check mle_retvals\", ConvergenceWarning)\n",
      "C:\\Users\\ritvik\\Anaconda3\\lib\\site-packages\\statsmodels\\base\\model.py:512: ConvergenceWarning: Maximum Likelihood optimization failed to converge. Check mle_retvals\n",
      "  \"Check mle_retvals\", ConvergenceWarning)\n",
      "C:\\Users\\ritvik\\Anaconda3\\lib\\site-packages\\statsmodels\\base\\model.py:512: ConvergenceWarning: Maximum Likelihood optimization failed to converge. Check mle_retvals\n",
      "  \"Check mle_retvals\", ConvergenceWarning)\n",
      "C:\\Users\\ritvik\\Anaconda3\\lib\\site-packages\\statsmodels\\base\\model.py:512: ConvergenceWarning: Maximum Likelihood optimization failed to converge. Check mle_retvals\n",
      "  \"Check mle_retvals\", ConvergenceWarning)\n"
     ]
    }
   ],
   "source": [
    "rolling_predictions = test_data.copy()\n",
    "for train_end in test_data.index:\n",
    "    train_data = adjusted_data[:train_end-timedelta(days=1)]\n",
    "    model = SARIMAX(train_data, order=my_order, seasonal_order=my_seasonal_order)\n",
    "    model_fit = model.fit()\n",
    "\n",
    "    pred = model_fit.forecast()\n",
    "    rolling_predictions[train_end] = pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "rolling_residuals = test_data - rolling_predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0, 0.5, 'Error')"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 29276 characters]\n",
      "text/plain": [
       "<Figure size 720x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(10,4))\n",
    "plt.plot(rolling_residuals)\n",
    "plt.axhline(0, linestyle='--', color='k')\n",
    "plt.title('Rolling Forecast Residuals from SARIMA Model', fontsize=20)\n",
    "plt.ylabel('Error', fontsize=16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 43480 characters]\n",
      "text/plain": [
       "<Figure size 720x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(10,4))\n",
    "\n",
    "plt.plot(lim_catfish_sales)\n",
    "plt.plot(rolling_predictions)\n",
    "\n",
    "plt.legend(('Data', 'Predictions'), fontsize=16)\n",
    "\n",
    "plt.title('Catfish Sales in 1000s of Pounds', fontsize=20)\n",
    "plt.ylabel('Production', fontsize=16)\n",
    "for year in range(start_date.year,end_date.year):\n",
    "    plt.axvline(pd.to_datetime(str(year)+'-01-01'), color='k', linestyle='--', alpha=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean Absolute Percent Error: 0.0459\n"
     ]
    }
   ],
   "source": [
    "print('Mean Absolute Percent Error:', round(np.mean(abs(rolling_residuals/test_data)),4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Root Mean Squared Error: 1321.1246722569683\n"
     ]
    }
   ],
   "source": [
    "print('Root Mean Squared Error:', np.sqrt(np.mean(rolling_residuals**2)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
