Initial version of SEC's 2020Q1 EDA
Václav Dekanovský
parent
6b3a7ab287
commit
bb2d8ec896
|
|
@ -0,0 +1,965 @@
|
|||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import pandas as pd\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"import os\n",
|
||||
"from collections import Counter\n",
|
||||
"from datetime import timedelta,datetime\n",
|
||||
"from calendar import monthrange\n",
|
||||
"import plotly.express as px\n",
|
||||
"pj = os.path.join"
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"pycharm": {
|
||||
"name": "#%%\n"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"# Loading the data"
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Folder with the unzipped data\n",
|
||||
"folder = r\"Data_Sec\\2020Q1\""
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"pycharm": {
|
||||
"name": "#%%\n"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"sub = pd.read_csv(pj(folder,\"sub.txt\"),sep=\"\\t\",dtype={\"cik\":str})"
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"pycharm": {
|
||||
"name": "#%%\n"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"sub.shape"
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"pycharm": {
|
||||
"name": "#%%\n"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"## Exploring the columns"
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def data_frame_stats(df):\n",
|
||||
" cnt = df.count()\n",
|
||||
" max_cnt = cnt.max()\n",
|
||||
" unique = df.nunique()\n",
|
||||
" \n",
|
||||
" return pd.DataFrame({\n",
|
||||
" \"notnulls\":cnt,\n",
|
||||
" \"notnulls%\":cnt/max_cnt,\n",
|
||||
" \"unique\":unique,\n",
|
||||
" \"unique%\":unique/max_cnt\n",
|
||||
" \n",
|
||||
" })\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"data_frame_stats(sub)"
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"pycharm": {
|
||||
"name": "#%%\n"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Let's have a look on one of the rows. Because it won't fit our line, we will transpose it to column\n",
|
||||
"sub.head(1).T"
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"pycharm": {
|
||||
"name": "#%%\n"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"## Date columns\n",
|
||||
"We see three date columns, let's transforma them into dates:"
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"sub[\"period\"] = pd.to_datetime(sub[\"period\"], format=\"%Y%m%d\")\n",
|
||||
"sub[\"filed\"] = pd.to_datetime(sub[\"filed\"], format=\"%Y%m%d\")\n",
|
||||
"sub[\"accepted\"] = pd.to_datetime(sub[\"accepted\"])"
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"pycharm": {
|
||||
"name": "#%%\n"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Let's check that they were correctly loaded\n",
|
||||
"sub.select_dtypes(\"datetime\").head()"
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"pycharm": {
|
||||
"name": "#%%\n"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"# Sumbission forms (type of the reports)\n",
|
||||
"The columns `\"form\"` contains information about the type of submissiong as described on the SEC page. https://www.sec.gov/forms. What are the most common submission forms?"
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# distribution of the different types of reports based on the analysis of the form field\n",
|
||||
"sub[\"form\"].value_counts().plot(kind=\"bar\")\n",
|
||||
"plt.title(\"Which fillings are reported the most\")\n",
|
||||
"plt.show()"
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"pycharm": {
|
||||
"name": "#%%\n"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from plotly.subplots import make_subplots\n",
|
||||
"import plotly.graph_objects as go\n",
|
||||
"\n",
|
||||
"fig = make_subplots(rows=1, cols=2, specs=[[{'type':'bar'}, {'type':'pie'}]],)\n",
|
||||
"\n",
|
||||
"# count different form types\n",
|
||||
"df = sub[\"form\"].value_counts().to_frame()\n",
|
||||
"fig.add_trace(\n",
|
||||
" go.Bar(y=df[\"form\"], x=df.index, text=df[\"form\"], name=\"Forms - Bar Chart\"),\n",
|
||||
" row=1, col=1,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# on the pie chart display only 5 most common categories and name the rest as \"Other\"\n",
|
||||
"df[\"label\"] = np.where(df[\"form\"].rank(ascending=False)<6, df.index, \"Other\")\n",
|
||||
"df = df.groupby(\"label\").sum()\n",
|
||||
"\n",
|
||||
"fig.add_trace(\n",
|
||||
" go.Pie(values=df[\"form\"], labels=df.index,textinfo='label+percent'),\n",
|
||||
" row=1, col=2\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"fig.show()"
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"pycharm": {
|
||||
"name": "#%%\n"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"We can see that the most common form is `8-K`. What does it contain and do we have some detailed data related to this submission type? "
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"## 8-K"
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# let's filter adsh, unique report identifier of the `8-K`s\n",
|
||||
"eight_k_filter = sub[sub[\"form\"]==\"8-K\"][[\"name\",\"adsh\"]]\n",
|
||||
"eight_k_filter.shape"
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"pycharm": {
|
||||
"name": "#%%\n"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# load the num.txt file containing detailed data\n",
|
||||
"num = pd.read_csv(pj(folder,\"num.txt\"),sep=\"\\t\")"
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"pycharm": {
|
||||
"name": "#%%\n"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# merge the file headers with the detailed data \n",
|
||||
"eight_k_nums = num.merge(eight_k_filter)\n",
|
||||
"eight_k_nums.shape"
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"pycharm": {
|
||||
"name": "#%%\n"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"Out of 7502 8-K records, there are only 2894 rows in the num file which contain the data values. But is it 2894 adsh? "
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"len(eight_k_nums[\"adsh\"].unique())"
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"pycharm": {
|
||||
"name": "#%%\n"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"No. From 7502,there are only 20 records in the dataset containing values. The rest is missing. Let's have a look what these values cover. "
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"eight_k_nums.loc[0].T"
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"pycharm": {
|
||||
"name": "#%%\n"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"On the url https://sec.report/Document/0001262976-20-000015/ you can see the details of this 8-K form. It's Item 8.01, other event, as desribec on the SEC page https://www.sec.gov/fast-answers/answersform8khtm.html. Other Events (The registrant can use this Item to report events that are not specifically called for by Form 8-K, that the registrant considers to be of importance to security holders.) This event can contain full or partial finance statement, like in this case of CIMPRESS company. The value \"Accounts payable\" appear on the page 5 of the report. "
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"# 10-K and 10-Q\n",
|
||||
"## Do companies report anual report in Q1?\n",
|
||||
"What interests us if not the most common form type? It should be the real financial statements. SEC explain the form types on this page: https://www.sec.gov/forms. When you study it a bit, you realize that we're concerned about: \n",
|
||||
"* `10-K` Anual report\n",
|
||||
"* `10-Q` Quarterly report\n",
|
||||
"and maybe\n",
|
||||
"* `20-F` Anual Reports of a foreign company\n",
|
||||
"* `40-F` Anual Reports of a foreign company\n",
|
||||
"\n",
|
||||
"Let's first have a look on US companies submitting `10-K` and `10-Q`. We have loaded the data for `2020Q1` and we expect that every company reports just a single submission, most likely `10-K` anual report for the end of the year 2019. "
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"tens = sub[sub[\"form\"].isin([\"10-Q\",\"10-K\"])]\n",
|
||||
"tens_counts = tens[\"form\"].value_counts().reset_index()\n",
|
||||
"f1 = px.bar(tens_counts,y=\"form\",x=\"index\",barmode='group', title=\"Number of forms in 2020Q1\")\n",
|
||||
"f1.show()"
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"pycharm": {
|
||||
"name": "#%%\n"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"## Does everyone report only one 10-K or 10-Q?"
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"As expected most of the submisison in 2020Q1 are anual records for 2019. However there are some quarterly records. We will look at those later. Let's first check if each company reports only one set of financial statements. "
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Is 10-Q reported once by each company or more times? \n",
|
||||
"sub[sub[\"form\"].isin([\"10-Q\",\"10-K\"])][\"cik\"].value_counts().value_counts().sort_index().plot(kind=\"bar\")\n",
|
||||
"plt.title(\"How many 10-Q reported each company\")\n",
|
||||
"plt.show()"
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"pycharm": {
|
||||
"name": "#%%\n"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"No. Some companies (even though only a few) are uploading more than one statetement. What could that mean?"
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"df = sub[sub[\"form\"].isin([\"10-Q\",\"10-K\"])][[\"form\",\"cik\"]].groupby([\"cik\",\"form\"]).size().to_frame('size').reset_index().groupby([\"form\",\"size\"]).size().to_frame('count').reset_index()\n",
|
||||
"\n",
|
||||
"f2 = px.bar(df[df[\"size\"]>1], x=\"size\", y=\"count\",barmode=\"group\", color=\"form\", title=\"Number of companies having more than 1 submission in 2020Q1\")\n",
|
||||
"f2.update_layout(\n",
|
||||
" xaxis = dict(\n",
|
||||
" tickmode = 'linear',\n",
|
||||
" tick0 = 1,\n",
|
||||
" dtick = 1\n",
|
||||
" )\n",
|
||||
")\n",
|
||||
"f2.show()"
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"pycharm": {
|
||||
"name": "#%%\n"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# attemps to make charts as subplots failed\n",
|
||||
"#fig = make_subplots(rows=2, cols=1,subplot_titles=(f1[\"layout\"].title.text, f2[\"layout\"].title.text))\n",
|
||||
"#fig.add_trace(f1['data'][0],row=1, col=1)\n",
|
||||
"#fig.add_trace(f2['data'][0],row=2, col=1)\n",
|
||||
"#fig.add_trace(f2['data'][1],row=2, col=1)\n",
|
||||
"#fig"
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"pycharm": {
|
||||
"name": "#%%\n"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# let's review which companies are having more than one submission by \n",
|
||||
"# exploring the [\"cik\"].value_counts a serie containing only count per each Central Index Key (cik)\n",
|
||||
"companies_reporting_more_tens = tens[\"cik\"].value_counts()[tens[\"cik\"].value_counts()>1]\n",
|
||||
"companies_reporting_more_tens.head()"
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"pycharm": {
|
||||
"name": "#%%\n"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# note that results of the value_counts is a serie\n",
|
||||
"type(tens[\"cik\"].value_counts()), type(companies_reporting_more_tenq)"
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"pycharm": {
|
||||
"name": "#%%\n"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# finally let's have a look on these companies. \n",
|
||||
"# there are too many column to see where they differ, but I would first check the dates\n",
|
||||
"# only two columns have date format - period and filled\n",
|
||||
"columns_to_see = [\"adsh\",\"cik\",\"name\",\"period\",\"fy\",\"fp\",\"filed\", \"form\"]\n",
|
||||
"tens[tens[\"cik\"].isin(companies_reporting_more_tens.index)].sort_values(by=[\"cik\",\"period\"])[columns_to_see]"
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"pycharm": {
|
||||
"name": "#%%\n"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"We can really see that mostly the filings are representing a different period (Note: period column contains - balance sheet date rounded to the nearest end of the month). To be sure, let's groupby accros both the cik and the period"
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# by applying max to the size, we would see if some company have more than one report in the same period\n",
|
||||
"# the maximum is two which mean that some companies are reporting more than one 10-Q for the same period, let's explore them\n",
|
||||
"tens.groupby([\"cik\",\"period\"]).size().max()"
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"pycharm": {
|
||||
"name": "#%%\n"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"Two companies have submited two reports in this quater, let's see how they differ?"
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# what are the companies and the periods? \n",
|
||||
"duplicates = tens.groupby([\"cik\",\"period\"]).size()[tens.groupby([\"cik\",\"period\"]).size()>1]\n",
|
||||
"duplicates"
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"pycharm": {
|
||||
"name": "#%%\n"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# groupby().size() will results in a serie with multiindex, which can be merged on the original dataframe after the index is reset\n",
|
||||
"duplicates = tens.merge(duplicates.reset_index(), on=[\"cik\",\"period\"]).sort_values(by=[\"cik\",\"period\"]).T\n",
|
||||
"duplicates"
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"pycharm": {
|
||||
"name": "#%%\n"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"So what are the differencesof those doubled lines? "
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# let's have a look at rows 2,3 which belongs to the company \"CHUN CAN CAPITAL GROUP\"\n",
|
||||
"duplicates[\"same23\"] = duplicates[2]==duplicates[3]\n",
|
||||
"duplicates.loc[duplicates[\"same23\"]==False,[3,2,\"same23\"]].dropna()"
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"pycharm": {
|
||||
"name": "#%%\n"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# let's have a look at rows 2,3 which belongs to the company \"ZOOMPASS HOLDINGS, INC.\"\"\n",
|
||||
"duplicates[\"same01\"] = duplicates[0]==duplicates[1]\n",
|
||||
"duplicates.loc[duplicates[\"same01\"]==False,[0,1,\"same01\"]].dropna()"
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"pycharm": {
|
||||
"name": "#%%\n"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"We see that they two records differe only in`filed` date and `accepted` time and adsh (which is unique for each records).The first example only differ in `accepted` Let's pick the latest filed and in case the filed is the same, the latest accepted data of the records."
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"### Filter out duplicates, only the latest updated recordb"
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# the shape of our file containing `10-K` and `10-Q` is 5212 rows and 36 columns\n",
|
||||
"tens.shape"
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"pycharm": {
|
||||
"name": "#%%\n"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"Because it's theoretically possible that a file which was accepted later has lower filled, the only way how to filter out the duplicated lines is:\n",
|
||||
"* sort data by the decisive fields - `filed` and `accepted`\n",
|
||||
"* group by partiotioning columns - `cik` and `period`\n",
|
||||
"* use `.cumcount()` to rank the rows 0,1,2 ...\n",
|
||||
"* filter only row which have this value `==0`\n",
|
||||
"\n",
|
||||
"Let's see an exmaple of the `cik` = `1191334`, we need to identify the index `3289` and filter out `3288`"
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# example cik 1191334 and period 2013-09-30\n",
|
||||
"tens[(tens[\"cik\"]==\"1191334\")&(tens[\"period\"]==\"20130930\")][[\"adsh\",\"cik\",\"period\",\"filed\",\"accepted\"]].sort_values(by=[\"filed\",\"accepted\"],ascending=True)"
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"pycharm": {
|
||||
"name": "#%%\n"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"tens[(tens[\"cik\"]==\"1191334\")&(tens[\"period\"]==\"20130930\")][[\"adsh\",\"cik\",\"period\",\"filed\",\"accepted\"]].sort_values(by=[\"filed\",\"accepted\"],ascending=False).groupby([\"cik\",\"period\"]).cumcount()"
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"pycharm": {
|
||||
"name": "#%%\n"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# for all the `10-K` and `10-Q` we reduce the number of rows from 5212 to 5210\n",
|
||||
"tens.loc[tens.sort_values(by=[\"filed\",\"accepted\"],ascending=False).groupby([\"cik\",\"period\"]).cumcount()==0]"
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"pycharm": {
|
||||
"name": "#%%\n"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"## Summision dates\n",
|
||||
"The 2020Q1 gets available in the beginning of April 2020. We would assume that it takes some time to process the data, so it won't probably contain financial statements from March, maybe not even Feb-2020. It could have data for Jan-2020, Dec-2019 and Nov-2019. What is the reality? "
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"tens[\"period\"].value_counts().reset_index().head()"
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"pycharm": {
|
||||
"name": "#%%\n"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"px.bar(tens[\"period\"].value_counts().reset_index(), y=\"period\", x=\"index\")\n",
|
||||
"fig.update_layout(yaxis_type=\"log\")"
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"pycharm": {
|
||||
"name": "#%%\n"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"Some of the companies have sent more than 1 submission. Let's filter only the latest period per `cik` to see what are the dates the companies have reported in 2020Q1. "
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"tens_latest_period = tens.loc[tens.sort_values(by=\"period\", ascending=False).groupby(\"cik\").cumcount()==0]"
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"pycharm": {
|
||||
"name": "#%%\n"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# using logarithim y-axis we can see the records in ranges of single-hits, tens, hundreds and thousands\n",
|
||||
"fig= px.bar(tens_latest_period[\"period\"].value_counts().reset_index(), y=\"period\", x=\"index\")\n",
|
||||
"fig.update_layout(yaxis_type=\"log\")"
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"pycharm": {
|
||||
"name": "#%%\n"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"Our dataset `tens_latest_period` contains only the latest submission `period` for each `cik`"
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# each cik contains maximum one submission in the set\n",
|
||||
"tens_latest_period.groupby([\"cik\",\"period\"]).size().max()"
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"pycharm": {
|
||||
"name": "#%%\n"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"## Average processing delay\n",
|
||||
"In this sesction we will review how much time elapse between the financial reporting period end and when the data are accepted into EDGAR system. The rules say that the companies should delivery quarterly report in 40-45 days and annual in 60-90 days depending on the size of the company (https://www.sec.gov/fast-answers/answers-form10khtm.html). But what is the reality."
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"pd.options.mode.chained_assignment = None # switch off the warnings\n",
|
||||
"tens_latest_period[\"delivery_time\"] = tens_latest_period[\"accepted\"] - tens_latest_period[\"period\"]\n",
|
||||
"tens_latest_period[\"delivery_time\"] = tens_latest_period[\"delivery_time\"].dt.days"
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"pycharm": {
|
||||
"name": "#%%\n"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"u = tens_latest_period[[\"form\",\"period\",\"accepted\",\"delivery_time\"]]"
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"pycharm": {
|
||||
"name": "#%%\n"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"tens_latest_period[\"delivery_time\"].agg([\"mean\",\"median\",\"min\",\"max\"])"
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"pycharm": {
|
||||
"name": "#%%\n"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"px.histogram(tens_latest_period, x=\"delivery_time\", color=\"form\")"
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"pycharm": {
|
||||
"name": "#%%\n"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"Most of the quarterly reports are delivered in 40 days and annual in sixty. Larger companies use their privilidge to extend the deadline to 90 days. In rare cases however the financial sheet arrives later."
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
}
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.7.4"
|
||||
},
|
||||
"toc": {
|
||||
"base_numbering": 1,
|
||||
"nav_menu": {},
|
||||
"number_sections": true,
|
||||
"sideBar": true,
|
||||
"skip_h1_title": false,
|
||||
"title_cell": "Table of Contents",
|
||||
"title_sidebar": "Contents",
|
||||
"toc_cell": false,
|
||||
"toc_position": {},
|
||||
"toc_section_display": true,
|
||||
"toc_window_display": true
|
||||
},
|
||||
"varInspector": {
|
||||
"cols": {
|
||||
"lenName": 16,
|
||||
"lenType": 16,
|
||||
"lenVar": 40
|
||||
},
|
||||
"kernels_config": {
|
||||
"python": {
|
||||
"delete_cmd_postfix": "",
|
||||
"delete_cmd_prefix": "del ",
|
||||
"library": "var_list.py",
|
||||
"varRefreshCmd": "print(var_dic_list())"
|
||||
},
|
||||
"r": {
|
||||
"delete_cmd_postfix": ") ",
|
||||
"delete_cmd_prefix": "rm(",
|
||||
"library": "var_list.r",
|
||||
"varRefreshCmd": "cat(var_dic_list()) "
|
||||
}
|
||||
},
|
||||
"types_to_exclude": [
|
||||
"module",
|
||||
"function",
|
||||
"builtin_function_or_method",
|
||||
"instance",
|
||||
"_Feature"
|
||||
],
|
||||
"window_display": false
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
Loading…
Reference in New Issue