first commit
ran
commit
b71edb9b58
|
|
@ -0,0 +1,5 @@
|
|||
__pycache__/*
|
||||
fix_yahoo_finance/__pycache__/*
|
||||
dist
|
||||
fix_yahoo_finance.egg-info
|
||||
*.pyc
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
Change Log
|
||||
===========
|
||||
|
||||
0.0.1
|
||||
-------
|
||||
- Initial release (alpha)
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
# This file contains a list of people who have made contributions
|
||||
# to the public version of QTPyLib.
|
||||
|
||||
Ran Aroussi, Creator <https://github.com/ranaroussi>
|
||||
|
|
@ -0,0 +1,165 @@
|
|||
GNU LESSER GENERAL PUBLIC LICENSE
|
||||
Version 3, 29 June 2007
|
||||
|
||||
Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
|
||||
Everyone is permitted to copy and distribute verbatim copies
|
||||
of this license document, but changing it is not allowed.
|
||||
|
||||
|
||||
This version of the GNU Lesser General Public License incorporates
|
||||
the terms and conditions of version 3 of the GNU General Public
|
||||
License, supplemented by the additional permissions listed below.
|
||||
|
||||
0. Additional Definitions.
|
||||
|
||||
As used herein, "this License" refers to version 3 of the GNU Lesser
|
||||
General Public License, and the "GNU GPL" refers to version 3 of the GNU
|
||||
General Public License.
|
||||
|
||||
"The Library" refers to a covered work governed by this License,
|
||||
other than an Application or a Combined Work as defined below.
|
||||
|
||||
An "Application" is any work that makes use of an interface provided
|
||||
by the Library, but which is not otherwise based on the Library.
|
||||
Defining a subclass of a class defined by the Library is deemed a mode
|
||||
of using an interface provided by the Library.
|
||||
|
||||
A "Combined Work" is a work produced by combining or linking an
|
||||
Application with the Library. The particular version of the Library
|
||||
with which the Combined Work was made is also called the "Linked
|
||||
Version".
|
||||
|
||||
The "Minimal Corresponding Source" for a Combined Work means the
|
||||
Corresponding Source for the Combined Work, excluding any source code
|
||||
for portions of the Combined Work that, considered in isolation, are
|
||||
based on the Application, and not on the Linked Version.
|
||||
|
||||
The "Corresponding Application Code" for a Combined Work means the
|
||||
object code and/or source code for the Application, including any data
|
||||
and utility programs needed for reproducing the Combined Work from the
|
||||
Application, but excluding the System Libraries of the Combined Work.
|
||||
|
||||
1. Exception to Section 3 of the GNU GPL.
|
||||
|
||||
You may convey a covered work under sections 3 and 4 of this License
|
||||
without being bound by section 3 of the GNU GPL.
|
||||
|
||||
2. Conveying Modified Versions.
|
||||
|
||||
If you modify a copy of the Library, and, in your modifications, a
|
||||
facility refers to a function or data to be supplied by an Application
|
||||
that uses the facility (other than as an argument passed when the
|
||||
facility is invoked), then you may convey a copy of the modified
|
||||
version:
|
||||
|
||||
a) under this License, provided that you make a good faith effort to
|
||||
ensure that, in the event an Application does not supply the
|
||||
function or data, the facility still operates, and performs
|
||||
whatever part of its purpose remains meaningful, or
|
||||
|
||||
b) under the GNU GPL, with none of the additional permissions of
|
||||
this License applicable to that copy.
|
||||
|
||||
3. Object Code Incorporating Material from Library Header Files.
|
||||
|
||||
The object code form of an Application may incorporate material from
|
||||
a header file that is part of the Library. You may convey such object
|
||||
code under terms of your choice, provided that, if the incorporated
|
||||
material is not limited to numerical parameters, data structure
|
||||
layouts and accessors, or small macros, inline functions and templates
|
||||
(ten or fewer lines in length), you do both of the following:
|
||||
|
||||
a) Give prominent notice with each copy of the object code that the
|
||||
Library is used in it and that the Library and its use are
|
||||
covered by this License.
|
||||
|
||||
b) Accompany the object code with a copy of the GNU GPL and this license
|
||||
document.
|
||||
|
||||
4. Combined Works.
|
||||
|
||||
You may convey a Combined Work under terms of your choice that,
|
||||
taken together, effectively do not restrict modification of the
|
||||
portions of the Library contained in the Combined Work and reverse
|
||||
engineering for debugging such modifications, if you also do each of
|
||||
the following:
|
||||
|
||||
a) Give prominent notice with each copy of the Combined Work that
|
||||
the Library is used in it and that the Library and its use are
|
||||
covered by this License.
|
||||
|
||||
b) Accompany the Combined Work with a copy of the GNU GPL and this license
|
||||
document.
|
||||
|
||||
c) For a Combined Work that displays copyright notices during
|
||||
execution, include the copyright notice for the Library among
|
||||
these notices, as well as a reference directing the user to the
|
||||
copies of the GNU GPL and this license document.
|
||||
|
||||
d) Do one of the following:
|
||||
|
||||
0) Convey the Minimal Corresponding Source under the terms of this
|
||||
License, and the Corresponding Application Code in a form
|
||||
suitable for, and under terms that permit, the user to
|
||||
recombine or relink the Application with a modified version of
|
||||
the Linked Version to produce a modified Combined Work, in the
|
||||
manner specified by section 6 of the GNU GPL for conveying
|
||||
Corresponding Source.
|
||||
|
||||
1) Use a suitable shared library mechanism for linking with the
|
||||
Library. A suitable mechanism is one that (a) uses at run time
|
||||
a copy of the Library already present on the user's computer
|
||||
system, and (b) will operate properly with a modified version
|
||||
of the Library that is interface-compatible with the Linked
|
||||
Version.
|
||||
|
||||
e) Provide Installation Information, but only if you would otherwise
|
||||
be required to provide such information under section 6 of the
|
||||
GNU GPL, and only to the extent that such information is
|
||||
necessary to install and execute a modified version of the
|
||||
Combined Work produced by recombining or relinking the
|
||||
Application with a modified version of the Linked Version. (If
|
||||
you use option 4d0, the Installation Information must accompany
|
||||
the Minimal Corresponding Source and Corresponding Application
|
||||
Code. If you use option 4d1, you must provide the Installation
|
||||
Information in the manner specified by section 6 of the GNU GPL
|
||||
for conveying Corresponding Source.)
|
||||
|
||||
5. Combined Libraries.
|
||||
|
||||
You may place library facilities that are a work based on the
|
||||
Library side by side in a single library together with other library
|
||||
facilities that are not Applications and are not covered by this
|
||||
License, and convey such a combined library under terms of your
|
||||
choice, if you do both of the following:
|
||||
|
||||
a) Accompany the combined library with a copy of the same work based
|
||||
on the Library, uncombined with any other library facilities,
|
||||
conveyed under the terms of this License.
|
||||
|
||||
b) Give prominent notice with the combined library that part of it
|
||||
is a work based on the Library, and explaining where to find the
|
||||
accompanying uncombined form of the same work.
|
||||
|
||||
6. Revised Versions of the GNU Lesser General Public License.
|
||||
|
||||
The Free Software Foundation may publish revised and/or new versions
|
||||
of the GNU Lesser General Public License from time to time. Such new
|
||||
versions will be similar in spirit to the present version, but may
|
||||
differ in detail to address new problems or concerns.
|
||||
|
||||
Each version is given a distinguishing version number. If the
|
||||
Library as you received it specifies that a certain numbered version
|
||||
of the GNU Lesser General Public License "or any later version"
|
||||
applies to it, you have the option of following the terms and
|
||||
conditions either of that published version or of any later version
|
||||
published by the Free Software Foundation. If the Library as you
|
||||
received it does not specify a version number of the GNU Lesser
|
||||
General Public License, you may choose any version of the GNU Lesser
|
||||
General Public License ever published by the Free Software Foundation.
|
||||
|
||||
If the Library as you received it specifies that a proxy can decide
|
||||
whether future versions of the GNU Lesser General Public License shall
|
||||
apply, that proxy's public statement of acceptance of any version is
|
||||
permanent authorization for you to choose that version for the
|
||||
Library.
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
# Include the license file
|
||||
include LICENSE.txt
|
||||
|
||||
# Include the data files
|
||||
recursive-include data *
|
||||
|
|
@ -0,0 +1,125 @@
|
|||
Yahoo! Finance Fix for Pandas Datareader
|
||||
========================================
|
||||
|
||||
.. image:: https://img.shields.io/pypi/pyversions/fix-yahoo-finance.svg?maxAge=60
|
||||
:target: https://pypi.python.org/pypi/fix-yahoo-finance
|
||||
:alt: Python version
|
||||
|
||||
.. image:: https://img.shields.io/travis/ranaroussi/fix-yahoo-finance/master.svg?
|
||||
:target: https://travis-ci.org/ranaroussi/fix-yahoo-finance
|
||||
:alt: Travis-CI build status
|
||||
|
||||
.. image:: https://img.shields.io/pypi/v/fix-yahoo-finance.svg?maxAge=60
|
||||
:target: https://pypi.python.org/pypi/fix-yahoo-finance
|
||||
:alt: PyPi version
|
||||
|
||||
.. image:: https://img.shields.io/pypi/status/fix-yahoo-finance.svg?maxAge=60
|
||||
:target: https://pypi.python.org/pypi/fix-yahoo-finance
|
||||
:alt: PyPi status
|
||||
|
||||
.. image:: https://img.shields.io/github/stars/ranaroussi/fix-yahoo-finance.svg?style=social&label=Star&maxAge=60
|
||||
:target: https://github.com/ranaroussi/fix-yahoo-finance
|
||||
:alt: Star this repo
|
||||
|
||||
.. image:: https://img.shields.io/twitter/follow/aroussi.svg?style=social&label=Follow%20Me&maxAge=60
|
||||
:target: https://twitter.com/aroussi
|
||||
:alt: Follow me on twitter
|
||||
|
||||
\
|
||||
|
||||
**fix-yahoo-finance** offers a **temporary fix** to the problem
|
||||
by using `Selenium <http://www.seleniumhq.org>`_ to scrape the data
|
||||
from Yahoo! finance and return a Pandas DataFrame/Panel in the same
|
||||
format as the original ``get_data_yahoo()`` method.
|
||||
|
||||
`Yahoo! finance <https://ichart.finance.yahoo.com>`_ has decommissioned their historical data API,
|
||||
causing many programs that relied on the ``get_data_yahoo()``
|
||||
to stop working.
|
||||
|
||||
By basically "hijacking" ``pandas_datareader.data.get_data_yahoo()`` method,
|
||||
**fix-yahoo-finance**'s implantation is easy and only requires to import
|
||||
``fix_yahoo_finance`` into your code.
|
||||
|
||||
|
||||
`Changelog » <./CHANGELOG.rst>`__
|
||||
|
||||
-----
|
||||
|
||||
Quick Start
|
||||
===========
|
||||
|
||||
.. code:: python
|
||||
|
||||
from pandas_datareader import data as pdr
|
||||
import fix_yahoo_finance # <== that's all it takes :-)
|
||||
|
||||
# download dataframe
|
||||
data = pdr.get_data_yahoo("SPY", start="2017-01-01", end="2017-04-30")
|
||||
|
||||
# download Panel
|
||||
data = pdr.get_data_yahoo(["SPY", "IWM"], start="2017-01-01", end="2017-04-30")
|
||||
|
||||
|
||||
I've also added some options to make life easier :)
|
||||
|
||||
Below is the full list of acceptable parameters:
|
||||
|
||||
.. code:: python
|
||||
|
||||
data = pdr.get_data_yahoo(
|
||||
# tickers list (single tickers accepts a string as well)
|
||||
tickers = ["SPY", "IWM", "..."],
|
||||
|
||||
# start date (YYYY-MM-DD / datetime.datetime object)
|
||||
# defaults to 1950-01-01
|
||||
start = "2017-01-01",
|
||||
|
||||
# end date (YYYY-MM-DD / datetime.datetime object)
|
||||
# defaults to Today
|
||||
end = "2017-04-30",
|
||||
|
||||
# return a multi-index dataframe
|
||||
# (default is Panel, which is deprecated)
|
||||
as_panel = False,
|
||||
|
||||
# group by ticker (to access via data['SPY'])
|
||||
# default is 'column'
|
||||
group_by = 'ticker',
|
||||
|
||||
# adjust all OHLC automatically
|
||||
# (default is False)
|
||||
auto_adjust = True
|
||||
)
|
||||
|
||||
|
||||
Installation
|
||||
------------
|
||||
|
||||
Install ``fix_yahoo_finance`` using ``pip``:
|
||||
|
||||
.. code:: bash
|
||||
|
||||
$ pip install fix_yahoo_finance --upgrade --no-cache-dir
|
||||
|
||||
|
||||
Requirements
|
||||
------------
|
||||
|
||||
* `Python <https://www.python.org>`_ >=3.4
|
||||
* `Pandas <https://github.com/pydata/pandas>`_ (tested to work with >=0.18.1)
|
||||
* `Numpy <http://www.numpy.org>`_ >= 1.11.1
|
||||
* `Selenium <http://www.seleniumhq.org>`_ >= 3.0.2
|
||||
* `PyVirtualDisplay <https://github.com/ponty/pyvirtualdisplay>`_ >= 0.2.1 (optional, to)
|
||||
|
||||
Legal Stuff
|
||||
------------
|
||||
|
||||
**fix-yahoo-finance** is distributed under the **GNU Lesser General Public License v3.0**. See the `LICENSE.txt <./LICENSE.txt>`_ file in the release for details.
|
||||
|
||||
|
||||
P.S.
|
||||
------------
|
||||
|
||||
Please drop me an note with any feedback you have.
|
||||
|
||||
**Ran Aroussi**
|
||||
Binary file not shown.
|
|
@ -0,0 +1,165 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# Yahoo! Finance Fix for Pandas Datareader
|
||||
# https://github.com/ranaroussi/yahoo-finance-fix
|
||||
#
|
||||
# Copyright 2017 Ran Aroussi
|
||||
#
|
||||
# Licensed under the GNU Lesser General Public License, v3.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# https://www.gnu.org/licenses/lgpl-3.0.en.html
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
__version__ = "0.0.1"
|
||||
__author__ = "Ran Aroussi"
|
||||
__all__ = ['get_data_yahoo']
|
||||
|
||||
import datetime
|
||||
import numpy as np
|
||||
import os
|
||||
import pandas as pd
|
||||
import time
|
||||
import warnings
|
||||
|
||||
from selenium import webdriver
|
||||
|
||||
try:
|
||||
from pyvirtualdisplay import Display
|
||||
virt_display = True
|
||||
except ImportError:
|
||||
virt_display = False
|
||||
|
||||
|
||||
def get_data_yahoo(tickers, start=None, end=None, as_panel=True,
|
||||
group_by='column', auto_adjust=False, *args, **kwargs):
|
||||
|
||||
libdir = os.path.dirname(os.path.realpath(__file__))
|
||||
|
||||
# format start
|
||||
if start is None:
|
||||
start = int(time.mktime(time.strptime('1950-01-01', '%Y-%m-%d')))
|
||||
elif isinstance(start, datetime.datetime):
|
||||
start = int(time.mktime(start.timetuple()))
|
||||
else:
|
||||
start = int(time.mktime(time.strptime(str(start), '%Y-%m-%d')))
|
||||
|
||||
# format end
|
||||
if end is None:
|
||||
end = int(time.mktime(datetime.datetime.now().timetuple()))
|
||||
elif isinstance(end, datetime.datetime):
|
||||
end = int(time.mktime(end.timetuple()))
|
||||
else:
|
||||
end = int(time.mktime(time.strptime(str(end), '%Y-%m-%d')))
|
||||
|
||||
# iterval
|
||||
interval = kwargs["interval"] if "interval" in kwargs else "1d"
|
||||
|
||||
# start browser
|
||||
if virt_display:
|
||||
display = Display(visible=0, size=(800, 600))
|
||||
display.start()
|
||||
|
||||
chromeOptions = webdriver.ChromeOptions()
|
||||
prefs = {"download.default_directory": "/tmp"}
|
||||
chromeOptions.add_experimental_option("prefs", prefs)
|
||||
|
||||
# add adblock to make page load faster
|
||||
try:
|
||||
chromeOptions.add_extension(libdir + '/Adblock-Plus_v1.11.crx')
|
||||
except:
|
||||
pass
|
||||
|
||||
driver = webdriver.Chrome(chrome_options=chromeOptions)
|
||||
|
||||
dfs = {}
|
||||
|
||||
# download tickers
|
||||
tickers = tickers if isinstance(tickers, list) else [tickers]
|
||||
tickers = [x.upper() for x in tickers]
|
||||
|
||||
for ticker in tickers:
|
||||
url = "https://finance.yahoo.com/quote/%s/history"
|
||||
url += "?period1=%s&period2=%s&interval=%s&filter=history&frequency=%s"
|
||||
url = url % (ticker, start, end, interval, interval)
|
||||
|
||||
driver.get(url)
|
||||
|
||||
tries = 0
|
||||
while tries < 3:
|
||||
tries += 1
|
||||
link = driver.find_elements_by_css_selector(
|
||||
'a[download="' + ticker + '.csv"]')
|
||||
|
||||
if len(link) == 0:
|
||||
time.sleep(.1)
|
||||
else:
|
||||
tries = 3
|
||||
hist = link[0].get_attribute('href')
|
||||
driver.get(hist)
|
||||
|
||||
dfs[ticker] = pd.read_csv('/tmp/' + ticker + '.csv', parse_dates=['Date'],
|
||||
index_col=['Date']).replace('null', np.nan).dropna()
|
||||
dfs[ticker] = dfs[ticker].apply(pd.to_numeric)
|
||||
dfs[ticker]['Volume'] = dfs[ticker][
|
||||
'Volume'].fillna(0).astype(int)
|
||||
|
||||
if auto_adjust:
|
||||
ratio = dfs[ticker]["Close"] / dfs[ticker]["Adj Close"]
|
||||
dfs[ticker]["Adj Open"] = dfs[ticker]["Open"] / ratio
|
||||
dfs[ticker]["Adj High"] = dfs[ticker]["High"] / ratio
|
||||
dfs[ticker]["Adj Low"] = dfs[ticker]["Low"] / ratio
|
||||
dfs[ticker].drop(
|
||||
["Open", "High", "Low", "Close"], axis=1, inplace=True)
|
||||
dfs[ticker].rename(columns={
|
||||
"Adj Open": "Open", "Adj High": "High",
|
||||
"Adj Low": "Low", "Adj Close": "Close"
|
||||
}, inplace=True)
|
||||
dfs[ticker] = dfs[ticker][
|
||||
['Open', 'High', 'Low', 'Close', 'Volume']]
|
||||
|
||||
# os.remove('/tmp/' + ticker + '.csv')
|
||||
|
||||
# close/stop browser
|
||||
driver.close()
|
||||
driver.quit()
|
||||
|
||||
if virt_display:
|
||||
display.stop()
|
||||
|
||||
# create pandl (derecated)
|
||||
if as_panel:
|
||||
with warnings.catch_warnings():
|
||||
warnings.filterwarnings("ignore", category=DeprecationWarning)
|
||||
data = pd.Panel(dfs)
|
||||
if group_by == 'column':
|
||||
data = data.swapaxes(0, 2)
|
||||
|
||||
# create multiIndex df
|
||||
else:
|
||||
data = pd.concat(dfs.values(), axis=1, keys=dfs.keys())
|
||||
if group_by == 'column':
|
||||
data.columns = data.columns.swaplevel(0, 1)
|
||||
data.sort_index(level=0, axis=1, inplace=True)
|
||||
if auto_adjust:
|
||||
data = data[['Open', 'High', 'Low', 'Close', 'Volume']]
|
||||
else:
|
||||
data = data[['Open', 'High', 'Low',
|
||||
'Close', 'Adj Close', 'Volume']]
|
||||
|
||||
# return single df if only one ticker
|
||||
if len(tickers) == 1:
|
||||
data = dfs[tickers[0]]
|
||||
|
||||
return data
|
||||
|
||||
|
||||
import pandas_datareader
|
||||
pandas_datareader.data.get_data_yahoo = get_data_yahoo
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
pandas>=0.18.1
|
||||
numpy>=1.11.1
|
||||
selenium>=3.0.2
|
||||
pyvirtualdisplay>=0.2.1
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
[bdist_wheel]
|
||||
# This flag says that the code is written to work on both Python 2 and Python
|
||||
# 3. If at all possible, it is good practice to do this. If you cannot, you
|
||||
# will need to generate wheels for each Python version that you support.
|
||||
universal=1
|
||||
|
|
@ -0,0 +1,58 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: UTF-8 -*-
|
||||
#
|
||||
# Yahoo! Finance Fix for Pandas Datareader
|
||||
# https://github.com/ranaroussi/fix-yahoo-finance
|
||||
|
||||
"""Yahoo! Finance Fix for Pandas Datareader"""
|
||||
|
||||
from setuptools import setup, find_packages
|
||||
from codecs import open
|
||||
from os import path
|
||||
|
||||
here = path.abspath(path.dirname(__file__))
|
||||
|
||||
# Get the long description from the README file
|
||||
with open(path.join(here, 'README.rst'), encoding='utf-8') as f:
|
||||
long_description = f.read()
|
||||
|
||||
setup(
|
||||
name='fix-yahoo-finance',
|
||||
version="0.0.1",
|
||||
description='Temporary fix for Pandas Datareader\'s get_data_yahoo()',
|
||||
long_description=long_description,
|
||||
url='https://github.com/ranaroussi/fix-yahoo-finance',
|
||||
author='Ran Aroussi',
|
||||
author_email='ran@aroussi.com',
|
||||
license='LGPL',
|
||||
classifiers=[
|
||||
'License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)',
|
||||
'Development Status :: 3 - Alpha',
|
||||
|
||||
'Operating System :: OS Independent',
|
||||
'Intended Audience :: Developers',
|
||||
'Topic :: Office/Business :: Financial',
|
||||
'Topic :: Office/Business :: Financial :: Investment',
|
||||
'Topic :: Scientific/Engineering :: Interface Engine/Protocol Translator',
|
||||
'Topic :: Software Development :: Libraries',
|
||||
'Topic :: Software Development :: Libraries :: Python Modules',
|
||||
|
||||
'Programming Language :: Python :: 3.4',
|
||||
'Programming Language :: Python :: 3.5',
|
||||
'Programming Language :: Python :: 3.6',
|
||||
],
|
||||
platforms = ['any'],
|
||||
keywords='pandas, yahoo finance, pandas datareader',
|
||||
packages=find_packages(exclude=['contrib', 'docs', 'tests', 'examples']),
|
||||
install_requires=['pandas', 'numpy', 'selenium', 'pyvirtualdisplay'],
|
||||
entry_points={
|
||||
'console_scripts': [
|
||||
'sample=sample:main',
|
||||
],
|
||||
},
|
||||
|
||||
include_package_data=True,
|
||||
package_data={
|
||||
'static': 'yahoo_finance_fix/Adblock-Plus_v1.11.crx'
|
||||
}
|
||||
)
|
||||
Loading…
Reference in New Issue