commit b71edb9b58dd41d4b2ac32c5088d838087e85898 Author: ran Date: Sun May 21 13:21:55 2017 +0300 first commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b22b90b --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +__pycache__/* +fix_yahoo_finance/__pycache__/* +dist +fix_yahoo_finance.egg-info +*.pyc diff --git a/CHANGELOG.rst b/CHANGELOG.rst new file mode 100644 index 0000000..a222ae1 --- /dev/null +++ b/CHANGELOG.rst @@ -0,0 +1,6 @@ +Change Log +=========== + +0.0.1 +------- +- Initial release (alpha) \ No newline at end of file diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt new file mode 100644 index 0000000..5f2df64 --- /dev/null +++ b/CONTRIBUTORS.txt @@ -0,0 +1,4 @@ +# This file contains a list of people who have made contributions +# to the public version of QTPyLib. + +Ran Aroussi, Creator \ No newline at end of file diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..02bbb60 --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,165 @@ + GNU LESSER GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + + This version of the GNU Lesser General Public License incorporates +the terms and conditions of version 3 of the GNU General Public +License, supplemented by the additional permissions listed below. + + 0. Additional Definitions. + + As used herein, "this License" refers to version 3 of the GNU Lesser +General Public License, and the "GNU GPL" refers to version 3 of the GNU +General Public License. + + "The Library" refers to a covered work governed by this License, +other than an Application or a Combined Work as defined below. + + An "Application" is any work that makes use of an interface provided +by the Library, but which is not otherwise based on the Library. +Defining a subclass of a class defined by the Library is deemed a mode +of using an interface provided by the Library. + + A "Combined Work" is a work produced by combining or linking an +Application with the Library. The particular version of the Library +with which the Combined Work was made is also called the "Linked +Version". + + The "Minimal Corresponding Source" for a Combined Work means the +Corresponding Source for the Combined Work, excluding any source code +for portions of the Combined Work that, considered in isolation, are +based on the Application, and not on the Linked Version. + + The "Corresponding Application Code" for a Combined Work means the +object code and/or source code for the Application, including any data +and utility programs needed for reproducing the Combined Work from the +Application, but excluding the System Libraries of the Combined Work. + + 1. Exception to Section 3 of the GNU GPL. + + You may convey a covered work under sections 3 and 4 of this License +without being bound by section 3 of the GNU GPL. + + 2. Conveying Modified Versions. + + If you modify a copy of the Library, and, in your modifications, a +facility refers to a function or data to be supplied by an Application +that uses the facility (other than as an argument passed when the +facility is invoked), then you may convey a copy of the modified +version: + + a) under this License, provided that you make a good faith effort to + ensure that, in the event an Application does not supply the + function or data, the facility still operates, and performs + whatever part of its purpose remains meaningful, or + + b) under the GNU GPL, with none of the additional permissions of + this License applicable to that copy. + + 3. Object Code Incorporating Material from Library Header Files. + + The object code form of an Application may incorporate material from +a header file that is part of the Library. You may convey such object +code under terms of your choice, provided that, if the incorporated +material is not limited to numerical parameters, data structure +layouts and accessors, or small macros, inline functions and templates +(ten or fewer lines in length), you do both of the following: + + a) Give prominent notice with each copy of the object code that the + Library is used in it and that the Library and its use are + covered by this License. + + b) Accompany the object code with a copy of the GNU GPL and this license + document. + + 4. Combined Works. + + You may convey a Combined Work under terms of your choice that, +taken together, effectively do not restrict modification of the +portions of the Library contained in the Combined Work and reverse +engineering for debugging such modifications, if you also do each of +the following: + + a) Give prominent notice with each copy of the Combined Work that + the Library is used in it and that the Library and its use are + covered by this License. + + b) Accompany the Combined Work with a copy of the GNU GPL and this license + document. + + c) For a Combined Work that displays copyright notices during + execution, include the copyright notice for the Library among + these notices, as well as a reference directing the user to the + copies of the GNU GPL and this license document. + + d) Do one of the following: + + 0) Convey the Minimal Corresponding Source under the terms of this + License, and the Corresponding Application Code in a form + suitable for, and under terms that permit, the user to + recombine or relink the Application with a modified version of + the Linked Version to produce a modified Combined Work, in the + manner specified by section 6 of the GNU GPL for conveying + Corresponding Source. + + 1) Use a suitable shared library mechanism for linking with the + Library. A suitable mechanism is one that (a) uses at run time + a copy of the Library already present on the user's computer + system, and (b) will operate properly with a modified version + of the Library that is interface-compatible with the Linked + Version. + + e) Provide Installation Information, but only if you would otherwise + be required to provide such information under section 6 of the + GNU GPL, and only to the extent that such information is + necessary to install and execute a modified version of the + Combined Work produced by recombining or relinking the + Application with a modified version of the Linked Version. (If + you use option 4d0, the Installation Information must accompany + the Minimal Corresponding Source and Corresponding Application + Code. If you use option 4d1, you must provide the Installation + Information in the manner specified by section 6 of the GNU GPL + for conveying Corresponding Source.) + + 5. Combined Libraries. + + You may place library facilities that are a work based on the +Library side by side in a single library together with other library +facilities that are not Applications and are not covered by this +License, and convey such a combined library under terms of your +choice, if you do both of the following: + + a) Accompany the combined library with a copy of the same work based + on the Library, uncombined with any other library facilities, + conveyed under the terms of this License. + + b) Give prominent notice with the combined library that part of it + is a work based on the Library, and explaining where to find the + accompanying uncombined form of the same work. + + 6. Revised Versions of the GNU Lesser General Public License. + + The Free Software Foundation may publish revised and/or new versions +of the GNU Lesser General Public License from time to time. Such new +versions will be similar in spirit to the present version, but may +differ in detail to address new problems or concerns. + + Each version is given a distinguishing version number. If the +Library as you received it specifies that a certain numbered version +of the GNU Lesser General Public License "or any later version" +applies to it, you have the option of following the terms and +conditions either of that published version or of any later version +published by the Free Software Foundation. If the Library as you +received it does not specify a version number of the GNU Lesser +General Public License, you may choose any version of the GNU Lesser +General Public License ever published by the Free Software Foundation. + + If the Library as you received it specifies that a proxy can decide +whether future versions of the GNU Lesser General Public License shall +apply, that proxy's public statement of acceptance of any version is +permanent authorization for you to choose that version for the +Library. \ No newline at end of file diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..75af3b0 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,5 @@ +# Include the license file +include LICENSE.txt + +# Include the data files +recursive-include data * \ No newline at end of file diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..b2f481b --- /dev/null +++ b/README.rst @@ -0,0 +1,125 @@ +Yahoo! Finance Fix for Pandas Datareader +======================================== + +.. image:: https://img.shields.io/pypi/pyversions/fix-yahoo-finance.svg?maxAge=60 + :target: https://pypi.python.org/pypi/fix-yahoo-finance + :alt: Python version + +.. image:: https://img.shields.io/travis/ranaroussi/fix-yahoo-finance/master.svg? + :target: https://travis-ci.org/ranaroussi/fix-yahoo-finance + :alt: Travis-CI build status + +.. image:: https://img.shields.io/pypi/v/fix-yahoo-finance.svg?maxAge=60 + :target: https://pypi.python.org/pypi/fix-yahoo-finance + :alt: PyPi version + +.. image:: https://img.shields.io/pypi/status/fix-yahoo-finance.svg?maxAge=60 + :target: https://pypi.python.org/pypi/fix-yahoo-finance + :alt: PyPi status + +.. image:: https://img.shields.io/github/stars/ranaroussi/fix-yahoo-finance.svg?style=social&label=Star&maxAge=60 + :target: https://github.com/ranaroussi/fix-yahoo-finance + :alt: Star this repo + +.. image:: https://img.shields.io/twitter/follow/aroussi.svg?style=social&label=Follow%20Me&maxAge=60 + :target: https://twitter.com/aroussi + :alt: Follow me on twitter + +\ + +**fix-yahoo-finance** offers a **temporary fix** to the problem +by using `Selenium `_ to scrape the data +from Yahoo! finance and return a Pandas DataFrame/Panel in the same +format as the original ``get_data_yahoo()`` method. + +`Yahoo! finance `_ has decommissioned their historical data API, +causing many programs that relied on the ``get_data_yahoo()`` +to stop working. + +By basically "hijacking" ``pandas_datareader.data.get_data_yahoo()`` method, +**fix-yahoo-finance**'s implantation is easy and only requires to import +``fix_yahoo_finance`` into your code. + + +`Changelog ยป <./CHANGELOG.rst>`__ + +----- + +Quick Start +=========== + +.. code:: python + + from pandas_datareader import data as pdr + import fix_yahoo_finance # <== that's all it takes :-) + + # download dataframe + data = pdr.get_data_yahoo("SPY", start="2017-01-01", end="2017-04-30") + + # download Panel + data = pdr.get_data_yahoo(["SPY", "IWM"], start="2017-01-01", end="2017-04-30") + + +I've also added some options to make life easier :) + +Below is the full list of acceptable parameters: + +.. code:: python + + data = pdr.get_data_yahoo( + # tickers list (single tickers accepts a string as well) + tickers = ["SPY", "IWM", "..."], + + # start date (YYYY-MM-DD / datetime.datetime object) + # defaults to 1950-01-01 + start = "2017-01-01", + + # end date (YYYY-MM-DD / datetime.datetime object) + # defaults to Today + end = "2017-04-30", + + # return a multi-index dataframe + # (default is Panel, which is deprecated) + as_panel = False, + + # group by ticker (to access via data['SPY']) + # default is 'column' + group_by = 'ticker', + + # adjust all OHLC automatically + # (default is False) + auto_adjust = True + ) + + +Installation +------------ + +Install ``fix_yahoo_finance`` using ``pip``: + +.. code:: bash + + $ pip install fix_yahoo_finance --upgrade --no-cache-dir + + +Requirements +------------ + +* `Python `_ >=3.4 +* `Pandas `_ (tested to work with >=0.18.1) +* `Numpy `_ >= 1.11.1 +* `Selenium `_ >= 3.0.2 +* `PyVirtualDisplay `_ >= 0.2.1 (optional, to) + +Legal Stuff +------------ + +**fix-yahoo-finance** is distributed under the **GNU Lesser General Public License v3.0**. See the `LICENSE.txt <./LICENSE.txt>`_ file in the release for details. + + +P.S. +------------ + +Please drop me an note with any feedback you have. + +**Ran Aroussi** diff --git a/fix_yahoo_finance/Adblock-Plus_v1.11.crx b/fix_yahoo_finance/Adblock-Plus_v1.11.crx new file mode 100644 index 0000000..4200c8f Binary files /dev/null and b/fix_yahoo_finance/Adblock-Plus_v1.11.crx differ diff --git a/fix_yahoo_finance/__init__.py b/fix_yahoo_finance/__init__.py new file mode 100644 index 0000000..3724a8a --- /dev/null +++ b/fix_yahoo_finance/__init__.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Yahoo! Finance Fix for Pandas Datareader +# https://github.com/ranaroussi/yahoo-finance-fix +# +# Copyright 2017 Ran Aroussi +# +# Licensed under the GNU Lesser General Public License, v3.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.gnu.org/licenses/lgpl-3.0.en.html +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__version__ = "0.0.1" +__author__ = "Ran Aroussi" +__all__ = ['get_data_yahoo'] + +import datetime +import numpy as np +import os +import pandas as pd +import time +import warnings + +from selenium import webdriver + +try: + from pyvirtualdisplay import Display + virt_display = True +except ImportError: + virt_display = False + + +def get_data_yahoo(tickers, start=None, end=None, as_panel=True, + group_by='column', auto_adjust=False, *args, **kwargs): + + libdir = os.path.dirname(os.path.realpath(__file__)) + + # format start + if start is None: + start = int(time.mktime(time.strptime('1950-01-01', '%Y-%m-%d'))) + elif isinstance(start, datetime.datetime): + start = int(time.mktime(start.timetuple())) + else: + start = int(time.mktime(time.strptime(str(start), '%Y-%m-%d'))) + + # format end + if end is None: + end = int(time.mktime(datetime.datetime.now().timetuple())) + elif isinstance(end, datetime.datetime): + end = int(time.mktime(end.timetuple())) + else: + end = int(time.mktime(time.strptime(str(end), '%Y-%m-%d'))) + + # iterval + interval = kwargs["interval"] if "interval" in kwargs else "1d" + + # start browser + if virt_display: + display = Display(visible=0, size=(800, 600)) + display.start() + + chromeOptions = webdriver.ChromeOptions() + prefs = {"download.default_directory": "/tmp"} + chromeOptions.add_experimental_option("prefs", prefs) + + # add adblock to make page load faster + try: + chromeOptions.add_extension(libdir + '/Adblock-Plus_v1.11.crx') + except: + pass + + driver = webdriver.Chrome(chrome_options=chromeOptions) + + dfs = {} + + # download tickers + tickers = tickers if isinstance(tickers, list) else [tickers] + tickers = [x.upper() for x in tickers] + + for ticker in tickers: + url = "https://finance.yahoo.com/quote/%s/history" + url += "?period1=%s&period2=%s&interval=%s&filter=history&frequency=%s" + url = url % (ticker, start, end, interval, interval) + + driver.get(url) + + tries = 0 + while tries < 3: + tries += 1 + link = driver.find_elements_by_css_selector( + 'a[download="' + ticker + '.csv"]') + + if len(link) == 0: + time.sleep(.1) + else: + tries = 3 + hist = link[0].get_attribute('href') + driver.get(hist) + + dfs[ticker] = pd.read_csv('/tmp/' + ticker + '.csv', parse_dates=['Date'], + index_col=['Date']).replace('null', np.nan).dropna() + dfs[ticker] = dfs[ticker].apply(pd.to_numeric) + dfs[ticker]['Volume'] = dfs[ticker][ + 'Volume'].fillna(0).astype(int) + + if auto_adjust: + ratio = dfs[ticker]["Close"] / dfs[ticker]["Adj Close"] + dfs[ticker]["Adj Open"] = dfs[ticker]["Open"] / ratio + dfs[ticker]["Adj High"] = dfs[ticker]["High"] / ratio + dfs[ticker]["Adj Low"] = dfs[ticker]["Low"] / ratio + dfs[ticker].drop( + ["Open", "High", "Low", "Close"], axis=1, inplace=True) + dfs[ticker].rename(columns={ + "Adj Open": "Open", "Adj High": "High", + "Adj Low": "Low", "Adj Close": "Close" + }, inplace=True) + dfs[ticker] = dfs[ticker][ + ['Open', 'High', 'Low', 'Close', 'Volume']] + + # os.remove('/tmp/' + ticker + '.csv') + + # close/stop browser + driver.close() + driver.quit() + + if virt_display: + display.stop() + + # create pandl (derecated) + if as_panel: + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=DeprecationWarning) + data = pd.Panel(dfs) + if group_by == 'column': + data = data.swapaxes(0, 2) + + # create multiIndex df + else: + data = pd.concat(dfs.values(), axis=1, keys=dfs.keys()) + if group_by == 'column': + data.columns = data.columns.swaplevel(0, 1) + data.sort_index(level=0, axis=1, inplace=True) + if auto_adjust: + data = data[['Open', 'High', 'Low', 'Close', 'Volume']] + else: + data = data[['Open', 'High', 'Low', + 'Close', 'Adj Close', 'Volume']] + + # return single df if only one ticker + if len(tickers) == 1: + data = dfs[tickers[0]] + + return data + + +import pandas_datareader +pandas_datareader.data.get_data_yahoo = get_data_yahoo diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..5a6500d --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +pandas>=0.18.1 +numpy>=1.11.1 +selenium>=3.0.2 +pyvirtualdisplay>=0.2.1 \ No newline at end of file diff --git a/setup.cgf b/setup.cgf new file mode 100644 index 0000000..c34b498 --- /dev/null +++ b/setup.cgf @@ -0,0 +1,5 @@ +[bdist_wheel] +# This flag says that the code is written to work on both Python 2 and Python +# 3. If at all possible, it is good practice to do this. If you cannot, you +# will need to generate wheels for each Python version that you support. +universal=1 \ No newline at end of file diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..f178ad7 --- /dev/null +++ b/setup.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +# +# Yahoo! Finance Fix for Pandas Datareader +# https://github.com/ranaroussi/fix-yahoo-finance + +"""Yahoo! Finance Fix for Pandas Datareader""" + +from setuptools import setup, find_packages +from codecs import open +from os import path + +here = path.abspath(path.dirname(__file__)) + +# Get the long description from the README file +with open(path.join(here, 'README.rst'), encoding='utf-8') as f: + long_description = f.read() + +setup( + name='fix-yahoo-finance', + version="0.0.1", + description='Temporary fix for Pandas Datareader\'s get_data_yahoo()', + long_description=long_description, + url='https://github.com/ranaroussi/fix-yahoo-finance', + author='Ran Aroussi', + author_email='ran@aroussi.com', + license='LGPL', + classifiers=[ + 'License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)', + 'Development Status :: 3 - Alpha', + + 'Operating System :: OS Independent', + 'Intended Audience :: Developers', + 'Topic :: Office/Business :: Financial', + 'Topic :: Office/Business :: Financial :: Investment', + 'Topic :: Scientific/Engineering :: Interface Engine/Protocol Translator', + 'Topic :: Software Development :: Libraries', + 'Topic :: Software Development :: Libraries :: Python Modules', + + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + ], + platforms = ['any'], + keywords='pandas, yahoo finance, pandas datareader', + packages=find_packages(exclude=['contrib', 'docs', 'tests', 'examples']), + install_requires=['pandas', 'numpy', 'selenium', 'pyvirtualdisplay'], + entry_points={ + 'console_scripts': [ + 'sample=sample:main', + ], + }, + + include_package_data=True, + package_data={ + 'static': 'yahoo_finance_fix/Adblock-Plus_v1.11.crx' + } +) \ No newline at end of file