Compare commits
49 Commits
main
...
release/0.
Author | SHA1 | Date |
---|---|---|
ValueRaider | dfffa6a551 | |
ValueRaider | 787b89c269 | |
ValueRaider | 882f8b3367 | |
ValueRaider | 338a94a8f3 | |
ValueRaider | e108a543fa | |
ValueRaider | 071c3937b5 | |
ValueRaider | a279d06810 | |
ValueRaider | 80db9dfe3c | |
ValueRaider | 6bb23e05c2 | |
ValueRaider | edf2f69b62 | |
ValueRaider | ce4c2e457d | |
ValueRaider | 4fc15251a0 | |
ValueRaider | c4600d6bd9 | |
ValueRaider | 14a839582d | |
ValueRaider | 3ae0434567 | |
ValueRaider | f24dab2f26 | |
ValueRaider | b47adf0a90 | |
ValueRaider | 3537ec3e4b | |
ValueRaider | 6a306b0353 | |
ValueRaider | 6e3282badb | |
ValueRaider | 829683ca02 | |
ValueRaider | 3011cb324d | |
ValueRaider | 366cfc0795 | |
ValueRaider | cbd4b924b8 | |
ValueRaider | 56759e3f3c | |
ValueRaider | c193428b38 | |
ValueRaider | a625d9e9c5 | |
ValueRaider | 36e80a73f7 | |
ValueRaider | cdae1cf226 | |
ValueRaider | bca569318e | |
ValueRaider | d11cd85a66 | |
ValueRaider | 2d32a6e204 | |
ValueRaider | bad6456a44 | |
ValueRaider | 1687ae66ab | |
ValueRaider | ddc34348d9 | |
ValueRaider | 1d74cfeb19 | |
ValueRaider | 1589d07b56 | |
ValueRaider | d261237320 | |
ValueRaider | 66af3080dd | |
ValueRaider | 9d396b9559 | |
ValueRaider | 23b6ad12c1 | |
ValueRaider | 22131e9fc7 | |
ValueRaider | e99e61f95a | |
ValueRaider | a3fe95ea27 | |
ValueRaider | 000cb70bcb | |
ValueRaider | c8d9d06e75 | |
ValueRaider | a5e07a0375 | |
ValueRaider | a0a12bcf4c | |
Jose Manuel | 42e5751705 |
|
@ -1,6 +1,60 @@
|
|||
Change Log
|
||||
===========
|
||||
|
||||
0.1.96
|
||||
------
|
||||
- Fix info[] not caching #1258
|
||||
|
||||
0.1.95
|
||||
------
|
||||
- Fix info[] bug #1257
|
||||
|
||||
0.1.94
|
||||
------
|
||||
- Fix delisted ticker info[]
|
||||
|
||||
0.1.93
|
||||
------
|
||||
- Fix Ticker.shares
|
||||
|
||||
0.1.92
|
||||
------
|
||||
- Decrypt new Yahoo encryption #1255
|
||||
|
||||
0.1.90
|
||||
------
|
||||
- Restore lxml req, increase min ver #1237
|
||||
|
||||
0.1.89
|
||||
------
|
||||
- Remove unused incompatible dependency #1222
|
||||
- Fix minimum Pandas version #1230
|
||||
|
||||
0.1.87
|
||||
------
|
||||
- Fix localizing midnight when non-existent (DST) #1176
|
||||
- Fix thread deadlock in bpython #1163
|
||||
|
||||
0.1.86
|
||||
------
|
||||
- Fix 'trailingPegRatio' #1141
|
||||
- Improve handling delisted tickers #1142
|
||||
- Fix corrupt tkr-tz-csv halting code #1162
|
||||
- Change default start to 1900-01-01 #1170
|
||||
|
||||
0.1.85
|
||||
------
|
||||
- Fix info['log_url'] #1062
|
||||
- Fix handling delisted ticker #1137
|
||||
|
||||
0.1.84
|
||||
------
|
||||
- Make tz-cache thread-safe
|
||||
|
||||
0.1.83
|
||||
------
|
||||
- Reduce spam-effect of tz-fetch
|
||||
|
||||
0.1.81
|
||||
------
|
||||
- Fix unhandled tz-cache exception #1107
|
||||
|
|
12
README.md
12
README.md
|
@ -274,12 +274,12 @@ To install `yfinance` using `conda`, see
|
|||
### Requirements
|
||||
|
||||
- [Python](https://www.python.org) \>= 2.7, 3.4+
|
||||
- [Pandas](https://github.com/pydata/pandas) (tested to work with
|
||||
\>=0.23.1)
|
||||
- [Numpy](http://www.numpy.org) \>= 1.11.1
|
||||
- [requests](http://docs.python-requests.org/en/master/) \>= 2.14.2
|
||||
- [lxml](https://pypi.org/project/lxml/) \>= 4.5.1
|
||||
- [appdirs](https://pypi.org/project/appdirs) \>=1.4.4
|
||||
- [Pandas](https://github.com/pydata/pandas) \>= 1.3.0
|
||||
- [Numpy](http://www.numpy.org) \>= 1.16.5
|
||||
- [requests](http://docs.python-requests.org/en/master/) \>= 2.26
|
||||
- [lxml](https://pypi.org/project/lxml/) \>= 4.9.1
|
||||
- [appdirs](https://pypi.org/project/appdirs) \>= 1.4.4
|
||||
- [cryptography](https://pypi.org/project/cryptography) \>=3.3.2
|
||||
|
||||
### Optional (if you want to use `pandas_datareader`)
|
||||
|
||||
|
|
16
meta.yaml
16
meta.yaml
|
@ -1,5 +1,5 @@
|
|||
{% set name = "yfinance" %}
|
||||
{% set version = "0.1.58" %}
|
||||
{% set version = "0.1.96" %}
|
||||
|
||||
package:
|
||||
name: "{{ name|lower }}"
|
||||
|
@ -16,22 +16,24 @@ build:
|
|||
|
||||
requirements:
|
||||
host:
|
||||
- pandas >=0.24.0
|
||||
- pandas >=1.3.0
|
||||
- numpy >=1.16.5
|
||||
- requests >=2.21
|
||||
- requests >=2.26
|
||||
- multitasking >=0.0.7
|
||||
- lxml >=4.5.1
|
||||
- lxml >=4.9.1
|
||||
- appdirs >= 1.4.4
|
||||
- cryptography >= 3.3.2
|
||||
- pip
|
||||
- python
|
||||
|
||||
run:
|
||||
- pandas >=0.24.0
|
||||
- pandas >=1.3.0
|
||||
- numpy >=1.16.5
|
||||
- requests >=2.21
|
||||
- requests >=2.26
|
||||
- multitasking >=0.0.7
|
||||
- lxml >=4.5.1
|
||||
- lxml >=4.9.1
|
||||
- appdirs >= 1.4.4
|
||||
- cryptography >= 3.3.2
|
||||
- python
|
||||
|
||||
test:
|
||||
|
|
|
@ -1,6 +1,7 @@
|
|||
pandas>=0.24.0
|
||||
pandas>=1.3.0
|
||||
numpy>=1.16.5
|
||||
requests>=2.26
|
||||
multitasking>=0.0.7
|
||||
lxml>=4.5.1
|
||||
lxml>=4.9.1
|
||||
appdirs>=1.4.4
|
||||
cryptography>=3.3.2
|
||||
|
|
5
setup.py
5
setup.py
|
@ -61,9 +61,10 @@ setup(
|
|||
platforms=['any'],
|
||||
keywords='pandas, yahoo finance, pandas datareader',
|
||||
packages=find_packages(exclude=['contrib', 'docs', 'tests', 'examples']),
|
||||
install_requires=['pandas>=0.24.0', 'numpy>=1.15',
|
||||
install_requires=['pandas>=1.3.0', 'numpy>=1.16.5',
|
||||
'requests>=2.26', 'multitasking>=0.0.7',
|
||||
'lxml>=4.5.1', 'appdirs>=1.4.4'],
|
||||
'lxml>=4.9.1', 'appdirs>=1.4.4',
|
||||
'cryptography>=3.3.2'],
|
||||
entry_points={
|
||||
'console_scripts': [
|
||||
'sample=sample:main',
|
||||
|
|
|
@ -15,21 +15,90 @@ Sanity check for most common library uses all working
|
|||
|
||||
import yfinance as yf
|
||||
import unittest
|
||||
import datetime
|
||||
|
||||
symbols = ['MSFT', 'IWO', 'VFINX', '^GSPC', 'BTC-USD']
|
||||
tickers = [yf.Ticker(symbol) for symbol in symbols]
|
||||
session = None
|
||||
import requests_cache ; session = requests_cache.CachedSession("yfinance.cache", expire_after=24*60*60)
|
||||
|
||||
# Good symbols = all attributes should work
|
||||
good_symbols = ['MSFT', 'IWO', 'VFINX', '^GSPC', 'BTC-USD']
|
||||
good_tickers = [yf.Ticker(symbol, session=session) for symbol in good_symbols]
|
||||
# Dodgy symbols = Yahoo data incomplete, so exclude from some tests
|
||||
dodgy_symbols = ["G7W.DU"]
|
||||
dodgy_tickers = [yf.Ticker(symbol, session=session) for symbol in dodgy_symbols]
|
||||
symbols = good_symbols + dodgy_symbols
|
||||
tickers = good_tickers + dodgy_tickers
|
||||
# Delisted = no data expected but yfinance shouldn't raise exception
|
||||
delisted_symbols = ["BRK.B", "SDLP"]
|
||||
delisted_tickers = [yf.Ticker(symbol, session=session) for symbol in delisted_symbols]
|
||||
|
||||
|
||||
class TestTicker(unittest.TestCase):
|
||||
def setUp(self):
|
||||
d_today = datetime.date.today()
|
||||
d_today -= datetime.timedelta(days=30)
|
||||
self.start_d = datetime.date(d_today.year, d_today.month, 1)
|
||||
|
||||
def test_info_history(self):
|
||||
# always should have info and history for valid symbols
|
||||
for ticker in tickers:
|
||||
# always should have info and history for valid symbols
|
||||
assert(ticker.info is not None and ticker.info != {})
|
||||
history = ticker.history(period="max")
|
||||
history = ticker.history(period="1mo")
|
||||
assert(history.empty is False and history is not None)
|
||||
histories = yf.download(symbols, period="1mo", session=session)
|
||||
assert(histories.empty is False and histories is not None)
|
||||
|
||||
for ticker in tickers:
|
||||
assert(ticker.info is not None and ticker.info != {})
|
||||
history = ticker.history(start=self.start_d)
|
||||
assert(history.empty is False and history is not None)
|
||||
histories = yf.download(symbols, start=self.start_d, session=session)
|
||||
assert(histories.empty is False and histories is not None)
|
||||
|
||||
def test_info_history_nofail(self):
|
||||
# should not throw Exception for delisted tickers, just print a message
|
||||
for ticker in delisted_tickers:
|
||||
history = ticker.history(period="1mo")
|
||||
histories = yf.download(delisted_symbols, period="1mo", session=session)
|
||||
histories = yf.download(delisted_symbols[0], period="1mo", session=session)
|
||||
histories = yf.download(delisted_symbols[1], period="1mo")#, session=session)
|
||||
for ticker in delisted_tickers:
|
||||
history = ticker.history(start=self.start_d)
|
||||
histories = yf.download(delisted_symbols, start=self.start_d, session=session)
|
||||
histories = yf.download(delisted_symbols[0], start=self.start_d, session=session)
|
||||
histories = yf.download(delisted_symbols[1], start=self.start_d, session=session)
|
||||
|
||||
def test_attributes(self):
|
||||
for ticker in tickers:
|
||||
ticker.isin
|
||||
ticker.major_holders
|
||||
ticker.institutional_holders
|
||||
ticker.mutualfund_holders
|
||||
ticker.dividends
|
||||
ticker.splits
|
||||
ticker.actions
|
||||
ticker.info
|
||||
ticker.info["trailingPegRatio"]
|
||||
ticker.calendar
|
||||
ticker.recommendations
|
||||
ticker.earnings
|
||||
ticker.quarterly_earnings
|
||||
ticker.financials
|
||||
ticker.quarterly_financials
|
||||
ticker.balance_sheet
|
||||
ticker.quarterly_balance_sheet
|
||||
ticker.cashflow
|
||||
ticker.quarterly_cashflow
|
||||
ticker.sustainability
|
||||
ticker.options
|
||||
ticker.news
|
||||
ticker.shares
|
||||
ticker.earnings_history
|
||||
ticker.earnings_dates
|
||||
|
||||
def test_attributes_nofail(self):
|
||||
# should not throw Exception for delisted tickers, just print a message
|
||||
for ticker in delisted_tickers:
|
||||
ticker.isin
|
||||
ticker.major_holders
|
||||
ticker.institutional_holders
|
||||
|
@ -56,8 +125,7 @@ class TestTicker(unittest.TestCase):
|
|||
ticker.earnings_dates
|
||||
|
||||
def test_holders(self):
|
||||
for ticker in tickers:
|
||||
assert(ticker.info is not None and ticker.info != {})
|
||||
for ticker in good_tickers:
|
||||
assert(ticker.major_holders is not None)
|
||||
assert(ticker.institutional_holders is not None)
|
||||
|
||||
|
|
197
yfinance/base.py
197
yfinance/base.py
|
@ -145,22 +145,22 @@ class TickerBase():
|
|||
debug_mode = True
|
||||
if "debug" in kwargs and isinstance(kwargs["debug"], bool):
|
||||
debug_mode = kwargs["debug"]
|
||||
if "many" in kwargs and kwargs["many"]:
|
||||
# Disable prints with threads, it deadlocks/throws
|
||||
debug_mode = False
|
||||
|
||||
err_msg = "No data found for this date range, symbol may be delisted"
|
||||
|
||||
if start or period is None or period.lower() == "max":
|
||||
# Check can get TZ. Fail => probably delisted
|
||||
try:
|
||||
tz = self._get_ticker_tz()
|
||||
except KeyError as e:
|
||||
if "exchangeTimezoneName" in str(e):
|
||||
shared._DFS[self.ticker] = utils.empty_df()
|
||||
shared._ERRORS[self.ticker] = err_msg
|
||||
if "many" not in kwargs and debug_mode:
|
||||
print('- %s: %s' % (self.ticker, err_msg))
|
||||
return utils.empty_df()
|
||||
else:
|
||||
raise
|
||||
tz = self._get_ticker_tz(debug_mode, proxy, timeout)
|
||||
if tz is None:
|
||||
# Every valid ticker has a timezone. Missing = problem
|
||||
shared._DFS[self.ticker] = utils.empty_df()
|
||||
shared._ERRORS[self.ticker] = err_msg
|
||||
if debug_mode:
|
||||
print('- %s: %s' % (self.ticker, err_msg))
|
||||
return utils.empty_df()
|
||||
|
||||
if end is None:
|
||||
end = int(_time.time())
|
||||
|
@ -170,7 +170,8 @@ class TickerBase():
|
|||
if interval == "1m":
|
||||
start = end - 604800 # Subtract 7 days
|
||||
else:
|
||||
start = -631159200
|
||||
#time stamp of 01/01/1900
|
||||
start = -2208994789
|
||||
else:
|
||||
start = utils._parse_user_dt(start, tz)
|
||||
params = {"period1": start, "period2": end}
|
||||
|
@ -218,7 +219,7 @@ class TickerBase():
|
|||
if data is None or not type(data) is dict or 'status_code' in data.keys():
|
||||
shared._DFS[self.ticker] = utils.empty_df()
|
||||
shared._ERRORS[self.ticker] = err_msg
|
||||
if "many" not in kwargs and debug_mode:
|
||||
if debug_mode:
|
||||
print('- %s: %s' % (self.ticker, err_msg))
|
||||
return utils.empty_df()
|
||||
|
||||
|
@ -226,7 +227,7 @@ class TickerBase():
|
|||
err_msg = data["chart"]["error"]["description"]
|
||||
shared._DFS[self.ticker] = utils.empty_df()
|
||||
shared._ERRORS[self.ticker] = err_msg
|
||||
if "many" not in kwargs and debug_mode:
|
||||
if debug_mode:
|
||||
print('- %s: %s' % (self.ticker, err_msg))
|
||||
return shared._DFS[self.ticker]
|
||||
|
||||
|
@ -234,7 +235,7 @@ class TickerBase():
|
|||
not data["chart"]["result"]:
|
||||
shared._DFS[self.ticker] = utils.empty_df()
|
||||
shared._ERRORS[self.ticker] = err_msg
|
||||
if "many" not in kwargs and debug_mode:
|
||||
if debug_mode:
|
||||
print('- %s: %s' % (self.ticker, err_msg))
|
||||
return shared._DFS[self.ticker]
|
||||
|
||||
|
@ -249,7 +250,7 @@ class TickerBase():
|
|||
except Exception:
|
||||
shared._DFS[self.ticker] = utils.empty_df()
|
||||
shared._ERRORS[self.ticker] = err_msg
|
||||
if "many" not in kwargs and debug_mode:
|
||||
if debug_mode:
|
||||
print('- %s: %s' % (self.ticker, err_msg))
|
||||
return shared._DFS[self.ticker]
|
||||
|
||||
|
@ -285,7 +286,7 @@ class TickerBase():
|
|||
err_msg = "back_adjust failed with %s" % e
|
||||
shared._DFS[self.ticker] = utils.empty_df()
|
||||
shared._ERRORS[self.ticker] = err_msg
|
||||
if "many" not in kwargs and debug_mode:
|
||||
if debug_mode:
|
||||
print('- %s: %s' % (self.ticker, err_msg))
|
||||
|
||||
if rounding:
|
||||
|
@ -315,7 +316,7 @@ class TickerBase():
|
|||
else:
|
||||
# If a midnight is during DST transition hour when clocks roll back,
|
||||
# meaning clock hits midnight twice, then use the 2nd (ambiguous=True)
|
||||
df.index = _pd.to_datetime(df.index.date).tz_localize(tz_exchange, ambiguous=True)
|
||||
df.index = _pd.to_datetime(df.index.date).tz_localize(tz_exchange, ambiguous=True, nonexistent='shift_forward')
|
||||
df.index.name = "Date"
|
||||
|
||||
# duplicates and missing rows cleanup
|
||||
|
@ -331,23 +332,79 @@ class TickerBase():
|
|||
|
||||
# ------------------------
|
||||
|
||||
def _get_ticker_tz(self):
|
||||
def _get_ticker_tz(self, debug_mode, proxy, timeout):
|
||||
if not self._tz is None:
|
||||
return self._tz
|
||||
|
||||
tkr_tz = utils.cache_lookup_tkr_tz(self.ticker)
|
||||
|
||||
if tkr_tz is not None:
|
||||
invalid_value = isinstance(tkr_tz, str)
|
||||
if not invalid_value:
|
||||
try:
|
||||
_tz.timezone(tz)
|
||||
except:
|
||||
invalid_value = True
|
||||
if invalid_value:
|
||||
# Clear from cache and force re-fetch
|
||||
utils.cache_store_tkr_tz(self.ticker, None)
|
||||
tkr_tz = None
|
||||
|
||||
if tkr_tz is None:
|
||||
tkr_tz = self.info["exchangeTimezoneName"]
|
||||
# info fetch is relatively slow so cache timezone
|
||||
try:
|
||||
utils.cache_store_tkr_tz(self.ticker, tkr_tz)
|
||||
except PermissionError:
|
||||
# System probably read-only, so cannot cache
|
||||
pass
|
||||
tkr_tz = self._fetch_ticker_tz(debug_mode, proxy, timeout)
|
||||
|
||||
if tkr_tz is not None:
|
||||
try:
|
||||
utils.cache_store_tkr_tz(self.ticker, tkr_tz)
|
||||
except PermissionError:
|
||||
# System probably read-only, so cannot cache
|
||||
pass
|
||||
|
||||
self._tz = tkr_tz
|
||||
return tkr_tz
|
||||
|
||||
|
||||
def _fetch_ticker_tz(self, debug_mode, proxy, timeout):
|
||||
# Query Yahoo for basic price data just to get returned timezone
|
||||
|
||||
params = {"range":"1d", "interval":"1d"}
|
||||
|
||||
# setup proxy in requests format
|
||||
if proxy is not None:
|
||||
if isinstance(proxy, dict) and "https" in proxy:
|
||||
proxy = proxy["https"]
|
||||
proxy = {"https": proxy}
|
||||
|
||||
# Getting data from json
|
||||
url = "{}/v8/finance/chart/{}".format(self._base_url, self.ticker)
|
||||
|
||||
session = self.session or _requests
|
||||
try:
|
||||
data = session.get(url=url, params=params, proxies=proxy, headers=utils.user_agent_headers, timeout=timeout)
|
||||
data = data.json()
|
||||
except Exception as e:
|
||||
if debug_mode:
|
||||
print("Failed to get ticker '{}' reason: {}".format(self.ticker, e))
|
||||
return None
|
||||
else:
|
||||
error = data.get('chart', {}).get('error', None)
|
||||
if error:
|
||||
# explicit error from yahoo API
|
||||
if debug_mode:
|
||||
print("Got error from yahoo api for ticker {}, Error: {}".format(self.ticker, error))
|
||||
else:
|
||||
try:
|
||||
return data["chart"]["result"][0]["meta"]["exchangeTimezoneName"]
|
||||
except Exception as err:
|
||||
if debug_mode:
|
||||
print("Could not get exchangeTimezoneName for ticker '{}' reason: {}".format(self.ticker, err))
|
||||
print("Got response: ")
|
||||
print("-------------")
|
||||
print(" {}".format(data))
|
||||
print("-------------")
|
||||
return None
|
||||
|
||||
|
||||
def _get_info(self, proxy=None):
|
||||
# setup proxy in requests format
|
||||
if proxy is not None:
|
||||
|
@ -382,7 +439,10 @@ class TickerBase():
|
|||
|
||||
self._sustainability = s[~s.index.isin(
|
||||
['maxAge', 'ratingYear', 'ratingMonth'])]
|
||||
else:
|
||||
self._sustainability = utils.empty_df()
|
||||
except Exception:
|
||||
self._sustainability = utils.empty_df()
|
||||
pass
|
||||
|
||||
# info (be nice to python 2)
|
||||
|
@ -425,9 +485,12 @@ class TickerBase():
|
|||
|
||||
self._info['logo_url'] = ""
|
||||
try:
|
||||
domain = self._info['website'].split(
|
||||
'://')[1].split('/')[0].replace('www.', '')
|
||||
self._info['logo_url'] = 'https://logo.clearbit.com/%s' % domain
|
||||
if not 'website' in self._info:
|
||||
self._info['logo_url'] = 'https://logo.clearbit.com/%s.com' % self._info['shortName'].split(' ')[0].split(',')[0]
|
||||
else:
|
||||
domain = self._info['website'].split(
|
||||
'://')[1].split('/')[0].replace('www.', '')
|
||||
self._info['logo_url'] = 'https://logo.clearbit.com/%s' % domain
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
@ -455,8 +518,32 @@ class TickerBase():
|
|||
self._recommendations = rec[[
|
||||
'Firm', 'To Grade', 'From Grade', 'Action']].sort_index()
|
||||
except Exception:
|
||||
self._recommendations = utils.empty_df()
|
||||
pass
|
||||
|
||||
# Complementary key-statistics. For now just want 'trailing PEG ratio'
|
||||
session = self.session or _requests
|
||||
keys = {"trailingPegRatio"}
|
||||
if len(keys)>0:
|
||||
# For just one/few variable is faster to query directly:
|
||||
url = "https://query1.finance.yahoo.com/ws/fundamentals-timeseries/v1/finance/timeseries/{}?symbol={}".format(self.ticker, self.ticker)
|
||||
for k in keys:
|
||||
url += "&type="+k
|
||||
# Request 6 months of data
|
||||
url += "&period1={}".format(int((_datetime.datetime.now()-_datetime.timedelta(days=365//2)).timestamp()))
|
||||
url += "&period2={}".format(int((_datetime.datetime.now()+_datetime.timedelta(days=1)).timestamp()))
|
||||
json_str = session.get(url=url, proxies=proxy, headers=utils.user_agent_headers).text
|
||||
json_data = _json.loads(json_str)
|
||||
key_stats = json_data["timeseries"]["result"][0]
|
||||
if k not in key_stats:
|
||||
# Yahoo website prints N/A, indicates Yahoo lacks necessary data to calculate
|
||||
v = None
|
||||
else:
|
||||
# Select most recent (last) raw value in list:
|
||||
v = key_stats[k][-1]["reportedValue"]["raw"]
|
||||
self._info[k] = v
|
||||
|
||||
|
||||
def _get_fundamentals(self, proxy=None):
|
||||
def cleanup(data):
|
||||
df = _pd.DataFrame(data).drop(columns=['maxAge'])
|
||||
|
@ -616,48 +703,6 @@ class TickerBase():
|
|||
except Exception:
|
||||
pass
|
||||
|
||||
# Complementary key-statistics (currently fetching the important trailingPegRatio which is the value shown in the website)
|
||||
res = {}
|
||||
try:
|
||||
my_headers = {'user-agent': 'curl/7.55.1', 'accept': 'application/json', 'content-type': 'application/json',
|
||||
'referer': 'https://finance.yahoo.com/', 'cache-control': 'no-cache', 'connection': 'close'}
|
||||
p = _re.compile(r'root\.App\.main = (.*);')
|
||||
r = _requests.session().get('https://finance.yahoo.com/quote/{}/key-statistics?p={}'.format(self.ticker,
|
||||
self.ticker), headers=my_headers)
|
||||
q_results = {}
|
||||
my_qs_keys = ['pegRatio'] # QuoteSummaryStore
|
||||
# , 'quarterlyPegRatio'] # QuoteTimeSeriesStore
|
||||
my_ts_keys = ['trailingPegRatio']
|
||||
|
||||
# Complementary key-statistics
|
||||
data = _json.loads(p.findall(r.text)[0])
|
||||
key_stats = data['context']['dispatcher']['stores']['QuoteTimeSeriesStore']
|
||||
q_results.setdefault(self.ticker, [])
|
||||
for i in my_ts_keys:
|
||||
# j=0
|
||||
try:
|
||||
# res = {i: key_stats['timeSeries'][i][1]['reportedValue']['raw']}
|
||||
# We need to loop over multiple items, if they exist: 0,1,2,..
|
||||
zzz = key_stats['timeSeries'][i]
|
||||
for j in range(len(zzz)):
|
||||
if key_stats['timeSeries'][i][j]:
|
||||
res = {i: key_stats['timeSeries']
|
||||
[i][j]['reportedValue']['raw']}
|
||||
q_results[self.ticker].append(res)
|
||||
|
||||
# print(res)
|
||||
# q_results[ticker].append(res)
|
||||
except:
|
||||
q_results[ticker].append({i: np.nan})
|
||||
|
||||
res = {'Company': ticker}
|
||||
q_results[ticker].append(res)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if 'trailingPegRatio' in res:
|
||||
self._info['trailingPegRatio'] = res['trailingPegRatio']
|
||||
|
||||
self._fundamentals = True
|
||||
|
||||
def get_recommendations(self, proxy=None, as_dict=False, *args, **kwargs):
|
||||
|
@ -803,6 +848,10 @@ class TickerBase():
|
|||
self.get_info(proxy=proxy)
|
||||
if "shortName" in self._info:
|
||||
q = self._info['shortName']
|
||||
if q is None:
|
||||
err_msg = "Cannot map to ISIN code, symbol may be delisted"
|
||||
print('- %s: %s' % (self.ticker, err_msg))
|
||||
return None
|
||||
|
||||
url = 'https://markets.businessinsider.com/ajax/' \
|
||||
'SearchController_Suggest?max_results=25&query=%s' \
|
||||
|
@ -901,8 +950,10 @@ class TickerBase():
|
|||
dates = _pd.concat([dates, data], axis=0)
|
||||
page_offset += page_size
|
||||
|
||||
if dates is None:
|
||||
raise Exception("No data found, symbol may be delisted")
|
||||
if (dates is None) or dates.shape[0]==0:
|
||||
err_msg = "No earnings dates found, symbol may be delisted"
|
||||
print('- %s: %s' % (self.ticker, err_msg))
|
||||
return None
|
||||
dates = dates.reset_index(drop=True)
|
||||
|
||||
# Drop redundant columns
|
||||
|
|
|
@ -198,7 +198,7 @@ def _download_one_threaded(ticker, start=None, end=None,
|
|||
|
||||
data = _download_one(ticker, start, end, auto_adjust, back_adjust,
|
||||
actions, period, interval, prepost, proxy, rounding,
|
||||
keepna, timeout)
|
||||
keepna, timeout, many=True)
|
||||
shared._DFS[ticker.upper()] = data
|
||||
if progress:
|
||||
shared._PROGRESS_BAR.animate()
|
||||
|
@ -208,11 +208,11 @@ def _download_one(ticker, start=None, end=None,
|
|||
auto_adjust=False, back_adjust=False,
|
||||
actions=False, period="max", interval="1d",
|
||||
prepost=False, proxy=None, rounding=False,
|
||||
keepna=False, timeout=None):
|
||||
keepna=False, timeout=None, many=False):
|
||||
|
||||
return Ticker(ticker).history(period=period, interval=interval,
|
||||
start=start, end=end, prepost=prepost,
|
||||
actions=actions, auto_adjust=auto_adjust,
|
||||
back_adjust=back_adjust, proxy=proxy,
|
||||
rounding=rounding, keepna=keepna, many=True,
|
||||
timeout=timeout)
|
||||
rounding=rounding, keepna=keepna, timeout=timeout,
|
||||
many=many)
|
||||
|
|
|
@ -31,6 +31,21 @@ import sys as _sys
|
|||
import os as _os
|
||||
import appdirs as _ad
|
||||
|
||||
from base64 import b64decode
|
||||
import hashlib
|
||||
usePycryptodome = False # slightly faster
|
||||
# usePycryptodome = True
|
||||
if usePycryptodome:
|
||||
# NOTE: if decide to use 'pycryptodome', set min version to 3.6.6
|
||||
from Crypto.Cipher import AES
|
||||
from Crypto.Util.Padding import unpad
|
||||
else:
|
||||
from cryptography.hazmat.primitives import padding
|
||||
from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
|
||||
|
||||
from threading import Lock
|
||||
mutex = Lock()
|
||||
|
||||
try:
|
||||
import ujson as _json
|
||||
except ImportError:
|
||||
|
@ -106,24 +121,112 @@ def get_html(url, proxy=None, session=None):
|
|||
return html
|
||||
|
||||
|
||||
|
||||
def decrypt_cryptojs_stores(data):
|
||||
"""
|
||||
Yahoo has started encrypting data stores, this method decrypts it.
|
||||
:param data: Python dict of the json data
|
||||
:return: The decrypted string data in data['context']['dispatcher']['stores']
|
||||
"""
|
||||
|
||||
_cs = data["_cs"]
|
||||
# Assumes _cr has format like: '{"words":[-449732894,601032952,157396918,2056341829],"sigBytes":16}';
|
||||
_cr = _json.loads(data["_cr"])
|
||||
_cr = b"".join(int.to_bytes(i, length=4, byteorder="big", signed=True) for i in _cr["words"])
|
||||
|
||||
password = hashlib.pbkdf2_hmac("sha1", _cs.encode("utf8"), _cr, 1, dklen=32).hex()
|
||||
|
||||
encrypted_stores = data['context']['dispatcher']['stores']
|
||||
encrypted_stores = b64decode(encrypted_stores)
|
||||
assert encrypted_stores[0:8] == b"Salted__"
|
||||
salt = encrypted_stores[8:16]
|
||||
encrypted_stores = encrypted_stores[16:]
|
||||
|
||||
key, iv = _EVPKDF(password, salt, keySize=32, ivSize=16, iterations=1, hashAlgorithm="md5")
|
||||
|
||||
if usePycryptodome:
|
||||
cipher = AES.new(key, AES.MODE_CBC, iv=iv)
|
||||
plaintext = cipher.decrypt(encrypted_stores)
|
||||
plaintext = unpad(plaintext, 16, style="pkcs7")
|
||||
else:
|
||||
cipher = Cipher(algorithms.AES(key), modes.CBC(iv))
|
||||
decryptor = cipher.decryptor()
|
||||
plaintext = decryptor.update(encrypted_stores) + decryptor.finalize()
|
||||
unpadder = padding.PKCS7(128).unpadder()
|
||||
plaintext = unpadder.update(plaintext) + unpadder.finalize()
|
||||
plaintext = plaintext.decode("utf-8")
|
||||
|
||||
return plaintext
|
||||
|
||||
def _EVPKDF(password, salt, keySize=32, ivSize=16, iterations=1, hashAlgorithm="md5") -> tuple:
|
||||
"""OpenSSL EVP Key Derivation Function
|
||||
Args:
|
||||
password (Union[str, bytes, bytearray]): Password to generate key from.
|
||||
salt (Union[bytes, bytearray]): Salt to use.
|
||||
keySize (int, optional): Output key length in bytes. Defaults to 32.
|
||||
ivSize (int, optional): Output Initialization Vector (IV) length in bytes. Defaults to 16.
|
||||
iterations (int, optional): Number of iterations to perform. Defaults to 1.
|
||||
hashAlgorithm (str, optional): Hash algorithm to use for the KDF. Defaults to 'md5'.
|
||||
Returns:
|
||||
key, iv: Derived key and Initialization Vector (IV) bytes.
|
||||
|
||||
Taken from: https://gist.github.com/rafiibrahim8/0cd0f8c46896cafef6486cb1a50a16d3
|
||||
OpenSSL original code: https://github.com/openssl/openssl/blob/master/crypto/evp/evp_key.c#L78
|
||||
"""
|
||||
|
||||
assert iterations > 0, "Iterations can not be less than 1."
|
||||
|
||||
if isinstance(password, str):
|
||||
password = password.encode("utf-8")
|
||||
|
||||
final_length = keySize + ivSize
|
||||
key_iv = b""
|
||||
block = None
|
||||
|
||||
while len(key_iv) < final_length:
|
||||
hasher = hashlib.new(hashAlgorithm)
|
||||
if block:
|
||||
hasher.update(block)
|
||||
hasher.update(password)
|
||||
hasher.update(salt)
|
||||
block = hasher.digest()
|
||||
for _ in range(1, iterations):
|
||||
block = hashlib.new(hashAlgorithm, block).digest()
|
||||
key_iv += block
|
||||
|
||||
key, iv = key_iv[:keySize], key_iv[keySize:final_length]
|
||||
return key, iv
|
||||
|
||||
|
||||
def get_json(url, proxy=None, session=None):
|
||||
session = session or _requests
|
||||
html = session.get(url=url, proxies=proxy, headers=user_agent_headers).text
|
||||
|
||||
if "QuoteSummaryStore" not in html:
|
||||
html = session.get(url=url, proxies=proxy).text
|
||||
if "QuoteSummaryStore" not in html:
|
||||
return {}
|
||||
if not "root.App.main =" in html:
|
||||
return {}
|
||||
|
||||
json_str = html.split('root.App.main =')[1].split(
|
||||
'(this)')[0].split(';\n}')[0].strip()
|
||||
data = _json.loads(json_str)[
|
||||
'context']['dispatcher']['stores']['QuoteSummaryStore']
|
||||
data = _json.loads(json_str)
|
||||
|
||||
if "_cs" in data and "_cr" in data:
|
||||
data_stores = _json.loads(decrypt_cryptojs_stores(data))
|
||||
else:
|
||||
if "context" in data and "dispatcher" in data["context"]:
|
||||
# Keep old code, just in case
|
||||
data_stores = data['context']['dispatcher']['stores']
|
||||
else:
|
||||
data_stores = data
|
||||
|
||||
if not 'QuoteSummaryStore' in data_stores:
|
||||
# Problem in data. Either delisted, or Yahoo spam triggered
|
||||
return {}
|
||||
|
||||
data = data_stores['QuoteSummaryStore']
|
||||
# add data about Shares Outstanding for companies' tickers if they are available
|
||||
try:
|
||||
data['annualBasicAverageShares'] = _json.loads(
|
||||
json_str)['context']['dispatcher']['stores'][
|
||||
'QuoteTimeSeriesStore']['timeSeries']['annualBasicAverageShares']
|
||||
data['annualBasicAverageShares'] = \
|
||||
data_stores['QuoteTimeSeriesStore']['timeSeries']['annualBasicAverageShares']
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
@ -332,27 +435,36 @@ def cache_lookup_tkr_tz(tkr):
|
|||
if not _os.path.isfile(fp):
|
||||
return None
|
||||
|
||||
df = _pd.read_csv(fp)
|
||||
f = df["Ticker"] == tkr
|
||||
if sum(f) == 0:
|
||||
mutex.acquire()
|
||||
df = _pd.read_csv(fp, index_col="Ticker", on_bad_lines="skip")
|
||||
mutex.release()
|
||||
if tkr in df.index:
|
||||
return df.loc[tkr,"Tz"]
|
||||
else:
|
||||
return None
|
||||
|
||||
return df["Tz"][f].iloc[0]
|
||||
def cache_store_tkr_tz(tkr,tz):
|
||||
df = _pd.DataFrame({"Ticker":[tkr], "Tz":[tz]})
|
||||
|
||||
dp = get_cache_dirpath()
|
||||
fp = _os.path.join(dp, "tkr-tz.csv")
|
||||
mutex.acquire()
|
||||
if not _os.path.isdir(dp):
|
||||
_os.makedirs(dp)
|
||||
fp = _os.path.join(dp, "tkr-tz.csv")
|
||||
if not _os.path.isfile(fp):
|
||||
df.to_csv(fp, index=False)
|
||||
return
|
||||
if (not _os.path.isfile(fp)) and (tz is not None):
|
||||
df = _pd.DataFrame({"Tz":[tz]}, index=[tkr])
|
||||
df.index.name = "Ticker"
|
||||
df.to_csv(fp)
|
||||
|
||||
df_all = _pd.read_csv(fp)
|
||||
f = df_all["Ticker"]==tkr
|
||||
if sum(f) > 0:
|
||||
raise Exception("Tkr {} tz already in cache".format(tkr))
|
||||
|
||||
_pd.concat([df_all,df]).to_csv(fp, index=False)
|
||||
else:
|
||||
df = _pd.read_csv(fp, index_col="Ticker", on_bad_lines="skip")
|
||||
if tz is None:
|
||||
# Delete if in cache:
|
||||
if tkr in df.index:
|
||||
df.drop(tkr).to_csv(fp)
|
||||
else:
|
||||
if tkr in df.index:
|
||||
raise Exception("Tkr {} tz already in cache".format(tkr))
|
||||
df.loc[tkr,"Tz"] = tz
|
||||
df.to_csv(fp)
|
||||
|
||||
mutex.release()
|
||||
|
||||
|
|
|
@ -1 +1 @@
|
|||
version = "0.1.81"
|
||||
version = "0.1.96"
|
||||
|
|
Loading…
Reference in New Issue