From 553bc5965aba2bdd405aeda1c48697709b2949d0 Mon Sep 17 00:00:00 2001 From: ValueRaider Date: Wed, 18 Jan 2023 18:13:56 +0000 Subject: [PATCH 01/21] Fix proxy arg passthrough --- yfinance/base.py | 3 ++- yfinance/scrapers/fundamentals.py | 12 ++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/yfinance/base.py b/yfinance/base.py index 440ca6f..de3731b 100644 --- a/yfinance/base.py +++ b/yfinance/base.py @@ -571,6 +571,7 @@ class TickerBase: data = get_fn( url=url, params=params, + proxy=proxy, timeout=timeout ) if "Will be right back" in data.text or data is None: @@ -1503,7 +1504,7 @@ class TickerBase: def get_shares_full(self, start=None, end=None, proxy=None): # Process dates - tz = self._get_ticker_tz(debug_mode=False, proxy=None, timeout=10) + tz = self._get_ticker_tz(debug_mode=False, proxy=proxy, timeout=10) dt_now = _pd.Timestamp.utcnow().tz_convert(tz) if start is not None: start_ts = utils._parse_user_dt(start, tz) diff --git a/yfinance/scrapers/fundamentals.py b/yfinance/scrapers/fundamentals.py index 2530e4c..6371392 100644 --- a/yfinance/scrapers/fundamentals.py +++ b/yfinance/scrapers/fundamentals.py @@ -110,19 +110,19 @@ class Financials: def get_income_time_series(self, freq="yearly", proxy=None) -> pd.DataFrame: res = self._income_time_series if freq not in res: - res[freq] = self._fetch_time_series("income", freq, proxy=None) + res[freq] = self._fetch_time_series("income", freq, proxy) return res[freq] def get_balance_sheet_time_series(self, freq="yearly", proxy=None) -> pd.DataFrame: res = self._balance_sheet_time_series if freq not in res: - res[freq] = self._fetch_time_series("balance-sheet", freq, proxy=None) + res[freq] = self._fetch_time_series("balance-sheet", freq, proxy) return res[freq] def get_cash_flow_time_series(self, freq="yearly", proxy=None) -> pd.DataFrame: res = self._cash_flow_time_series if freq not in res: - res[freq] = self._fetch_time_series("cash-flow", freq, proxy=None) + res[freq] = self._fetch_time_series("cash-flow", freq, proxy) return res[freq] def _fetch_time_series(self, name, timescale, proxy=None): @@ -235,19 +235,19 @@ class Financials: def get_income_scrape(self, freq="yearly", proxy=None) -> pd.DataFrame: res = self._income_scraped if freq not in res: - res[freq] = self._scrape("income", freq, proxy=None) + res[freq] = self._scrape("income", freq, proxy) return res[freq] def get_balance_sheet_scrape(self, freq="yearly", proxy=None) -> pd.DataFrame: res = self._balance_sheet_scraped if freq not in res: - res[freq] = self._scrape("balance-sheet", freq, proxy=None) + res[freq] = self._scrape("balance-sheet", freq, proxy) return res[freq] def get_cash_flow_scrape(self, freq="yearly", proxy=None) -> pd.DataFrame: res = self._cash_flow_scraped if freq not in res: - res[freq] = self._scrape("cash-flow", freq, proxy=None) + res[freq] = self._scrape("cash-flow", freq, proxy) return res[freq] def _scrape(self, name, timescale, proxy=None): From 2e6d3d0e60635375aaacccba5693c6a3832c5865 Mon Sep 17 00:00:00 2001 From: ValueRaider Date: Wed, 1 Feb 2023 17:06:23 +0000 Subject: [PATCH 02/21] Fix proxy in 'history()' --- yfinance/base.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/yfinance/base.py b/yfinance/base.py index de3731b..7138001 100644 --- a/yfinance/base.py +++ b/yfinance/base.py @@ -506,6 +506,12 @@ class TickerBase: exceptions instead of printing to console. """ + # setup proxy in requests format + if proxy is not None: + if isinstance(proxy, dict) and "https" in proxy: + proxy = proxy["https"] + # proxy = {"https": proxy} + if start or period is None or period.lower() == "max": # Check can get TZ. Fail => probably delisted tz = self._get_ticker_tz(debug, proxy, timeout) @@ -545,12 +551,6 @@ class TickerBase: if params["interval"] == "30m": params["interval"] = "15m" - # setup proxy in requests format - if proxy is not None: - if isinstance(proxy, dict) and "https" in proxy: - proxy = proxy["https"] - proxy = {"https": proxy} - #if the ticker is MUTUALFUND or ETF, then get capitalGains events params["events"] = "div,splits,capitalGains" From 563a1a3448d460a33ef09dde22ab3ae06d6efb6f Mon Sep 17 00:00:00 2001 From: ValueRaider Date: Wed, 1 Feb 2023 17:28:57 +0000 Subject: [PATCH 03/21] Add Ticker test for proxy --- tests/ticker.py | 125 +++++++++++++++++++++++++++++++++++++++++++++ yfinance/base.py | 21 ++++---- yfinance/ticker.py | 4 ++ 3 files changed, 141 insertions(+), 9 deletions(-) diff --git a/tests/ticker.py b/tests/ticker.py index 1aa89b6..043a932 100644 --- a/tests/ticker.py +++ b/tests/ticker.py @@ -32,6 +32,8 @@ class TestTicker(unittest.TestCase): def setUpClass(cls): cls.session = requests_cache.CachedSession(backend='memory') + cls.proxy = None + @classmethod def tearDownClass(cls): if cls.session is not None: @@ -129,6 +131,129 @@ class TestTicker(unittest.TestCase): dat.history(start="2022-01-01", end="2022-03-01") yf.download([tkr], period="1wk") + def test_goodTicker_withProxy(self): + # that yfinance works when full api is called on same instance of ticker + + tkr = "IBM" + dat = yf.Ticker(tkr, session=self.session) + + dat._fetch_ticker_tz(debug_mode=False, proxy=self.proxy, timeout=5) + dat._get_ticker_tz(debug_mode=False, proxy=self.proxy, timeout=5) + dat.history(period="1wk", proxy=self.proxy) + + v = dat.stats(proxy=self.proxy) + self.assertIsNotNone(v) + self.assertTrue(len(v) > 0) + + v = dat.get_recommendations(proxy=self.proxy) + self.assertIsNotNone(v) + self.assertFalse(v.empty) + + v = dat.get_calendar(proxy=self.proxy) + self.assertIsNotNone(v) + self.assertFalse(v.empty) + + v = dat.get_major_holders(proxy=self.proxy) + self.assertIsNotNone(v) + self.assertFalse(v.empty) + + v = dat.get_institutional_holders(proxy=self.proxy) + self.assertIsNotNone(v) + self.assertFalse(v.empty) + + v = dat.get_mutualfund_holders(proxy=self.proxy) + self.assertIsNotNone(v) + self.assertFalse(v.empty) + + v = dat.get_info(proxy=self.proxy) + self.assertIsNotNone(v) + self.assertTrue(len(v) > 0) + + v = dat.get_sustainability(proxy=self.proxy) + self.assertIsNotNone(v) + self.assertFalse(v.empty) + + v = dat.get_recommendations_summary(proxy=self.proxy) + self.assertIsNotNone(v) + self.assertFalse(v.empty) + + v = dat.get_analyst_price_target(proxy=self.proxy) + self.assertIsNotNone(v) + self.assertFalse(v.empty) + + v = dat.get_rev_forecast(proxy=self.proxy) + self.assertIsNotNone(v) + self.assertFalse(v.empty) + + v = dat.get_earnings_forecast(proxy=self.proxy) + self.assertIsNotNone(v) + self.assertFalse(v.empty) + + v = dat.get_trend_details(proxy=self.proxy) + self.assertIsNotNone(v) + self.assertFalse(v.empty) + + v = dat.get_earnings_trend(proxy=self.proxy) + self.assertIsNotNone(v) + self.assertFalse(v.empty) + + v = dat.get_earnings(proxy=self.proxy) + self.assertIsNotNone(v) + self.assertFalse(v.empty) + + v = dat.get_income_stmt(proxy=self.proxy) + self.assertIsNotNone(v) + self.assertFalse(v.empty) + + v = dat.get_incomestmt(proxy=self.proxy) + self.assertIsNotNone(v) + self.assertFalse(v.empty) + + v = dat.get_financials(proxy=self.proxy) + self.assertIsNotNone(v) + self.assertFalse(v.empty) + + v = dat.get_balance_sheet(proxy=self.proxy) + self.assertIsNotNone(v) + self.assertFalse(v.empty) + + v = dat.get_balancesheet(proxy=self.proxy) + self.assertIsNotNone(v) + self.assertFalse(v.empty) + + v = dat.get_cash_flow(proxy=self.proxy) + self.assertIsNotNone(v) + self.assertFalse(v.empty) + + v = dat.get_cashflow(proxy=self.proxy) + self.assertIsNotNone(v) + self.assertFalse(v.empty) + + v = dat.get_shares(proxy=self.proxy) + self.assertIsNotNone(v) + self.assertFalse(v.empty) + + v = dat.get_shares_full(proxy=self.proxy) + self.assertIsNotNone(v) + self.assertFalse(v.empty) + + v = dat.get_isin(proxy=self.proxy) + self.assertIsNotNone(v) + self.assertTrue(v != "") + + v = dat.get_news(proxy=self.proxy) + self.assertIsNotNone(v) + self.assertTrue(len(v) > 0) + + v = dat.get_earnings_dates(proxy=self.proxy) + self.assertIsNotNone(v) + self.assertFalse(v.empty) + + # TODO: enable after merge + # dat.get_history_metadata(proxy=self.proxy) + # self.assertIsNotNone(v) + # self.assertTrue(len(v) > 0) + class TestTickerHistory(unittest.TestCase): session = None diff --git a/yfinance/base.py b/yfinance/base.py index 7138001..63ef140 100644 --- a/yfinance/base.py +++ b/yfinance/base.py @@ -50,8 +50,9 @@ _ROOT_URL_ = 'https://finance.yahoo.com' class FastInfo: # Contain small subset of info[] items that can be fetched faster elsewhere. # Imitates a dict. - def __init__(self, tickerBaseObject): + def __init__(self, tickerBaseObject, proxy=None): self._tkr = tickerBaseObject + self.proxy = proxy self._prices_1y = None self._md = None @@ -124,8 +125,8 @@ class FastInfo: def _get_1y_prices(self, fullDaysOnly=False): if self._prices_1y is None: - self._prices_1y = self._tkr.history(period="380d", auto_adjust=False, debug=False) - self._md = self._tkr.get_history_metadata() + self._prices_1y = self._tkr.history(period="380d", auto_adjust=False, debug=False, proxy=self.proxy) + self._md = self._tkr.get_history_metadata(proxy=self.proxy) try: ctp = self._md["currentTradingPeriod"] self._today_open = pd.to_datetime(ctp["regular"]["start"], unit='s', utc=True).tz_convert(self.timezone) @@ -152,7 +153,7 @@ class FastInfo: return self._md self._get_1y_prices() - self._md = self._tkr.get_history_metadata() + self._md = self._tkr.get_history_metadata(proxy=self.proxy) return self._md def _exchange_open_now(self): @@ -185,7 +186,7 @@ class FastInfo: if self._tkr._history_metadata is None: self._get_1y_prices() - md = self._tkr.get_history_metadata() + md = self._tkr.get_history_metadata(proxy=self.proxy) self._currency = md["currency"] return self._currency @@ -448,7 +449,7 @@ class TickerBase: self._quote = Quote(self._data) self._fundamentals = Fundamentals(self._data) - self._fast_info = FastInfo(self) + self._fast_info = None def stats(self, proxy=None): ticker_url = "{}/{}".format(self._scrape_url, self.ticker) @@ -1269,8 +1270,9 @@ class TickerBase: data = self._quote.info return data - @property - def fast_info(self): + def get_fast_info(self, proxy=None): + if self._fast_info is None: + self._fast_info = FastInfo(self, proxy=proxy) return self._fast_info @property @@ -1699,8 +1701,9 @@ class TickerBase: return dates - def get_history_metadata(self) -> dict: + def get_history_metadata(self, proxy=None) -> dict: if self._history_metadata is None: raise RuntimeError("Metadata was never retrieved so far, " "call history() to retrieve it") + # TODO after merge: use proxy here return self._history_metadata diff --git a/yfinance/ticker.py b/yfinance/ticker.py index 498303a..fc521a9 100644 --- a/yfinance/ticker.py +++ b/yfinance/ticker.py @@ -141,6 +141,10 @@ class Ticker(TickerBase): def info(self) -> dict: return self.get_info() + @property + def fast_info(self): + return self.get_fast_info() + @property def calendar(self) -> _pd.DataFrame: return self.get_calendar() From e89d390824d84db4c626ff0b21a4928a4ae65b55 Mon Sep 17 00:00:00 2001 From: ValueRaider Date: Wed, 1 Feb 2023 18:09:51 +0000 Subject: [PATCH 04/21] Potential fix for proxy --- yfinance/data.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/yfinance/data.py b/yfinance/data.py index 1fb81d0..5f3ad4c 100644 --- a/yfinance/data.py +++ b/yfinance/data.py @@ -200,6 +200,8 @@ class TickerData: if isinstance(proxy, dict) and "https" in proxy: proxy = proxy["https"] proxy = {"https": proxy} + if isinstance(proxy, frozendict.frozendict): + proxy = dict(proxy) return proxy def _get_decryption_keys_from_yahoo_js(self, soup): From ec3dfaf305c15ffb26099012c70db94e15ef2a05 Mon Sep 17 00:00:00 2001 From: ValueRaider Date: Wed, 1 Feb 2023 18:10:45 +0000 Subject: [PATCH 05/21] Potential fix for proxy - revert --- yfinance/data.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/yfinance/data.py b/yfinance/data.py index 5f3ad4c..0b14be9 100644 --- a/yfinance/data.py +++ b/yfinance/data.py @@ -200,8 +200,8 @@ class TickerData: if isinstance(proxy, dict) and "https" in proxy: proxy = proxy["https"] proxy = {"https": proxy} - if isinstance(proxy, frozendict.frozendict): - proxy = dict(proxy) + # if isinstance(proxy, frozendict.frozendict): + # proxy = dict(proxy) return proxy def _get_decryption_keys_from_yahoo_js(self, soup): From 2d6b6b26ed6d208aa3a64203cb7f2cd509083cb0 Mon Sep 17 00:00:00 2001 From: ValueRaider Date: Wed, 1 Feb 2023 19:04:47 +0000 Subject: [PATCH 06/21] Potential fix for proxy - enable --- yfinance/data.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/yfinance/data.py b/yfinance/data.py index 0b14be9..5f3ad4c 100644 --- a/yfinance/data.py +++ b/yfinance/data.py @@ -200,8 +200,8 @@ class TickerData: if isinstance(proxy, dict) and "https" in proxy: proxy = proxy["https"] proxy = {"https": proxy} - # if isinstance(proxy, frozendict.frozendict): - # proxy = dict(proxy) + if isinstance(proxy, frozendict.frozendict): + proxy = dict(proxy) return proxy def _get_decryption_keys_from_yahoo_js(self, soup): From 4eae728a0683f084546bbdf7f6cc55a56c81dfa3 Mon Sep 17 00:00:00 2001 From: ValueRaider Date: Wed, 1 Feb 2023 19:17:18 +0000 Subject: [PATCH 07/21] Potential fix for proxy - enable #2 --- yfinance/data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yfinance/data.py b/yfinance/data.py index 5f3ad4c..f469105 100644 --- a/yfinance/data.py +++ b/yfinance/data.py @@ -200,7 +200,7 @@ class TickerData: if isinstance(proxy, dict) and "https" in proxy: proxy = proxy["https"] proxy = {"https": proxy} - if isinstance(proxy, frozendict.frozendict): + if isinstance(proxy, frozendict): proxy = dict(proxy) return proxy From 141ce7e47192fa7a231ba361c48f131695571549 Mon Sep 17 00:00:00 2001 From: ValueRaider Date: Wed, 1 Feb 2023 21:19:54 +0000 Subject: [PATCH 08/21] Fix proxy + cache_get. Improve error propagation --- tests/ticker.py | 6 +++--- yfinance/base.py | 46 +++++++++++++++++++++++++--------------------- yfinance/data.py | 4 +--- 3 files changed, 29 insertions(+), 27 deletions(-) diff --git a/tests/ticker.py b/tests/ticker.py index 043a932..4af8b5c 100644 --- a/tests/ticker.py +++ b/tests/ticker.py @@ -47,7 +47,7 @@ class TestTicker(unittest.TestCase): # Test: dat = yf.Ticker(tkr, session=self.session) - tz = dat._get_ticker_tz(debug_mode=False, proxy=None, timeout=None) + tz = dat._get_ticker_tz(proxy=None, timeout=None, debug_mode=False, raise_errors=False) self.assertIsNotNone(tz) @@ -137,8 +137,8 @@ class TestTicker(unittest.TestCase): tkr = "IBM" dat = yf.Ticker(tkr, session=self.session) - dat._fetch_ticker_tz(debug_mode=False, proxy=self.proxy, timeout=5) - dat._get_ticker_tz(debug_mode=False, proxy=self.proxy, timeout=5) + dat._fetch_ticker_tz(proxy=self.proxy, timeout=5, debug_mode=False, raise_errors=False) + dat._get_ticker_tz(proxy=self.proxy, timeout=5, debug_mode=False, raise_errors=False) dat.history(period="1wk", proxy=self.proxy) v = dat.stats(proxy=self.proxy) diff --git a/yfinance/base.py b/yfinance/base.py index 63ef140..37d4246 100644 --- a/yfinance/base.py +++ b/yfinance/base.py @@ -507,15 +507,9 @@ class TickerBase: exceptions instead of printing to console. """ - # setup proxy in requests format - if proxy is not None: - if isinstance(proxy, dict) and "https" in proxy: - proxy = proxy["https"] - # proxy = {"https": proxy} - if start or period is None or period.lower() == "max": # Check can get TZ. Fail => probably delisted - tz = self._get_ticker_tz(debug, proxy, timeout) + tz = self._get_ticker_tz(proxy, timeout, debug, raise_errors) if tz is None: # Every valid ticker has a timezone. Missing = problem err_msg = "No timezone found, symbol may be delisted" @@ -1172,7 +1166,7 @@ class TickerBase: return df2 - def _get_ticker_tz(self, debug_mode, proxy, timeout): + def _get_ticker_tz(self, proxy, timeout, debug_mode, raise_errors): if self._tz is not None: return self._tz cache = utils.get_tz_cache() @@ -1184,7 +1178,7 @@ class TickerBase: tz = None if tz is None: - tz = self._fetch_ticker_tz(debug_mode, proxy, timeout) + tz = self._fetch_ticker_tz(proxy, timeout, debug_mode, raise_errors) if utils.is_valid_timezone(tz): # info fetch is relatively slow so cache timezone @@ -1195,7 +1189,7 @@ class TickerBase: self._tz = tz return tz - def _fetch_ticker_tz(self, debug_mode, proxy, timeout): + def _fetch_ticker_tz(self, proxy, timeout, debug_mode, raise_errors): # Query Yahoo for fast price data just to get returned timezone params = {"range": "1d", "interval": "1d"} @@ -1208,24 +1202,34 @@ class TickerBase: data = data.json() except Exception as e: if debug_mode: - print("Failed to get ticker '{}' reason: {}".format(self.ticker, e)) + err_msg = "GET request failed with error: " + str(e) + if raise_errors: + # raise Exception('%s: %s' % (self.ticker, err_msg)) + raise + else: + print('- %s: %s' % (self.ticker, err_msg)) return None else: error = data.get('chart', {}).get('error', None) if error: # explicit error from yahoo API if debug_mode: - print("Got error from yahoo api for ticker {}, Error: {}".format(self.ticker, error)) + err_msg = "Received error from Yahoo: " + str(error) + if raise_errors: + raise Exception('%s: %s' % (self.ticker, err_msg)) + else: + print('- %s: %s' % (self.ticker, err_msg)) else: try: return data["chart"]["result"][0]["meta"]["exchangeTimezoneName"] - except Exception as err: + except Exception as e: if debug_mode: - print("Could not get exchangeTimezoneName for ticker '{}' reason: {}".format(self.ticker, err)) - print("Got response: ") - print("-------------") - print(" {}".format(data)) - print("-------------") + err_msg = "Failed to extract 'exchangeTimezoneName' from metadata: " + str(e) + if raise_errors: + # raise Exception('%s: %s' % (self.ticker, err_msg)) + raise + else: + print('- %s: %s' % (self.ticker, err_msg)) return None def get_recommendations(self, proxy=None, as_dict=False): @@ -1506,7 +1510,7 @@ class TickerBase: def get_shares_full(self, start=None, end=None, proxy=None): # Process dates - tz = self._get_ticker_tz(debug_mode=False, proxy=proxy, timeout=10) + tz = self._get_ticker_tz(proxy=proxy, timeout=10, debug_mode=False, raise_errors=False) dt_now = _pd.Timestamp.utcnow().tz_convert(tz) if start is not None: start_ts = utils._parse_user_dt(start, tz) @@ -1692,7 +1696,7 @@ class TickerBase: dates[cn] = _pd.to_datetime(dates[cn], format="%b %d, %Y, %I %p") # - instead of attempting decoding of ambiguous timezone abbreviation, just use 'info': self._quote.proxy = proxy - tz = self._get_ticker_tz(debug_mode=False, proxy=proxy, timeout=30) + tz = self._get_ticker_tz(proxy=proxy, timeout=30, debug_mode=False, raise_errors=False) dates[cn] = dates[cn].dt.tz_localize(tz) dates = dates.set_index("Earnings Date") @@ -1705,5 +1709,5 @@ class TickerBase: if self._history_metadata is None: raise RuntimeError("Metadata was never retrieved so far, " "call history() to retrieve it") - # TODO after merge: use proxy here + # TODO: after future dev->main merge, update this function to pass-through proxy return self._history_metadata diff --git a/yfinance/data.py b/yfinance/data.py index f469105..7ab77b7 100644 --- a/yfinance/data.py +++ b/yfinance/data.py @@ -197,11 +197,9 @@ class TickerData: def _get_proxy(self, proxy): # setup proxy in requests format if proxy is not None: - if isinstance(proxy, dict) and "https" in proxy: + if isinstance(proxy, (dict, frozendict)) and "https" in proxy: proxy = proxy["https"] proxy = {"https": proxy} - if isinstance(proxy, frozendict): - proxy = dict(proxy) return proxy def _get_decryption_keys_from_yahoo_js(self, soup): From 8a5ca71f52904f8c84ad2ca174ce2a7718d83717 Mon Sep 17 00:00:00 2001 From: ValueRaider Date: Sun, 5 Feb 2023 00:06:49 +0000 Subject: [PATCH 09/21] Fix holders.py proxy pass-through --- yfinance/scrapers/holders.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yfinance/scrapers/holders.py b/yfinance/scrapers/holders.py index 76faad7..4e950e0 100644 --- a/yfinance/scrapers/holders.py +++ b/yfinance/scrapers/holders.py @@ -34,7 +34,7 @@ class Holders: def _scrape(self, proxy): ticker_url = "{}/{}".format(self._SCRAPE_URL_, self._data.ticker) try: - resp = self._data.cache_get(ticker_url + '/holders', proxy) + resp = self._data.cache_get(ticker_url + '/holders', proxy=proxy) holders = pd.read_html(resp.text) except Exception: holders = [] From ba634fad0ef6717b65d0e27d87ede5a34f2e0697 Mon Sep 17 00:00:00 2001 From: Marco Vidal <124048111+vidalmarco@users.noreply.github.com> Date: Sun, 5 Feb 2023 09:17:22 +0100 Subject: [PATCH 10/21] get_shares_full does not work with proxy Error: "Yahoo web request for share count failed" updated cache_get call by adding proxy parameter and by calling it by keyword --- yfinance/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/yfinance/base.py b/yfinance/base.py index 37d4246..63a82f8 100644 --- a/yfinance/base.py +++ b/yfinance/base.py @@ -1534,8 +1534,8 @@ class TickerBase: ts_url_base = "https://query2.finance.yahoo.com/ws/fundamentals-timeseries/v1/finance/timeseries/{0}?symbol={0}".format(self.ticker) shares_url = ts_url_base + "&period1={}&period2={}".format(int(start.timestamp()), int(end.timestamp())) try: - json_str = self._data.cache_get(shares_url).text - json_data = _json.loads(json_str) + json_data = self._data.cache_get(url=shares_url, proxy=proxy) + json_data = json_data.json() except: print(f"{self.ticker}: Yahoo web request for share count failed") return None From 835dbd9629712203b479c1fcb25406e663a5285c Mon Sep 17 00:00:00 2001 From: Ricardo Prins Date: Mon, 17 Jul 2023 08:49:39 -0600 Subject: [PATCH 11/21] Fix failure when using single ISIN as a ticker --- yfinance/multi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yfinance/multi.py b/yfinance/multi.py index 7a3d3f4..6ec8e4a 100644 --- a/yfinance/multi.py +++ b/yfinance/multi.py @@ -207,7 +207,7 @@ def download(tickers, start=None, end=None, actions=False, threads=True, ignore_ if len(tickers) == 1: ticker = tickers[0] - return shared._DFS[shared._ISINS.get(ticker, ticker)] + return shared._DFS[ticker] try: data = _pd.concat(shared._DFS.values(), axis=1, sort=True, From 46f53f9957abe9a4f0f72fd1f52e648b9f36668d Mon Sep 17 00:00:00 2001 From: ValueRaider Date: Mon, 17 Jul 2023 18:34:00 +0100 Subject: [PATCH 12/21] Port proxy fix to relocated 'FastInfo' --- yfinance/scrapers/quote.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/yfinance/scrapers/quote.py b/yfinance/scrapers/quote.py index 6fc3c41..746130b 100644 --- a/yfinance/scrapers/quote.py +++ b/yfinance/scrapers/quote.py @@ -77,8 +77,9 @@ class InfoDictWrapper(MutableMapping): class FastInfo: # Contain small subset of info[] items that can be fetched faster elsewhere. # Imitates a dict. - def __init__(self, tickerBaseObject): + def __init__(self, tickerBaseObject, proxy=None): self._tkr = tickerBaseObject + self.proxy = proxy self._prices_1y = None self._prices_1wk_1h_prepost = None @@ -174,9 +175,9 @@ class FastInfo: if self._prices_1y is None: # Temporarily disable error printing logging.disable(logging.CRITICAL) - self._prices_1y = self._tkr.history(period="380d", auto_adjust=False, keepna=True) + self._prices_1y = self._tkr.history(period="380d", auto_adjust=False, keepna=True, proxy=self.proxy) logging.disable(logging.NOTSET) - self._md = self._tkr.get_history_metadata() + self._md = self._tkr.get_history_metadata(proxy=self.proxy) try: ctp = self._md["currentTradingPeriod"] self._today_open = pd.to_datetime(ctp["regular"]["start"], unit='s', utc=True).tz_convert(self.timezone) @@ -203,7 +204,7 @@ class FastInfo: if self._prices_1wk_1h_prepost is None: # Temporarily disable error printing logging.disable(logging.CRITICAL) - self._prices_1wk_1h_prepost = self._tkr.history(period="1wk", interval="1h", auto_adjust=False, prepost=True) + self._prices_1wk_1h_prepost = self._tkr.history(period="1wk", interval="1h", auto_adjust=False, prepost=True, proxy=self.proxy) logging.disable(logging.NOTSET) return self._prices_1wk_1h_prepost @@ -211,7 +212,7 @@ class FastInfo: if self._prices_1wk_1h_reg is None: # Temporarily disable error printing logging.disable(logging.CRITICAL) - self._prices_1wk_1h_reg = self._tkr.history(period="1wk", interval="1h", auto_adjust=False, prepost=False) + self._prices_1wk_1h_reg = self._tkr.history(period="1wk", interval="1h", auto_adjust=False, prepost=False, proxy=self.proxy) logging.disable(logging.NOTSET) return self._prices_1wk_1h_reg @@ -220,7 +221,7 @@ class FastInfo: return self._md self._get_1y_prices() - self._md = self._tkr.get_history_metadata() + self._md = self._tkr.get_history_metadata(proxy=self.proxy) return self._md def _exchange_open_now(self): @@ -253,7 +254,7 @@ class FastInfo: if self._tkr._history_metadata is None: self._get_1y_prices() - md = self._tkr.get_history_metadata() + md = self._tkr.get_history_metadata(proxy=self.proxy) self._currency = md["currency"] return self._currency @@ -264,7 +265,7 @@ class FastInfo: if self._tkr._history_metadata is None: self._get_1y_prices() - md = self._tkr.get_history_metadata() + md = self._tkr.get_history_metadata(proxy=self.proxy) self._quote_type = md["instrumentType"] return self._quote_type @@ -289,7 +290,7 @@ class FastInfo: if self._shares is not None: return self._shares - shares = self._tkr.get_shares_full(start=pd.Timestamp.utcnow().date()-pd.Timedelta(days=548)) + shares = self._tkr.get_shares_full(start=pd.Timestamp.utcnow().date()-pd.Timedelta(days=548), proxy=self.proxy) # if shares is None: # # Requesting 18 months failed, so fallback to shares which should include last year # shares = self._tkr.get_shares() From 693565a85b7172cfcc2bb91b8be27bdfb728742b Mon Sep 17 00:00:00 2001 From: ValueRaider Date: Tue, 18 Jul 2023 13:45:55 +0100 Subject: [PATCH 13/21] Bump version to 0.2.25 --- CHANGELOG.rst | 5 +++++ meta.yaml | 2 +- yfinance/version.py | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index c64ce29..f3b39a4 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,11 @@ Change Log =========== +0.2.25 +------ +Fix single ISIN as ticker #1611 +Fix 'Only 100 years allowed' error #1576 + 0.2.24 ------ Fix info[] missing values #1603 diff --git a/meta.yaml b/meta.yaml index b07cb54..b0c4a0f 100644 --- a/meta.yaml +++ b/meta.yaml @@ -1,5 +1,5 @@ {% set name = "yfinance" %} -{% set version = "0.2.24" %} +{% set version = "0.2.25" %} package: name: "{{ name|lower }}" diff --git a/yfinance/version.py b/yfinance/version.py index e8cade5..ca7bdab 100644 --- a/yfinance/version.py +++ b/yfinance/version.py @@ -1 +1 @@ -version = "0.2.25b1" +version = "0.2.25" From 97b13dfa8c7a38a132326d1908dfc19fdc957630 Mon Sep 17 00:00:00 2001 From: ValueRaider Date: Tue, 18 Jul 2023 12:36:46 +0100 Subject: [PATCH 14/21] Convert issue template to yaml + improve --- .github/ISSUE_TEMPLATE/bug_report.md | 44 ------------- .github/ISSUE_TEMPLATE/bug_report.yaml | 88 ++++++++++++++++++++++++++ 2 files changed, 88 insertions(+), 44 deletions(-) delete mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/bug_report.yaml diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md deleted file mode 100644 index 3da8f1b..0000000 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ /dev/null @@ -1,44 +0,0 @@ ---- -name: Bug report -about: Create a report to help us improve -title: '' -labels: '' -assignees: '' - ---- - -# IMPORTANT - -# Read and follow these instructions carefully. Help us help you. - -### Are you up-to-date? - -Upgrade to the latest version and confirm the issue/bug is still there. - -`$ pip install yfinance --upgrade --no-cache-dir` - -Confirm by running: - -`import yfinance as yf ; print(yf.__version__)` - -and comparing against [PIP](https://pypi.org/project/yfinance/#history). - -### Does Yahoo actually have the data? - -Are you spelling symbol *exactly* same as Yahoo? - -Then visit `finance.yahoo.com` and confirm they have the data you want. Maybe your symbol was delisted, or your expectations of `yfinance` are wrong. - -### Are you spamming Yahoo? - -Yahoo Finance free service has rate-limiting depending on request type - roughly 60/minute for prices, 10/minute for info. Once limit hit, Yahoo can delay, block, or return bad data -> not a `yfinance` bug. - -### Still think it's a bug? - -**Delete these instructions** and replace with your bug report, providing the following as best you can: - -- Simple code that reproduces your problem, that we can copy-paste-run. -- Run code with [debug logging enabled](https://github.com/ranaroussi/yfinance#logging) and post the full output. -- If you think `yfinance` returning bad data, give us proof. -- `yfinance` version and Python version. -- Operating system type. diff --git a/.github/ISSUE_TEMPLATE/bug_report.yaml b/.github/ISSUE_TEMPLATE/bug_report.yaml new file mode 100644 index 0000000..f0a4232 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yaml @@ -0,0 +1,88 @@ +name: Bug report +description: Report a bug in our project +labels: ["bug"] + +body: + - type: markdown + attributes: + value: | + # IMPORTANT - Read and follow these instructions carefully. Help us help you. + + ### Does issue already exist? + + Use the search tool. Don't annoy everyone by duplicating existing Issues. + + ### Are you up-to-date? + + Upgrade to the latest version and confirm the issue/bug is still there. + + `$ pip install yfinance --upgrade --no-cache-dir` + + Confirm by running: + + `import yfinance as yf ; print(yf.__version__)` + + and comparing against [PIP](https://pypi.org/project/yfinance/#history). + + ### Does Yahoo actually have the data? + + Are you spelling symbol *exactly* same as Yahoo? + + Then visit `finance.yahoo.com` and confirm they have the data you want. Maybe your symbol was delisted, or your expectations of `yfinance` are wrong. + + ### Are you spamming Yahoo? + + Yahoo Finance free service has rate-limiting https://github.com/ranaroussi/yfinance/discussions/1513. Once limit hit, Yahoo can delay, block, or return bad data -> not a `yfinance` bug. + + - type: markdown + attributes: + value: | + --- + ## Still think it's a bug? + + Provide the following as best you can: + + - type: textarea + id: code + attributes: + label: "Simple code that reproduces your problem" + description: "Provide a snippet of code that we can copy-paste-run. Wrap code in Python Markdown code blocks for proper formatting (```python ... ```)." + validations: + required: true + + - type: textarea + id: debug-log + attributes: + label: "Debug log" + description: "Run code with debug logging enabled and post the full output. Instructions: https://github.com/ranaroussi/yfinance/tree/main#logging" + validations: + required: true + + - type: textarea + id: bad-data-proof + attributes: + label: "Bad data proof" + description: "If you think `yfinance` returning bad data, provide your proof here." + validations: + required: false + + - type: input + id: version-yfinance + attributes: + label: "`yfinance` version" + validations: + required: true + + - type: input + id: version-python + attributes: + label: "Python version" + validations: + required: false + + - type: input + id: os + attributes: + label: "Operating system" + validations: + required: false From 45f1c88460ce6b1c2498ccd3448ea22130526b38 Mon Sep 17 00:00:00 2001 From: ValueRaider Date: Wed, 19 Jul 2023 18:09:20 +0100 Subject: [PATCH 15/21] yaml issue template - escape some backticks --- .github/ISSUE_TEMPLATE/bug_report.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.yaml b/.github/ISSUE_TEMPLATE/bug_report.yaml index f0a4232..aa0a222 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yaml +++ b/.github/ISSUE_TEMPLATE/bug_report.yaml @@ -46,7 +46,7 @@ body: id: code attributes: label: "Simple code that reproduces your problem" - description: "Provide a snippet of code that we can copy-paste-run. Wrap code in Python Markdown code blocks for proper formatting (```python ... ```)." + description: "Provide a snippet of code that we can copy-paste-run. Wrap code in Python Markdown code blocks for proper formatting (\`\`\`python ... \`\`\`)." validations: required: true From 39527d24d445739efe5889e6a61a6f4745cf7e86 Mon Sep 17 00:00:00 2001 From: ValueRaider Date: Wed, 19 Jul 2023 18:15:50 +0100 Subject: [PATCH 16/21] Fix yaml issue template rendering --- .github/ISSUE_TEMPLATE/bug_report.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.yaml b/.github/ISSUE_TEMPLATE/bug_report.yaml index f0a4232..050ecce 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yaml +++ b/.github/ISSUE_TEMPLATE/bug_report.yaml @@ -46,7 +46,7 @@ body: id: code attributes: label: "Simple code that reproduces your problem" - description: "Provide a snippet of code that we can copy-paste-run. Wrap code in Python Markdown code blocks for proper formatting (```python ... ```)." + description: "Provide a snippet of code that we can copy-paste-run. Wrap code in Python Markdown code blocks for proper formatting (```` ```python ... ``` ````)." validations: required: true From 274f309052ff50b86d934370d795a0c97a0fc2ab Mon Sep 17 00:00:00 2001 From: Ricardo Prins Date: Thu, 20 Jul 2023 17:17:44 -0600 Subject: [PATCH 17/21] Bump requests to 2.31 and removes cryptography. --- README.md | 3 +-- meta.yaml | 6 ++---- requirements.txt | 3 +-- setup.py | 2 +- 4 files changed, 5 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 234ceca..cac2449 100644 --- a/README.md +++ b/README.md @@ -251,14 +251,13 @@ To install `yfinance` using `conda`, see - [Python](https://www.python.org) \>= 2.7, 3.4+ - [Pandas](https://github.com/pydata/pandas) \>= 1.3.0 - [Numpy](http://www.numpy.org) \>= 1.16.5 -- [requests](http://docs.python-requests.org/en/master) \>= 2.26 +- [requests](http://docs.python-requests.org/en/master) \>= 2.31 - [lxml](https://pypi.org/project/lxml) \>= 4.9.1 - [appdirs](https://pypi.org/project/appdirs) \>= 1.4.4 - [pytz](https://pypi.org/project/pytz) \>=2022.5 - [frozendict](https://pypi.org/project/frozendict) \>= 2.3.4 - [beautifulsoup4](https://pypi.org/project/beautifulsoup4) \>= 4.11.1 - [html5lib](https://pypi.org/project/html5lib) \>= 1.1 -- [cryptography](https://pypi.org/project/cryptography) \>= 3.3.2 #### Optional (if you want to use `pandas_datareader`) diff --git a/meta.yaml b/meta.yaml index b0c4a0f..1984126 100644 --- a/meta.yaml +++ b/meta.yaml @@ -18,7 +18,7 @@ requirements: host: - pandas >=1.3.0 - numpy >=1.16.5 - - requests >=2.26 + - requests >=2.31 - multitasking >=0.0.7 - lxml >=4.9.1 - appdirs >=1.4.4 @@ -27,14 +27,13 @@ requirements: - beautifulsoup4 >=4.11.1 - html5lib >=1.1 # - pycryptodome >=3.6.6 - - cryptography >=3.3.2 - pip - python run: - pandas >=1.3.0 - numpy >=1.16.5 - - requests >=2.26 + - requests >=2.31 - multitasking >=0.0.7 - lxml >=4.9.1 - appdirs >=1.4.4 @@ -43,7 +42,6 @@ requirements: - beautifulsoup4 >=4.11.1 - html5lib >=1.1 # - pycryptodome >=3.6.6 - - cryptography >=3.3.2 - python test: diff --git a/requirements.txt b/requirements.txt index 08eceaf..1fe6cab 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ pandas>=1.3.0 numpy>=1.16.5 -requests>=2.26 +requests>=2.31 multitasking>=0.0.7 lxml>=4.9.1 appdirs>=1.4.4 @@ -8,4 +8,3 @@ pytz>=2022.5 frozendict>=2.3.4 beautifulsoup4>=4.11.1 html5lib>=1.1 -cryptography>=3.3.2 diff --git a/setup.py b/setup.py index 0b12c31..315d6b2 100644 --- a/setup.py +++ b/setup.py @@ -60,7 +60,7 @@ setup( keywords='pandas, yahoo finance, pandas datareader', packages=find_packages(exclude=['contrib', 'docs', 'tests', 'examples']), install_requires=['pandas>=1.3.0', 'numpy>=1.16.5', - 'requests>=2.26', 'multitasking>=0.0.7', + 'requests>=2.31', 'multitasking>=0.0.7', 'lxml>=4.9.1', 'appdirs>=1.4.4', 'pytz>=2022.5', 'frozendict>=2.3.4', 'beautifulsoup4>=4.11.1', 'html5lib>=1.1'], From ddf0cf19cd42fdc6bd379ad9b4c69915a651c93b Mon Sep 17 00:00:00 2001 From: ValueRaider Date: Fri, 21 Jul 2023 12:56:10 +0100 Subject: [PATCH 18/21] Bump version to 0.2.26 --- CHANGELOG.rst | 6 ++++++ meta.yaml | 2 +- yfinance/version.py | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index f3b39a4..a5b3baa 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,12 @@ Change Log =========== +0.2.26 +------ +Proxy improvements +- bug fixes #1371 +- security fix #1625 + 0.2.25 ------ Fix single ISIN as ticker #1611 diff --git a/meta.yaml b/meta.yaml index 1984126..0fbb483 100644 --- a/meta.yaml +++ b/meta.yaml @@ -1,5 +1,5 @@ {% set name = "yfinance" %} -{% set version = "0.2.25" %} +{% set version = "0.2.26" %} package: name: "{{ name|lower }}" diff --git a/yfinance/version.py b/yfinance/version.py index ca7bdab..22af852 100644 --- a/yfinance/version.py +++ b/yfinance/version.py @@ -1 +1 @@ -version = "0.2.25" +version = "0.2.26" From d0b20700365c71583cebcd8c164b2d7c5a0a67a9 Mon Sep 17 00:00:00 2001 From: ValueRaider Date: Sun, 23 Jul 2023 15:20:57 +0100 Subject: [PATCH 19/21] Fix merging 1d-prices with out-of-range divs/splits --- tests/prices.py | 9 +++++++++ yfinance/const.py | 2 ++ yfinance/utils.py | 37 +++++++++++++++++++++++-------------- 3 files changed, 34 insertions(+), 14 deletions(-) diff --git a/tests/prices.py b/tests/prices.py index 3fdb84f..2273493 100644 --- a/tests/prices.py +++ b/tests/prices.py @@ -223,6 +223,15 @@ class TestPriceHistory(unittest.TestCase): print("{}-without-events missing these dates: {}".format(tkr, missing_from_df2)) raise + # Reproduce issue #1634 - 1d dividend out-of-range, should be prepended to prices + div_dt = _pd.Timestamp(2022, 7, 21).tz_localize("America/New_York") + df_dividends = _pd.DataFrame(data={"Dividends":[1.0]}, index=[div_dt]) + df_prices = _pd.DataFrame(data={c:[1.0] for c in yf.const.price_colnames}|{'Volume':0}, index=[div_dt+_dt.timedelta(days=1)]) + df_merged = yf.utils.safe_merge_dfs(df_prices, df_dividends, '1d') + self.assertEqual(df_merged.shape[0], 2) + self.assertTrue(df_merged[df_prices.columns].iloc[1:].equals(df_prices)) + self.assertEqual(df_merged.index[0], div_dt) + def test_intraDayWithEvents(self): tkrs = ["BHP.AX", "IMP.JO", "BP.L", "PNL.L", "INTC"] test_run = False diff --git a/yfinance/const.py b/yfinance/const.py index 9eedcdc..dabcb2f 100644 --- a/yfinance/const.py +++ b/yfinance/const.py @@ -6,3 +6,5 @@ fundamentals_keys['financials'] = ["TaxEffectOfUnusualItems","TaxRateForCalcs"," fundamentals_keys['balance-sheet'] = ["TreasurySharesNumber","PreferredSharesNumber","OrdinarySharesNumber","ShareIssued","NetDebt","TotalDebt","TangibleBookValue","InvestedCapital","WorkingCapital","NetTangibleAssets","CapitalLeaseObligations","CommonStockEquity","PreferredStockEquity","TotalCapitalization","TotalEquityGrossMinorityInterest","MinorityInterest","StockholdersEquity","OtherEquityInterest","GainsLossesNotAffectingRetainedEarnings","OtherEquityAdjustments","FixedAssetsRevaluationReserve","ForeignCurrencyTranslationAdjustments","MinimumPensionLiabilities","UnrealizedGainLoss","TreasuryStock","RetainedEarnings","AdditionalPaidInCapital","CapitalStock","OtherCapitalStock","CommonStock","PreferredStock","TotalPartnershipCapital","GeneralPartnershipCapital","LimitedPartnershipCapital","TotalLiabilitiesNetMinorityInterest","TotalNonCurrentLiabilitiesNetMinorityInterest","OtherNonCurrentLiabilities","LiabilitiesHeldforSaleNonCurrent","RestrictedCommonStock","PreferredSecuritiesOutsideStockEquity","DerivativeProductLiabilities","EmployeeBenefits","NonCurrentPensionAndOtherPostretirementBenefitPlans","NonCurrentAccruedExpenses","DuetoRelatedPartiesNonCurrent","TradeandOtherPayablesNonCurrent","NonCurrentDeferredLiabilities","NonCurrentDeferredRevenue","NonCurrentDeferredTaxesLiabilities","LongTermDebtAndCapitalLeaseObligation","LongTermCapitalLeaseObligation","LongTermDebt","LongTermProvisions","CurrentLiabilities","OtherCurrentLiabilities","CurrentDeferredLiabilities","CurrentDeferredRevenue","CurrentDeferredTaxesLiabilities","CurrentDebtAndCapitalLeaseObligation","CurrentCapitalLeaseObligation","CurrentDebt","OtherCurrentBorrowings","LineOfCredit","CommercialPaper","CurrentNotesPayable","PensionandOtherPostRetirementBenefitPlansCurrent","CurrentProvisions","PayablesAndAccruedExpenses","CurrentAccruedExpenses","InterestPayable","Payables","OtherPayable","DuetoRelatedPartiesCurrent","DividendsPayable","TotalTaxPayable","IncomeTaxPayable","AccountsPayable","TotalAssets","TotalNonCurrentAssets","OtherNonCurrentAssets","DefinedPensionBenefit","NonCurrentPrepaidAssets","NonCurrentDeferredAssets","NonCurrentDeferredTaxesAssets","DuefromRelatedPartiesNonCurrent","NonCurrentNoteReceivables","NonCurrentAccountsReceivable","FinancialAssets","InvestmentsAndAdvances","OtherInvestments","InvestmentinFinancialAssets","HeldToMaturitySecurities","AvailableForSaleSecurities","FinancialAssetsDesignatedasFairValueThroughProfitorLossTotal","TradingSecurities","LongTermEquityInvestment","InvestmentsinJointVenturesatCost","InvestmentsInOtherVenturesUnderEquityMethod","InvestmentsinAssociatesatCost","InvestmentsinSubsidiariesatCost","InvestmentProperties","GoodwillAndOtherIntangibleAssets","OtherIntangibleAssets","Goodwill","NetPPE","AccumulatedDepreciation","GrossPPE","Leases","ConstructionInProgress","OtherProperties","MachineryFurnitureEquipment","BuildingsAndImprovements","LandAndImprovements","Properties","CurrentAssets","OtherCurrentAssets","HedgingAssetsCurrent","AssetsHeldForSaleCurrent","CurrentDeferredAssets","CurrentDeferredTaxesAssets","RestrictedCash","PrepaidAssets","Inventory","InventoriesAdjustmentsAllowances","OtherInventories","FinishedGoods","WorkInProcess","RawMaterials","Receivables","ReceivablesAdjustmentsAllowances","OtherReceivables","DuefromRelatedPartiesCurrent","TaxesReceivable","AccruedInterestReceivable","NotesReceivable","LoansReceivable","AccountsReceivable","AllowanceForDoubtfulAccountsReceivable","GrossAccountsReceivable","CashCashEquivalentsAndShortTermInvestments","OtherShortTermInvestments","CashAndCashEquivalents","CashEquivalents","CashFinancial"] fundamentals_keys['cash-flow'] = ["ForeignSales","DomesticSales","AdjustedGeographySegmentData","FreeCashFlow","RepurchaseOfCapitalStock","RepaymentOfDebt","IssuanceOfDebt","IssuanceOfCapitalStock","CapitalExpenditure","InterestPaidSupplementalData","IncomeTaxPaidSupplementalData","EndCashPosition","OtherCashAdjustmentOutsideChangeinCash","BeginningCashPosition","EffectOfExchangeRateChanges","ChangesInCash","OtherCashAdjustmentInsideChangeinCash","CashFlowFromDiscontinuedOperation","FinancingCashFlow","CashFromDiscontinuedFinancingActivities","CashFlowFromContinuingFinancingActivities","NetOtherFinancingCharges","InterestPaidCFF","ProceedsFromStockOptionExercised","CashDividendsPaid","PreferredStockDividendPaid","CommonStockDividendPaid","NetPreferredStockIssuance","PreferredStockPayments","PreferredStockIssuance","NetCommonStockIssuance","CommonStockPayments","CommonStockIssuance","NetIssuancePaymentsOfDebt","NetShortTermDebtIssuance","ShortTermDebtPayments","ShortTermDebtIssuance","NetLongTermDebtIssuance","LongTermDebtPayments","LongTermDebtIssuance","InvestingCashFlow","CashFromDiscontinuedInvestingActivities","CashFlowFromContinuingInvestingActivities","NetOtherInvestingChanges","InterestReceivedCFI","DividendsReceivedCFI","NetInvestmentPurchaseAndSale","SaleOfInvestment","PurchaseOfInvestment","NetInvestmentPropertiesPurchaseAndSale","SaleOfInvestmentProperties","PurchaseOfInvestmentProperties","NetBusinessPurchaseAndSale","SaleOfBusiness","PurchaseOfBusiness","NetIntangiblesPurchaseAndSale","SaleOfIntangibles","PurchaseOfIntangibles","NetPPEPurchaseAndSale","SaleOfPPE","PurchaseOfPPE","CapitalExpenditureReported","OperatingCashFlow","CashFromDiscontinuedOperatingActivities","CashFlowFromContinuingOperatingActivities","TaxesRefundPaid","InterestReceivedCFO","InterestPaidCFO","DividendReceivedCFO","DividendPaidCFO","ChangeInWorkingCapital","ChangeInOtherWorkingCapital","ChangeInOtherCurrentLiabilities","ChangeInOtherCurrentAssets","ChangeInPayablesAndAccruedExpense","ChangeInAccruedExpense","ChangeInInterestPayable","ChangeInPayable","ChangeInDividendPayable","ChangeInAccountPayable","ChangeInTaxPayable","ChangeInIncomeTaxPayable","ChangeInPrepaidAssets","ChangeInInventory","ChangeInReceivables","ChangesInAccountReceivables","OtherNonCashItems","ExcessTaxBenefitFromStockBasedCompensation","StockBasedCompensation","UnrealizedGainLossOnInvestmentSecurities","ProvisionandWriteOffofAssets","AssetImpairmentCharge","AmortizationOfSecurities","DeferredTax","DeferredIncomeTax","DepreciationAmortizationDepletion","Depletion","DepreciationAndAmortization","AmortizationCashFlow","AmortizationOfIntangibles","Depreciation","OperatingGainsLosses","PensionAndEmployeeBenefitExpense","EarningsLossesFromEquityInvestments","GainLossOnInvestmentSecurities","NetForeignCurrencyExchangeGainLoss","GainLossOnSaleOfPPE","GainLossOnSaleOfBusiness","NetIncomeFromContinuingOperations","CashFlowsfromusedinOperatingActivitiesDirect","TaxesRefundPaidDirect","InterestReceivedDirect","InterestPaidDirect","DividendsReceivedDirect","DividendsPaidDirect","ClassesofCashPayments","OtherCashPaymentsfromOperatingActivities","PaymentsonBehalfofEmployees","PaymentstoSuppliersforGoodsandServices","ClassesofCashReceiptsfromOperatingActivities","OtherCashReceiptsfromOperatingActivities","ReceiptsfromGovernmentGrants","ReceiptsfromCustomers"] + +price_colnames = ['Open', 'High', 'Low', 'Close', 'Adj Close'] diff --git a/yfinance/utils.py b/yfinance/utils.py index c95879a..8faf627 100644 --- a/yfinance/utils.py +++ b/yfinance/utils.py @@ -21,6 +21,8 @@ from __future__ import print_function +from yfinance import const + import datetime as _datetime import dateutil as _dateutil from typing import Dict, Union, List, Optional @@ -689,21 +691,28 @@ def safe_merge_dfs(df_main, df_sub, interval): f_outOfRange = indices == -1 if f_outOfRange.any() and not intraday: - # If dividend is occuring in next interval after last price row, - # add a new row of NaNs - last_dt = df_main.index[-1] - next_interval_start_dt = last_dt + td + empty_row_data = {c:[_np.nan] for c in const.price_colnames}|{'Volume':[0]} if interval == '1d': - # Allow for weekends & holidays - next_interval_end_dt = last_dt+7*_pd.Timedelta(days=7) - else: - next_interval_end_dt = next_interval_start_dt + td - for i in _np.where(f_outOfRange)[0]: - dt = df_sub.index[i] - if dt >= next_interval_start_dt and dt < next_interval_end_dt: - new_dt = dt if interval == '1d' else next_interval_start_dt + # For 1d, add all out-of-range event dates + for i in _np.where(f_outOfRange)[0]: + dt = df_sub.index[i] get_yf_logger().debug(f"Adding out-of-range {data_col} @ {dt.date()} in new prices row of NaNs") - df_main.loc[new_dt] = _np.nan + empty_row = _pd.DataFrame(data=empty_row_data, index=[dt]) + df_main = _pd.concat([df_main, empty_row], sort=True) + else: + # Else, only add out-of-range event dates if occurring in interval + # immediately after last pricfe row + last_dt = df_main.index[-1] + next_interval_start_dt = last_dt + td + next_interval_end_dt = next_interval_start_dt + td + for i in _np.where(f_outOfRange)[0]: + dt = df_sub.index[i] + if dt >= next_interval_start_dt and dt < next_interval_end_dt: + new_dt = next_interval_start_dt + get_yf_logger().debug(f"Adding out-of-range {data_col} @ {dt.date()} in new prices row of NaNs") + empty_row = _pd.DataFrame(data=empty_row_data, index=[dt]) + df_main = _pd.concat([df_main, empty_row], sort=True) + df_main = df_main.sort_index() # Re-calculate indices indices = _np.searchsorted(_np.append(df_main.index, df_main.index[-1]+td), df_sub.index, side='right') @@ -718,7 +727,7 @@ def safe_merge_dfs(df_main, df_sub, interval): f_outOfRange = indices == -1 if f_outOfRange.any(): if intraday or interval in ['1d', '1wk']: - raise Exception(f"The following '{data_col}' events are out-of-range, did not expect with interval {interval}: {df_sub.index}") + raise Exception(f"The following '{data_col}' events are out-of-range, did not expect with interval {interval}: {df_sub.index[f_outOfRange]}") get_yf_logger().debug(f'Discarding these {data_col} events:' + '\n' + str(df_sub[f_outOfRange])) df_sub = df_sub[~f_outOfRange].copy() indices = indices[~f_outOfRange] From cc876088247b4537739bf80db1b54fcd835a7cf3 Mon Sep 17 00:00:00 2001 From: Value Raider Date: Wed, 2 Aug 2023 19:29:06 +0100 Subject: [PATCH 20/21] Fix multithread error 'tz already in cache' --- tests/utils.py | 51 +++++++++++++++++++++++++++++++++++++++++++++++ yfinance/utils.py | 10 +++++++--- 2 files changed, 58 insertions(+), 3 deletions(-) create mode 100644 tests/utils.py diff --git a/tests/utils.py b/tests/utils.py new file mode 100644 index 0000000..3798c2d --- /dev/null +++ b/tests/utils.py @@ -0,0 +1,51 @@ +""" +Tests for utils + +To run all tests in suite from commandline: + python -m unittest tests.utils + +Specific test class: + python -m unittest tests.utils.TestTicker + +""" +# import pandas as pd +# import numpy as np + +from .context import yfinance as yf +from .context import session_gbl + +import unittest +# import requests_cache +import tempfile + + +class TestUtils(unittest.TestCase): + session = None + + @classmethod + def setUpClass(cls): + cls.tempCacheDir = tempfile.TemporaryDirectory() + yf.set_tz_cache_location(cls.tempCacheDir.name) + + @classmethod + def tearDownClass(cls): + cls.tempCacheDir.cleanup() + + def test_storeTzNoRaise(self): + # storing TZ to cache should never raise exception + tkr = 'AMZN' + tz1 = "America/New_York" + tz2 = "London/Europe" + cache = yf.utils.get_tz_cache() + cache.store(tkr, tz1) + cache.store(tkr, tz2) + + +def suite(): + suite = unittest.TestSuite() + suite.addTest(TestUtils('Test utils')) + return suite + + +if __name__ == '__main__': + unittest.main() diff --git a/yfinance/utils.py b/yfinance/utils.py index c95879a..25aca21 100644 --- a/yfinance/utils.py +++ b/yfinance/utils.py @@ -980,10 +980,14 @@ class _TzCache: def store(self, tkr, tz): if tz is None: self.tz_db.delete(tkr) - elif self.tz_db.get(tkr) is not None: - raise Exception("Tkr {} tz already in cache".format(tkr)) else: - self.tz_db.set(tkr, tz) + tz_db = self.tz_db.get(tkr) + if tz_db is not None: + if tz != tz_db: + get_yf_logger().debug(f'{tkr}: Overwriting cached TZ "{tz_db}" with different TZ "{tz}"') + self.tz_db.set(tkr, tz) + else: + self.tz_db.set(tkr, tz) @property def _db_dir(self): From 5b0cb60cf5cc1d1a0afbfea9f20fe0da7933b6af Mon Sep 17 00:00:00 2001 From: Value Raider Date: Thu, 3 Aug 2023 21:24:07 +0100 Subject: [PATCH 21/21] Bump version to 0.2.27 --- CHANGELOG.rst | 6 ++++++ meta.yaml | 2 +- yfinance/version.py | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index a5b3baa..e498423 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,12 @@ Change Log =========== +0.2.27 +------ +Bug fixes: +- fix merging 1d-prices with out-of-range divs/splits #1635 +- fix multithread error 'tz already in cache' #1648 + 0.2.26 ------ Proxy improvements diff --git a/meta.yaml b/meta.yaml index 0fbb483..f07cb28 100644 --- a/meta.yaml +++ b/meta.yaml @@ -1,5 +1,5 @@ {% set name = "yfinance" %} -{% set version = "0.2.26" %} +{% set version = "0.2.27" %} package: name: "{{ name|lower }}" diff --git a/yfinance/version.py b/yfinance/version.py index 22af852..f0a9e5e 100644 --- a/yfinance/version.py +++ b/yfinance/version.py @@ -1 +1 @@ -version = "0.2.26" +version = "0.2.27"