From 0521428f693276b090b9dc3a68f5e4e5a6dadb52 Mon Sep 17 00:00:00 2001 From: Mike Reiche Date: Mon, 9 Oct 2023 09:53:57 +0200 Subject: [PATCH 01/13] Fixed typing bug when series are empty --- yfinance/base.py | 21 ++++++++++++--------- yfinance/ticker.py | 2 +- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/yfinance/base.py b/yfinance/base.py index 21af2bb51..ecc973aa7 100644 --- a/yfinance/base.py +++ b/yfinance/base.py @@ -45,6 +45,9 @@ from .const import _BASE_URL_, _ROOT_URL_ +_empty_series = pd.Series() + + class TickerBase: def __init__(self, ticker, session=None): self.ticker = ticker.upper() @@ -1910,31 +1913,31 @@ def get_cash_flow(self, proxy=None, as_dict=False, pretty=False, freq="yearly"): def get_cashflow(self, proxy=None, as_dict=False, pretty=False, freq="yearly"): return self.get_cash_flow(proxy, as_dict, pretty, freq) - def get_dividends(self, proxy=None): + def get_dividends(self, proxy=None) -> pd.Series: if self._history is None: self.history(period="max", proxy=proxy) if self._history is not None and "Dividends" in self._history: dividends = self._history["Dividends"] return dividends[dividends != 0] - return [] + return pd.Series() - def get_capital_gains(self, proxy=None): + def get_capital_gains(self, proxy=None) -> pd.Series: if self._history is None: self.history(period="max", proxy=proxy) if self._history is not None and "Capital Gains" in self._history: capital_gains = self._history["Capital Gains"] return capital_gains[capital_gains != 0] - return [] + return _empty_series - def get_splits(self, proxy=None): + def get_splits(self, proxy=None) -> pd.Series: if self._history is None: self.history(period="max", proxy=proxy) if self._history is not None and "Stock Splits" in self._history: splits = self._history["Stock Splits"] return splits[splits != 0] - return [] + return _empty_series - def get_actions(self, proxy=None): + def get_actions(self, proxy=None) -> pd.Series: if self._history is None: self.history(period="max", proxy=proxy) if self._history is not None and "Dividends" in self._history and "Stock Splits" in self._history: @@ -1943,7 +1946,7 @@ def get_actions(self, proxy=None): action_columns.append("Capital Gains") actions = self._history[action_columns] return actions[actions != 0].dropna(how='all').fillna(0) - return [] + return _empty_series def get_shares(self, proxy=None, as_dict=False): self._fundamentals.proxy = proxy @@ -2044,7 +2047,7 @@ def get_isin(self, proxy=None) -> Optional[str]: self._isin = data.split(search_str)[1].split('"')[0].split('|')[0] return self._isin - def get_news(self, proxy=None): + def get_news(self, proxy=None) -> list: if self._news: return self._news diff --git a/yfinance/ticker.py b/yfinance/ticker.py index 241638a5c..8d20a1d27 100644 --- a/yfinance/ticker.py +++ b/yfinance/ticker.py @@ -240,7 +240,7 @@ def options(self) -> tuple: return tuple(self._expirations.keys()) @property - def news(self): + def news(self) -> list: return self.get_news() @property From 9a3d60105ca8b0148a320806a33c07769be8235a Mon Sep 17 00:00:00 2001 From: Mike Reiche Date: Mon, 9 Oct 2023 09:59:25 +0200 Subject: [PATCH 02/13] Minor typing fixes --- yfinance/base.py | 12 ++++++------ yfinance/ticker.py | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/yfinance/base.py b/yfinance/base.py index ecc973aa7..8deaea3b4 100644 --- a/yfinance/base.py +++ b/yfinance/base.py @@ -26,7 +26,7 @@ import logging import time as _time import warnings -from typing import Optional +from typing import Optional, Union, List from urllib.parse import quote as urlencode import dateutil as _dateutil @@ -1883,7 +1883,7 @@ def get_balance_sheet(self, proxy=None, as_dict=False, pretty=False, freq="yearl def get_balancesheet(self, proxy=None, as_dict=False, pretty=False, freq="yearly"): return self.get_balance_sheet(proxy, as_dict, pretty, freq) - def get_cash_flow(self, proxy=None, as_dict=False, pretty=False, freq="yearly"): + def get_cash_flow(self, proxy=None, as_dict=False, pretty=False, freq="yearly") -> Union[pd.DataFrame, dict]: """ :Parameters: as_dict: bool @@ -1919,7 +1919,7 @@ def get_dividends(self, proxy=None) -> pd.Series: if self._history is not None and "Dividends" in self._history: dividends = self._history["Dividends"] return dividends[dividends != 0] - return pd.Series() + return _empty_series def get_capital_gains(self, proxy=None) -> pd.Series: if self._history is None: @@ -1937,7 +1937,7 @@ def get_splits(self, proxy=None) -> pd.Series: return splits[splits != 0] return _empty_series - def get_actions(self, proxy=None) -> pd.Series: + def get_actions(self, proxy=None) -> pd.DataFrame: if self._history is None: self.history(period="max", proxy=proxy) if self._history is not None and "Dividends" in self._history and "Stock Splits" in self._history: @@ -1946,9 +1946,9 @@ def get_actions(self, proxy=None) -> pd.Series: action_columns.append("Capital Gains") actions = self._history[action_columns] return actions[actions != 0].dropna(how='all').fillna(0) - return _empty_series + return pd.DataFrame() - def get_shares(self, proxy=None, as_dict=False): + def get_shares(self, proxy=None, as_dict=False) -> Union[pd.DataFrame, dict]: self._fundamentals.proxy = proxy data = self._fundamentals.shares if as_dict: diff --git a/yfinance/ticker.py b/yfinance/ticker.py index 8d20a1d27..09f55db48 100644 --- a/yfinance/ticker.py +++ b/yfinance/ticker.py @@ -134,7 +134,7 @@ def actions(self) -> _pd.DataFrame: return self.get_actions() @property - def shares(self) -> _pd.DataFrame : + def shares(self) -> _pd.DataFrame: return self.get_shares() @property From ba977a16a26cb97dc77fb4c3346e6d437952948c Mon Sep 17 00:00:00 2001 From: Mike Reiche Date: Tue, 10 Oct 2023 08:25:47 +0200 Subject: [PATCH 03/13] Added tests --- requirements.txt | 4 +++- tests/ticker.py | 22 +++++++++++++++++++--- yfinance/ticker.py | 2 +- 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/requirements.txt b/requirements.txt index 427c16ebb..bbe2495ef 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,8 @@ pandas>=1.3.0 numpy>=1.16.5 requests>=2.31 +requests_cache==1.1.0 +requests_ratelimiter==0.4.2 multitasking>=0.0.7 lxml>=4.9.1 appdirs>=1.4.4 @@ -8,4 +10,4 @@ pytz>=2022.5 frozendict>=2.3.4 beautifulsoup4>=4.11.1 html5lib>=1.1 -peewee>=3.16.2 \ No newline at end of file +peewee>=3.16.2 diff --git a/tests/ticker.py b/tests/ticker.py index 681f0387d..8587a7ed6 100644 --- a/tests/ticker.py +++ b/tests/ticker.py @@ -102,8 +102,24 @@ def test_badTicker(self): dat.fast_info[k] for attribute_name, attribute_type in ticker_attributes: - assert_attribute_type(self, dat, attribute_name, attribute_type) - + assert_attribute_type(self, dat, attribute_name, attribute_type) + + with self.assertRaises(YFNotImplementedError): + assert isinstance(dat.earnings, pd.Series) + assert dat.earnings.empty + assert isinstance(dat.dividends, pd.Series) + assert dat.dividends.empty + assert isinstance(dat.splits, pd.Series) + assert dat.splits.empty + assert isinstance(dat.capital_gains, pd.Series) + assert dat.capital_gains.empty + with self.assertRaises(YFNotImplementedError): + assert isinstance(dat.shares, pd.DataFrame) + assert dat.shares.empty + assert isinstance(dat.actions, pd.DataFrame) + assert dat.actions.empty + + def test_goodTicker(self): # that yfinance works when full api is called on same instance of ticker @@ -912,7 +928,7 @@ def test_info(self): def suite(): suite = unittest.TestSuite() suite.addTest(TestTicker('Test ticker')) - suite.addTest(TestTickerEarnings('Test earnings')) + #suite.addTest(TestTickerEarnings('Test earnings')) suite.addTest(TestTickerHolders('Test holders')) suite.addTest(TestTickerHistory('Test Ticker history')) suite.addTest(TestTickerMiscFinancials('Test misc financials')) diff --git a/yfinance/ticker.py b/yfinance/ticker.py index 09f55db48..e6e2fa2b3 100644 --- a/yfinance/ticker.py +++ b/yfinance/ticker.py @@ -122,7 +122,7 @@ def dividends(self) -> _pd.Series: return self.get_dividends() @property - def capital_gains(self): + def capital_gains(self) -> _pd.Series: return self.get_capital_gains() @property From a3095d2a40800573011ce0f8cf47a638be62bc48 Mon Sep 17 00:00:00 2001 From: Manlai Amar <70603274+amanlai@users.noreply.github.com> Date: Thu, 21 Dec 2023 00:02:53 -0800 Subject: [PATCH 04/13] explicitly name the column levels --- yfinance/multi.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/yfinance/multi.py b/yfinance/multi.py index 99357c8a2..e2ae84b5b 100644 --- a/yfinance/multi.py +++ b/yfinance/multi.py @@ -217,11 +217,11 @@ def download(tickers, start=None, end=None, actions=False, threads=True, ignore_ try: data = _pd.concat(shared._DFS.values(), axis=1, sort=True, - keys=shared._DFS.keys()) + keys=shared._DFS.keys(), names=['Ticker', 'Price']) except Exception: _realign_dfs() data = _pd.concat(shared._DFS.values(), axis=1, sort=True, - keys=shared._DFS.keys()) + keys=shared._DFS.keys(), names=['Ticker', 'Price']) data.index = _pd.to_datetime(data.index) # switch names back to isins if applicable data.rename(columns=shared._ISINS, inplace=True) From ac8a9172884940275c2381313acd69c3e46f9587 Mon Sep 17 00:00:00 2001 From: Mike Reiche Date: Tue, 9 Jan 2024 08:43:54 +0100 Subject: [PATCH 05/13] Revert adding explicit requirements --- requirements.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index bbe2495ef..b8768416b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,6 @@ pandas>=1.3.0 numpy>=1.16.5 requests>=2.31 -requests_cache==1.1.0 -requests_ratelimiter==0.4.2 multitasking>=0.0.7 lxml>=4.9.1 appdirs>=1.4.4 From 4c34487149ee95b8efb78644d7ee6cac5d760740 Mon Sep 17 00:00:00 2001 From: Mike Reiche Date: Tue, 9 Jan 2024 08:50:00 +0100 Subject: [PATCH 06/13] Revert disabling earnings test --- tests/ticker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ticker.py b/tests/ticker.py index 54ed7b85c..ff0d8e6c4 100644 --- a/tests/ticker.py +++ b/tests/ticker.py @@ -926,7 +926,7 @@ def test_complementary_info(self): def suite(): suite = unittest.TestSuite() suite.addTest(TestTicker('Test ticker')) - #suite.addTest(TestTickerEarnings('Test earnings')) + suite.addTest(TestTickerEarnings('Test earnings')) suite.addTest(TestTickerHolders('Test holders')) suite.addTest(TestTickerHistory('Test Ticker history')) suite.addTest(TestTickerMiscFinancials('Test misc financials')) From 223f5337a8f995c7f4623c425a49ece0ba56e193 Mon Sep 17 00:00:00 2001 From: Mike Reiche Date: Tue, 9 Jan 2024 08:50:31 +0100 Subject: [PATCH 07/13] Remove empty static series --- yfinance/base.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/yfinance/base.py b/yfinance/base.py index af32ddd85..6cb3e751d 100644 --- a/yfinance/base.py +++ b/yfinance/base.py @@ -45,9 +45,6 @@ from .const import _BASE_URL_, _ROOT_URL_ -_empty_series = pd.Series() - - class TickerBase: def __init__(self, ticker, session=None, proxy=None): self.ticker = ticker.upper() @@ -1955,7 +1952,7 @@ def get_dividends(self, proxy=None) -> pd.Series: if self._history is not None and "Dividends" in self._history: dividends = self._history["Dividends"] return dividends[dividends != 0] - return _empty_series + return pd.Series() def get_capital_gains(self, proxy=None) -> pd.Series: if self._history is None: @@ -1963,7 +1960,7 @@ def get_capital_gains(self, proxy=None) -> pd.Series: if self._history is not None and "Capital Gains" in self._history: capital_gains = self._history["Capital Gains"] return capital_gains[capital_gains != 0] - return _empty_series + return pd.Series() def get_splits(self, proxy=None) -> pd.Series: if self._history is None: @@ -1971,7 +1968,7 @@ def get_splits(self, proxy=None) -> pd.Series: if self._history is not None and "Stock Splits" in self._history: splits = self._history["Stock Splits"] return splits[splits != 0] - return _empty_series + return pd.Series() def get_actions(self, proxy=None) -> pd.DataFrame: if self._history is None: From dbc55e55960d685eac0e31365f94fd38b52bac0a Mon Sep 17 00:00:00 2001 From: Mike Reiche Date: Tue, 9 Jan 2024 21:08:46 +0100 Subject: [PATCH 08/13] Remove unused List import --- yfinance/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yfinance/base.py b/yfinance/base.py index 6cb3e751d..c852df245 100644 --- a/yfinance/base.py +++ b/yfinance/base.py @@ -27,7 +27,7 @@ import logging import time as _time import warnings -from typing import Optional, Union, List +from typing import Optional, Union from urllib.parse import quote as urlencode import dateutil as _dateutil From 47bc46c804b27622e55edf451a7b09ae5dbaa747 Mon Sep 17 00:00:00 2001 From: molpcs Date: Fri, 12 Jan 2024 11:57:58 -0800 Subject: [PATCH 09/13] Update README.md Wrap yfinance[optional] code snippet with quotes to avoid conflict with zsh globbing. Remains compatible with bash. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 356a28416..2ba663d7f 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ $ pip install yfinance --upgrade --no-cache-dir To install with optional dependencies, replace `optional` with: `nospam` for [caching-requests](#smarter-scraping), `repair` for [price repair](https://github.com/ranaroussi/yfinance/wiki/Price-repair), or `nospam,repair` for both: ``` {.sourceCode .bash} -$ pip install yfinance[optional] +$ pip install "yfinance[optional]" ``` [Required dependencies](./requirements.txt) , [all dependencies](./setup.py#L62). From 6686258e669f868ae1af3a17b05099ea4f7de97b Mon Sep 17 00:00:00 2001 From: Value Raider Date: Sat, 13 Jan 2024 13:19:44 +0000 Subject: [PATCH 10/13] Fix history() keepna=False with repair=True --- yfinance/base.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/yfinance/base.py b/yfinance/base.py index c852df245..c4f8fd61e 100644 --- a/yfinance/base.py +++ b/yfinance/base.py @@ -42,7 +42,7 @@ from .scrapers.holders import Holders from .scrapers.quote import Quote, FastInfo -from .const import _BASE_URL_, _ROOT_URL_ +from .const import _BASE_URL_, _ROOT_URL_, price_colnames class TickerBase: @@ -426,7 +426,9 @@ def history(self, period="1mo", interval="1d", if not actions: df = df.drop(columns=["Dividends", "Stock Splits", "Capital Gains"], errors='ignore') if not keepna: - mask_nan_or_zero = (df.isna() | (df == 0)).all(axis=1) + data_colnames = price_colnames + ['Volume'] + ['Dividends', 'Stock Splits', 'Capital Gains'] + data_colnames = [c for c in data_colnames if c in df.columns] + mask_nan_or_zero = (df[data_colnames].isna() | (df[data_colnames] == 0)).all(axis=1) df = df.drop(mask_nan_or_zero.index[mask_nan_or_zero]) logger.debug(f'{self.ticker}: yfinance returning OHLC: {df.index[0]} -> {df.index[-1]}') @@ -455,7 +457,7 @@ def _reconstruct_intervals_batch(self, df, interval, prepost, tag=-1): else: intraday = True - price_cols = [c for c in ["Open", "High", "Low", "Close", "Adj Close"] if c in df] + price_cols = [c for c in price_colnames if c in df] data_cols = price_cols + ["Volume"] # If interval is weekly then can construct with daily. But if smaller intervals then @@ -1011,7 +1013,7 @@ def _fix_zeroes(self, df, interval, tz_exchange, prepost): elif df2.index.tz != tz_exchange: df2.index = df2.index.tz_convert(tz_exchange) - price_cols = [c for c in ["Open", "High", "Low", "Close", "Adj Close"] if c in df2.columns] + price_cols = [c for c in price_colnames if c in df2.columns] f_prices_bad = (df2[price_cols] == 0.0) | df2[price_cols].isna() df2_reserve = None if intraday: From ffaf200562eb88e57623a0919033617c47c74edd Mon Sep 17 00:00:00 2001 From: Value Raider Date: Sat, 13 Jan 2024 23:00:59 +0000 Subject: [PATCH 11/13] Handle peewee with old sqlite --- yfinance/cache.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/yfinance/cache.py b/yfinance/cache.py index d625a2ab2..e34254909 100644 --- a/yfinance/cache.py +++ b/yfinance/cache.py @@ -145,7 +145,14 @@ def initialise(self): db.connect() tz_db_proxy.initialize(db) - db.create_tables([_KV]) + try: + db.create_tables([_KV]) + except _peewee.OperationalError as e: + if 'WITHOUT' in str(e): + _KV._meta.without_rowid = False + db.create_tables([_KV]) + else: + raise self.initialised = 1 # success def lookup(self, key): @@ -344,7 +351,14 @@ def initialise(self): db.connect() Cookie_db_proxy.initialize(db) - db.create_tables([_CookieSchema]) + try: + db.create_tables([_CookieSchema]) + except _peewee.OperationalError as e: + if 'WITHOUT' in str(e): + _CookieSchema._meta.without_rowid = False + db.create_tables([_CookieSchema]) + else: + raise self.initialised = 1 # success def lookup(self, strategy): From 91f468e4d3b2bcdc3d22d4043d3e69c75a4227e7 Mon Sep 17 00:00:00 2001 From: Mike Reiche Date: Mon, 9 Oct 2023 09:53:57 +0200 Subject: [PATCH 12/13] Fix JSON access to prevent KeyError --- yfinance/base.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/yfinance/base.py b/yfinance/base.py index c852df245..196a8c21a 100644 --- a/yfinance/base.py +++ b/yfinance/base.py @@ -45,6 +45,9 @@ from .const import _BASE_URL_, _ROOT_URL_ +_empty_series = pd.Series() + + class TickerBase: def __init__(self, ticker, session=None, proxy=None): self.ticker = ticker.upper() @@ -1960,7 +1963,7 @@ def get_capital_gains(self, proxy=None) -> pd.Series: if self._history is not None and "Capital Gains" in self._history: capital_gains = self._history["Capital Gains"] return capital_gains[capital_gains != 0] - return pd.Series() + return _empty_series def get_splits(self, proxy=None) -> pd.Series: if self._history is None: @@ -1970,7 +1973,7 @@ def get_splits(self, proxy=None) -> pd.Series: return splits[splits != 0] return pd.Series() - def get_actions(self, proxy=None) -> pd.DataFrame: + def get_actions(self, proxy=None) -> pd.Series: if self._history is None: self.history(period="max", proxy=proxy) if self._history is not None and "Dividends" in self._history and "Stock Splits" in self._history: @@ -1979,7 +1982,7 @@ def get_actions(self, proxy=None) -> pd.DataFrame: action_columns.append("Capital Gains") actions = self._history[action_columns] return actions[actions != 0].dropna(how='all').fillna(0) - return pd.DataFrame() + return _empty_series def get_shares(self, proxy=None, as_dict=False) -> Union[pd.DataFrame, dict]: self._fundamentals.proxy = proxy or self.proxy From 06fd35121a01365794fd7ca4d5fa2737e889d24a Mon Sep 17 00:00:00 2001 From: Ange Daumal Date: Fri, 19 Jan 2024 22:51:02 +0100 Subject: [PATCH 13/13] Fix JSON access to prevent KeyError --- yfinance/scrapers/quote.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/yfinance/scrapers/quote.py b/yfinance/scrapers/quote.py index 1daed5327..1d018359b 100644 --- a/yfinance/scrapers/quote.py +++ b/yfinance/scrapers/quote.py @@ -733,10 +733,11 @@ def _fetch_complementary(self, proxy): json_str = self._data.cache_get(url=url, proxy=proxy).text json_data = json.loads(json_str) - if json_data["timeseries"]["error"] is not None: - raise YFinanceException("Failed to parse json response from Yahoo Finance: " + json_data["error"]) + json_result = json_data.get("timeseries") or json_data.get("finance") + if json_result["error"] is not None: + raise YFinanceException("Failed to parse json response from Yahoo Finance: " + str(json_result["error"])) for k in keys: - keydict = json_data["timeseries"]["result"][0] + keydict = json_result["result"][0] if k in keydict: self._info[k] = keydict[k][-1]["reportedValue"]["raw"] else: