Merge pull request #1147 from fredrik-corneliusson/request_optimization

Request optimization

.gitignore

@@ -9,3 +9,10 @@ build/
 *.html
 *.css
 *.png
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/

CHANGELOG.rst

@@ -1,6 +1,16 @@
 Change Log
 ===========
 
+0.2.0rc1
+------
+Jumping to 0.2 for this big update. 0.1.* will continue to receive bug-fixes
+- timezone cache performance massively improved. Thanks @fredrik-corneliusson #1113 #1112 #1109 #1105 #1099
+- price repair feature #1110
+- fix merging of dividends/splits with prices #1069 #1086 #1102
+- fix Yahoo returning latest price interval across 2 rows #1070
+- optional: raise errors as exceptions: raise_errors=True #1104
+- add proper unit tests #1069
+
 0.1.81
 ------
 - Fix unhandled tz-cache exception #1107

requirements.txt

@@ -4,4 +4,7 @@ requests>=2.26
 multitasking>=0.0.7
 lxml>=4.5.1
 appdirs>=1.4.4
-pytz>=2022.5
+pytz>=2022.5
+frozendict>=2.3.4
+beautifulsoup4>=4.11.1
+html5lib>=1.1

setup.py

@@ -38,8 +38,8 @@
     classifiers=[
         'License :: OSI Approved :: Apache Software License',
         # 'Development Status :: 3 - Alpha',
-        # 'Development Status :: 4 - Beta',
-        'Development Status :: 5 - Production/Stable',
+        'Development Status :: 4 - Beta',
+        #'Development Status :: 5 - Production/Stable',
 
 
         'Operating System :: OS Independent',
@@ -50,20 +50,20 @@
         'Topic :: Software Development :: Libraries',
         'Topic :: Software Development :: Libraries :: Python Modules',
 
-        'Programming Language :: Python :: 2.7',
-        'Programming Language :: Python :: 3.4',
-        'Programming Language :: Python :: 3.5',
-        # 'Programming Language :: Python :: 3.6',
+        'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
         'Programming Language :: Python :: 3.9',
+        'Programming Language :: Python :: 3.10',
     ],
     platforms=['any'],
     keywords='pandas, yahoo finance, pandas datareader',
     packages=find_packages(exclude=['contrib', 'docs', 'tests', 'examples']),
-    install_requires=['pandas>=0.24.0', 'numpy>=1.15',
+    install_requires=['pandas>=1.1.0', 'numpy>=1.15',
                       'requests>=2.26', 'multitasking>=0.0.7',
-                      'lxml>=4.5.1', 'appdirs>=1.4.4', 'pytz>=2022.5'],
+                      'lxml>=4.5.1', 'appdirs>=1.4.4', 'pytz>=2022.5',
+                      'frozendict>=2.3.4',
+                      'beautifulsoup4>=4.11.1', 'html5lib>=1.1'],
     entry_points={
         'console_scripts': [
             'sample=sample:main',

tests/prices.py

@@ -6,19 +6,19 @@
 import pytz as _tz
 import numpy as _np
 import pandas as _pd
+import os
 
 # Create temp session
 import requests_cache, tempfile
 
 td = tempfile.TemporaryDirectory()
-cache_fp = td.name+'/'+"yfinance.cache"
 
 
 class TestPriceHistory(unittest.TestCase):
     def setUp(self):
         global td
         self.td = td
-        self.session = requests_cache.CachedSession(self.td.name + '/' + "yfinance.cache")
+        self.session = requests_cache.CachedSession(os.path.join(self.td.name, "yfinance.cache"))
 
     def tearDown(self):
         self.session.close()
@@ -116,8 +116,8 @@ def test_dailyWithEvents(self):
         end_d = "2020-11-29"
         df1 = yf.Ticker(tkr1).history(start=start_d, end=end_d, interval="1d", actions=True)
         df2 = yf.Ticker(tkr2).history(start=start_d, end=end_d, interval="1d", actions=True)
-        self.assertTrue(((df1["Dividends"]>0)|(df1["Stock Splits"]>0)).any())
-        self.assertTrue(((df2["Dividends"]>0)|(df2["Stock Splits"]>0)).any())
+        self.assertTrue(((df1["Dividends"] > 0) | (df1["Stock Splits"] > 0)).any())
+        self.assertTrue(((df2["Dividends"] > 0) | (df2["Stock Splits"] > 0)).any())
         try:
             self.assertTrue(df1.index.equals(df2.index))
         except:
@@ -132,7 +132,7 @@ def test_dailyWithEvents(self):
         for tkr in tkrs:
             df1 = yf.Ticker(tkr, session=self.session).history(start=start_d, end=end_d, interval="1d", actions=True)
             df2 = yf.Ticker(tkr, session=self.session).history(start=start_d, end=end_d, interval="1d", actions=False)
-            self.assertTrue(((df1["Dividends"]>0)|(df1["Stock Splits"]>0)).any())
+            self.assertTrue(((df1["Dividends"] > 0) | (df1["Stock Splits"] > 0)).any())
             try:
                 self.assertTrue(df1.index.equals(df2.index))
             except:
@@ -150,8 +150,8 @@ def test_weeklyWithEvents(self):
         end_d = "2020-11-29"
         df1 = yf.Ticker(tkr1).history(start=start_d, end=end_d, interval="1wk", actions=True)
         df2 = yf.Ticker(tkr2).history(start=start_d, end=end_d, interval="1wk", actions=True)
-        self.assertTrue(((df1["Dividends"]>0)|(df1["Stock Splits"]>0)).any())
-        self.assertTrue(((df2["Dividends"]>0)|(df2["Stock Splits"]>0)).any())
+        self.assertTrue(((df1["Dividends"] > 0) | (df1["Stock Splits"] > 0)).any())
+        self.assertTrue(((df2["Dividends"] > 0) | (df2["Stock Splits"] > 0)).any())
         try:
             self.assertTrue(df1.index.equals(df2.index))
         except:
@@ -166,7 +166,7 @@ def test_weeklyWithEvents(self):
         for tkr in tkrs:
             df1 = yf.Ticker(tkr, session=self.session).history(start=start_d, end=end_d, interval="1wk", actions=True)
             df2 = yf.Ticker(tkr, session=self.session).history(start=start_d, end=end_d, interval="1wk", actions=False)
-            self.assertTrue(((df1["Dividends"]>0)|(df1["Stock Splits"]>0)).any())
+            self.assertTrue(((df1["Dividends"] > 0) | (df1["Stock Splits"] > 0)).any())
             try:
                 self.assertTrue(df1.index.equals(df2.index))
             except:
@@ -183,8 +183,8 @@ def test_monthlyWithEvents(self):
         end_d = "2020-11-29"
         df1 = yf.Ticker(tkr1).history(start=start_d, end=end_d, interval="1mo", actions=True)
         df2 = yf.Ticker(tkr2).history(start=start_d, end=end_d, interval="1mo", actions=True)
-        self.assertTrue(((df1["Dividends"]>0)|(df1["Stock Splits"]>0)).any())
-        self.assertTrue(((df2["Dividends"]>0)|(df2["Stock Splits"]>0)).any())
+        self.assertTrue(((df1["Dividends"] > 0) | (df1["Stock Splits"] > 0)).any())
+        self.assertTrue(((df2["Dividends"] > 0) | (df2["Stock Splits"] > 0)).any())
         try:
             self.assertTrue(df1.index.equals(df2.index))
         except:
@@ -199,7 +199,7 @@ def test_monthlyWithEvents(self):
         for tkr in tkrs:
             df1 = yf.Ticker(tkr, session=self.session).history(start=start_d, end=end_d, interval="1mo", actions=True)
             df2 = yf.Ticker(tkr, session=self.session).history(start=start_d, end=end_d, interval="1mo", actions=False)
-            self.assertTrue(((df1["Dividends"]>0)|(df1["Stock Splits"]>0)).any())
+            self.assertTrue(((df1["Dividends"] > 0) | (df1["Stock Splits"] > 0)).any())
             try:
                 self.assertTrue(df1.index.equals(df2.index))
             except:
@@ -232,25 +232,24 @@ def test_dst_fix(self):
 
         interval = "1d"
         df = dat.history(start=start, end=end, interval=interval)
-        self.assertTrue(((df.index.weekday>=0) & (df.index.weekday<=4)).all())
+        self.assertTrue(((df.index.weekday >= 0) & (df.index.weekday <= 4)).all())
 
         interval = "1wk"
         df = dat.history(start=start, end=end, interval=interval)
         try:
-            self.assertTrue((df.index.weekday==0).all())
+            self.assertTrue((df.index.weekday == 0).all())
         except:
             print("Weekly data not aligned to Monday")
             raise
 
     def test_weekly_2rows_fix(self):
         tkr = "AMZN"
-        start = _dt.date.today()-_dt.timedelta(days=14)
+        start = _dt.date.today() - _dt.timedelta(days=14)
         start -= _dt.timedelta(days=start.weekday())
 
         dat = yf.Ticker(tkr)
         df = dat.history(start=start, interval="1wk")
-        self.assertTrue((df.index.weekday==0).all())
-
+        self.assertTrue((df.index.weekday == 0).all())
 
     def test_repair_weekly(self):
         # Sometimes, Yahoo returns prices 100x the correct value.
@@ -501,9 +500,11 @@ def test_repair_daily(self):
         f_1 = ratio == 1
         self.assertTrue((f_100 | f_1).all())
 
-
-if __name__ == '__main__':
-    unittest.main()
+try:
+    if __name__ == '__main__':
+        unittest.main()
+finally:
+    td.cleanup()
 
 # # Run tests sequentially:
 # import inspect
@@ -513,4 +514,3 @@ def test_repair_daily(self):
 # )
 # unittest.main(verbosity=2)
 
-td.cleanup()