data loading

lib/tsetmc_api/__init__.py

@@ -3,4 +3,4 @@
 from .watch import Watch, Filter, AtiFilter, SahamFilter, WatchTick, SandoghFilter, KalaForoushFilter, \
     EkhtiarForoushFilter, HaghTaghaddomFilter, OraghMosharekatFilter, PayeFarabourseFilter
 
-__version__ = '4.0.2'
+__version__ = '4.1.0'

lib/tsetmc_api/bin/tsetmc_loader.py

@@ -0,0 +1,55 @@
+import sys
+
+from jdatetime import datetime as jdatetime
+
+from tsetmc_api.data_file import SymbolDataFile
+from tsetmc_api.types import SymbolId
+
+
+def generate(symbol_id: SymbolId, start_time: jdatetime, end_time: jdatetime):
+    file_name = f'{symbol_id}_{start_time.strftime("%Y%m%d")}_{end_time.strftime("%Y%m%d")}.zip'
+    print(f'Generating Data File {file_name} ...')
+    SymbolDataFile.generate_data_file(symbol_id=symbol_id,
+                                      start_time=start_time,
+                                      end_time=end_time,
+                                      file_location=file_name)
+
+
+def install(file_location: str):
+    SymbolDataFile.install_data_file(file_location)
+
+
+def main():
+    if len(sys.argv) < 2:
+        print('not enough arguments')
+        sys.exit(1)
+
+    action = sys.argv[1]
+    if action == 'generate':
+        if len(sys.argv) <= 2:
+            print('symbol id was not provided')
+            sys.exit(1)
+
+        symbol_id = sys.argv[2]
+
+        if len(sys.argv) > 3:
+            start_time = jdatetime.strptime(sys.argv[3], '%Y-%m-%d')
+        else:
+            start_time = jdatetime(1396, 1, 1)
+
+        if len(sys.argv) > 4:
+            end_time = jdatetime.strptime(sys.argv[4], '%Y-%m-%d')
+        else:
+            end_time = jdatetime.now()
+
+        generate(symbol_id=symbol_id, start_time=start_time, end_time=end_time)
+    elif action == 'install':
+        if len(sys.argv) <= 2:
+            print('file location was not provided')
+            sys.exit(1)
+
+        file_location = sys.argv[2]
+        install(file_location=file_location)
+    else:
+        print('invalid action')
+        sys.exit(1)

lib/tsetmc_api/data_file/__init__.py

@@ -0,0 +1 @@
+from .symbol_data_file import SymbolDataFile

lib/tsetmc_api/data_file/symbol_data_file.py

@@ -0,0 +1,82 @@
+from __future__ import annotations
+
+import pickle
+import zipfile
+from dataclasses import dataclass
+
+from jdatetime import datetime as jdatetime
+
+from tsetmc_api.cache import PersistentCache
+from tsetmc_api.day_details.day_details import SymbolDayDetails
+from tsetmc_api.day_details.exceptions import NoDataError
+from tsetmc_api.symbol import Symbol
+from tsetmc_api.types import SymbolId
+from tsetmc_api.utils import jalali_daterange
+
+
+@dataclass
+class SymbolDataFileInformation:
+    symbol_id: SymbolId
+    start_time: jdatetime
+    end_time: jdatetime
+
+
+class SymbolDataFile:
+    @staticmethod
+    def get_data_file_information(file_location: str) -> SymbolDataFileInformation:
+        with zipfile.ZipFile(file_location, 'r') as zp:
+            with zp.open('information.pickle', 'r') as infop:
+                return pickle.load(infop)
+
+    @staticmethod
+    def install_data_file(file_location: str):
+        file_information = SymbolDataFile.get_data_file_information(file_location=file_location)
+        with zipfile.ZipFile(file_location, 'r') as zp:
+            for jdate in jalali_daterange(start_time=file_information.start_time, end_time=file_information.end_time):
+                print(f'Installing {jdate.year}/{jdate.month}/{jdate.day}')
+                file_name = f'{jdate.year}-{jdate.month}-{jdate.day}.pickle'
+                try:
+                    with zp.open(file_name, 'r') as pfp:
+                        day_details = pickle.load(pfp)
+                        if type(day_details) == SymbolDayDetails:
+                            day_details._save_to_cache()
+                        else:
+                            PersistentCache.store(
+                                'symbol_day_details',
+                                f'{file_information.symbol_id}-{jdate.year}-{jdate.month}-{jdate.day}',
+                                {
+                                    'no_data': True,
+                                }
+                            )
+                except KeyError:
+                    pass
+                except Exception as ex:
+                    print(type(ex))
+                    print(ex)
+
+    @staticmethod
+    def generate_data_file(symbol_id: SymbolId, start_time: jdatetime, end_time: jdatetime, file_location: str):
+        symbol = Symbol(symbol_id=symbol_id)
+        with zipfile.ZipFile(file_location, 'w') as zp:
+            for jdate in jalali_daterange(start_time=start_time, end_time=end_time):
+                while True:
+                    print(f'Loading {jdate.year}/{jdate.month}/{jdate.day}')
+                    file_name = f'{jdate.year}-{jdate.month}-{jdate.day}.pickle'
+                    try:
+                        day_details = symbol.get_day_details(jyear=jdate.year, jmonth=jdate.month, jday=jdate.day)
+                        with zp.open(file_name, 'w') as pfp:
+                            pickle.dump(day_details, pfp)
+                        break
+                    except NoDataError as ex:
+                        with zp.open(file_name, 'w') as pfp:
+                            pickle.dump(None, pfp)
+                        break
+                    except Exception as ex:
+                        print(ex)
+
+            with zp.open('information.pickle', 'w') as infop:
+                pickle.dump(SymbolDataFileInformation(
+                    symbol_id=symbol_id,
+                    start_time=start_time,
+                    end_time=end_time,
+                ), infop)

lib/tsetmc_api/day_details/core.py

@@ -5,6 +5,8 @@
 import requests
 from bs4 import BeautifulSoup
 
+from .exceptions import NoDataError
+
 _intraday_pattern = re.compile("var (?P<name>.*)=(?P<content>\[[^;]*\]);")
 
 
@@ -156,7 +158,7 @@ def load_intraday_data(symbol_id, year, month, day):
     price_data = _extract_price_data(script_vars)
 
     if not price_data:
-        raise ValueError('there is no data for this symbol in the specified date')
+        raise NoDataError('there is no data for this symbol in the specified date')
 
     previous_shareholders, shareholders = _extract_share_holders_data(script_vars)
     trades = _extract_trade_data(script_vars)

lib/tsetmc_api/day_details/day_details.py

@@ -7,30 +7,71 @@
 from tsetmc_api.shareholder import SymbolMajorShareholder
 from tsetmc_api.utils import get_timezone
 from . import core as day_details_core
+from .exceptions import NoDataError
 from .snapshot import Snapshot
 from .trade import Trade
 
 
 class SymbolDayDetails:
-    def __init__(self, symbol, jyear: int, jmonth: int, jday: int):
+    def __init__(self, symbol, jyear: int, jmonth: int, jday: int,
+                 load_from_cache: bool = True, save_to_cache: bool = True):
         self.symbol = symbol
         self.date = date(year=jyear, month=jmonth, day=jday)
         self.greg_date = self.date.togregorian()
+        self.load_from_cache = load_from_cache
+        self.save_to_cache = save_to_cache
+        self._is_no_data = False
 
         self._load()
 
-    def _load(self):
-        cache_minor = f'{self.symbol.id}-{self.date.year}-{self.date.month}-{self.date.day}'
-        if PersistentCache.exists('symbol_day_details', cache_minor):
-            f = PersistentCache.fetch('symbol_day_details', cache_minor)
+    def _get_cache_minor(self):
+        return f'{self.symbol.id}-{self.date.year}-{self.date.month}-{self.date.day}'
+
+    def _save_to_cache(self):
+        if not self.save_to_cache:
+            return
+
+        if self._is_no_data:
+            PersistentCache.store('symbol_day_details', self._get_cache_minor(), {
+                'no_data': True,
+            })
+            return
+
+        PersistentCache.store('symbol_day_details', self._get_cache_minor(), {
+            'final_shareholders': self._final_shareholders,
+            'initial_shareholders': self._initial_shareholders,
+            'trades': self._trades,
+            'snapshots': self._snapshots,
+        })
+
+    def _load_from_cache(self) -> bool:
+        if not self.load_from_cache:
+            return False
+
+        if PersistentCache.exists('symbol_day_details', self._get_cache_minor()):
+            f = PersistentCache.fetch('symbol_day_details', self._get_cache_minor())
+
+            if 'no_data' in f:
+                self._is_no_data = True
+                raise NoDataError('there is no data for this symbol in the specified date')
+
             self._final_shareholders = f['final_shareholders']
             self._initial_shareholders = f['initial_shareholders']
             self._trades = f['trades']
             self._snapshots = f['snapshots']
-            return
+            print('loaded from cache')
+            return True
+
+        return False
 
+    def _load_from_tsetmc(self):
         mdate = self.date.togregorian()
-        data = day_details_core.load_intraday_data(self.symbol.id, mdate.year, mdate.month, mdate.day)
+        try:
+            data = day_details_core.load_intraday_data(self.symbol.id, mdate.year, mdate.month, mdate.day)
+        except NoDataError as ex:
+            self._is_no_data = True
+            self._save_to_cache()
+            raise ex
         symbol_details = self.symbol.get_details()
 
         self._final_shareholders = []
@@ -59,12 +100,13 @@ def _load(self):
                                                                  data['orders_data'])
         )
 
-        PersistentCache.store('asset_day_details', cache_minor, {
-            'final_shareholders': self._final_shareholders,
-            'initial_shareholders': self._initial_shareholders,
-            'trades': self._trades,
-            'snapshots': self._snapshots,
-        })
+    def _load(self):
+        if self._load_from_cache():
+            return
+
+        self._load_from_tsetmc()
+
+        self._save_to_cache()
 
     def get_final_major_shareholders(self) -> List[SymbolMajorShareholder]:
         """

lib/tsetmc_api/day_details/exceptions.py

@@ -0,0 +1,2 @@
+class NoDataError(Exception):
+    pass

lib/tsetmc_api/utils.py

@@ -1,6 +1,7 @@
 from datetime import datetime, time, date
 
 from dateutil import tz as timezoneutil
+from jdatetime import timedelta as jtimedelta, datetime as jdatetime
 
 _timezone = timezoneutil.gettz('Asia/Tehran')
 
@@ -44,3 +45,8 @@ def get_timestamp(date: date, i: int) -> int:
 
     return int(datetime(year=date.year, month=date.month, day=date.day,
                         hour=h, minute=m, second=s, tzinfo=get_timezone()).timestamp())
+
+
+def jalali_daterange(start_time: jdatetime, end_time: jdatetime):
+    for n in range(int((end_time - start_time).days)):
+        yield start_time + jtimedelta(n)

setup.py

@@ -3,14 +3,14 @@
 from setuptools import setup, find_packages
 
 here = path.abspath(path.dirname(__file__))
-lib = path.join(here, 'lib')
+lib = path.join(here, "lib")
 
 with open(path.join(here, "README.md"), encoding="utf-8") as f:
     long_description = f.read()
 
 setup(
     name="tsetmc-api",
-    version="4.0.2",
+    version="4.1.0",
     python_requires=">=3.7",
     install_requires=[
         "beautifulsoup4==4.9.3",
@@ -78,7 +78,8 @@
         ]
     },
     package_dir={"": "lib"},
-    packages=find_packages(where=lib, exclude=["scripts"]),
+    packages=find_packages(where=lib, exclude=["tsetmc_api.bin", "scripts"]),
+    entry_points={"console_scripts": ["tsetmc-loader = tsetmc_api.bin.tsetmc_loader:main"]},
     url="https://github.com/mahs4d/tsetmc-api",
     license="MIT",
     author="Mahdi Sadeghi",