-
Notifications
You must be signed in to change notification settings - Fork 1
/
Copy pathmain.py
136 lines (98 loc) · 4.4 KB
/
main.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
import pandas as pd
import chardet
import yfinance as yf
import logging
import os
from pykrx import stock
from kos import KOS_StockDownloader
from loader import NAS_StockDownloader, NasdaqTickerLoader, InterceptHandler
hoynet_home = './'
nasdaqdata_path = './NASDAQ'
kospidata_path = './KOSPI'
indicator_path = './IND'
### Naming csv files ###
# Input Stock items Information
stock_list = 'NASDAQ_FC_STK_IEM_IFO.csv'
# Data without Indicators
nasdaq_output_file = 'NASDAQ_DATA.csv'
merged_nasdaq_output_file = 'MERGED_NASDAQ_DATA.csv'
kospi_output_file = 'KOSPI_DATA.csv'
kospi_ticker_code = '000020'
# Data with Indicators
ind_nasdaq_output_file = 'E_NASDAQ_DATA.csv'
### ( change the name in this section ) ###
# Assigning Start / End date
start_date = '2013-01-01'
end_date = '2023-08-31'
def nasdaq_crawl(stock_list, output_csv):
# loading stock list
loader = NasdaqTickerLoader(file_path)
ticker_list = loader.load_tickers()
# yfinance logger error handling (Failed data)
logger = logging.getLogger('yfinance')
logger.setLevel(logging.INFO)
handler = InterceptHandler()
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
logger.addHandler(handler)
# download Stock Data
downloader = NAS_StockDownloader(ticker_list, output_csv, start=start_date, end=end_date)
downloader.execute()
downloader.save_to_csv(output_csv, 'exception_stock_data.csv')
def indicator_merge(origianl_csv, merged_csv):
# uploading origianl data
merged_stock_data = pd.read_csv(origianl_csv, parse_dates=['Date'])
# gold_value 100 troy ounce, 10Y_american_bond data, USD/KRW, dow_jonse
gold_data = yf.download('GC=F', start=start_date, end=end_date)
bond_data = yf.download('^TNX', start=start_date, end=end_date)
dow_jones = yf.download("^DJI", start=start_date, end=end_date)
# change row name
gold_data.rename(columns={"Open": "gold"}, inplace=True)
bond_data.rename(columns={"Open": "bond"}, inplace=True)
dow_jones.rename(columns={"Open": "dow"}, inplace=True)
# setting rows for matching Date index
merged_stock_data.set_index('Date', inplace=True)
gold_data.reset_index(inplace=True)
bond_data.reset_index(inplace=True)
dow_jones.reset_index(inplace=True)
gold_data.set_index('Date', inplace=True)
bond_data.set_index('Date', inplace=True)
dow_jones.set_index('Date', inplace=True)
# merging with original_stock_data
merged_stock_data = merged_stock_data.merge(gold_data[['gold']], on='Date', how='left')
merged_stock_data = merged_stock_data.merge(bond_data[['bond']], on='Date', how='left')
merged_stock_data = merged_stock_data.merge(dow_jones[['dow']], on='Date', how='left')
# Replacing NaN's with final_stock_data
merged_stock_data.fillna(0, inplace=True)
merged_stock_data.reset_index(inplace=True)
merged_stock_data.to_csv(merged_csv, index=False)
gold_path = os.path.join(indicator_path, 'gold_data.csv')
bond_path = os.path.join(indicator_path, 'amreican_bond_data.csv')
dow_path = os.path.join(indicator_path, 'dow_jones.csv')
gold_data.to_csv(gold_path)
bond_data.to_csv(bond_path)
dow_jones.to_csv(dow_path)
def nasdaq_merge(file_path):
total_stock = pd.DataFrame()
csv_files = [file for file in os.listdir(nasdaqdata_path) if file.endswith('.csv')]
for csv_file in csv_files:
merge_file = os.path.join(nasdaqdata_path, csv_file)
ind_stock_data = pd.read_csv(merge_file)
total_stock = pd.concat([total_stock, ind_stock_data])
total_stock.to_csv(file_path, index=False)
if __name__ == "__main__":
# Making path for Data
file_path = os.path.join(hoynet_home, stock_list)
nas_file_path = os.path.join(nasdaqdata_path, nasdaq_output_file)
ind_nas_file_path = os.path.join(nasdaqdata_path, ind_nasdaq_output_file)
merged_nas_file_path = os.path.join(nasdaqdata_path, merged_nasdaq_output_file)
kos_file_path = os.path.join(kospidata_path, kospi_output_file)
# NASDAQ Crawling implement
nasdaq_crawl(file_path, nas_file_path)
indicator_merge(nas_file_path, ind_nas_file_path)
# # KOSPI Crawling implement
kospi_crawl = KOS_StockDownloader(stock, start_date, end_date)
kospi_crawl.execute(kospi_ticker_code)
kospi_crawl.save_data(kos_file_path)
# NASDAQ Data Merging
# nasdaq_merge(merged_nas_file_path)