Added date_time column

.gitignore

@@ -1,3 +1,4 @@
 *.pyc
 *.csv
-.vscode/
+.vscode/
+.vs/

.vs/PythonSettings.json

@@ -0,0 +1,3 @@
+{
+  "SuppressEnvironmentCreationPrompt": true
+}

.vs/VSWorkspaceState.json

@@ -0,0 +1,8 @@
+{
+  "ExpandedNodes": [
+    "",
+    "\\util"
+  ],
+  "SelectedNode": "\\weather_scraper.py",
+  "PreviewInSolutionExplorer": false
+}

README.md

@@ -15,7 +15,7 @@ python weather_scraper.py
 ```  
 
 ### How to run TWS?
-First, find  the weather stations you are looking for.  
+First, find the weather stations you are looking for.  
 Then you just have to update 2 config files before running TWS. 
 
 1. Go to https://www.wunderground.com/wundermap and zoom in to your location  
@@ -44,12 +44,12 @@ UNIT_SYSTEM = "metric"
 FIND_FIRST_DATE = False
 ```
 
-Now you are read to run your downloads:
+Now you are ready to run your downloads:
 ```sh
 $ python weather_scraper.py
 ```
 Wait until TWS finishes writing your data to files with this naming pattern ***station_name.csv***!  
 
-You resulting CSV file will look something like this (if you give it a nice format)  
+You resulting CSV file will look something like this (if you format it nicely)  
 
 ![CSV example](https://raw.githubusercontent.com/Karlheinzniebuhr/the-weather-scraper/master/resources/csv.JPG)

TODO

@@ -10,4 +10,4 @@ Todo:
   ✔ add error handling to UnitConverter @done(20-07-03 15:11)
   ✔ Write Dates in YYYY MM DD format @done(20-07-04 11:54)
   ☐ See if there is a way to start downloading from first available date
-  ☐ Write one more column with date_time
+  ✔ Write one more column with date_time

util/Parser.py

@@ -28,8 +28,10 @@ def parse_html_table(date_string: str, history_table: list) -> dict:
                 if i == 0:
                     date = datetime.strptime(date_string, "%Y-%m-%d")
                     time = datetime.strptime(td_content, "%I:%M %p")
+                    blank = datetime.strptime('','')
                     row_dict['Date'] = date.strftime('%Y/%m/%d')
                     row_dict['Time'] = time.strftime('%I:%M %p')
+                    row_dict['Date_time'] = ((time-blank)+date).strftime('%Y/%m/%d %I:%M:%S')
                 else:
                     row_dict[Parser.format_key(headers_list[i])] = td_content
 

util/UnitConverter.py

@@ -24,16 +24,7 @@ def temperature(self, temp_string: str):
         except Exception as e:
             print(f'{e}! probably caused by an empty row in the data')
             return 'NA'
-
-    def dew_point(self, dew_point_string: str):
-        try:
-            fahrenheit = float(re.findall(self.extract_numbers_pattern, dew_point_string)[0]) if dew_point_string else 'NA'
-            if self.system == "metric":
-                celsius = (fahrenheit - 32) * 5/9
-                return round(celsius, self.round_to_decimals)
-            else:
-                return fahrenheit
-
+
         except Exception as e:
             print(f'{e}! probably caused by an empty row in the data')
             return 'NA'
@@ -111,29 +102,31 @@ def clean_and_convert(self, dict_list: list):
             for key, value in dict.items():
                 if key == 'Date':
                     converted_dict['Date'] = value
-                if key == 'Time':
+                elif key == 'Time':
                     converted_dict['Time'] = value
-                if key ==  'Temperature':
+                elif key == 'Date_time':
+                    converted_dict['Date_time'] = value
+                elif key ==  'Temperature':
                     converted_dict['Temperature'] = self.temperature(value)
-                if key ==  'Dew_Point':
-                    converted_dict['Dew_Point'] = self.dew_point(value)
-                if key ==  'Humidity':
+                elif key ==  'Dew_Point':
+                    converted_dict['Dew_Point'] = self.temperature(value)
+                elif key ==  'Humidity':
                     converted_dict['Humidity'] = self.humidity(value)
-                if key ==  'Wind':
+                elif key ==  'Wind':
                     converted_dict['Wind'] = value
-                if key ==  'Speed':
+                elif key ==  'Speed':
                     converted_dict['Speed'] = self.speed(value)
-                if key ==  'Gust':
+                elif key ==  'Gust':
                     converted_dict['Gust'] = self.speed(value)
-                if key ==  'Pressure':
+                elif key ==  'Pressure':
                     converted_dict['Pressure'] = self.pressure(value)
-                if key ==  'Precip_Rate':
+                elif key ==  'Precip_Rate':
                     converted_dict['Precip_Rate'] = self.precipitation(value)
-                if key ==  'Precip_Accum':
+                elif key ==  'Precip_Accum':
                     converted_dict['Precip_Accum'] = self.precipitation(value)
-                if key ==  'UV':
+                elif key ==  'UV':
                     converted_dict['UV'] = self.uv(value)
-                if key ==  'Solar':
+                elif key ==  'Solar':
                     converted_dict['Solar'] = self.solar(value)
 
             converted_dict_list.append(converted_dict)

weather_scraper.py

@@ -24,7 +24,7 @@
 FIND_FIRST_DATE = config.FIND_FIRST_DATE
 
 
-def scrap_station(weather_station_url):
+def scrape_station(weather_station_url):
 
     session = requests.Session()
     timeout = 5
@@ -87,4 +87,4 @@ def scrap_station(weather_station_url):
 for url in URLS:
     url = url.strip()
     print(url)
-    scrap_station(url)
+    scrape_station(url)