Merge pull request #1 from argysamo/update

WIP

.gitignore

@@ -5,3 +5,228 @@ venv
 .idea/
 !/edam/viewer/app/templates/
 /projectFilesBackup/
+
+# Created by https://www.gitignore.io/api/macos,python,pycharm
+# Edit at https://www.gitignore.io/?templates=macos,python,pycharm
+
+### macOS ###
+# General
+.DS_Store
+.AppleDouble
+.LSOverride
+
+# Icon must end with two \r
+Icon
+
+# Thumbnails
+._*
+
+# Files that might appear in the root of a volume
+.DocumentRevisions-V100
+.fseventsd
+.Spotlight-V100
+.TemporaryItems
+.Trashes
+.VolumeIcon.icns
+.com.apple.timemachine.donotpresent
+
+# Directories potentially created on remote AFP share
+.AppleDB
+.AppleDesktop
+Network Trash Folder
+Temporary Items
+.apdisk
+
+### PyCharm ###
+# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm
+# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
+
+# User-specific stuff
+.idea/**/workspace.xml
+.idea/**/tasks.xml
+.idea/**/usage.statistics.xml
+.idea/**/dictionaries
+.idea/**/shelf
+
+# Generated files
+.idea/**/contentModel.xml
+
+# Sensitive or high-churn files
+.idea/**/dataSources/
+.idea/**/dataSources.ids
+.idea/**/dataSources.local.xml
+.idea/**/sqlDataSources.xml
+.idea/**/dynamic.xml
+.idea/**/uiDesigner.xml
+.idea/**/dbnavigator.xml
+
+# Gradle
+.idea/**/gradle.xml
+.idea/**/libraries
+
+# Gradle and Maven with auto-import
+# When using Gradle or Maven with auto-import, you should exclude module files,
+# since they will be recreated, and may cause churn.  Uncomment if using
+# auto-import.
+# .idea/modules.xml
+# .idea/*.iml
+# .idea/modules
+# *.iml
+# *.ipr
+
+# CMake
+cmake-build-*/
+
+# Mongo Explorer plugin
+.idea/**/mongoSettings.xml
+
+# File-based project format
+*.iws
+
+# IntelliJ
+out/
+
+# mpeltonen/sbt-idea plugin
+.idea_modules/
+
+# JIRA plugin
+atlassian-ide-plugin.xml
+
+# Cursive Clojure plugin
+.idea/replstate.xml
+
+# Crashlytics plugin (for Android Studio and IntelliJ)
+com_crashlytics_export_strings.xml
+crashlytics.properties
+crashlytics-build.properties
+fabric.properties
+
+# Editor-based Rest Client
+.idea/httpRequests
+
+# Android studio 3.1+ serialized cache file
+.idea/caches/build_file_checksums.ser
+
+### PyCharm Patch ###
+# Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721
+
+# *.iml
+# modules.xml
+# .idea/misc.xml
+# *.ipr
+
+# Sonarlint plugin
+.idea/**/sonarlint/
+
+# SonarQube Plugin
+.idea/**/sonarIssues.xml
+
+# Markdown Navigator plugin
+.idea/**/markdown-navigator.xml
+.idea/**/markdown-navigator/
+
+### Python ###
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# Mr Developer
+.mr.developer.cfg
+.project
+.pydevproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+# End of https://www.gitignore.io/api/macos,python,pycharm

MANIFEST.in

@@ -1,6 +1,6 @@
 include README.md
 recursive-include edam/resources *
-recursive-include edam/resources/configurations *
+recursive-include edam/resources/metadata *
 recursive-include edam/resources/flask_related *
 recursive-include edam/resources/inputs *
 recursive-include edam/resources/templates *

README.md

@@ -340,7 +340,7 @@ Units of Measurement:
 Following command downloads and stores data from 27 weather stations. 
 It is executed in approximately 9 seconds.   
 
-`edam --input "http://www.metoffice.gov.uk/pub/data/weather/uk/climate/stationdata/{\$var}data.txt" --template uk.tmpl --metadata uk.yaml --var "aberporth,armagh, ballypatrick, camborne, cambridge, cardiff, chivenor, cwmystwyth, dunstaffnage, durham, eastbourne, eskdalemuir" --drop yes`
+`edam --input "http://www.metoffice.gov.uk/pub/data/weather/uk/climate/stationdata/{{ var }}data.txt" --template uk.tmpl --metadata uk.yaml --var "aberporth,armagh, ballypatrick, camborne, cambridge, cardiff, chivenor, cwmystwyth, dunstaffnage, durham, eastbourne, eskdalemuir" --drop yes`
 
 ## **Australian Bureau of Meteorology (Online)**
 

bin/edam.py → bin/read.py

@@ -12,13 +12,13 @@
 @click.option('--input', required=True, help='input string')
 @click.option('--query', required=False, help='SQL query')
 @click.option('--template', required=True, help='template file to parse data with')
-@click.option('--config', required=True, help='configuration file to annotate data with')
+@click.option('--metadata', required=True, help='configuration file to annotate data with')
 @click.option('--var', required=False, default="", help='Extra variables for URI generation')
 @click.option('--storage', required=False, default='file', type=click.Choice(['file', 'memory']),
               help="Whether input files to be stored or not")
 @click.option('--drop', required=False, default='no', type=click.Choice(['yes', 'no']),
               help="Whether to drop stored data or not")
-def cli(input, template, query, config, var, storage, drop):
+def cli(input, template, query, metadata, var, storage, drop):
     now = datetime.now()
     if drop == "yes":
         # Dropping database
@@ -27,7 +27,7 @@ def cli(input, template, query, config, var, storage, drop):
         DatabaseInstantiation(drop=False)
 
     template_path, template_object = handle_input_files(template)
-    config_path, config_object = handle_input_files(config)
+    config_path, config_object = handle_input_files(metadata)
 
     success, inputs_path, file_type = identify_input_type(input_file=input, extra_variables=var,
                                                           template=template_path, template_object=template_object,
@@ -36,26 +36,26 @@ def cli(input, template, query, config, var, storage, drop):
     if success:
         if not inputs_path:
             # It means template and input file does not match
-            click.echo("I can't match template: %s with input: %s" % (template_path, input))
+            click.echo(f"I can't match template: {template_path} with input: {input}")
             exit(3)
         elif inputs_path:
             if template_path and config_path:
                 mid_time = datetime.now()
                 Workflow(input_list=inputs_path, template_file=template_object, configuration_file=config_object)
                 end_time = datetime.now()
-                print("Download all data: %s" % (mid_time - now))
+                print(f"Download all data: {mid_time - now}")
 
-                print("Store all data: %s" % (end_time - mid_time))
-                print("Total time: %s" % (end_time - now))
+                print(f"Store all data: {end_time - mid_time}")
+                print(f"Total time: {end_time - now}")
                 # run()
             else:
                 click.echo("No template or config were given")
         else:
-            click.echo("%s does not exist" % input)
+            click.echo(f"{input} does not exist")
             exit(2)
         run()
     else:
-        click.echo("%s does not exist" % input)
+        click.echo(f"{input} does not exist")
         exit(4)
 
 
@@ -68,7 +68,7 @@ def handle_input_files(filename):
         if exists:
             return file_path, file_object
         else:
-            click.echo("%s does not exist" % filename)
+            click.echo(f"{filename} does not exist")
             raise SystemExit(0)
             # raise Exception("File does not exit")
 

bin/viewer.py

@@ -7,5 +7,6 @@ def run():
     app.config['TEMPLATES_AUTO_RELOAD'] = True
     app.run(host=SERVER['host'], port=SERVER['port'], debug=SERVER['debug'])
 
+
 if __name__ == "__main__":
     run()

edam/reader/SourceConfiguration.py

@@ -63,7 +63,7 @@ def __init__(self, input_yaml, input_file_data=io.StringIO(), input_preamble=io.
 
     def check_yaml(self):
         try:
-            return yaml.load(self.input_yaml)
+            return yaml.load(self.input_yaml, Loader=yaml.FullLoader)
         except yaml.YAMLError as exc:
             return exc
 
@@ -375,4 +375,4 @@ def __check_type_of_field(self, field_name):
 
 
 if __name__ == "__main__":
-    test = SourceConfiguration('configurations/knmi.yaml', 'inputs/Yucheng.csv')
+    test = SourceConfiguration('metadata/knmi.yaml', 'inputs/Yucheng.csv')

edam/reader/TemplateReader.py

@@ -2,6 +2,7 @@
 import io
 import logging
 import re
+from datetime import datetime
 
 import numpy as np
 import pandas as pd
@@ -288,7 +289,7 @@ def __create_dataframe_from_csv__(self):
             if self.parse_dates['timestamp']['format']:
                 def date_parser(x):
                     try:
-                        return pd.datetime.strptime(x, ' '.join(self.parse_dates['timestamp']['format']))
+                        return datetime.strptime(x, ' '.join(self.parse_dates['timestamp']['format']))
                     except:
                         # This exception catches the case where in datetime column we have litter (e.g. Site closed)
                         return x
@@ -549,7 +550,7 @@ def __update_helper_observable_id(self, helper_observable_id: HelperTemplateIDs,
 
 
 if __name__ == "__main__":
-    conf = SourceConfiguration(input_yaml="/Users/argyris/Documents/git/templateFramework/configurations/knmi.yaml",
+    conf = SourceConfiguration(input_yaml="/Users/argyris/Documents/git/templateFramework/metadata/knmi.yaml",
                                input_file_data="/Users/argyris/Documents/git/templateFramework/inputs/knmi_alldata_data.txt")
     t = TemplateReader(config=conf,
                        input_file="/Users/argyris/Documents/git/templateFramework/inputs/knmi_alldata_data.txt",