Code

.gitignore

@@ -0,0 +1,91 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*,cover
+.hypothesis/
+tests/*.log
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+documentation/
+
+# PyBuilder
+target/
+
+# IPython Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# dotenv
+.env
+
+# virtualenv
+venv/
+ENV/
+
+# Spyder project settings
+.spyderproject
+
+# Rope project settings
+.ropeproject

LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 Michael Rans
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1 @@
+HXL to Frictionless data package conversion

config/project_configuration.yml

@@ -0,0 +1,2 @@
+# Collector specific configuration
+hxl_schema_dataset: "hxl-validation-schemas"

fts_funding_requirements_afg.csv

@@ -0,0 +1,19 @@
+country,id,name,code,startDate,endDate,year,revisedRequirements,totalFunding,percentFunded
+#country+name,#activity+appeal+id+fts_internal,#activity+appeal+name,#activity+appeal+id+external,#date+start,#date+end,#date+year,#value+funding+required+usd,#value+funding+total+usd,#value+funding+pct
+Afghanistan,544,Afghanistan 2017,HAFG17,2017-01-01,2017-12-31,2017,550236457.0,150258483,27
+Afghanistan,512,Afghanistan 2016,HAFG16,2016-01-01,2016-12-31,2016,338833024.0,287033060,84
+Afghanistan,525,Afghanistan Flash Appeal: One million people on the move (September - December 2016),FAFG16,2016-09-01,2016-12-31,2016,152100000.0,67497498,44
+Afghanistan,462,Afghanistan 2015,HAFG15,2015-01-01,2015-12-31,2015,416666132.0,275560305,66
+Afghanistan,426,Afghanistan 2014,HAFG14,2014-01-01,2014-12-31,2014,406429895.0,314388805,77
+Afghanistan,399,Afghanistan 2013,CAFG13,2013-01-01,2013-12-31,2013,474428380.0,345061832,72
+Afghanistan,370,Afghanistan 2012,CAFG1112,2011-11-30,2012-12-31,2012,448551322.0,187201606,41
+Afghanistan,354,Afghanistan 2011,CAFG11,2011-01-01,2011-12-31,2011,582318627.0,420001448,72
+Afghanistan,325,Afghanistan Humanitarian Action Plan 2010,CAFG10,2010-01-01,2010-12-31,2010,774508310.0,460333684,59
+Afghanistan,307,Afghanistan Humanitarian Action Plan 2009,CAFG09,2009-01-01,2009-12-31,2009,664923055.0,384431132,57
+Afghanistan,283,Afghanistan Joint Emergency Appeal 2008: High Food Price and Drought Crisis (July 2008 - June 2009),OAFG0809,2008-07-01,2009-06-30,2008,404319728.0,201952864,49
+Afghanistan,272,Afghanistan Joint Appeal 2008: Humanitarian Consequences of Rise in Food Prices (February - June 2008),OAFG08,2008-02-01,2008-06-30,2008,81321254.0,75038024,92
+Afghanistan,213,Afghanistan Drought Joint Appeal 2006,OAFG06,2006-07-01,2006-12-31,2006,119769805.0,63620911,53
+Afghanistan,176,Afghanistan UN-Government Drought Appeal 2004,OAFG04,2004-09-01,2004-12-31,2004,72841021.0,26511074,36
+Afghanistan,121,Afghanistan TAPA 2003,OAFG03,2003-01-01,2003-12-31,2003,270590000.0,313948366,116
+Afghanistan,92,Afghanistan 2002 (ITAP for the Afghan People),CAFG0102,2001-10-01,2002-12-31,2002,1780509639.0,1196693956,67
+Afghanistan,70,Afghanistan 2001,CAFG01,2001-01-01,2001-09-30,2001,,-81522932,

requirements.txt

@@ -0,0 +1,4 @@
+awesome-slugify==1.6.5
+# Update version below to latest from https://pypi.python.org/pypi/hdx-python-api/
+hdx-python-api==2.5.2
+libhxl==4.3

run.py

@@ -0,0 +1,72 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+"""
+Top level script. Calls other functions that generate datasets that this script then creates in HDX.
+
+"""
+import logging
+from collections import OrderedDict
+from os.path import join
+
+import hxl
+from hdx.data.dataset import Dataset
+from hdx.hdx_configuration import Configuration
+
+from hdx.facades.hdx_scraperwiki import facade
+
+logger = logging.getLogger(__name__)
+
+
+class HXLFrictionlessError(Exception):
+    pass
+
+
+def main():
+    """Read HXL schema"""
+    hxl_schema_dataset = Configuration.read()['hxl_schema_dataset']
+    dataset = Dataset.read_from_hdx(hxl_schema_dataset) # type: Dataset
+    if dataset is None:
+        raise HXLFrictionlessError('HXL Schema Dataset: %s does not exist!' % hxl_schema_dataset)
+    resources = dataset.get_resources()
+    hashtag_schema = None
+    attribute_schema = None
+    for resource in resources:
+        if 'hashtag' in resource['name']:
+            hashtag_schema = resource['url']
+        elif 'attribute' in resource['name']:
+            attribute_schema = resource['url']
+    logger.info('hashtag_schema = %s, attribute schema = %s')
+    if not hashtag_schema or not attribute_schema:
+        raise HXLFrictionlessError('Missing schema!')
+    dataset = hxl.data(hashtag_schema)
+    hashtag_types = dict()
+    for row in dataset:
+        hashtag_types[row.get('#valid_tag')] = row.get('#valid_datatype')
+
+    file = 'fts_funding_requirements_afg.csv'
+    dataset = hxl.data(file, allow_local=True)
+    print(dataset.headers)
+    print(dataset.display_tags)
+    jsonoutput = OrderedDict()
+    jsonoutput['name'] = file
+    jsonoutput['datapackage_version'] = '1.0-beta'
+    jsonoutput['title'] = 'Afghanistan - Requirements and Funding Data'
+    resource = OrderedDict()
+    jsonoutput['resources'] = [resource]
+    resource['url'] = 'https://data.humdata.org/dataset/6a60da4e-253f-474f-8683-7c9ed9a20bf9/resource/96cc8cda-120f-483a-a7ce-5f89af4d99ee/download/fts_funding_requirements_afg.csv'
+    fields = list()
+    resource['schema'] = {'fields': fields}
+    for i, header in enumerate(dataset.headers):
+        tag = dataset.display_tags[i]
+        hashtag = tag.split('+')[0]
+        hashtag_type = hashtag_types.get(hashtag)
+        field = OrderedDict([('name', header)])
+        if hashtag_type is not None:
+            field['type'] = hashtag_type
+        fields.append(field)
+    print(jsonoutput)
+
+
+if __name__ == '__main__':
+    facade(main, hdx_site='prod', hdx_read_only=True, project_config_yaml=join('config', 'project_configuration.yml'))
+