Skip to content
This repository has been archived by the owner on Jun 10, 2023. It is now read-only.

Commit

Permalink
issue #7 - some new data from RSpoor added, and first shot at Dataflo…
Browse files Browse the repository at this point in the history 
…w-C (AQD_AssessmentRegime) ETL
  • Loading branch information
@justb4
justb4 committed Oct 7, 2014
1 parent 40b447a commit 63a23a3
Show file tree
Hide file tree
Showing 27 changed files with 96,265 additions and 5,236 deletions.
Binary file added data/rivm-lml/raw/xls/AssessmentRegimes_NL-005.xls
View file
Binary file not shown.
Binary file added data/rivm-lml/raw/xls/Attainments_NL-003.xls
View file
Binary file not shown.
Binary file added data/rivm-lml/raw/xls/DEM_NL_SamplingPoint-004.xls
View file
Binary file not shown.
Binary file added data/rivm-lml/raw/xls/DEM_NL_SamplingPointForCompliance-003.xls
View file
Binary file not shown.
Binary file added data/rivm-lml/raw/xls/DEM_NL_networks-002.xls
View file
Binary file not shown.
Binary file added data/rivm-lml/raw/xls/DEM_NL_stations-003.xls
View file
Binary file not shown.
Binary file added data/rivm-lml/raw/xls/Zones_NL-003.xls
View file
Binary file not shown.
2 changes: 1 addition & 1 deletion src/aq-report/dataflow-B.sh
100644 → 100755
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,6 @@ logdir=/var/log/sospilot
stetl_cmd=stetl

# debugging
# stetl_cmd=../../../../stetl/git/stetl/main.py
stetl_cmd=../../../../stetl/git/stetl/main.py

$stetl_cmd -c dataflow-B.cfg
39 changes: 39 additions & 0 deletions src/aq-report/dataflow-C.cfg
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
# Example dataflow-C (Assessment Regimes) report generation using Stetl with Jinja2 templating filter.
# The input data comes directly from a WFS providing station info.
# The template file is under the templates folder and global variables (globals.json) are under input.
#
# All rendered output under output/
#
[etl]
chains = input_json|records_to_struct|dataflow-C_template|output_dataflow-C_file
# chains = input_json|records_to_struct|dataflow-C_template|output_std

[input_json]
class = inputs.fileinput.CsvFileInput
file_path = input/rspoor/AssessmentRegimes_NL-005.csv
output_format = record_array
delimiter = ;

# Convert CSV records to struct (dict with array of records)
[records_to_struct]
class = filters.formatconverter.FormatConverter
input_format = record_array
output_format = struct
converter_args = {
'top_name': 'records'
}

# Advanced templating with Jinja2
[dataflow-C_template]
class = filters.templatingfilter.Jinja2TemplatingFilter
input_format = struct
template_file = templates/dataflow-C.jinja2
template_globals_path = input/globals-rivm.json

[output_dataflow-C_file]
class = outputs.fileoutput.FileOutput
file_path = output/dataflow-C.xml

# for testing/debugging only
[output_std]
class = outputs.standardoutput.StandardOutput
16 changes: 16 additions & 0 deletions src/aq-report/dataflow-C.sh
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
#!/bin/bash
#
# Usually required in order to have Python find your package
# export PYTHONPATH=.:$PYTHONPATH

# set postgres credentials and options in $pg_options
# . pgcreds.sh

logdir=/var/log/sospilot

stetl_cmd=stetl

# debugging
stetl_cmd=../../../../stetl/git/stetl/main.py

$stetl_cmd -c dataflow-C.cfg
1,056 changes: 558 additions & 498 deletions src/aq-report/input/globals-rivm.json
View file

Large diffs are not rendered by default.

138 changes: 138 additions & 0 deletions src/aq-report/input/rspoor/AssessmentRegimes_NL-005.csv
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,138 @@
change;class_year;zone_code;ENV_pollutant;assessment_method_type;assessment_threshold_exceedance;assessment_method_type_descr;link_class_doc;link_model
;2013;NL0100;As;fixed;belowLAT;;;
;2013;NL0100;BaP;fixed;belowLAT;;;
;2013;NL0100;Benzene;model;belowLAT;GCN kaarten;Grootschalige_concentratie_en_depositiekaarten_Nederland_Rapportage_2014;http://rivm.nl/Documenten_en_publicaties/Wetenschappelijk/Rapporten
;2013;NL0100;Cd;fixed;belowLAT;;;
;2013;NL0100;CO;model;belowLAT;GCN kaarten;Grootschalige_concentratie_en_depositiekaarten_Nederland_Rapportage_2014;http://rivm.nl/Documenten_en_publicaties/Wetenschappelijk/Rapporten
;2013;NL0100;Ni;fixed;belowLAT;;;
;2013;NL0100;NO2_all;fixed;aboveUAT;;;
;2013;NL0100;NO2_hour;fixed;belowLAT;;;
;2013;NL0100;NOx;fixed;belowLAT;;;
;2013;NL0100;O3-H_LTO;fixed;aboveLTO;;;
;2013;NL0100;O3-V_LTO;fixed;aboveLTO;;;
;2013;NL0100;Pb;fixed;belowLAT;;;
;2013;NL0100;PM10_all;fixed;LAT-UAT;;;
;2013;NL0100;PM10_day;fixed;aboveUAT;;;
;2013;NL0100;PM2.5_LV;fixed;LAT-UAT;;;
;2013;NL0100;SO2-H_day;model;belowLAT;GCN kaarten;Grootschalige_concentratie_en_depositiekaarten_Nederland_Rapportage_2014;http://rivm.nl/Documenten_en_publicaties/Wetenschappelijk/Rapporten
;2013;NL0100;SO2-V_winter;model;belowLAT;GCN kaarten;Grootschalige_concentratie_en_depositiekaarten_Nederland_Rapportage_2014;http://rivm.nl/Documenten_en_publicaties/Wetenschappelijk/Rapporten
;2013;NL0201;As;fixed;belowLAT;;;
;2013;NL0201;BaP;fixed;belowLAT;;;
;2013;NL0201;Benzene;fixed;belowLAT;;;
;2013;NL0201;Cd;fixed;belowLAT;;;
;2013;NL0201;CO;fixed;belowLAT;;;
;2013;NL0201;Ni;fixed;belowLAT;;;
;2013;NL0201;NO2_all;fixed;aboveUAT;;;
;2013;NL0201;NO2_hour;fixed;LAT-UAT;;;
;2013;NL0201;O3-H_LTO;fixed;aboveLTO;;;
;2013;NL0201;O3-V_LTO;fixed;aboveLTO;;;
;2013;NL0201;Pb;fixed;belowLAT;;;
;2013;NL0201;PM10_all;fixed;LAT-UAT;;;
;2013;NL0201;PM10_day;fixed;aboveUAT;;;
;2013;NL0201;PM2.5_LV;fixed;aboveUAT;;;
;2013;NL0201;SO2-H_day;fixed;belowLAT;;;
;2013;NL0210;As;objective;belowLAT;Report RIVM 680704001;Prelimanary assessment of heavy metals and benzo(a)pyrene in ambient air;http://rivm.nl/Documenten_en_publicaties/Wetenschappelijk/Rapporten
;2013;NL0210;BaP;fixed;LAT-UAT;;;
;2013;NL0210;Benzene;fixed;belowLAT;;;
;2013;NL0210;Cd;objective;belowLAT;Report RIVM 680704001;Prelimanary assessment of heavy metals and benzo(a)pyrene in ambient air;http://rivm.nl/Documenten_en_publicaties/Wetenschappelijk/Rapporten
;2013;NL0210;CO;fixed;belowLAT;;;
;2013;NL0210;Ni;objective;belowLAT;Report RIVM 680704001;Prelimanary assessment of heavy metals and benzo(a)pyrene in ambient air;http://rivm.nl/Documenten_en_publicaties/Wetenschappelijk/Rapporten
;2013;NL0210;NO2_all;fixed;aboveUAT;;;
;2013;NL0210;NO2_hour;fixed;aboveUAT;;;
;2013;NL0210;O3-H_LTO;fixed;aboveLTO;;;
;2013;NL0210;O3-V_LTO;fixed;aboveLTO;;;
;2013;NL0210;Pb;objective;belowLAT;Report RIVM 680704001;Prelimanary assessment of heavy metals and benzo(a)pyrene in ambient air;http://rivm.nl/Documenten_en_publicaties/Wetenschappelijk/Rapporten
;2013;NL0210;PM10_all;fixed;aboveUAT;;;
;2013;NL0210;PM10_day;fixed;aboveUAT;;;
;2013;NL0210;PM2.5_LV;fixed;aboveUAT;;;
;2013;NL0210;SO2-H_day;fixed;belowLAT;;;
;2013;NL0221;As;objective;belowLAT;Report RIVM 680704001;Prelimanary assessment of heavy metals and benzo(a)pyrene in ambient air;http://rivm.nl/Documenten_en_publicaties/Wetenschappelijk/Rapporten
;2013;NL0221;BaP;objective;belowLAT;Report RIVM 680704001;Prelimanary assessment of heavy metals and benzo(a)pyrene in ambient air;http://rivm.nl/Documenten_en_publicaties/Wetenschappelijk/Rapporten
;2013;NL0221;Benzene;fixed;belowLAT;;;
;2013;NL0221;Cd;objective;belowLAT;Report RIVM 680704001;Prelimanary assessment of heavy metals and benzo(a)pyrene in ambient air;http://rivm.nl/Documenten_en_publicaties/Wetenschappelijk/Rapporten
;2013;NL0221;CO;model;belowLAT;GCN kaarten;Grootschalige_concentratie_en_depositiekaarten_Nederland_Rapportage_2014;http://rivm.nl/Documenten_en_publicaties/Wetenschappelijk/Rapporten
;2013;NL0221;Ni;objective;belowLAT;Report RIVM 680704001;Prelimanary assessment of heavy metals and benzo(a)pyrene in ambient air;http://rivm.nl/Documenten_en_publicaties/Wetenschappelijk/Rapporten
;2013;NL0221;NO2_all;fixed;aboveUAT;;;
;2013;NL0221;NO2_hour;fixed;LAT-UAT;;;
;2013;NL0221;O3-H_LTO;fixed;aboveLTO;;;
;2013;NL0221;O3-V_LTO;fixed;aboveLTO;;;
;2013;NL0221;Pb;objective;belowLAT;Report RIVM 680704001;Prelimanary assessment of heavy metals and benzo(a)pyrene in ambient air;http://rivm.nl/Documenten_en_publicaties/Wetenschappelijk/Rapporten
;2013;NL0221;PM10_all;fixed;LAT-UAT;;;
;2013;NL0221;PM10_day;fixed;aboveUAT;;;
;2013;NL0221;PM2.5_LV;fixed;aboveUAT;;;
;2013;NL0221;SO2-H_day;model;belowLAT;GCN kaarten;Grootschalige_concentratie_en_depositiekaarten_Nederland_Rapportage_2014;http://rivm.nl/Documenten_en_publicaties/Wetenschappelijk/Rapporten
;2013;NL0231;As;objective;belowLAT;Report RIVM 680704001;Prelimanary assessment of heavy metals and benzo(a)pyrene in ambient air;http://rivm.nl/Documenten_en_publicaties/Wetenschappelijk/Rapporten
;2013;NL0231;BaP;objective;belowLAT;Report RIVM 680704001;Prelimanary assessment of heavy metals and benzo(a)pyrene in ambient air;http://rivm.nl/Documenten_en_publicaties/Wetenschappelijk/Rapporten
;2013;NL0231;Benzene;model;belowLAT;GCN kaarten;Grootschalige_concentratie_en_depositiekaarten_Nederland_Rapportage_2014;http://rivm.nl/Documenten_en_publicaties/Wetenschappelijk/Rapporten
;2013;NL0231;Cd;objective;belowLAT;Report RIVM 680704001;Prelimanary assessment of heavy metals and benzo(a)pyrene in ambient air;http://rivm.nl/Documenten_en_publicaties/Wetenschappelijk/Rapporten
;2013;NL0231;CO;model;belowLAT;GCN kaarten;Grootschalige_concentratie_en_depositiekaarten_Nederland_Rapportage_2014;http://rivm.nl/Documenten_en_publicaties/Wetenschappelijk/Rapporten
;2013;NL0231;Ni;objective;belowLAT;Report RIVM 680704001;Prelimanary assessment of heavy metals and benzo(a)pyrene in ambient air;http://rivm.nl/Documenten_en_publicaties/Wetenschappelijk/Rapporten
;2013;NL0231;NO2_all;fixed;aboveUAT;;;
;2013;NL0231;NO2_hour;fixed;LAT-UAT;;;
;2013;NL0231;O3-H_LTO;fixed;aboveLTO;;;
;2013;NL0231;O3-V_LTO;fixed;belowLTO;;;
;2013;NL0231;Pb;objective;belowLAT;Report RIVM 680704001;Prelimanary assessment of heavy metals and benzo(a)pyrene in ambient air;http://rivm.nl/Documenten_en_publicaties/Wetenschappelijk/Rapporten
;2013;NL0231;PM10_all;fixed;aboveUAT;;;
;2013;NL0231;PM10_day;fixed;aboveUAT;;;
;2013;NL0231;PM2.5_LV;fixed;LAT-UAT;;;
;2013;NL0231;SO2-H_day;model;belowLAT;GCN kaarten;Grootschalige_concentratie_en_depositiekaarten_Nederland_Rapportage_2014;http://rivm.nl/Documenten_en_publicaties/Wetenschappelijk/Rapporten
;2013;NL0241;As;fixed;belowLAT;;;
;2013;NL0241;BaP;fixed;LAT-UAT;;;
;2013;NL0241;Benzene;fixed;belowLAT;;;
;2013;NL0241;Cd;fixed;belowLAT;;;
;2013;NL0241;CO;fixed;belowLAT;;;
;2013;NL0241;Ni;fixed;belowLAT;;;
;2013;NL0241;NO2_all;fixed;aboveUAT;;;
;2013;NL0241;NO2_hour;fixed;aboveUAT;;;
;2013;NL0241;O3-H_LTO;fixed;aboveLTO;;;
;2013;NL0241;O3-V_LTO;fixed;aboveLTO;;;
;2013;NL0241;Pb;fixed;belowLAT;;;
;2013;NL0241;PM10_all;fixed;LAT-UAT;;;
;2013;NL0241;PM10_day;fixed;aboveUAT;;;
;2013;NL0241;PM2.5_LV;fixed;aboveUAT;;;
;2013;NL0241;SO2-H_day;fixed;belowLAT;;;
;2013;NL0300;As;fixed;belowLAT;;;
;2013;NL0300;BaP;fixed;belowLAT;;;
;2013;NL0300;Benzene;model;belowLAT;GCN kaarten;Grootschalige_concentratie_en_depositiekaarten_Nederland_Rapportage_2014;http://rivm.nl/Documenten_en_publicaties/Wetenschappelijk/Rapporten
;2013;NL0300;Cd;fixed;belowLAT;;;
;2013;NL0300;CO;model;belowLAT;GCN kaarten;Grootschalige_concentratie_en_depositiekaarten_Nederland_Rapportage_2014;http://rivm.nl/Documenten_en_publicaties/Wetenschappelijk/Rapporten
;2013;NL0300;Ni;fixed;belowLAT;;;
;2013;NL0300;NO2_all;fixed;belowLAT;;;
;2013;NL0300;NO2_hour;fixed;belowLAT;;;
;2013;NL0300;O3-H_LTO;fixed;aboveLTO;;;
;2013;NL0300;O3-V_LTO;fixed;aboveLTO;;;
;2013;NL0300;Pb;fixed;belowLAT;;;
;2013;NL0300;PM10_all;fixed;aboveUAT;;;
;2013;NL0300;PM10_day;fixed;aboveUAT;;;
;2013;NL0300;PM2.5_LV;fixed;aboveUAT;;;
;2013;NL0300;SO2-H_day;fixed;belowLAT;;;
;2013;NL0310;As;objective;belowLAT;Report RIVM 680704001;Prelimanary assessment of heavy metals and benzo(a)pyrene in ambient air;http://rivm.nl/Documenten_en_publicaties/Wetenschappelijk/Rapporten
;2013;NL0310;BaP;objective;belowLAT;Report RIVM 680704001;Prelimanary assessment of heavy metals and benzo(a)pyrene in ambient air;http://rivm.nl/Documenten_en_publicaties/Wetenschappelijk/Rapporten
;2013;NL0310;Benzene;model;belowLAT;GCN kaarten;Grootschalige_concentratie_en_depositiekaarten_Nederland_Rapportage_2014;http://rivm.nl/Documenten_en_publicaties/Wetenschappelijk/Rapporten
;2013;NL0310;Cd;objective;belowLAT;Report RIVM 680704001;Prelimanary assessment of heavy metals and benzo(a)pyrene in ambient air;http://rivm.nl/Documenten_en_publicaties/Wetenschappelijk/Rapporten
;2013;NL0310;CO;model;belowLAT;GCN kaarten;Grootschalige_concentratie_en_depositiekaarten_Nederland_Rapportage_2014;http://rivm.nl/Documenten_en_publicaties/Wetenschappelijk/Rapporten
;2013;NL0310;Ni;objective;belowLAT;Report RIVM 680704001;Prelimanary assessment of heavy metals and benzo(a)pyrene in ambient air;http://rivm.nl/Documenten_en_publicaties/Wetenschappelijk/Rapporten
;2013;NL0310;NO2_all;fixed;aboveUAT;;;
;2013;NL0310;NO2_hour;fixed;LAT-UAT;;;
;2013;NL0310;O3-H_LTO;fixed;aboveLTO;;;
;2013;NL0310;O3-V_LTO;fixed;aboveLTO;;;
;2013;NL0310;Pb;objective;belowLAT;Report RIVM 680704001;Prelimanary assessment of heavy metals and benzo(a)pyrene in ambient air;http://rivm.nl/Documenten_en_publicaties/Wetenschappelijk/Rapporten
;2013;NL0310;PM10_all;fixed;aboveUAT;;;
;2013;NL0310;PM10_day;fixed;aboveUAT;;;
;2013;NL0310;PM2.5_LV;fixed;LAT-UAT;;;
;2013;NL0310;SO2-H_day;model;belowLAT;GCN kaarten;Grootschalige_concentratie_en_depositiekaarten_Nederland_Rapportage_2014;http://rivm.nl/Documenten_en_publicaties/Wetenschappelijk/Rapporten
;2013;NL0320;As;objective;belowLAT;Report RIVM 680704001;Prelimanary assessment of heavy metals and benzo(a)pyrene in ambient air;http://rivm.nl/Documenten_en_publicaties/Wetenschappelijk/Rapporten
;2013;NL0320;BaP;objective;belowLAT;Report RIVM 680704001;Prelimanary assessment of heavy metals and benzo(a)pyrene in ambient air;http://rivm.nl/Documenten_en_publicaties/Wetenschappelijk/Rapporten
;2013;NL0320;Benzene;model;belowLAT;GCN kaarten;Grootschalige_concentratie_en_depositiekaarten_Nederland_Rapportage_2014;http://rivm.nl/Documenten_en_publicaties/Wetenschappelijk/Rapporten
;2013;NL0320;Cd;objective;belowLAT;Report RIVM 680704001;Prelimanary assessment of heavy metals and benzo(a)pyrene in ambient air;http://rivm.nl/Documenten_en_publicaties/Wetenschappelijk/Rapporten
;2013;NL0320;CO;model;belowLAT;GCN kaarten;Grootschalige_concentratie_en_depositiekaarten_Nederland_Rapportage_2014;http://rivm.nl/Documenten_en_publicaties/Wetenschappelijk/Rapporten
;2013;NL0320;Ni;objective;belowLAT;Report RIVM 680704001;Prelimanary assessment of heavy metals and benzo(a)pyrene in ambient air;http://rivm.nl/Documenten_en_publicaties/Wetenschappelijk/Rapporten
;2013;NL0320;NO2_all;fixed;aboveUAT;;;
;2013;NL0320;NO2_hour;fixed;LAT-UAT;;;
;2013;NL0320;O3-H_LTO;fixed;aboveLTO;;;
;2013;NL0320;O3-V_LTO;fixed;aboveLTO;;;
;2013;NL0320;Pb;objective;belowLAT;Report RIVM 680704001;Prelimanary assessment of heavy metals and benzo(a)pyrene in ambient air;http://rivm.nl/Documenten_en_publicaties/Wetenschappelijk/Rapporten
;2013;NL0320;PM10_all;fixed;LAT-UAT;;;
;2013;NL0320;PM10_day;fixed;aboveUAT;;;
;2013;NL0320;PM2.5_LV;fixed;aboveUAT;;;
;2013;NL0320;SO2-H_day;fixed;belowLAT;;;
Loading

0 comments on commit 63a23a3

Please sign in to comment.