From 3eb6c4042ac23e9f133e3d7c0addb4c0711cf8c7 Mon Sep 17 00:00:00 2001 From: Avsar2001 Date: Sat, 27 Apr 2024 19:34:09 +0200 Subject: [PATCH] exercise 1 submission - data pipeline for airports data extraction --- .gitignore | 3 ++- exercises/exercise1.jv | 55 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 exercises/exercise1.jv diff --git a/.gitignore b/.gitignore index d6d425f68..12bae38e7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ .DS_Store /data/* -!/data/.gitkeep \ No newline at end of file +!/data/.gitkeep +.vscode \ No newline at end of file diff --git a/exercises/exercise1.jv b/exercises/exercise1.jv new file mode 100644 index 000000000..e670f503c --- /dev/null +++ b/exercises/exercise1.jv @@ -0,0 +1,55 @@ +// pipeline to dump airports data into sqlite +pipeline AirportsPipeline { + AirportsExtractor + -> AirportsFileInterpreter + -> AirportsCSVInterpreter + -> NameHeaderWriter + -> AirportsTableInterpreter + -> AirportsLoader; + + // Extract remote airports data (null -> file) + block AirportsExtractor oftype HttpExtractor { + url: "https://opendata.rhein-kreis-neuss.de/api/explore/v2.1/catalog/datasets/rhein-kreis-neuss-flughafen-weltweit/exports/csv?lang=en&timezone=Europe%2FBerlin&use_labels=true&delimiter=%3B"; + } + + // Interprete into text file (file -> text_file) + block AirportsFileInterpreter oftype TextFileInterpreter { + } + + // Interprete into csv file (text_file -> csv_file) + block AirportsCSVInterpreter oftype CSVInterpreter { + delimiter: ";"; + } + + // Write Headers into table + block NameHeaderWriter oftype CellWriter { + at: range A1:A13; + write: ["Lfd. Nummer", "Name des Flughafens", "Ort", "Land", "IATA", "ICAO", "Latitude", "Longitude", "Altitude", "Zeitzone", "DST", "Zeitzonen-Datenbank", "geo_punkt"]; + } + + // Define columns of table with datatypes + block AirportsTableInterpreter oftype TableInterpreter { + header: true; + columns: [ + "Lfd. Nummer" oftype integer, + "Name des Flughafens" oftype text, + "Ort" oftype text, + "Land" oftype text, + "IATA" oftype text, + "ICAO" oftype text, + "Latitude" oftype decimal, + "Longitude" oftype decimal, + "Altitude" oftype integer, + "Zeitzone" oftype decimal, + "DST" oftype text, + "Zeitzonen-Datenbank" oftype text, + "geo_punkt" oftype text + ]; + } + + // Extract into sqlite + block AirportsLoader oftype SQLiteLoader { + table: "airports"; + file: "airports.sqlite"; + } +} \ No newline at end of file