Skip to content

Commit

Permalink
exercise 1 submission
Browse files Browse the repository at this point in the history
- data pipeline for airports data extraction
  • Loading branch information
Avsar2001 committed Apr 27, 2024
1 parent afb8ef7 commit 3eb6c40
Show file tree
Hide file tree
Showing 2 changed files with 57 additions and 1 deletion.
3 changes: 2 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
.DS_Store
/data/*
!/data/.gitkeep
!/data/.gitkeep
.vscode
55 changes: 55 additions & 0 deletions exercises/exercise1.jv
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
// pipeline to dump airports data into sqlite
pipeline AirportsPipeline {
AirportsExtractor
-> AirportsFileInterpreter
-> AirportsCSVInterpreter
-> NameHeaderWriter
-> AirportsTableInterpreter
-> AirportsLoader;

// Extract remote airports data (null -> file)
block AirportsExtractor oftype HttpExtractor {
url: "https://opendata.rhein-kreis-neuss.de/api/explore/v2.1/catalog/datasets/rhein-kreis-neuss-flughafen-weltweit/exports/csv?lang=en&timezone=Europe%2FBerlin&use_labels=true&delimiter=%3B";
}

// Interprete into text file (file -> text_file)
block AirportsFileInterpreter oftype TextFileInterpreter {
}

// Interprete into csv file (text_file -> csv_file)
block AirportsCSVInterpreter oftype CSVInterpreter {
delimiter: ";";
}

// Write Headers into table
block NameHeaderWriter oftype CellWriter {
at: range A1:A13;
write: ["Lfd. Nummer", "Name des Flughafens", "Ort", "Land", "IATA", "ICAO", "Latitude", "Longitude", "Altitude", "Zeitzone", "DST", "Zeitzonen-Datenbank", "geo_punkt"];
}

// Define columns of table with datatypes
block AirportsTableInterpreter oftype TableInterpreter {
header: true;
columns: [
"Lfd. Nummer" oftype integer,
"Name des Flughafens" oftype text,
"Ort" oftype text,
"Land" oftype text,
"IATA" oftype text,
"ICAO" oftype text,
"Latitude" oftype decimal,
"Longitude" oftype decimal,
"Altitude" oftype integer,
"Zeitzone" oftype decimal,
"DST" oftype text,
"Zeitzonen-Datenbank" oftype text,
"geo_punkt" oftype text
];
}

// Extract into sqlite
block AirportsLoader oftype SQLiteLoader {
table: "airports";
file: "airports.sqlite";
}
}

0 comments on commit 3eb6c40

Please sign in to comment.