Update to newest HAFAS format (hrdf 5.4) (#232)

* support for commented lines * support multiple operator codes * new length of Fahrtnummer * new filename for BFKOORD * additional updates in the format for FPLAN * updates in the hrdf format for BFKOORD * ignore comments in BITFELD file * fix substring length * update test files to match new format * fix git pull issue
matsim-org · Jul 2, 2024 · 44a2bb1 · 44a2bb1
1 parent a7b68d3
commit 44a2bb1
Show file tree

Hide file tree

Showing 17 changed files with 118 additions and 79 deletions.
diff --git a/src/main/java/org/matsim/pt2matsim/hafas/HafasConverter.java b/src/main/java/org/matsim/pt2matsim/hafas/HafasConverter.java
@@ -91,7 +91,7 @@ public static void run(String hafasFolder, TransitSchedule schedule, CoordinateT
 
 		// 1. Read and create stop facilities
 		log.info("  Read transit stops...");
-		StopReader.run(schedule, transformation, hafasFolder + "BFKOORD_GEO");
+		StopReader.run(schedule, transformation, hafasFolder + "BFKOORD_WGS");
 		log.info("  Read transit stops... done.");
 
 		// 1.a Read minimal transfer times

diff --git a/src/main/java/org/matsim/pt2matsim/hafas/lib/BitfeldAnalyzer.java b/src/main/java/org/matsim/pt2matsim/hafas/lib/BitfeldAnalyzer.java
@@ -47,8 +47,11 @@ public static Set<Integer> findBitfeldnumbersOfBusiestDay(String FPLAN, String B
 		final Set<Integer> bitfeldNummern = new HashSet<>();
 		final int posMaxFVals = find4DayBlockWithMostFVals(FPLAN, BITFELD);
 			BufferedReader readsLines = new BufferedReader(new InputStreamReader(new FileInputStream(BITFELD), "latin1"));
-			String newLine = readsLines.readLine();
-			while (newLine != null) {
+			String newLine;
+			while ((newLine = readsLines.readLine()) != null) {
+				if (newLine.startsWith("*")) {
+					continue;
+				}
 				/*Spalte Typ Bedeutung
 				1−6 INT32 Bitfeldnummer
 				8−103 CHAR Bitfeld (Binärkodierung der Tage, an welchen Fahrt, in Hexadezimalzahlen notiert.)*/
@@ -69,7 +72,6 @@ public static Set<Integer> findBitfeldnumbersOfBusiestDay(String FPLAN, String B
 				if (matches >= 1) {
 					bitfeldNummern.add(bitfeldnummer);
 				}
-				newLine = readsLines.readLine();
 			}
 			readsLines.close();
 		bitfeldNummern.add(0);
@@ -117,8 +119,11 @@ private static int find4DayBlockWithMostFVals(String FPLAN, String BITFELD) {
 		int[] bitfeldStats = new int[96];
 		try {
 			BufferedReader readsLines = new BufferedReader(new InputStreamReader(new FileInputStream(BITFELD), "latin1"));
-			String newLine = readsLines.readLine();
-			while (newLine != null) {
+			String newLine;
+			while ((newLine = readsLines.readLine()) != null) {
+				if (newLine.startsWith("*")) {
+					continue;
+				}
 				/*Spalte Typ Bedeutung
 				1−6 INT32 Bitfeldnummer
 				8−103 CHAR Bitfeld (Binärkodierung der Tage, an welchen Fahrt, in Hexadezimalzahlen notiert.)*/
@@ -132,7 +137,6 @@ private static int find4DayBlockWithMostFVals(String FPLAN, String BITFELD) {
 						bitfeldStats[i] += bitFeldValue;
 					}
 				}
-				newLine = readsLines.readLine();
 			}
 			readsLines.close();
 		} catch (IOException e) {
@@ -163,12 +167,14 @@ public static Set<Integer> getBitfieldsAtValidDay(final int dayNr, final String
 		Set<Integer> validBitfields = new HashSet<>();
 
 		BufferedReader readsLines = new BufferedReader(new InputStreamReader(new FileInputStream(pathFile), "utf-8"));
-		String newLine = readsLines.readLine();
-		while (newLine != null) {
+		String newLine;
+		while ((newLine = readsLines.readLine()) != null) {
+			if (newLine.startsWith("*")) {
+				continue;
+			}
 			int id = Integer.parseInt(newLine.substring(0, 6));
 			String bitfield = new BigInteger(newLine.substring(7), 16).toString(2).substring(offset_bitstring);
 			if (bitfield.charAt(dayNr)== '1') validBitfields.add(id);
-			newLine = readsLines.readLine();
 		}
 		readsLines.close();
         // TODO this error-prone and should be removed by a stable solution

diff --git a/src/main/java/org/matsim/pt2matsim/hafas/lib/ECKDATENReader.java b/src/main/java/org/matsim/pt2matsim/hafas/lib/ECKDATENReader.java
@@ -20,7 +20,17 @@ public class ECKDATENReader {
     public static LocalDate getFahrPlanStart(String pathToHafasFolder) throws IOException {
         if (new File(pathToHafasFolder, ECKDATEN).exists()) {
             BufferedReader readsLines = new BufferedReader(new InputStreamReader(new FileInputStream(pathToHafasFolder + ECKDATEN), "utf-8"));
-            LocalDate startDate = getDate(readsLines.readLine());
+            String line;
+            String firstLineAfterComments = null;
+
+            while ((line = readsLines.readLine()) != null) {
+                if (line.startsWith("*")) {
+                    continue;
+                }
+                firstLineAfterComments = line;
+                break;
+            }
+            LocalDate startDate = getDate(firstLineAfterComments);
             readsLines.close();
             return startDate;
         } else {
@@ -32,8 +42,17 @@ public static LocalDate getFahrPlanStart(String pathToHafasFolder) throws IOExce
     public static LocalDate getFahrPlanEnd(String pathToHafasFolder) throws IOException {
         if (new File(pathToHafasFolder, ECKDATEN).exists()) {
             BufferedReader readsLines = new BufferedReader(new InputStreamReader(new FileInputStream(pathToHafasFolder + ECKDATEN), "utf-8"));
-            readsLines.readLine();
-            LocalDate endDate = getDate(readsLines.readLine());
+            String line;
+            String secondLineAfterComments = null;
+
+            while ((line = readsLines.readLine()) != null) {
+                if (line.startsWith("*")) {
+                    continue;
+                }
+                secondLineAfterComments = readsLines.readLine();
+                break;
+            }
+            LocalDate endDate = getDate(secondLineAfterComments);
 
             readsLines.close();
             return endDate;

diff --git a/src/main/java/org/matsim/pt2matsim/hafas/lib/FPLANReader.java b/src/main/java/org/matsim/pt2matsim/hafas/lib/FPLANReader.java
@@ -65,24 +65,25 @@ public static List<FPLANRoute> parseFPLAN(Set<Integer> bitfeldNummern, Map<Strin
 					/*
 					 Initialzeile neue Fahrt
 					 1−2 	CHAR 	*Z
-					 4−8 	INT32 	Fahrtnummer
-					 10−15 	CHAR 	Verwaltung (6-stellig); Die Verwaltungsangabe darf keine Leerzeichen enthalten.
-					 17−21 	INT16 	leer // Tatsächlich unterscheidet dieser Eintrag noch verschiedene Fahrtvarianten...
-					 23−25 	INT16 	Taktanzahl; gibt die Anzahl der noch folgenden Takte an.
-					 27−29 	INT16 	Taktzeit in Minuten (Abstand zwischen zwei Fahrten).
+					 4−9 	INT32 	Fahrtnummer (6-stellig)
+					 11−16 	CHAR 	Verwaltung (6-stellig); Die Verwaltungsangabe darf keine Leerzeichen enthalten.
+					 17−19 	INT16 	leer
+					 20-22 	INT16 	Nummer der Variante des Verkehrsmittels (Kein Standard Feld von HRDF). Hat aber keine fachliche Bedeutung.
+					 24−26 	INT16 	Taktanzahl; gibt die Anzahl der noch folgenden Takte an.
+					 28−30 	INT16 	Taktzeit in Minuten (Abstand zwischen zwei Fahrten).
 					 */
 					if(newLine.charAt(1) == 'Z') {
 						// get operator
-						String operator = operators.get(newLine.substring(9, 15).trim());
+						String operator = operators.get(newLine.substring(10, 16).trim());
 
 						// get the fahrtnummer
-						String fahrtnummer = newLine.substring(3, 8).trim();
+						String fahrtnummer = newLine.substring(3, 9).trim();
 
 						int numberOfDepartures = 0;
 						int cycleTime = 0;
 						try {
-							numberOfDepartures = Integer.parseInt(newLine.substring(22, 25));
-							cycleTime = Integer.parseInt(newLine.substring(26, 29));
+							numberOfDepartures = Integer.parseInt(newLine.substring(23, 26));
+							cycleTime = Integer.parseInt(newLine.substring(27, 30));
 						} catch (Exception ignored) {
 						}
 						currentFPLANRoute = new FPLANRoute(operator, fahrtnummer, numberOfDepartures, cycleTime);

diff --git a/src/main/java/org/matsim/pt2matsim/hafas/lib/MinimalTransferTimesReader.java b/src/main/java/org/matsim/pt2matsim/hafas/lib/MinimalTransferTimesReader.java
@@ -26,8 +26,11 @@ public static void run(TransitSchedule schedule, String pathToHafasFolder, Strin
         // read from UMSTEIGB
         if (new File(pathToHafasFolder, UMSTEIGB).exists()) {
             BufferedReader readsLines = new BufferedReader(new InputStreamReader(new FileInputStream(pathToHafasFolder + UMSTEIGB), "utf-8"));
-            String newLine = readsLines.readLine();
-            while (newLine != null) {
+            String newLine;
+            while ((newLine = readsLines.readLine()) != null) {
+                if (newLine.startsWith("*")) {
+                    continue;
+                }
                 /*
                 1-7 INT32 Die Nummer der Haltestelle.
                 9-10 INT16 Umsteigezeit IC-IC
@@ -37,7 +40,6 @@ public static void run(TransitSchedule schedule, String pathToHafasFolder, Strin
                 Id<TransitStopFacility> stopId = Id.create(newLine.substring(0, 7), TransitStopFacility.class);
                 double transferTime = Integer.parseInt(newLine.substring(11, 13)) * 60;
                 minimalTransferTimes.set(stopId, stopId, transferTime);
-                newLine = readsLines.readLine();
             }
             readsLines.close();
         } else {

diff --git a/src/main/java/org/matsim/pt2matsim/hafas/lib/OperatorReader.java b/src/main/java/org/matsim/pt2matsim/hafas/lib/OperatorReader.java
@@ -38,15 +38,18 @@ public class OperatorReader {
 	public static Map<String, String> readOperators(String BETRIEB_DE) throws IOException {
 		Map<String, String> operators = new HashMap<>();
 		try(BufferedReader readsLines = new BufferedReader(new InputStreamReader(new FileInputStream(BETRIEB_DE), "utf-8"))) {
-			String newLine = readsLines.readLine();
-			while (newLine != null) {
+			String newLine;
+			while ((newLine = readsLines.readLine()) != null) {
+				if (newLine.startsWith("*")) {
+					continue;
+				}
 				String abbrevationOperator = newLine.split("\"")[1].replace(" ","");
 				newLine = readsLines.readLine();
 				if (newLine == null) break;
-				String operatorId = newLine.substring(8, 14).trim();
-				operators.put(operatorId, abbrevationOperator);
-				// read the next operator:
-				newLine = readsLines.readLine();
+				String[] operatorIds = newLine.substring(8).trim().split("\\s+");
+				for (String operatorId : operatorIds) {
+					operators.put(operatorId.trim(), abbrevationOperator);
+				}
 			}
 		}
 		return operators;

diff --git a/src/main/java/org/matsim/pt2matsim/hafas/lib/StopReader.java b/src/main/java/org/matsim/pt2matsim/hafas/lib/StopReader.java
@@ -38,7 +38,7 @@
 import java.util.Map;
 
 /**
- * Reads all stops from HAFAS-BFKOORD_GEO and adds them as TransitStopFacilities
+ * Reads all stops from HAFAS-BFKOORD_WGS and adds them as TransitStopFacilities
  * to the provided TransitSchedule.
  *
  * @author boescpa
@@ -50,41 +50,43 @@ public class StopReader {
 	private final TransitSchedule schedule;
 	private final TransitScheduleFactory scheduleBuilder;
 	private final Map<Coord, String> usedCoordinates = new HashMap<>();
-	private final String pathToBFKOORD_GEOFile;
+	private final String pathToBFKOORD_WGSFile;
 
-	public StopReader(TransitSchedule schedule, CoordinateTransformation transformation, String pathToBFKOORD_GEOFile) {
+	public StopReader(TransitSchedule schedule, CoordinateTransformation transformation, String pathToBFKOORD_WGSFile) {
 		this.schedule = schedule;
 		this.transformation = transformation;
 		this.scheduleBuilder = this.schedule.getFactory();
-		this.pathToBFKOORD_GEOFile = pathToBFKOORD_GEOFile;
+		this.pathToBFKOORD_WGSFile = pathToBFKOORD_WGSFile;
 	}
 
-	public static void run(TransitSchedule schedule, CoordinateTransformation transformation, String pathToBFKOORD_GEOFile) throws IOException {
-		new StopReader(schedule, transformation, pathToBFKOORD_GEOFile).createStops();
+	public static void run(TransitSchedule schedule, CoordinateTransformation transformation, String pathToBFKOORD_WGSFile) throws IOException {
+		new StopReader(schedule, transformation, pathToBFKOORD_WGSFile).createStops();
 	}
 
 	private void createStops() throws IOException {
 		log.info("  Read transit stops...");
-			BufferedReader readsLines = new BufferedReader(new InputStreamReader(new FileInputStream(pathToBFKOORD_GEOFile), "utf-8"));
-			String newLine = readsLines.readLine();
-			while (newLine != null) {
+			BufferedReader readsLines = new BufferedReader(new InputStreamReader(new FileInputStream(pathToBFKOORD_WGSFile), "utf-8"));
+			String newLine;
+			while ((newLine = readsLines.readLine()) != null) {
+				if (newLine.startsWith("*")) {
+					continue;
+				}
 				/*
 				1−7 INT32 Nummer der Haltestelle
-				9−18 FLOAT X-Koordinate
-				20−29 FLOAT Y-Koordinate
-				31−36 INT16 Z-Koordinate (Tunnel und andere Streckenelemente ohne eigentliche Haltestelle haben keine Z-Koordinate)
-				38ff CHAR Kommentarzeichen "%"gefolgt vom Klartext des Haltestellennamens (optional zur besseren Lesbarkeit)
+				9−19 FLOAT X-Koordinate
+				21−31 FLOAT Y-Koordinate
+				33−38 INT16 Z-Koordinate (Tunnel und andere Streckenelemente ohne eigentliche Haltestelle haben keine Z-Koordinate)
+				40ff CHAR Kommentarzeichen "%"gefolgt vom Klartext des Haltestellennamens (optional zur besseren Lesbarkeit)
 				 */
 				Id<TransitStopFacility> stopId = Id.create(newLine.substring(0, 7), TransitStopFacility.class);
-				double xCoord = Double.parseDouble(newLine.substring(8, 18));
-				double yCoord = Double.parseDouble(newLine.substring(19, 29));
+				double xCoord = Double.parseDouble(newLine.substring(8, 19));
+				double yCoord = Double.parseDouble(newLine.substring(20, 31));
 				Coord coord = new Coord(xCoord, yCoord);
 				if (this.transformation != null) {
 					coord = this.transformation.transform(coord);
 				}
-				String stopName = newLine.substring(39, newLine.length());
+				String stopName = newLine.substring(41);
 				createStop(stopId, coord, stopName);
-				newLine = readsLines.readLine();
 			}
 			readsLines.close();
 		log.info("  Read transit stops... done.");

diff --git a/test/BrienzRothornBahn-HAFAS/BETRIEB_DE b/test/BrienzRothornBahn-HAFAS/BETRIEB_DE
@@ -1,2 +1,3 @@
+* Kommentarzeile
 00075 K "BRB" L "BRB" V "Brienz Rothorn Bahn AG"
 00075 : 000104
diff --git a/test/BrienzRothornBahn-HAFAS/BFKOORD_GEO b/test/BrienzRothornBahn-HAFAS/BFKOORD_GEO
diff --git a/test/BrienzRothornBahn-HAFAS/BFKOORD_WGS b/test/BrienzRothornBahn-HAFAS/BFKOORD_WGS
@@ -0,0 +1,4 @@
+* Kommentarzeile
+8508350   8.0380890  46.7552140 566    % Brienz BRB
+8508351   8.0198450  46.7723730 1346   % Planalp
+8508352   8.0384470  46.7873990 2252   % Brienzer Rothorn
diff --git a/test/BrienzRothornBahn-HAFAS/BITFELD b/test/BrienzRothornBahn-HAFAS/BITFELD
@@ -1 +1,2 @@
+* Kommentarzeile
 003499 F00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
diff --git a/test/BrienzRothornBahn-HAFAS/FPLAN b/test/BrienzRothornBahn-HAFAS/FPLAN
@@ -1,4 +1,4 @@
-*Z 00001 000104   001                                     % 00001 000104   001 (001)
+*Z 000001 000104   001                                    % 00001 000104   001 (001)
 *G R   8508350 8508352  00730  00825                      % 00001 000104   001 (002)
 *A VE 8508350 8508352 003499  00730  00825                % 00001 000104   001 (003)
 *A 2  8508350 8508352         00730  00825                % 00001 000104   001 (004)
@@ -7,7 +7,7 @@
 8508350 Brienz BRB                   00730                % 00001 000104   001 (007)
 8508351 Planalp               00756  00756                % 00001 000104   001 (008)
 8508352 Brienzer Rothorn      00825                       % 00001 000104   001 (009)
-*Z 00002 000104   001                                     % 00002 000104   001 (001)
+*Z 000002 000104   001                                    % 00002 000104   001 (001)
 *G R   8508352 8508350  00830  00930                      % 00002 000104   001 (002)
 *A VE 8508352 8508350 003499  00830  00930                % 00002 000104   001 (003)
 *A 2  8508352 8508350         00830  00930                % 00002 000104   001 (004)

diff --git a/test/BrienzRothornBahn-HAFAS/readme.txt b/test/BrienzRothornBahn-HAFAS/readme.txt
@@ -1,12 +1,13 @@
 Extract of http://www.fahrplanfelder.ch/de/fahrplandaten.html => testdaten.zip (accessed 02.11.2016), which is publicly available.
+Note 2024: test files are no longer available in the link above and the HAFAS-Format has changed since. The test files have been manually edited to reflect the changes.
 
 Extracted the files
 	- BETRIEB_DE
-	- BFKOORD_GEO
+	- BFKOORD_WGS
 	- BITFELD
 	- FPLAN
 
 Modifications:
 	- FPLAN is reduced to the first two public transport lines (Brienz BRB to Rothorn via Planalp and back).
-	- BETRIEB_DE and BFKOORD_GEO are reduced to only the values required by the reduced FPLAN.
+	- BETRIEB_DE and BFKOORD_WGS are reduced to only the values required by the reduced FPLAN.
 	- BITFELD is also reduced to the line required by the reduced FPLAN, but additionally the BITFELD-Stream of this line was changed and simplified for testing purposes.
diff --git a/test/FPLAN_HAFAS/BETRIEB_DE b/test/FPLAN_HAFAS/BETRIEB_DE
@@ -1,2 +1,3 @@
+* Kommentarzeile
 00343 K "RhB" L "RhB" V "Rhätische Bahn"
 00343 : 000072
diff --git a/test/FPLAN_HAFAS/BFKOORD_GEO b/test/FPLAN_HAFAS/BFKOORD_GEO
diff --git a/test/FPLAN_HAFAS/BFKOORD_WGS b/test/FPLAN_HAFAS/BFKOORD_WGS
@@ -0,0 +1,22 @@
+* Kommentarzeile
+8509002   9.5540280  46.9674390 523    % Landquart
+8509056   9.5608660  46.9571990 523    % Landquart Ried
+8509055   9.5647540  46.9498290 523    % Igis
+8509054   9.5595890  46.9348430 531    % Zizers
+8509053   9.5591730  46.9189870 537    % Untervaz-Trimmis
+8509051   9.5331220  46.8769840 562    % Haldenstein
+8509006   9.5314450  46.8618980 577    % Chur Wiesental
+8509000   9.5289250  46.8530800 585    % Chur
+8509183   9.4125510  46.8236640 604    % Reichenau-Tamins
+8509167   9.3595210  46.8165780 609    % Trin
+8509168   9.3102990  46.8073800 635    % Versam-Safien
+8509169   9.2754590  46.7917940 669    % Valendas-Sagogn
+8509170   9.2330800  46.7784940 655    % Castrisch
+8509171   9.2074370  46.7753590 698    % Ilanz
+8509173   9.1456140  46.7746980 733    % Rueun
+8509174   9.1201520  46.7695600 744    % Waltensburg/Vuorz
+8509175   9.0624300  46.7547680 788    % Tavanasa-Breil/Brigels
+8509176   8.9904440  46.7425430 852    % Trun
+8509177   8.9574960  46.7325770 928    % Rabius-Surrein
+8509178   8.9313260  46.7240780 982    % Sumvitg-Cumpadials
+8509179   8.8550210  46.7049790 1130   % Disentis/Mustér
diff --git a/test/FPLAN_HAFAS/FPLAN b/test/FPLAN_HAFAS/FPLAN
@@ -1,4 +1,4 @@
-*Z 01729 000072   001                                     % -- 6100720007 --
+*Z 001729 000072   001                                    % -- 6100720007 --
 *G RE  8509002 8509179                                    %
 *A VE 8509002 8509179                                     %
 *A X  8509056 8509056                                     %
@@ -35,7 +35,7 @@
 8509177 Rabius-Surrein        01057  01057                %
 8509178 Sumvitg-Cumpadials    01100  01100                %
 8509179 Disentis/Mustér       01111                       %
-*Z 99999 000072   001                                     % -- 6100720007 --
+*Z 099999 000072   001                                    % -- 6100720007 --
 *G RE  8509002 8509179                                    %
 *A VE 8509002 8509179                                     %
 *A X  8509056 8509056                                     %
@@ -72,7 +72,7 @@
 8509177 Rabius-Surrein        01057  01057                %
 8509178 Sumvitg-Cumpadials    01100  01100                %
 8509179 Disentis/Mustér       01111                       %
-*Z 01729 000072   001                                     % -- 6100720007 --
+*Z 001729 000072   001                                    % -- 6100720007 --
 *G RE  8509002 8509179                                    %
 *A VE 8509002 8509179                                     %
 *A X  8509056 8509056                                     %
Original file line number	Diff line number	Diff line change
		@@ -1 +1,2 @@
		* Kommentarzeile
		003499 F00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000