Skip to content

Commit

Permalink
package structure + debug dashboard color ramps (#89)
Browse files Browse the repository at this point in the history 
* package structure + debug dashboard color ramps

* fix typo

* fix typo consistently

* * use MapPlot instead of Link panel
* emissions -> airPollution
* delete default for base-run

* checkstyle

* * only dump 4 decimal digits
* always set projection per yaml for GridMaps

---------

Co-authored-by: sime94 <[email protected]>
  • Loading branch information
@tschlenther @simei94
tschlenther and simei94 authored Sep 26, 2024
1 parent 9233116 commit 6697326
Show file tree
Hide file tree
Showing 10 changed files with 96 additions and 59 deletions.
20 changes: 10 additions & 10 deletions src/main/java/org/matsim/analysis/postAnalysis/NoiseAverageAnalysis.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,8 +57,8 @@ public class NoiseAverageAnalysis implements MATSimAppCommand {
private static final String ANALYSIS_DIR = "/analysis/noise";
private static final String LINK_ID = "Link Id";
private static final String VALUE = "value";
private List<GenericRecord> imissionsPerDay = new ArrayList<>();
private List<GenericRecord> imissionsPerHour = new ArrayList<>();
private List<GenericRecord> immissionsPerDay = new ArrayList<>();
private List<GenericRecord> immissionsPerHour = new ArrayList<>();
private Map<String, List<Double>> emissionsPerDay = new HashMap<>();
private Map<String, Double> meanEmissionsPerDay = new HashMap<>();
private Map<String, List<Double>> totalStats = new HashMap<>();
Expand Down Expand Up @@ -86,8 +86,8 @@ public Integer call() throws Exception {
for (String folder : foldersSeeded) {
final Path analysisDir = Path.of(folder + ANALYSIS_DIR);
String emissionsCsv = globFile(analysisDir, "*emission_per_day.csv*").toString();
String imissionsPerDayAvro = globFile(analysisDir, "*immission_per_day.avro*").toString();
String imissionsPerHourAvro = globFile(analysisDir, "*immission_per_hour.avro*").toString();
String immissionsPerDayAvro = globFile(analysisDir, "*immission_per_day.avro*").toString();
String immissionsPerHourAvro = globFile(analysisDir, "*immission_per_hour.avro*").toString();
String totalStatsCsv = globFile(analysisDir, "*noise_stats.csv*").toString();
String damagesPerDayAvro = globFile(analysisDir, "*damages_receiverPoint_per_day.avro*").toString();
String damagesPerHourAvro = globFile(analysisDir, "*damages_receiverPoint_per_hour.avro*").toString();
Expand All @@ -105,8 +105,8 @@ public Integer call() throws Exception {
.separator(CsvOptions.detectDelimiter(totalStatsCsv)).build());

// read avro files
readAvroFile(imissionsPerDayAvro, imissionsPerDay);
readAvroFile(imissionsPerHourAvro, imissionsPerHour);
readAvroFile(immissionsPerDayAvro, immissionsPerDay);
readAvroFile(immissionsPerHourAvro, immissionsPerHour);
readAvroFile(damagesPerDayAvro, damagesPerDay);
readAvroFile(damagesPerHourAvro, damagesPerHour);

Expand Down Expand Up @@ -137,8 +137,8 @@ public Integer call() throws Exception {
calcCsvMeans(totalStats, meanTotalStatsPerDay);

// calc avro means
XYTData imissionsPerDayMean = calcAvroMeans(imissionsPerDay, "imissions");
XYTData imissionsPerHourMean = calcAvroMeans(imissionsPerHour, "imissions");
XYTData immissionsPerDayMean = calcAvroMeans(immissionsPerDay, "immission");
XYTData immissionsPerHourMean = calcAvroMeans(immissionsPerHour, "immission");
XYTData damagesPerDayMean = calcAvroMeans(damagesPerDay, "damages_receiverPoint");
XYTData damagesPerHourMean = calcAvroMeans(damagesPerHour, "damages_receiverPoint");

Expand All @@ -161,8 +161,8 @@ public Integer call() throws Exception {
}

// write avro mean files
writeAvro(imissionsPerDayMean, new File(output.getPath("mean_immission_per_day.avro").toString()));
writeAvro(imissionsPerHourMean, new File(output.getPath("mean_immission_per_hour.avro").toString()));
writeAvro(immissionsPerDayMean, new File(output.getPath("mean_immission_per_day.avro").toString()));
writeAvro(immissionsPerHourMean, new File(output.getPath("mean_immission_per_hour.avro").toString()));
writeAvro(damagesPerDayMean, new File(output.getPath("mean_damages_receiverPoint_per_day.avro").toString()));
writeAvro(damagesPerHourMean, new File(output.getPath("mean_damages_receiverPoint_per_hour.avro").toString()));

Expand Down
10 changes: 5 additions & 5 deletions ...issionsPostProcessingAverageAnalysis.java → ...issionsPostProcessingAverageAnalysis.java
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package org.matsim.analysis.postAnalysis;
package org.matsim.analysis.postAnalysis.emissions;

import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVPrinter;
Expand Down Expand Up @@ -198,8 +198,8 @@ public Integer call() throws Exception {
}

// write grid mean stats
writeGridFile("mean_emissions_grid_per_day.xyt.csv", meanGridPerDay);
writeGridFile("mean_emissions_grid_per_hour.csv", meanGridPerHour);
writeGridFile("mean_emissions_grid_per_day.xyt.csv", meanGridPerDay, nf);
writeGridFile("mean_emissions_grid_per_hour.csv", meanGridPerHour, nf);

return 0;
}
Expand All @@ -223,15 +223,15 @@ private void getGridData(Table gridTable, Map<Map.Entry<Double, Coord>, List<Dou
}
}

private void writeGridFile(String fileName, Map<Map.Entry<Double, Coord>, Double> values) throws IOException {
private void writeGridFile(String fileName, Map<Map.Entry<Double, Coord>, Double> values, NumberFormat numberFormat) throws IOException {
try (CSVPrinter printer = new CSVPrinter(Files.newBufferedWriter(output.getPath(fileName)), CSVFormat.DEFAULT)) {

//set the projection in the YAML instead, as this is put out with a quote atm...
//printer.printRecord("# EPSG:25832");
printer.printRecord("time", "x", "y", VALUE);

for (Map.Entry<Map.Entry<Double, Coord>, Double> e : values.entrySet()) {
printer.printRecord(e.getKey().getKey(), e.getKey().getValue().getX(), e.getKey().getValue().getY(), e.getValue());
printer.printRecord(e.getKey().getKey(), e.getKey().getValue().getX(), e.getKey().getValue().getY(), numberFormat.format(e.getValue()));
}
}
}
Expand Down
22 changes: 14 additions & 8 deletions ...PollutionAnalysisByEngineInformation.java → ...PollutionAnalysisByEngineInformation.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
* *
* *********************************************************************** */

package org.matsim.analysis.emissions;
package org.matsim.analysis.postAnalysis.emissions;

import it.unimi.dsi.fastutil.objects.Object2DoubleLinkedOpenHashMap;
import it.unimi.dsi.fastutil.objects.Object2DoubleMap;
Expand Down Expand Up @@ -428,23 +428,25 @@ private void writeTotal(Network network, EmissionsOnLinkEventHandler emissionsEv
//so we need to do some stupid filtering afterwards, which means that we produce and calculate more data than we dump out....
private void writeRaster(Network fullNetwork, Network filteredNetwork, Config config, EmissionsOnLinkEventHandler emissionsEventHandler) {



Map<Pollutant, Raster> rasterMap = FastEmissionGridAnalyzer.processHandlerEmissions(emissionsEventHandler.getLink2pollutants(), fullNetwork, gridSize, 20);

Raster raster = rasterMap.values().stream().findFirst().orElseThrow();

try (CSVPrinter printer = new CSVPrinter(Files.newBufferedWriter(output.getPath("emissions_grid_per_day.xyt.csv")),
CSVFormat.DEFAULT.builder().setCommentMarker('#').build())) {

NumberFormat nf = NumberFormat.getInstance(Locale.US);
nf.setMaximumFractionDigits(4);
nf.setGroupingUsed(false);

String crs = ProjectionUtils.getCRS(fullNetwork);
if (crs == null)
crs = config.network().getInputCRS();
if (crs == null)
crs = config.global().getCoordinateSystem();

// print coordinate system
printer.printComment(crs);
// printer.printComment(crs);

// print header
printer.print("time");
Expand Down Expand Up @@ -478,7 +480,7 @@ private void writeRaster(Network fullNetwork, Network filteredNetwork, Config co
printer.print(coord.getY());

double value = rasterMap.get(Pollutant.CO2_TOTAL).getValueByIndex(xi, yi);
printer.print(value);
printer.print(nf.format(value));

printer.println();
}
Expand Down Expand Up @@ -515,14 +517,18 @@ private void writeTimeDependentRaster(Network fullNetwork, Network filteredNetwo
try (CSVPrinter printer = new CSVPrinter(IOUtils.getBufferedWriter(output.getPath("emissions_grid_per_hour.csv").toString()),
CSVFormat.DEFAULT.builder().setCommentMarker('#').build())) {

NumberFormat nf = NumberFormat.getInstance(Locale.US);
nf.setMaximumFractionDigits(4);
nf.setGroupingUsed(false);

String crs = ProjectionUtils.getCRS(fullNetwork);
if (crs == null)
crs = config.network().getInputCRS();
if (crs == null)
crs = config.global().getCoordinateSystem();

// print coordinate system
printer.printComment(crs);
// print coordinate system
// printer.printComment(crs);

// print header
printer.print("time");
Expand Down Expand Up @@ -560,7 +566,7 @@ private void writeTimeDependentRaster(Network fullNetwork, Network filteredNetwo
printer.print(coord.getX());
printer.print(coord.getY());

printer.print(value);
printer.print(nf.format(value));

printer.println();
}
Expand Down
56 changes: 42 additions & 14 deletions src/main/java/org/matsim/dashboard/AverageKelheimEmissionsDashboard.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,19 +20,18 @@

package org.matsim.dashboard;

import org.matsim.analysis.postAnalysis.EmissionsPostProcessingAverageAnalysis;
import org.matsim.analysis.postAnalysis.emissions.EmissionsPostProcessingAverageAnalysis;
import org.matsim.simwrapper.Dashboard;
import org.matsim.simwrapper.Data;
import org.matsim.simwrapper.Header;
import org.matsim.simwrapper.Layout;
import org.matsim.simwrapper.viz.GridMap;
import org.matsim.simwrapper.viz.Links;
import org.matsim.simwrapper.viz.Table;
import org.matsim.simwrapper.viz.*;

import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;

import static org.matsim.dashboard.AverageKelheimNoiseDashboard.*;

/**
* Average emissions dashboard for several runs with the same config but a different random seed.
*/
Expand All @@ -51,10 +50,14 @@ public AverageKelheimEmissionsDashboard(List<String> dirs, Integer noRuns, Strin
this.dirs = dirs;
this.noRuns = noRuns;

if (!pathToBaseRun.endsWith("/")) {
pathToBaseRun += "/";
if (pathToBaseRun == null || pathToBaseRun.equals("null")){
this.pathToCsvBase = null;
} else {
if (!pathToBaseRun.endsWith("/")) {
pathToBaseRun += "/";
}
this.pathToCsvBase = pathToBaseRun + "analysis/emissions/emissions_per_link_per_m.csv";
}
this.pathToCsvBase = pathToBaseRun + "analysis/emissions/emissions_per_link_per_m.csv";
}

private String postProcess(Data data, String outputFile) {
Expand All @@ -68,8 +71,8 @@ private String postProcess(Data data, String outputFile) {
* Produces the dashboard.
*/
public void configure(Header header, Layout layout) {
header.title = "Average Emissions";
header.description = "Shows the average emissions footprint and spatial distribution for several simulation runs.";
header.title = "Average Air Pollution";
header.description = "Shows the average air pollution and spatial distribution for several simulation runs.";

String linkDescription = "Displays the emissions for each link per meter. Be aware that emission values are provided in the simulation sample size!";
if (pathToCsvBase != null){
Expand All @@ -86,32 +89,57 @@ public void configure(Header header, Layout layout) {
viz.showAllRows = true;
viz.width = 1.0;
})
.el(Links.class, (viz, data) -> {
viz.title = "Emissions per Link per Meter";
/*
* Commented out link panel, because the MapPlot can show a legend and seems to be the development head.
* However, it doesn't seem to have the pointer to the base case
*/
/*.el(Links.class, (viz, data) -> {
viz.title = "Emitted Pollutant in Gram per Meter";
viz.description = finalLinkDescription;
viz.height = 12.0;
viz.datasets.csvFile = postProcess(data, "mean_emissions_per_link_per_m.csv");
viz.datasets.csvBase = Path.of(this.dirs.get(0)).getParent().relativize(Path.of(pathToCsvBase)).toString();
viz.datasets.csvBase = pathToCsvBase == null ? "" : Path.of(this.dirs.get(0)).getParent().relativize(Path.of(pathToCsvBase)).toString();
viz.network = new CreateAverageDashboards().copyVizNetwork(dirs, ".avro");
viz.display.color.columnName = "CO2_TOTAL [g/m]";
viz.display.color.dataset = "csvFile";
viz.display.width.scaleFactor = 100;
viz.display.width.columnName = "CO2_TOTAL [g/m]";
viz.display.width.dataset = "csvFile";
viz.center = data.context().getCenter();
viz.width = 1.0;
})*/
.el(MapPlot.class, (viz, data) -> {
viz.title = "Emitted Pollutant in Gram per Meter";
viz.description = finalLinkDescription;
viz.height = 12.0;
viz.center = data.context().getCenter();
viz.zoom = data.context().mapZoomLevel;
viz.setShape(new CreateAverageDashboards().copyVizNetwork(dirs, ".avro"), "id");
viz.addDataset("emissions_per_m", postProcess(data, "mean_emissions_per_link_per_m.csv"));
viz.display.lineColor.dataset = "emissions_per_m";
viz.display.lineColor.columnName = "CO2_TOTAL [g/m]";
viz.display.lineColor.join = "linkId";
viz.display.lineColor.setColorRamp(ColorScheme.Oranges, 8, false, "35, 45, 55, 65, 75, 85, 95");
viz.display.lineWidth.dataset = "emissions_per_m";
viz.display.lineWidth.columnName = "CO2_TOTAL [g/m]";
viz.display.lineWidth.scaleFactor = 100d;
viz.display.lineWidth.join = "linkId";
viz.width = 3.0;
});
});

layout.row("second").el(GridMap.class, (viz, data) -> {
viz.title = "CO₂ Emissions";
viz.description = "per day. Be aware that CO2 values are provided in the simulation sample size!";
viz.projection = "EPSG:25832";
viz.setColorRamp(new double[]{30, 40, 50, 60, 70}, new String[]{DARK_BLUE, LIGHT_BLUE, YELLOW, SAND, ORANGE, RED});
viz.height = 12.0;
viz.file = postProcess(data, "mean_emissions_grid_per_day.xyt.csv");
});
layout.row("third")
.el(GridMap.class, (viz, data) -> {
viz.title = "CO₂ Emissions";
viz.projection = "EPSG:25832";
viz.setColorRamp(new double[]{30, 40, 50, 60, 70}, new String[]{DARK_BLUE, LIGHT_BLUE, YELLOW, SAND, ORANGE, RED});
viz.description = "per hour. Be aware that CO2 values are provided in the simulation sample size!";
viz.height = 12.;
viz.file = postProcess(data, "mean_emissions_grid_per_hour.csv");
Expand Down
14 changes: 7 additions & 7 deletions src/main/java/org/matsim/dashboard/AverageKelheimNoiseDashboard.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
import java.util.List;

/**
* Shows emission in the scenario.
* Averages and displays the noise outcome of single runs.
*/
public class AverageKelheimNoiseDashboard implements Dashboard {

Expand All @@ -23,12 +23,12 @@ public class AverageKelheimNoiseDashboard implements Dashboard {
private final List<String> dirs;
private final Integer noRuns;
private static final String NOISE = "noise";
private static final String DARK_BLUE = "#1175b3";
private static final String LIGHT_BLUE = "#95c7df";
private static final String ORANGE = "#f4a986";
private static final String RED = "#cc0c27";
private static final String SAND = "#dfb095";
private static final String YELLOW = "#dfdb95";
static final String DARK_BLUE = "#1175b3";
static final String LIGHT_BLUE = "#95c7df";
static final String ORANGE = "#f4a986";
static final String RED = "#cc0c27";
static final String SAND = "#dfb095";
static final String YELLOW = "#dfdb95";

public AverageKelheimNoiseDashboard(List<String> dirs, Integer noRuns) {
this.dirs = dirs;
Expand Down
8 changes: 2 additions & 6 deletions src/main/java/org/matsim/dashboard/CreateAverageDashboards.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,18 +24,14 @@ final class CreateAverageDashboards implements MATSimAppCommand {
private String inputPath;
@CommandLine.Option(names = "--no-runs", defaultValue = "5", description = "Number of simulation runs to be averaged.")
private Integer noRuns;
@CommandLine.Option(names = "--base-run", description = "Path to directory base run.", defaultValue = "/net/ils/matsim-kelheim/v3.0-release/output-base/25pct")
@CommandLine.Option(names = "--base-run", description = "Path to directory base run.")
private String pathToBaseRun;

public static void main(String[] args) {
new CreateAverageDashboards().execute(args);
}

CreateAverageDashboards() {

}


CreateAverageDashboards() {}

@Override
public Integer call() throws Exception {
Expand Down
1 change: 0 additions & 1 deletion src/main/java/org/matsim/dashboard/KelheimDashboardProvider.java
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
package org.matsim.dashboard;

import org.matsim.analysis.emissions.KelheimEmissionsDashboard;
import org.matsim.core.config.Config;
import org.matsim.core.utils.io.IOUtils;
import org.matsim.simwrapper.Dashboard;
Expand Down
Loading

0 comments on commit 6697326

Please sign in to comment.