Merge pull request #15 from matsim-vsp/kaibranch2

Kaibranch2

src/main/java/org/tub/vsp/bvwp/RunLocalCsvScraping.java

@@ -0,0 +1,36 @@
+package org.tub.vsp.bvwp;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.tub.vsp.bvwp.data.container.analysis.StreetAnalysisDataContainer;
+import org.tub.vsp.bvwp.io.StreetCsvWriter;
+import org.tub.vsp.bvwp.scraping.StreetScraper;
+
+import java.util.List;
+
+public class RunLocalCsvScraping {
+    private static final Logger logger = LogManager.getLogger(RunLocalCsvScraping.class);
+
+    public static void main(String[] args) {
+        logger.warn("(vermutl. weitgehend gelöst) Teilweise werden die Hauptprojekte bewertet und nicht die " +
+                "Teilprojekte (A20); teilweise werden die Teilprojekte " +
+                "bewertet aber nicht das Hauptprojekt (A2).  Müssen aufpassen, dass nichts unter den Tisch fällt.");
+        logger.warn("Bei https://www.bvwp-projekte.de/strasse/A559-G10-NW/A559-G10-NW.html hat evtl. die Veränderung " +
+                "Betriebsleistung PV falsches VZ.  Nutzen (positiv) dann wieder richtig.");
+        logger.warn("Wieso geht bei der https://www.bvwp-projekte.de/strasse/A14-G20-ST-BB/A14-G20-ST-BB.html der " +
+                "Nutzen mit impl und co2Price sogar nach oben?");
+
+
+        StreetScraper scraper = new StreetScraper();
+
+        logger.info("Starting scraping");
+        List<StreetAnalysisDataContainer> allStreetBaseData = scraper.extractAllLocalBaseData("./data/street", "A", ".*" )
+                                                                     .stream()
+                                                                     .map(StreetAnalysisDataContainer::new)
+                                                                     .toList();
+
+        logger.info("Writing csv");
+        StreetCsvWriter csvWriter = new StreetCsvWriter("output/street_data.csv");
+        csvWriter.writeCsv(allStreetBaseData);
+    }
+}

src/main/java/org/tub/vsp/bvwp/computation/ComputationKN.java

@@ -242,7 +242,7 @@ public static Double b_co2(Modifications modifications, Amounts amounts, Benefit
 
         // co2 Bau
         {
-            co2 += benefits.co2_infra;
+			co2 += benefits.co2_infra / 145. * modifications.co2Price() ;
         }
 
         // co2 Betrieb
@@ -265,9 +265,14 @@ public static Double b_co2(Modifications modifications, Amounts amounts, Benefit
         }
         {
             co2 += b_co2_induz / 145. * modifications.co2Price();
+
             co2 += modifications.mehrFzkm() * amounts.co2_per_pkwkm * b_per_co2 / 145. * modifications.co2Price();
+			// mehrFzkm are those which are on top of PRINS.  We do same calculation as with the other co2 contribs, except that this one here is currently on one line.
         }
 
-        return -co2;
+		return co2;
+		// The result will be negative.  We return positive so we can logplot it. yyyyyy really??? -- no longer
+
+		// Note that this really says nothing about old vs new co2 price, or old vs new addl traffic.  That all depends on the settings in "modifications".
     }
 }

src/main/java/org/tub/vsp/bvwp/data/Headers.java

@@ -1,33 +1,40 @@
 package org.tub.vsp.bvwp.data;
 
 public final class Headers{
+	public static final String ADDTL_LANE_KM = "additional_lane_km";
+	public static final String ADDTL_PKWKM_DIFF = "addtl_pkwkm_diff";
+	public static final String ADDTL_PKWKM_NEU = "addtl_pkwkm_neu";
+	public static final String ADDTL_PKWKM_ORIG = "addtl_pkwkm_orig";
+	public static final String BAUTYP = "bautyp";
+	public static final String B_CO_2_EQUIVALENTS_ORIG = "b_co2-equivalents-overall";
+	public static final String B_FZKM = "b_fzkm";
+	public static final String B_OVERALL = "b_overall";
 	public static final String B_PER_KM = "b/km";
-	public static final String NKV_NO_CHANGE = "nkv";
+	public static final String CO2_COST_NEU = "cost_co2_neu";
+	public static final String CO2_COST_ORIG = "cost_co2_orig";
+	public static final String COST_OVERALL = "cost";
+	public static final String CO_2_EQUIVALENTS_EMISSIONS = "co2-equivalents-emissions";
+	public static final String EINSTUFUNG = "Einstufung";
+	public static final String LENGTH = "length";
+	public static final String LINK = "link";
+	public static final String NKV_CAPPED = "nkv_old_capped";
 	public static final String NKV_CO2 = "nkv_co2";
+	public static final String NKV_CO2_2000_EN = "BCR_co2_2000";
+	public static final String NKV_CO2_680_EN = "BCR_co2_680";
+	public static final String NKV_DIFF = "nkvDiff";
 	public static final String NKV_INDUZ = "nkv_induz";
 	public static final String NKV_INDUZ_CO2 = "nkv_induzCo2";
-	public static final String NKV_NO_CHANGE_EN = "BCR";
-	public static final String NKV_CO2_680_EN = "BCR_co2_680";
-	public static final String NKV_CO2_2000_EN = "BCR_co2_2000";
-	public static final String NKV_INDUZ_EN = "BCR_induz";
+	public static final String NKV_INDUZ_CO2_CAPPED = "nkv_induzCo2_capped";
 	public static final String NKV_INDUZ_CO2_EN = "BCR_induzCo2";
-	public static final String PROJECT_NAME = "project name";
-	public static final String LINK = "link";
-	public static final String PRIORITY = "priority";
-	public static final String BAUTYP = "bautyp";
-	public static final String LENGTH = "length";
-	public static final String PKWKM_ALL = "pkwkm_all";
-	public static final String B_FZKM = "b_fzkm";
-	public static final String CO_2_EQUIVALENTS_EMISSIONS = "co2-equivalents-emissions";
-	public static final String B_CO_2_EQUIVALENTS_ANNUAL = "b_co2-equivalents-annual";
-	public static final String B_CO_2_EQUIVALENTS_OVERALL = "b_co2-equivalents-overall";
-	public static final String B_OVERALL = "b_overall";
-	public static final String COST_OVERALL = "cost";
+	public static final String NKV_INDUZ_EN = "BCR_induz";
+	public static final String NKV_NO_CHANGE = "nkv";
+	public static final String NKV_NO_CHANGE_EN = "BCR";
+	public static final String NKV_ORIG = "nkv";
 	public static final String PKWKM_INDUZ = "pkwkm_induz";
-	public static final String PKWKM_ALL_NEU = "pkwkm_all_neu";
-	public static final String B_CO2_NEU = "b_co2_neu";
-	public static final String ADDITIONAL_LANE_KM = "additional_lane_km";
-	public static final String NKV_DIFF = "nkvDiff";
-	public static final String VERKEHRSBELASTUNG_2030 = "DTV 2030";
+	public static final String PRIORITY = "priority";
+	public static final String PRIO_AS_NUMBER = "prioAsNumber";
+	public static final String PROJECT_NAME = "project name";
+	public static final String VERKEHRSBELASTUNG_PLANFALL = "Verkehrsbelastung Planfall";
+//	public static final String B_CO_2_EQUIVALENTS_ANNUAL = "b_co2-equivalents-annual";
 	private Headers(){} // do not instantiate
 }

src/main/java/org/tub/vsp/bvwp/data/container/analysis/StreetAnalysisDataContainer.java

@@ -77,7 +77,7 @@ private void addComputations() {
                                                                                               .getBautyp());
         }
 
-        entries.put(Headers.ADDITIONAL_LANE_KM, additionalLaneKm);
+        entries.put( Headers.ADDTL_LANE_KM, additionalLaneKm );
 
         double mehrFzkmFromElasticity = additionalLaneKm / ComputationKN.LANE_KM_AB * 0.6 * ComputationKN.FZKM_AB;
         final double INFLATION_Factor2022to2012 = 0.917; // Zinse Wert von 2020 auf BVWP Zeitpunkt 2012 ab.
@@ -94,7 +94,7 @@ private void addComputations() {
         entries.put(Headers.B_PER_KM, streetBaseDataContainer.getCostBenefitAnalysis().getOverallBenefit()
                                                              .overall() / streetBaseDataContainer.getProjectInformation()
                                                                                                  .getLength());
-        entries.put(Headers.NKV_NO_CHANGE, NkvCalculator.calculateNkv(Modifications.NO_CHANGE,
+        entries.put(Headers.NKV_ORIG, NkvCalculator.calculateNkv(Modifications.NO_CHANGE,
                 streetBaseDataContainer));
         entries.put(Headers.NKV_CO2, NkvCalculator.calculateNkv(Modifications.CO2_PRICE_5FACH,
                 streetBaseDataContainer));
@@ -109,11 +109,11 @@ private void addComputations() {
         entries.put(Headers.NKV_INDUZ_CO2,
                 NkvCalculator.calculateNkv(Modifications.createInducedAndCo2WithMehrFzkm(additionalFzkm),
                         streetBaseDataContainer));
-        entries.put(Headers.PKWKM_ALL_NEU, mehrFzkmFromElasticity);
-        entries.put(Headers.B_CO2_NEU,
-                NkvCalculator.calculateB_CO2(Modifications.createInducedAndCo2WithMehrFzkm(additionalFzkm),
+        entries.put(Headers.ADDTL_PKWKM_NEU, mehrFzkmFromElasticity );
+	    entries.put( Headers.CO2_COST_ORIG, - NkvCalculator.calculateB_CO2( Modifications.NO_CHANGE, streetBaseDataContainer ) );
+	    entries.put( Headers.CO2_COST_NEU, - NkvCalculator.calculateB_CO2(Modifications.createInducedAndCo2WithMehrFzkm(additionalFzkm),
                         streetBaseDataContainer));
-        entries.put(Headers.VERKEHRSBELASTUNG_2030, streetBaseDataContainer.getProjectInformation()
+        entries.put(Headers.VERKEHRSBELASTUNG_PLANFALL, streetBaseDataContainer.getProjectInformation()
                                                                            .getVerkehrsbelastung2030());
 
         if (streetBaseDataContainer.getProjectInformation().getProjectNumber().contains("A1-G50-NI")) {

src/main/java/org/tub/vsp/bvwp/data/container/base/street/StreetProjectInformationDataContainer.java

@@ -85,8 +85,7 @@ public StreetProjectInformationDataContainer setLength(Double length) {
     public Double getLength() {
         return length;
     }
-
-    public StreetProjectInformationDataContainer setVerkehrsbelastung2030(Double verkehrsbelastung2030) {
+    public StreetProjectInformationDataContainer setVerkehrsbelastungPlanfall( Double verkehrsbelastung2030 ){
         this.verkehrsbelastung2030 = verkehrsbelastung2030;
         return this;
     }

src/main/java/org/tub/vsp/bvwp/data/mapper/ProjectInformationMapper.java

@@ -0,0 +1,84 @@
+package org.tub.vsp.bvwp.data.mapper;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.jsoup.nodes.Document;
+import org.tub.vsp.bvwp.JSoupUtils;
+import org.tub.vsp.bvwp.data.container.base.street.StreetProjectInformationDataContainer;
+import org.tub.vsp.bvwp.data.type.Bautyp;
+import org.tub.vsp.bvwp.data.type.Priority;
+
+import java.text.ParseException;
+import java.util.Optional;
+
+public class ProjectInformationMapper {
+    private static final Logger logger = LogManager.getLogger(ProjectInformationMapper.class);
+
+    public StreetProjectInformationDataContainer mapDocument( Document document ) {
+        StreetProjectInformationDataContainer projectInformation = new StreetProjectInformationDataContainer();
+
+        String projectNumber = extractInformation(document, 0, "Projektnummer");
+        String street = extractInformation(document, 0, "Straße");
+        String length = extractInformation( document, 0, "Länge" );
+        if ( length!= null ) {
+            length = length.replace( " km", "" );
+        }
+        String bautyp = extractInformation( document, 0, "Bautyp(en), Bauziel(e)" );
+
+        String verkehrsbelastungPlanfall = extractInformation( document, 0, "im Planfall 2030" );
+        if ( verkehrsbelastungPlanfall!=null ) {
+            verkehrsbelastungPlanfall = verkehrsbelastungPlanfall.replace( " Kfz/24h", "" );
+        } else {
+            // ( other format for Knotenpunkte )
+            verkehrsbelastungPlanfall = extractInformation( document, 0, "/ Planfall 2030" );
+            verkehrsbelastungPlanfall = verkehrsbelastungPlanfall.replaceAll( "^.* / ", "" );
+            verkehrsbelastungPlanfall = verkehrsbelastungPlanfall.replace( " Kfz/24h", "" );
+        }
+        if ( verkehrsbelastungPlanfall==null ) {
+            verkehrsbelastungPlanfall = "0.";
+        }
+
+        String priority = extractInformation(document, 1, "Dringlichkeitseinstufung");
+
+        if ( projectNumber.startsWith( "A008-G010" ) ) {
+            // NKV ist im Hauptprojekt, aber Einstufungen sind in den Teilprojekten.  3 davon VBE, 1 WBP --> VBE.
+
+            priority = Priority.VBE.name();
+            logger.warn( "priority=" + priority );
+        }
+
+        if ( priority.equalsIgnoreCase( "siehe Teilprojekte" ) ) {
+            logger.warn("project=" + projectNumber ) ;
+        }
+
+        try{
+            return projectInformation.setProjectNumber(projectNumber)
+                                     .setStreet(street)
+                                     .setLength( JSoupUtils.parseDouble( length ) )
+                                     .setBautyp( Bautyp.getFromString( bautyp ) )
+                                     .setPriority( Priority.getFromString(priority ) )
+                                     .setVerkehrsbelastungPlanfall( JSoupUtils.parseDouble( verkehrsbelastungPlanfall ) )
+                            ;
+        } catch( ParseException e ){
+            throw new RuntimeException( e );
+        }
+    }
+
+    //mapping information from the grunddaten table. There are two tables with the same class, so we need to specify
+    // the index
+    public static String extractInformation(Document document, int tableIndex, String key) {
+        Optional<String> info = document.select("table.table_grunddaten")
+                                        .get(tableIndex)
+                                        .select("tr")
+                                        .stream()
+                                        .filter(r -> r.text()
+                                                      .contains(key))
+                                        .findFirst()
+                                        .map(r -> r.child(1)
+                                                   .text());
+        if (info.isEmpty()) {
+            logger.warn("Could not find information for key {} in table {}", key, tableIndex);
+        }
+        return info.orElse(null);
+    }
+}

src/main/java/org/tub/vsp/bvwp/data/type/Priority.java

@@ -1,23 +1,34 @@
 package org.tub.vsp.bvwp.data.type;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
 public enum Priority{
     VBE("Vordringlicher Bedarf - Engpassbeseitigung (VB-E)"),
     VB("Vordringlicher Bedarf (VB)"),
     WBP("Weiterer Bedarf mit Planungsrecht (WB*)"),
     WB( "Weiterer Bedarf (WB)"),
+    @Deprecated // yyyyyy lieber aufklären!!
     UNDEFINED("undefined");
 
+    private static final Logger log = LogManager.getLogger( Priority.class );
+
     public final String description;
 
     Priority( String description ) {
         this.description = description;
     }
 
     public static Priority getFromString( String description ) {
-        for ( Priority v : values())
-            if (v.description.equalsIgnoreCase(description)) {
+        for ( Priority v : values()){
+            if( v.description.equalsIgnoreCase( description ) ){
+                return v;
+            }
+            if ( v.name().equalsIgnoreCase( description ) ) {
                 return v;
             }
+        }
+        log.warn( "description=" + description );
         return UNDEFINED;
     }
 }

src/main/java/org/tub/vsp/bvwp/io/StreetCsvWriter.java

@@ -171,12 +171,12 @@ private static List<Object> getCsvRecord(StreetAnalysisDataContainer analysisDat
         //general info
         record.add(Headers.PROJECT_NAME, baseDataContainer.getProjectInformation().getProjectNumber());
         record.add(Headers.LINK, baseDataContainer.getUrl());
-        record.add(Headers.PRIORITY, baseDataContainer.getProjectInformation().getPriority().name());
+        record.add( Headers.EINSTUFUNG, baseDataContainer.getProjectInformation().getPriority().name() );
         record.add(Headers.BAUTYP, baseDataContainer.getProjectInformation().getBautyp().name());
 
         record.add(Headers.LENGTH, baseDataContainer.getProjectInformation().getLength());
 
-        record.add(Headers.PKWKM_ALL, baseDataContainer.getPhysicalEffect().getVehicleKilometers().overall());
+        record.add(Headers.ADDTL_PKWKM_ORIG, baseDataContainer.getPhysicalEffect().getVehicleKilometers().overall());
         record.add(Headers.PKWKM_INDUZ, Optional.ofNullable(baseDataContainer.getPhysicalEffect().getVehicleKilometers()
                                                                              .induced()).orElse(0.));
 
@@ -185,10 +185,10 @@ private static List<Object> getCsvRecord(StreetAnalysisDataContainer analysisDat
         //co2 equivalents
         record.add(Headers.CO_2_EQUIVALENTS_EMISSIONS, baseDataContainer.getPhysicalEffect().getEmissionsDataContainer()
                                                                         .co2Overall());
-        record.add(Headers.B_CO_2_EQUIVALENTS_ANNUAL, baseDataContainer.getCostBenefitAnalysis()
-                                                                       .getCo2EquivalentBenefit()
-                                                                       .annual()); // yyyy not needed
-        record.add(Headers.B_CO_2_EQUIVALENTS_OVERALL, baseDataContainer.getCostBenefitAnalysis()
+//        record.add(Headers.B_CO_2_EQUIVALENTS_ANNUAL, baseDataContainer.getCostBenefitAnalysis()
+//                                                                       .getCo2EquivalentBenefit()
+//                                                                       .annual()); // yyyy not needed
+        record.add(Headers.B_CO_2_EQUIVALENTS_ORIG, baseDataContainer.getCostBenefitAnalysis()
                                                                         .getCo2EquivalentBenefit().overall());
 
         //emissions
@@ -257,18 +257,18 @@ private static List<String> getHeaders(List<StreetAnalysisDataContainer> analysi
         headers.addStringColumn(Headers.PROJECT_NAME);
 
         headers.addStringColumn(Headers.LINK);
-        headers.addStringColumn(Headers.PRIORITY);
+        headers.addStringColumn( Headers.EINSTUFUNG );
         headers.addStringColumn(Headers.BAUTYP);
         headers.addDoubleColumn(Headers.LENGTH);
 
-        headers.addDoubleColumn(Headers.PKWKM_ALL);
+        headers.addDoubleColumn( Headers.ADDTL_PKWKM_ORIG );
         headers.addDoubleColumn(Headers.PKWKM_INDUZ);
 //        headers.addDoubleColumn( Headers.PKWKM_INDUZ_NEU ); // added by automagic
         headers.addDoubleColumn(Headers.B_FZKM);
 
         headers.addDoubleColumn(Headers.CO_2_EQUIVALENTS_EMISSIONS);
-        headers.addDoubleColumn(Headers.B_CO_2_EQUIVALENTS_ANNUAL);
-        headers.addDoubleColumn(Headers.B_CO_2_EQUIVALENTS_OVERALL);
+//        headers.addDoubleColumn( Headers.B_CO_2_EQUIVALENTS_ANNUAL );
+        headers.addDoubleColumn( Headers.B_CO_2_EQUIVALENTS_ORIG );
 
 //        for (String colName : EMISSION_COLUMNS.values()) {
 //            headers.add(colName + "-emissions");

src/main/java/org/tub/vsp/bvwp/plot/MultiPlotExample.java

@@ -49,6 +49,14 @@ class MultiPlotExample {
 					+ "<div id='plot6'>"
 					+ System.lineSeparator()
 					+ "<div id='plot7'>"
+					+ System.lineSeparator()
+					+ "<div id='plot8'>"
+					+ System.lineSeparator()
+					+ "<div id='plotA'>"
+					+ System.lineSeparator()
+					+ "<div id='plotB'>"
+					+ System.lineSeparator()
+					+ "<div id='plotC'>"
 					+ System.lineSeparator();
 
 	private static final String pageBottom = "</body>" + System.lineSeparator() + "</html>";

src/main/java/org/tub/vsp/bvwp/users/kmt/FiguresKMT.java

@@ -164,8 +164,8 @@ static Figure createFigureNkv(Axis xAxis, int plotWidth, Table table, String xNa
   }
 
   static Figure createFigureCO2(Axis xAxis, int plotWidth, Table table, String xName){
-      String yName = Headers.B_CO2_NEU;
-      String y2Name = Headers.B_CO2_NEU;
+      String yName = Headers.CO2_COST_NEU;
+      String y2Name = Headers.CO2_COST_NEU;
 
       Axis yAxis = Axis.builder()
           .title( yName )