matsim-vsp · kainagel · Mar 24, 2024 · Mar 24, 2024 · Mar 24, 2024
diff --git a/pom.xml b/pom.xml
@@ -14,21 +14,20 @@
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     </properties>
     <dependencies>
-        <!-- https://mvnrepository.com/artifact/org.bytedeco/openblas -->
+
         <dependency>
             <groupId>org.bytedeco</groupId>
-            <artifactId>openblas</artifactId>
-            <version>0.3.6-1.5.1</version>
+            <artifactId>openblas-platform</artifactId>
+            <version>0.3.26-1.5.10</version>
         </dependency>
 
-
         <dependency>
             <groupId>com.github.haifengl</groupId>
             <artifactId>smile-core</artifactId>
             <version>3.0.1</version>
         </dependency>
 
-        <!-- https://mvnrepository.com/artifact/io.pebbletemplates/pebble -->
+<!--         https://mvnrepository.com/artifact/io.pebbletemplates/pebble-->
         <dependency>
             <groupId>io.pebbletemplates</groupId>
             <artifactId>pebble</artifactId>

diff --git a/src/main/java/org/tub/vsp/bvwp/RunSaveRawHtmlData.java b/src/main/java/org/tub/vsp/bvwp/RunSaveRawHtmlData.java
@@ -7,6 +7,7 @@
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
 import org.tub.vsp.bvwp.scraping.RailScraper;
+import org.tub.vsp.bvwp.scraping.StreetScraper;
 
 import java.io.File;
 import java.io.IOException;
@@ -16,11 +17,21 @@
 public class RunSaveRawHtmlData {
     private static final Logger logger = LogManager.getLogger(RunSaveRawHtmlData.class);
 
-    public static void main(String[] args) throws IOException {
-//        StreetScraper scraper = new StreetScraper();
-//        List<String> projectUrls = scraper.getProjectUrls();
+    private enum Mode { ROAD, RAIL }
+
+    private static final Mode mode = Mode.ROAD;
 
-        List<String> projectUrls = new RailScraper().getProjectUrls();
+    public static void main(String[] args) throws IOException {
+        List<String> projectUrls;
+        switch( mode ) {
+            case ROAD -> {
+                projectUrls = new StreetScraper().getProjectUrls();
+            }
+            case RAIL -> {
+                projectUrls = new RailScraper().getProjectUrls();
+            }
+            default -> throw new IllegalStateException( "Unexpected value: " + mode );
+        }
 
         for (String projectUrl : projectUrls) {
             logger.info("Downloading and saving html for {}", projectUrl);
@@ -32,7 +43,17 @@ private static void downloadAndSaveHtml(String projectUrl) throws IOException {
         final Connection.Response response = Jsoup.connect(projectUrl).execute();
         final Document doc = response.parse();
 
-        final File f = new File("data/rail/all/" + projectUrl.substring(projectUrl.lastIndexOf("/") + 1));
-        FileUtils.writeStringToFile(f, doc.outerHtml(), StandardCharsets.UTF_8);
+        final File file;
+        switch( mode ) {
+            case ROAD -> {
+                file = new File("data/street/all" + projectUrl.substring(projectUrl.lastIndexOf("/") + 1));
+            }
+            case RAIL -> {
+                file = new File("data/rail/all/" + projectUrl.substring(projectUrl.lastIndexOf("/") + 1));
+            }
+            default -> throw new IllegalStateException( "Unexpected value: " + mode );
+        }
+
+        FileUtils.writeStringToFile(file, doc.outerHtml(), StandardCharsets.UTF_8);
     }
 }
diff --git a/src/main/java/org/tub/vsp/bvwp/data/Headers.java b/src/main/java/org/tub/vsp/bvwp/data/Headers.java
@@ -91,7 +91,7 @@ public final class Headers{
 	public static final String NKV_CO2_2000_EN = "BCR_co2_2000";
 	public static final String NKV_CO2_680_EN = "BCR_co2_680";
 	/**
-	 * {@link #NKV_EL03_CO2_215_CONSTRUCTION} - {@link #NKV_ORIG}
+	 * {@link #NKV_EL03_CARBON215_INVCOST50} - {@link #NKV_ORIG}
 	 */
 	public static final String NKV_EL03_DIFF = "NKV_el03_Diff";
 	/**
@@ -101,19 +101,19 @@ public final class Headers{
 	/**
 	 * Neues NKV mit höherem Mehrverkehr, CO2-Preis 215, sowie höheren Investitionskosten.
 	 */
-	public static final String NKV_EL03_CO2_215_CONSTRUCTION = "NKV_el03Co2_215Invkosten";
+	public static final String NKV_EL03_CARBON215_INVCOST50 = "NKV_el03_carbon215_invcost50";
 	/**
-	 * min( 5, {@link #NKV_EL03_CO2_215_CONSTRUCTION})
+	 * min( 5, {@link #NKV_EL03_CARBON215_INVCOST50})
 	 */
-	public static final String NKV_EL03_CO2_215_CONSTRUCTION_CAPPED5 = "NKV_el03Co2_215Invkosten_capped5";
-	/**
-	 * Neues NKV mit höherem Mehrverkehr, höherem CO2-Preis, höheren Investitionskosten.
-	 */
-	public static final String NKV_EL03_CO2_CONSTRUCTION = "NKV_el03Co2Invkosten";
-	/**
-	 * min( 5, {@link #NKV_EL03_CO2_CONSTRUCTION} )
-	 */
-	public static final String NKV_EL03_CO2_CONSTRUCTION_CAPPED5 = "NKV_el03Co2Invkosten_capped5";
+	public static final String NKV_EL03_CARBON215_INVCOST50_CAPPED5 = NKV_EL03_CARBON215_INVCOST50 + "_capped5";
+//	/**
+//	 * Neues NKV mit höherem Mehrverkehr, höherem CO2-Preis, höheren Investitionskosten.
+//	 */
+//	public static final String NKV_EL03_CO2_INVCOST50 = "NKV_el03_Co2_Invcost50";
+//	/**
+//	 * min( 5, {@link #NKV_EL03_CO2_INVCOST50} )
+//	 */
+//	public static final String NKV_EL03_CO2_INVCOST50_CAPPED5 = "NKV_el03_Co2_invcost50_capped5";
 	public static final String NKV_INDUZ_CO2_EN = "BCR_induzCo2";
 	public static final String NKV_INDUZ_EN = "BCR_induz";
 	public static final String NKV_NO_CHANGE_EN = "BCR";
@@ -126,6 +126,7 @@ public final class Headers{
 	 */
 	public static final String EINSTUFUNG_AS_NUMBER = "einstufungAsNumber";
 	public static final String PROJECT_NAME = "project _name";
-	public static final String VERKEHRSBELASTUNG_PLANFALL = "Verkehrsbelastung_Planfall";
+	public static final String VERKEHRSBELASTUNG_PLANFALL = "DTV_Planfall";
+	public static final String NKV_EL03_CARBON215_INVCOST50_CAPPED10 = NKV_EL03_CARBON215_INVCOST50 + "_capped10";
 	private Headers(){} // do not instantiate
 }
diff --git a/src/main/java/org/tub/vsp/bvwp/data/container/analysis/StreetAnalysisDataContainer.java b/src/main/java/org/tub/vsp/bvwp/data/container/analysis/StreetAnalysisDataContainer.java
@@ -78,8 +78,7 @@ private void addComputations() {
             case BLANK:
                 break;
             default:
-                throw new IllegalStateException("Unexpected value: " + streetBaseData.getProjectInformation()
-                                                                                     .getBautyp());
+//                throw new IllegalStateException("Unexpected value: " + streetBaseData.getProjectInformation().getBautyp());
         }
 
         final double INFLATION_Factor2022to2012 = 0.917; // Zinse Wert von 2020 auf BVWP Zeitpunkt 2012 ab.
@@ -108,9 +107,9 @@ private void addComputations() {
         entries.put(Headers.NKV_CO2_680_EN, NkvCalculator.calculateNkv( new Modifications(co2Price680, 0., 1 ), streetBaseData ) );
         entries.put(Headers.NKV_CO2_2000_EN, NkvCalculator.calculateNkv( new Modifications( 2000 * INFLATION_Factor2022to2012, 0, 1 ), streetBaseData ) );
         entries.put(Headers.NKV_EL03, NkvCalculator.calculateNkv( new Modifications( co2PriceBVWP, addtlFzkmBeyondPrinsEl03, 1 ), streetBaseData ) );
-        entries.put(Headers.NKV_EL03_CO2_215_CONSTRUCTION,
+        entries.put(Headers.NKV_EL03_CARBON215_INVCOST50,
                         NkvCalculator.calculateNkv( new Modifications( 215, addtlFzkmBeyondPrinsEl03, constructionCostFactor ), streetBaseData ) );
-        entries.put(Headers.NKV_EL03_CO2_CONSTRUCTION, NkvCalculator.calculateNkv( new Modifications( co2Price680, addtlFzkmBeyondPrinsEl03, constructionCostFactor ), streetBaseData ) );
+//        entries.put(Headers.NKV_EL03_CO2_INVCOST50, NkvCalculator.calculateNkv( new Modifications( co2Price680, addtlFzkmBeyondPrinsEl03, constructionCostFactor ), streetBaseData ) );
         entries.put(Headers.ADDTL_PKWKM_EL03, addtlFzkmFromElasticity03 );
         entries.put(Headers.CO2_COST_ORIG, Math.max( 1., NkvCalculator.calculateCost_CO2( NO_CHANGE, streetBaseData ) ) );
         entries.put(Headers.CO2_COST_NEU,

diff --git a/...va/org/tub/vsp/bvwp/data/container/base/street/StreetProjectInformationDataContainer.java b/...va/org/tub/vsp/bvwp/data/container/base/street/StreetProjectInformationDataContainer.java
@@ -35,7 +35,7 @@ public Einstufung getPriority() {
         return priority;
     }
 
-    public StreetProjectInformationDataContainer setPriority( Einstufung priority ) {
+    public StreetProjectInformationDataContainer setEinstufung( Einstufung priority ) {
         this.priority = priority;
         return this;
     }

diff --git a/.../java/org/tub/vsp/bvwp/data/mapper/projectInformation/StreetProjectInformationMapper.java b/.../java/org/tub/vsp/bvwp/data/mapper/projectInformation/StreetProjectInformationMapper.java
@@ -2,6 +2,7 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.core.util.Assert;
 import org.jsoup.nodes.Document;
 import org.tub.vsp.bvwp.JSoupUtils;
 import org.tub.vsp.bvwp.data.container.base.street.StreetProjectInformationDataContainer;
@@ -26,10 +27,10 @@ public static StreetProjectInformationDataContainer mapDocument(Document documen
         }
         String bautyp = extractInformation(document, 0, "Bautyp(en), Bauziel(e)");
 
-        String priority = extractInformation(document, 1, "Dringlichkeitseinstufung");
+        String einstufung = extractInformation(document, 1, "Dringlichkeitseinstufung");
 
         String verkehrsbelastungPlanfall = extractInformation( document, 0, "im Planfall 2030" );
-        if ( verkehrsbelastungPlanfall!=null ) {
+        if ( verkehrsbelastungPlanfall != null ) {
             verkehrsbelastungPlanfall = verkehrsbelastungPlanfall.replace( " Kfz/24h", "" );
         } else {
             // ( other format for Knotenpunkte )
@@ -43,15 +44,36 @@ public static StreetProjectInformationDataContainer mapDocument(Document documen
 
 //        logger.warn( ConsoleColors.TEXT_RED + "project=" + projectNumber + "; verkehrsbelastungPlanfall=" + verkehrsbelastungPlanfall + ConsoleColors.TEXT_BLACK ) ;
 
+        if ( projectNumber.contains( "A008-G010-BY" ) ){
+            logger.warn( "projectNumber=" + projectNumber + "; einstufung=" + einstufung );
+            logger.warn( "Project has NKV in main project but the Einstufungen VBE, VBE, VBE, WBP in the subprojects.  Setting the main project to VBE." );
+            einstufung = Einstufung.VBE.name();
+        }
+        if ( projectNumber.contains( "A21-G20-SH-NI" ) ) {
+            logger.warn( "projectNumber=" + projectNumber + "; einstufung=" + einstufung );
+            logger.warn( "Project has NKV in main project but the Einstufungen VB, WBP, WBP, WBP in the subprojects.  Setting the main project to VB (!)." );
+            einstufung = Einstufung.VB.name();
+        }
+        if ( projectNumber.contains( "A98-G110-BW" ) ) {
+            logger.warn( "projectNumber=" + projectNumber + "; einstufung=" + einstufung );
+            logger.warn( "Project has NKV in main project but the Einstufungen VB, WBP in the subprojects.  Setting the main project to VB." );
+            einstufung = Einstufung.VB.name();
+        }
+        if ( projectNumber.contains( "A006-G015-BY" ) ) {
+            logger.warn( "projectNumber=" + projectNumber + "; einstufung=" + einstufung );
+            logger.warn( "Project has NKV in main project but the Einstufungen VB, WBP in the subprojects.  Setting the main project to VB." );
+            einstufung = Einstufung.VB.name();
+        }
 
         try {
             return projectInformation.setProjectNumber(projectNumber)
                                      .setStreet(street)
                                      .setLength(JSoupUtils.parseDouble(length))
                                      .setBautyp(Bautyp.getFromString(bautyp))
-                                     .setPriority( Einstufung.getFromString(priority ) )
+                                     .setEinstufung( Einstufung.getFromString(einstufung ) )
                                      .setVerkehrsbelastungPlanfall( JSoupUtils.parseDouble( verkehrsbelastungPlanfall ) );
         } catch (ParseException e) {
+            logger.error( "projectNumber=" + projectNumber );
             throw new RuntimeException(e);
         }
     }

diff --git a/src/main/java/org/tub/vsp/bvwp/data/type/Bautyp.java b/src/main/java/org/tub/vsp/bvwp/data/type/Bautyp.java
@@ -3,17 +3,23 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
-public enum Bautyp{
+public enum Bautyp {
 //	NB2_4 ("2-streifiger Neubau / 4-streifiger Neubau"),
 	// z.B. https://bvwp-projekte.de/strasse/A25_B5-G20-SH/A25_B5-G20-SH.html .
 	// Eher eine Ortsumgehung ("B5").
 //	NB3_4 ("3-streifiger Neubau / 4-streifiger Neubau"),
 	// z.B. https://bvwp-projekte.de/strasse/A46-B7-G41-NW/A46-B7-G41-NW.html .
 	// Lückenschluss, der teilweise als Bundesstrasse ("B7") realisiert wird.
+NB2( "2-streifiger Neubau"),
+	NB2_NB3( "2-streifiger Neubau / 3-streifiger Neubau"),
+	NB2_NB4( "2-streifiger Neubau / 4-streifiger Neubau"),
+	NB3( "3-streifiger Neubau"),
 	NB4( "4-streifiger Neubau"),
 	NB6( "6-streifiger Neubau"),
 //	NB2_EW4( "2-streifiger Neubau / Erweiterung auf 4 Fahrstreifen"),
+NB3_EW4( "3-streifiger Neubau / Erweiterung auf 4 Fahrstreifen"),
 	NB4_EW4( "4-streifiger Neubau / Erweiterung auf 4 Fahrstreifen"),
+	NB4_EW6( "4-streifiger Neubau / Erweiterung auf 6 Fahrstreifen"),
 	NB6_EW6( "6-streifiger Neubau / Erweiterung auf 6 Fahrstreifen"),
 	EW4("Erweiterung auf 4 Fahrstreifen"),
 	EW6("Erweiterung auf 6 Fahrstreifen"),
@@ -54,6 +60,18 @@ public static Bautyp getFromString( String description ) {
 		if ( description.equals( "Erweiterung auf 8 Fahrstreifen / Erweiterung auf 9 Fahrstreifen" ) ) {
 			return EW8;
 		}
+		if ( description.equals( "Erweiterung auf 4 Fahrstreifen / 4-streifiger Neubau" ) ) {
+			return NB4_EW4;
+		}
+		if ( description.equals( "2/3-streifiger Neubau / 4-streifiger Neubau" ) ) {
+			return NB3;
+		}
+		if ( description.equals( "3/4-streifiger Neubau / Erweiterung auf 4 Fahrstreifen" ) ) {
+			return NB4_EW4;
+		}
+		if ( description.equals( "3-streifiger Neubau / 4-streifiger Neubau" ) ) {
+			return NB3;
+		}
 		log.error( "cannot find Bautyp for description=" + description );
 //		Thread.dumpStack();
 		throw new RuntimeException( "cannot find Bautyp for description=" + description );

diff --git a/src/main/java/org/tub/vsp/bvwp/data/type/Einstufung.java b/src/main/java/org/tub/vsp/bvwp/data/type/Einstufung.java
@@ -7,9 +7,9 @@ public enum Einstufung{
     VBE("Vordringlicher Bedarf - Engpassbeseitigung (VB-E)"),
     VB("Vordringlicher Bedarf (VB)"),
     WBP("Weiterer Bedarf mit Planungsrecht (WB*)"),
-    WB( "Weiterer Bedarf (WB)"),
-    @Deprecated // yyyyyy lieber aufklären!!
-    UNDEFINED("undefined");
+    WB( "Weiterer Bedarf (WB)")
+    ,@Deprecated /* yyyyyy lieber aufklären!! */ UNDEFINED("undefined")
+    ;
 
     private static final Logger log = LogManager.getLogger( Einstufung.class );
 
@@ -29,6 +29,7 @@ public static Einstufung getFromString( String description ) {
             }
         }
         log.warn( "description=" + description );
+//        throw new RuntimeException( "unknown Einstufung=" + description );
         return UNDEFINED;
     }
 }
diff --git a/src/main/java/org/tub/vsp/bvwp/plot/MultiPlotUtils.java b/src/main/java/org/tub/vsp/bvwp/plot/MultiPlotUtils.java
@@ -41,6 +41,10 @@ public class MultiPlotUtils {
 					+ "<div id='plotC'>"
 					+ System.lineSeparator()
 					+ "<div id='plotD'>"
+					+ System.lineSeparator()
+					+ "<div id='plotE'>"
+					+ System.lineSeparator()
+					+ "<div id='plotF'>"
 					+ System.lineSeparator();
 
 	public static final String pageBottom = "</body>" + System.lineSeparator() + "</html>";

diff --git a/src/main/java/org/tub/vsp/bvwp/scraping/Scraper.java b/src/main/java/org/tub/vsp/bvwp/scraping/Scraper.java
@@ -38,7 +38,7 @@ public List<String> getProjectUrls() throws IOException {
 
         //Todo: Sollte dieses Aufräumen nicht irgendwo in den Streetscraper? Denn das ist ja die **straßen**spezifische Filterung, KMT Feb24
         //A = Autobahn; B = Bundesstraße -> Diese sollen behalten werden.
-        projectUrls.removeIf( link -> (!link.startsWith( "A" ) ));
+//        projectUrls.removeIf( link -> (!link.startsWith( "A" ) ));
 //        projectUrls.removeIf( link -> (!link.startsWith( "A" ) && !link.startsWith( "B" )));
 
         logger.info("Found {} projects", projectUrls.size());

diff --git a/src/main/java/org/tub/vsp/bvwp/scraping/StreetScraper.java b/src/main/java/org/tub/vsp/bvwp/scraping/StreetScraper.java
@@ -25,7 +25,7 @@ protected String getBaseUrl() {
     @Override
     public List<String> getProjectUrls() throws IOException {
         List<String> result = super.getProjectUrls();
-        result.removeIf(link -> !link.startsWith("A"));
+//        result.removeIf(link -> !link.startsWith("A"));
         return result;
     }
 
@@ -55,9 +55,9 @@ public List<StreetBaseDataContainer> extractAllLocalBaseData(String path, String
         return files.stream()
                     .filter(file -> file.getName().startsWith(prefix))
                     .filter(file -> file.getName().matches(regexToMatch))
-                    .filter(file -> !file.getName()
-                                         .matches("A20-G10-SH.html")) // gibt es nochmal mit A20-G10-SH-NI.  Muss man
-                    // beide zusammenzählen?  kai, feb'23
+                    .filter(file -> !file.getName().matches("A20-G10-SH.html")) // gibt es nochmal mit A20-G10-SH-NI.  Muss man beide zusammenzählen?  kai, feb'24
+//                    .filter(file -> !file.getName().matches("A57-G10-NW.html")) // sehr hohes DTV für 4 Spuren.  ??  kai, mar'24
+//                    .filter(file -> !file.getName().matches("A81-G50-BW.html")) // sehr hohes DTV für 4 Spuren.  ??  kai, mar'24
                     .filter(file -> !file.getName().matches("A...B.*")) // Ortsumgehungen, die an AB angrenzen.
                     .map(this::extractLocalBaseData)
                     .filter(Optional::isPresent)
@@ -116,12 +116,8 @@ public Optional<StreetBaseDataContainer> extractBaseData(Document doc, String ur
     }
 
     private boolean checkIfProjectIsScrapable(Document doc) {
-        boolean sieheHauptprojekt = ProjectInformationMapperUtils.extractInformation(doc, 2, "Nutzen-Kosten-Verh" +
-                                                                         "ältnis")
-                                                                 .contains("siehe Hauptprojekt");
-
-        String extractedInformation = ProjectInformationMapperUtils.extractInformation(doc, 2, "Nutzen-Kosten-Verh" +
-                "ältnis");
+        boolean sieheHauptprojekt = ProjectInformationMapperUtils.extractInformation(doc, 2, "Nutzen-Kosten-Verhältnis").contains("siehe Hauptprojekt");
+        String extractedInformation = ProjectInformationMapperUtils.extractInformation(doc, 2, "Nutzen-Kosten-Verhältnis");
         boolean sieheTeilprojekt = extractedInformation.contains("siehe Teilprojekt");
         logger.warn("extractedInformation=" + extractedInformation + "; sieheTeilprojekt=" + sieheTeilprojekt);
 

diff --git a/src/main/java/org/tub/vsp/bvwp/users/kmt/FiguresKMT.java b/src/main/java/org/tub/vsp/bvwp/users/kmt/FiguresKMT.java
@@ -83,8 +83,8 @@ static Figure createFigureCostByPriority(int plotWidth, Table table, String xNam
 	static Figure createFigureNkvByPriority( Axis xAxis, int plotWidth, Table table, String xName){
 		Figure figure2;
 		String yName = Headers.NKV_ORIG;
-		String y3Name = Headers.NKV_CO2_680_EN;
-		String y2Name = Headers.NKV_EL03_CO2_CONSTRUCTION;
+//		String y3Name = Headers.NKV_CO2_680_EN;
+		String y2Name = Headers.NKV_EL03_CARBON215_INVCOST50;
 
 		Axis yAxis = Axis.builder()
 //			     .type( Axis.Type.LOG ) // wirft NKV < 0 raus!
@@ -107,19 +107,21 @@ static Figure createFigureNkvByPriority( Axis xAxis, int plotWidth, Table table,
 		final Trace traceVbe = getPriorityTrace( table, Einstufung.VBE , xName, y2Name, "red" );
 
 
-		Trace trace3 = ScatterTrace.builder( table.numberColumn( xName ), table.numberColumn( y3Name ) )
-					   .text( table.stringColumn( Headers.PROJECT_NAME ).asObjectArray() )
-					   .name( String.format( legendFormat, y3Name ) )
-					   .marker( Marker.builder().color( "gray" ).build() )
-					   .build();
+//		Trace trace3 = ScatterTrace.builder( table.numberColumn( xName ), table.numberColumn( y3Name ) )
+//					   .text( table.stringColumn( Headers.PROJECT_NAME ).asObjectArray() )
+//					   .name( String.format( legendFormat, y3Name ) )
+//					   .marker( Marker.builder().color( "gray" ).build() )
+//					   .build();
 
 		double[] xx = new double[]{0., 1.1* table.numberColumn( xName ).max() };
 		double[] yy = new double[]{1., 1.};
 		Trace trace4 = ScatterTrace.builder( xx, yy )
 					   .mode( ScatterTrace.Mode.LINE )
 					   .build();
 
-		figure2 = new Figure( layout, trace, trace3, traceWb,traceWbp,  traceVb, traceVbe, trace4 );
+		figure2 = new Figure( layout, trace
+//				, trace3
+				, traceWb,traceWbp,  traceVb, traceVbe, trace4 );
 		return figure2;
 	}