diff --git a/aws/src/main/python/updateDependencies/lambda_function.py b/aws/src/main/python/updateDependencies/lambda_function.py index 4a29a541d14..a8fc4b568f0 100644 --- a/aws/src/main/python/updateDependencies/lambda_function.py +++ b/aws/src/main/python/updateDependencies/lambda_function.py @@ -37,15 +37,19 @@ - sudo apt-get install gcc-8 g++-8 -y - sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-8 60 --slave /usr/bin/g++ g++ /usr/bin/g++-8 - sudo apt install jq -y + - sudo add-apt-repository ppa:git-core/ppa -y + - sudo apt-get update + - sudo apt-get install git -y - echo "-------------------Finished updating Beam dependencies----------------------" - cd /home/ubuntu/git/beam - echo "send notification ..." - /home/ubuntu/git/glip.sh -i "http://icons.iconarchive.com/icons/uiconstock/socialmedia/32/AWS-icon.png" -a "Updating Dependencies" -b "Beam automated deployment image update started on $(ec2metadata --instance-id)." - echo "git checkout ..." - - sudo git reset origin/HEAD + - sudo git reset --hard origin/HEAD - sudo git checkout -- . - sudo git clean -df - sudo git checkout develop + - sudo git reset --hard origin/develop - sudo git pull - sudo git fetch - sudo git fetch --prune @@ -101,6 +105,7 @@ def init_ec2(region): def deploy(script, instance_type, region_prefix, shutdown_behaviour, instance_name, en_vars): res = ec2.run_instances(ImageId=en_vars[region_prefix + 'IMAGE_ID'], + InstanceType=instance_type, UserData=script, KeyName=en_vars[region_prefix + 'KEY_NAME'], @@ -114,7 +119,7 @@ def deploy(script, instance_type, region_prefix, shutdown_behaviour, instance_na 'Tags': [ { 'Key': 'Name', 'Value': instance_name - } ] + }] } ]) return res['Instances'][0]['InstanceId'] diff --git a/build.gradle b/build.gradle index f0199365510..248bf1880b3 100755 --- a/build.gradle +++ b/build.gradle @@ -136,7 +136,7 @@ configurations.all { dependencies { - implementation(group: 'com.github.LBNL-UCB-STI', name: 'beam-utilities', version: 'v0.2.15') { + implementation(group: 'com.github.LBNL-UCB-STI', name: 'beam-utilities', version: 'v0.2.17') { exclude group: 'com.github.LBNL-UCB-STI', module: 'r5' exclude group: 'org.matsim', module: 'matsim' } diff --git a/src/main/R/common/helpers.R b/src/main/R/common/helpers.R index 5c53f2af497..63f96a50f46 100644 --- a/src/main/R/common/helpers.R +++ b/src/main/R/common/helpers.R @@ -11,18 +11,18 @@ mergefiles <- function(directory){ for (file in list.files(directory)){ # if the merged dataset doesn't exist, create it filepath <- paste(directory, file, sep="/") - + if (!exists("dataset")){ dataset <- readCsv(filepath) } - + # if the merged dataset does exist, append to it if (exists("dataset")){ temp_dataset <-readCsv(filepath) dataset<-rbind(dataset, temp_dataset) rm(temp_dataset) } - + } return(dataset) } @@ -66,3 +66,141 @@ sankeyDiagram <- function(source, target, value, title) { } +clusteringFreightBy <- function(data,cols,dataCbg,numClusters,labelData) { + data[,hour:=as.integer(arrivalTime/3600)%%24] + data_asSf <- st_as_sf(data,coords=cols,crs=4326,agr="constant") + data_withCBG_asSf <- st_intersection(data_asSf,dataCbg) + data_withCBG_asSf$X <- st_coordinates(data_withCBG_asSf$geometry)[,1] + data_withCBG_asSf$Y <- st_coordinates(data_withCBG_asSf$geometry)[,2] + data_withCBG <- data.table::as.data.table(data_withCBG_asSf) + data_withCBG[,cluster:=kmeans(data_withCBG[,.(X,Y)],numClusters)$cluster] + result <- data_withCBG[,.(count=.N,x2=mean(X),y2=mean(Y)),by=.(hour,cluster)] + result$label <- labelData + result +} + +assignPostMilesGeometries <- function(TRUCK_DATA, POSTMILES_SHP) { + getElementWithoutFlag <- function(list_elements, flag) { + i <- 1 + while(list_elements[i] == '') { + i <- i+1 + } + return(list_elements[i]) + } + TRUCK_DATA$X <- 0.0 + TRUCK_DATA$Y <- 0.0 + ### GEOCODING ### + #TRUCK_DATA$lon <- 0.0 + #TRUCK_DATA$lat <- 0.0 + #TRUCK_DATA$geoAddress <- "" + #nrow(TRUCK_DATA) + postmiles <- st_read(POSTMILES_SHP) + for(i in 1:nrow(TRUCK_DATA)) { + county <- TRUCK_DATA$COUNTY[i] + cnty <- TRUCK_DATA$CNTY[i] + dist <- TRUCK_DATA$DIST[i] + rte <- TRUCK_DATA$RTE[i] + leg <- TRUCK_DATA$LEG[i] + pm <- TRUCK_DATA$POST_MILE[i] + filteredPM <- postmiles %>% filter(County == cnty, + District == dist, + Route == rte, + startsWith(as.character(PMc), as.character(pm))) + if(nrow(filteredPM) == 0) { + filteredPM <- postmiles %>% filter(County == cnty, District == dist, Route == rte) + } + if(nrow(filteredPM) > 1) { + pm_numeric <- 0 + if(!is.na(as.numeric(pm))) { + pm_numeric <- as.numeric(pm) + } else { + element <- getElementWithoutFlag(unlist(strsplit(pm, "R")), "R") + element <- getElementWithoutFlag(unlist(strsplit(element, "L")), "L") + element <- getElementWithoutFlag(unlist(strsplit(element, "T")), "T") + element <- getElementWithoutFlag(unlist(strsplit(element, "M")), "M") + pm_numeric <- as.numeric(element) + } + filteredPM <- filteredPM %>% rowwise() %>% mutate(diff = abs(as.numeric(PM) - pm_numeric)) + filteredPM <- filteredPM %>% filter(diff == min(filteredPM$diff)) + } + if(nrow(filteredPM) > 0) { + TRUCK_DATA$X[i] <- st_coordinates(filteredPM$geometry[1])[1] + TRUCK_DATA$Y[i] <- st_coordinates(filteredPM$geometry[1])[2] + } + # address <- paste(county, " County, California, USA", sep="") + # descriptions <- unlist(strsplit(TRUCK_DATA$DESCRIPTION[i],",")) + # if(length(descriptions) > 1) { + # address <- paste(str_trim(descriptions[2]), ", ", str_trim(descriptions[1]), ", ", address, sep="") + # } else { + # address <- paste(str_trim(descriptions), ", ", address, sep="") + # } + # address <- ", CA" + # address <- paste(TRUCK_DATA$DESCRIPTION[i], ", ", address, sep="") + # address <- gsub("RTE.", "route", address) + # address <- gsub("JCT.", "", address) + # print(address) + # result <- geocode(address, output = "latlona", source = "google") + # TRUCK_DATA$lon[i] <- as.numeric(result[1]) + # TRUCK_DATA$lat[i] <- as.numeric(result[2]) + # TRUCK_DATA$geoAddress[i] <- as.character(result[3]) + # print(TRUCK_DATA[i]) + } + data.table::fwrite(TRUCK_DATA, normalizePath(pp(freightDir,"/validation/2017_truck_aadtt_geocoded.csv")), quote=T) + print("END OF assignPostMilesGeometries") + return(TRUCK_DATA) +} + +assignLinkIdToTruckAADTT <- function(NETWORK_CLEANED, NETWORK_CRS, TRUCK_AADTT, MAX_DISTANCE_IN_METER, EXPANSION_FACTOR) { + network_sf <- st_transform(st_as_sf( + NETWORK_CLEANED, + coords = c("fromLocationX", "fromLocationY"), + crs = NETWORK_CRS, + agr = "constant"), 4326) + + truck_aadtt_sf <- st_as_sf( + TRUCK_AADTT, + coords = c("X", "Y"), + crs = 4326, + agr = "constant") + + network_sf$tempDistance <- NA + truck_aadtt_sf$linkId <- NA + expansionFactor <- EXPANSION_FACTOR + maxDistanceToSearch <- MAX_DISTANCE_IN_METER + counter1 <- 0 + counter2 <- 0 + for (row in 1:nrow(truck_aadtt_sf)) { + if(row %% 100 == 0) { + print(paste(row, " entries have been processed so far!")) + } + current_pm <- truck_aadtt_sf[row,] + currentDist <- 20 + current_pm_links <- st_is_within_distance(current_pm, network_sf, dist = currentDist) + list_of_links <- current_pm_links[1][[1]] + while(length(list_of_links) < 1 & currentDist < maxDistanceToSearch) { + currentDist <- currentDist * 2 + current_pm_links <- st_is_within_distance(current_pm, network_sf, dist = currentDist) + list_of_links = current_pm_links[1][[1]] + } + selected_links <- network_sf[list_of_links,] + if (length(list_of_links) == 0){ + counter1 <- counter1 + 1 + print("no links!!!") + } else if (length(list_of_links) == 1){ + truck_aadtt_sf$linkId[row] <- network_sf[list_of_links,]$linkId + network_sf <- network_sf[-list_of_links,] + counter2 <- counter2 + 1 + } else { + selected_pm <- truck_aadtt_sf[row,] + selected_network <- network_sf[selected_links,] + selected_network$tempDistance <- st_distance(selected_network, selected_pm) + selected_network_filtered <- slice(selected_network, which.min(tempDistance)) + truck_aadtt_sf$linkId[row] <- selected_network_filtered$linkId + network_sf <- network_sf %>% filter(linkId != selected_network_filtered$linkId) + counter2 <- counter2 + 1 + } + } + print(paste("# unmatched links ", counter1)) + print(paste("# matched links ", counter2)) + return(truck_aadtt_sf) +} \ No newline at end of file diff --git a/src/main/R/freight/freight-processing.R b/src/main/R/freight/freight-processing.R index 4e83f661795..ba0dd4d1ede 100644 --- a/src/main/R/freight/freight-processing.R +++ b/src/main/R/freight/freight-processing.R @@ -10,23 +10,116 @@ library(ggmap) library(sf) library(stringr) -workDir <- normalizePath("~/Data/SMART") -activitySimDir <- normalizePath("~/Data/ACTIVITYSIM") -events <- readCsv(pp(workDir, "/5.events.csv.gz")) -unloading <- events[actType=="Unloading"] -nrow(unloading[type=="actstart"]) -nrow(unloading[type=="actend"]) -warehouse <- events[actType=="Home"] -nrow(warehouse[type=="actstart"]) -nrow(warehouse[type=="actend"]) -pt <- events[type=="PathTraversal"][,c("time","type","vehicleType","vehicle","secondaryFuelLevel", +activitySimDir <- normalizePath("~/Data/ACTIVITYSIM") +freightDir <- normalizePath("~/Data/FREIGHT") +validationDir <- normalizePath("~/Data/FREIGHT/validation") +freightWorkDir <- normalizePath(paste(validationDir,"/beam",sep="")) + +# events <- readCsv(pp(freightWorkDir, "/0.events.csv")) +# events_filtered <- events[(actType %in% c("Warehouse", "Unloading", "Loading")) | (type=="PathTraversal" & startsWith(vehicle,"freight"))] +# write.csv( +# events_filtered, +# file = pp(freightWorkDir, "/filtered.0.events.csv"), +# row.names=F, +# quote=T) +events_filtered <- readCsv(pp(freightWorkDir, "/filtered.0.events.new.csv")) +pt <- events_filtered[type=="PathTraversal"][,c("time","type","vehicleType","vehicle","secondaryFuelLevel", "primaryFuelLevel","driver","mode","seatingCapacity","startX", "startY", "endX", "endY", "capacity", "arrivalTime", "departureTime", "secondaryFuel", "secondaryFuelType", "primaryFuelType", "numPassengers", "length", "primaryFuel")] freight_pt <- pt[startsWith(vehicle,"freight")] -nrow(freight_pt) + +if (nrow(freight_pt[grepl("-emergency-",vehicle)]) > 0) { + println("This is a bug") +} +print(paste("# vehicles: ", length(unique(freight_pt$vehicle)))) +print("By category: ") +freight_pt[,.N,by=.(vehicle, vehicleType)][,.(count=.N),by=.(vehicleType)] + +# nrow(freight_pt) +# all_pt_x <- data.table::as.data.table(rbind(b2b_pt,b2c_pt)[,c("time","vehicle","departureTime","arrivalTime","label")]) +# all_pt_1 <- all_pt_x[,-c("arrivalTime")][order(time),`:=`(IDX = 1:.N),by=vehicle] +# all_pt_2 <- all_pt_x[,-c("departureTime")][order(time),`:=`(IDX = 1+1:.N),by=vehicle] +# all_pt <- all_pt_1[all_pt_2, on=c("vehicle", "IDX", "label")][!is.na(arrivalTime)&!is.na(departureTime)] +# all_pt[,`:=`(stopDuration = departureTime - arrivalTime)] +# all_pt[,.(mean(stopDuration)),by=.(label)] +# unloading <- events[actType=="Unloading"] +# warehouse <- events[actType=="Warehouse"] +# nrow(unloading[type=="actstart"]) +# nrow(unloading[type=="actend"]) +# nrow(warehouse[type=="actstart"]) +# nrow(warehouse[type=="actend"]) +## + +## *************************** +#FRISM +## *************************** +freightWorkDir <- normalizePath(paste(freightDir,"/Outputs(All SF)_0322_2022_merged",sep="")) +carriers <- readCsv(pp(freightWorkDir, "/freight-merged-carriers.csv")) +payload <- readCsv(pp(freightWorkDir, "/freight-merged-payload-plans.csv")) +tours <- readCsv(pp(freightWorkDir, "/freight-merged-tours.csv")) + +print(paste("# carriers:", length(unique(carriers$carrierId)))) +print(paste("# tours:", length(unique(carriers$tourId)))) +print(paste("# vehicles:", length(unique(carriers$vehicleId)))) +print("By category: ") +carriers[,.N,by=.(vehicleId, vehicleTypeId)][,.(count=.N),by=.(vehicleTypeId)] + +plans <- rbind(plansb2b,plansb2c)[,c("payloadId","sequenceRank","tourId","payloadType","estimatedTimeOfArrivalInSec","arrivalTimeWindowInSec_lower","business")] +tours <- rbind(toursb2b,toursb2c)[,c("tourId","departureTimeInSec","business")] + +p <- plans[tours, on=c("tourId","business")] +pRank2 <- p[sequenceRank==2] +pRank2$negativeTravelTime <- pRank2$estimatedTimeOfArrivalInSec - pRank2$departureTimeInSec - pRank2$arrivalTimeWindowInSec_lower +p2 <- p[,diff:=estimatedTimeOfArrivalInSec-arrivalTimeWindowInSec_lower-departureTimeInSec] + +write.csv( + pRank2[negativeTravelTime<0], + file = pp(freightDir, "/10percent/negativeTravelTime.csv"), + row.names=FALSE, + quote=FALSE, + na="0") + +toursb2$hour <- toursb2c$departureTimeInSec/3600.0 +ggplot(toursb2b[,.N,by=hour]) + geom_line(aes(hour, N)) + +payload$business <- "B2B" +payload[startsWith(payloadId,"b2c")]$business <- "B2C" +payload[,time24:=estimatedTimeOfArrivalInSec%%(24*3600),][,.N,by=.(timeBin=as.POSIXct(cut(toDateTime(time24),"30 min")), business)] %>% + ggplot(aes(timeBin, N, colour=business)) + + geom_line() + + scale_x_datetime("Hour", + breaks=scales::date_breaks("2 hour"), + labels=scales::date_format("%H", tz = dateTZ)) + + scale_y_continuous("Activity", breaks = scales::pretty_breaks()) + + scale_colour_manual("Supply Chain", values = c("#eca35b", "#20b2aa")) + + theme_marain() + + theme(legend.title = element_text(size = 10), + legend.text = element_text(size = 10), + axis.text.x = element_text(angle = 0, hjust = 1)) + + +payload_carriers <- payload[carriers, on="tourId"] +payload_carriers$vehicleCategory <- "LightDutyTruck" +payload_carriers[vehicleTypeId == "FREIGHT-2"]$vehicleCategory <- "HeavyDutyTruck" +p <- payload_carriers[,time24:=estimatedTimeOfArrivalInSec%%(24*3600),][,.N,by=.(timeBin=as.POSIXct(cut(toDateTime(time24),"30 min")), vehicleCategory)] %>% + ggplot(aes(timeBin, N, colour=vehicleCategory)) + + geom_line() + + scale_x_datetime("Hour", + breaks=scales::date_breaks("2 hour"), + labels=scales::date_format("%H", tz = dateTZ)) + + scale_y_continuous("Activity", breaks = scales::pretty_breaks()) + + scale_colour_manual("Vehicle Category", values = c("#eca35b", "#20b2aa")) + + theme_marain() + + theme(legend.title = element_text(size = 10), + legend.text = element_text(size = 10), + axis.text.x = element_text(angle = 0, hjust = 1)) +ggsave(pp(freightWorkDir,'/output/frism-activity-by-category.png'),p,width=6,height=3,units='in') + + +## MODE SPLIT # pt$mode2 <- "Transit" # pt[mode=="car"]$mode2 <- "Car" # pt[mode=="car"&startsWith(vehicle,"rideHailVehicle")]$mode2 <- "Ridehail" @@ -34,76 +127,41 @@ nrow(freight_pt) # pt[mode=="walk"]$mode2 <- "Walk" # pt[mode=="bike"]$mode2 <- "Bike" # summary <- pt[,.(VMTInMiles=mean(length)/1609.34,fuelInKW=(mean(primaryFuel+secondaryFuel))*2.77778e-7),by=.(mode2)] -# - - -factor.remap <- c('Walk'='Walk','Bike'='Bike','Ridehail'='Ridehail','Car'='Car','Transit'='Public Transit', 'Freight'='Freight') -factor.colors <- c('Walk'='#669900','Bike'='#FFB164','Ridehail'='#B30C0C','Car'='#8A8A8A','Transit'='#0066CC','Freight'="#660099") -factor.colors.remapped <- factor.colors -ggplot(summary, aes(x="",y=VMTInMiles,fill=mode2))+ - geom_bar(stat='identity')+ - labs(y='',x='Scenario',fill='Mode',title='Mobility Metrics')+ - theme_marain()+ - theme(axis.text.x = element_text(angle = 0, hjust=0.5),strip.text = element_text(size=rel(1.2)))+ - scale_fill_manual(values = factor.colors.remapped) - - -source("~/Documents/Workspace/scripts/common/keys.R") -register_google(key = google_api_key_1) -oaklandMap <- ggmap::get_googlemap("oakland california", zoom = 13, maptype = "roadmap") -shpFile <- pp(activitySimDir, "/__San_Francisco_Bay_Region_2010_Census_Block_Groups-shp/641aa0d4-ce5b-4a81-9c30-8790c4ab8cfb202047-1-wkkklf.j5ouj.shp") -sfBayCbg <- st_read(shpFile) -# ggplot(data = oaklandCbg) + geom_sf()+ -# coord_sf( xlim = c(-130, -60),ylim = c(20, 50)) - - - -freight_pt[,hour:=as.integer(arrivalTime/3600)%%24] -##1 -# freight_pt_withCBG <- data.table::as.data.table(st_intersection(freight_pt_asSf,sfBayCbg)) -# freight_pt_summary <- freight_pt_withCBG[,.(count=.N),by=.(blkgrpid)] -# freight_pt_withCBG_asSf <- st_join(sfBayCbg, freight_pt_asSf) -# data <- freight_pt_withCBG_asSf %>% -# group_by(blkgrpid) %>% -# summarize(geometry = st_union(geometry)) -# ggplot() + -# geom_sf(data = data, aes(fill = blkgrpid)) + -# theme(legend.position = "none") +# factor.remap <- c('Walk'='Walk','Bike'='Bike','Ridehail'='Ridehail','Car'='Car','Transit'='Public Transit', 'Freight'='Freight') +# factor.colors <- c('Walk'='#669900','Bike'='#FFB164','Ridehail'='#B30C0C','Car'='#8A8A8A','Transit'='#0066CC','Freight'="#660099") +# factor.colors.remapped <- factor.colors +# ggplot(summary, aes(x="",y=VMTInMiles,fill=mode2))+ +# geom_bar(stat='identity')+ +# labs(y='',x='Scenario',fill='Mode',title='Mobility Metrics')+ +# theme_marain()+ +# theme(axis.text.x = element_text(angle = 0, hjust=0.5),strip.text = element_text(size=rel(1.2)))+ +# scale_fill_manual(values = factor.colors.remapped) + + +# *************************** +# B2B vs B2C +# *************************** +b2b_pt <- freight_pt[grepl("b2b",vehicle)][,label:="B2B"] +b2c_pt <- freight_pt[grepl("b2c",vehicle)][,label:="B2C"] -##2 +## FREIGHT B2C GEO DISTRIBUTION OF STOPS #countyNames <- c('Alameda County','Contra Costa County','Marin County','Napa County','Santa Clara County','San Francisco County','San Mateo County','Sonoma County','Solano County') source("~/Documents/Workspace/scripts/common/keys.R") register_google(key = google_api_key_1) oakland_map <- ggmap::get_googlemap("alameda california", zoom = 9, maptype = "roadmap",color = "bw", style = c(feature = "all", element = "labels", visibility = "off")) - -clusteringFreightBy <- function(data,cols,dataCbg,numClusters,labelData) { - data_asSf <- st_as_sf(data,coords=cols,crs=4326,agr="constant") - data_withCBG_asSf <- st_intersection(data_asSf,dataCbg) - data_withCBG_asSf$X <- st_coordinates(data_withCBG_asSf$geometry)[,1] - data_withCBG_asSf$Y <- st_coordinates(data_withCBG_asSf$geometry)[,2] - data_withCBG <- data.table::as.data.table(data_withCBG_asSf) - data_withCBG[,cluster:=kmeans(data_withCBG[,.(X,Y)],numClusters)$cluster] - result <- data_withCBG[,.(count=.N,x2=mean(X),y2=mean(Y)),by=.(hour,cluster)] - result$label <- labelData - result -} - -b2b_pt <- freight_pt[grepl("b2b",vehicle)][,label:="B2B"] -b2c_pt <- freight_pt[grepl("b2c",vehicle)][,label:="B2C"] - +shpFile <- pp(activitySimDir, "/__San_Francisco_Bay_Region_2010_Census_Block_Groups-shp/641aa0d4-ce5b-4a81-9c30-8790c4ab8cfb202047-1-wkkklf.j5ouj.shp") +sfBayCbg <- st_read(shpFile) b2b_pt_stops <- clusteringFreightBy(b2b_pt,c("endX","endY"),sfBayCbg,50,"B2B") b2c_pt_stops <- clusteringFreightBy(b2c_pt,c("endX","endY"),sfBayCbg,50,"B2C") - -# Plot it to_plot <- rbind(b2c_pt_stops) -hours_to_show <- c(8, 12, 16) +hours_to_show <- c(8, 14, 20) toplot <- to_plot[hour %in% hours_to_show] toplot$hour.label <- "" toplot[hour==8]$hour.label <- "8am" -toplot[hour==12]$hour.label <- "12pm" -toplot[hour==16]$hour.label <- "4pm" +toplot[hour==14]$hour.label <- "2pm" +toplot[hour==20]$hour.label <- "8pm" #toplot[hour==20]$hour.label <- "8pm" -hour.label_order <- c("8am", "12pm", "4pm") +hour.label_order <- c("8am", "2pm", "8pm") # counties <- data.table(urbnmapr::counties)[county_name%in%countyNames] #,xlim=c(-122.36,-122.20),ylim=c(37.70,37.81) p <- oakland_map %>% @@ -123,9 +181,10 @@ p <- oakland_map %>% strip.text.x = element_text(size = 10)) + facet_wrap(~factor(hour.label, levels=hour.label_order)) + guides(color= guide_legend(), size=guide_legend()) -ggsave(pp(workDir,'/b2c-stops.png'),p,width=9,height=5,units='in') +ggsave(pp(freightWorkDir,'/output/b2c-stops.png'),p,width=9,height=5,units='in') +## FREIGHT B2B GEO DISTRIBUTION OF STOPS toplot <- rbind(b2b_pt_stops) p <- oakland_map %>% ggmap() + @@ -142,18 +201,12 @@ p <- oakland_map %>% axis.ticks.x = element_blank(), axis.ticks.y = element_blank(), strip.text.x = element_text(size = 10)) -ggsave(pp(workDir,'/b2b-stops.png'),p,width=4,height=5,units='in') - - -to_plot <- rbind(b2c_pt_stops,b2b_pt_stops)[,.(stopsPerHour=sum(cluster)),by=.(label,hour)] -ggplot(to_plot) + geom_line(aes(hour, stopsPerHour, colour=label)) - -to_plot <- rbind(b2b_pt,b2c_pt) -to_plot$timeBin <- as.integer(to_plot$time/1800) +ggsave(pp(freightWorkDir,'/b2b-stops.png'),p,width=4,height=5,units='in') +## FREIGHT ACTIVITY to_plot <- rbind(b2b_pt,b2c_pt) -p <- to_plot[,.N,by=.(timeBin=as.POSIXct(cut(toDateTime(arrivalTime),"15 min")), label)] %>% +p <- to_plot[,time24:=arrivalTime%%(24*3600),][,.N,by=.(timeBin=as.POSIXct(cut(toDateTime(time24),"30 min")), label)] %>% ggplot(aes(timeBin, N, colour=label)) + geom_line() + scale_x_datetime("Hour", @@ -165,12 +218,10 @@ p <- to_plot[,.N,by=.(timeBin=as.POSIXct(cut(toDateTime(arrivalTime),"15 min")), theme(legend.title = element_text(size = 10), legend.text = element_text(size = 10), axis.text.x = element_text(angle = 0, hjust = 1)) -ggsave(pp(workDir,'/freight-activity.png'),p,width=6,height=3,units='in') - +ggsave(pp(freightWorkDir,'/freight-activity.png'),p,width=6,height=3,units='in') +## FREIGHT AVG VMT to_plot <- rbind(b2b_pt,b2c_pt)[,.(VMT=mean(length)/1609.3),by=.(label)] -ggplot(to_plot, ) - p <- ggplot(to_plot, aes(x=label,y=VMT,fill=label))+ geom_bar(stat='identity')+ labs(y='Miles',x='',title='Avg VMT')+ @@ -180,30 +231,150 @@ p <- ggplot(to_plot, aes(x=label,y=VMT,fill=label))+ axis.text.x = element_blank(), legend.title = element_text(size = 10), legend.text = element_text(size = 10)) -ggsave(pp(workDir,'/freight-avg-vmt.png'),p,width=4,height=3,units='in') +ggsave(pp(freightWorkDir,'/freight-avg-vmt.png'),p,width=4,height=3,units='in') + + +# *************************** +# LDT vs HDT +# *************************** +ldt_pt <- freight_pt[vehicleType == "FREIGHT-1"][,category:="LightDutyTruck"] +hdt_pt <- freight_pt[vehicleType == "FREIGHT-2"][,category:="HeavyDutyTruck"] + +## FREIGHT ACTIVITY BY TRUCK CATEGORY +to_plot <- rbind(ldt_pt,hdt_pt) +p <- to_plot[,time24:=arrivalTime%%(24*3600),][,.N,by=.(timeBin=as.POSIXct(cut(toDateTime(time24),"30 min")), category)] %>% + ggplot(aes(timeBin, N, colour=category)) + + geom_line() + + scale_x_datetime("Hour", + breaks=scales::date_breaks("2 hour"), + labels=scales::date_format("%H", tz = dateTZ)) + + scale_y_continuous("Activity", breaks = scales::pretty_breaks()) + + scale_colour_manual("Vehicle Category", values = c("#eca35b", "#20b2aa")) + + theme_marain() + + theme(legend.title = element_text(size = 10), + legend.text = element_text(size = 10), + axis.text.x = element_text(angle = 0, hjust = 1)) +ggsave(pp(freightWorkDir,'/freight-activity-by-category.png'),p,width=6,height=3,units='in') + +## FREIGHT AVG VMT BY TRUCK CATEGORY +to_plot <- rbind(ldt_pt,hdt_pt)[,.(VMT=mean(length)/1609.3),by=.(category)] +p <- ggplot(to_plot, aes(x=category,y=VMT,fill=category))+ + geom_bar(stat='identity')+ + labs(y='Miles',x='',title='Avg VMT')+ + scale_fill_manual("Vehicle Category", values = c("#eca35b", "#20b2aa")) + + theme_marain()+ + theme(strip.text = element_text(size = 10), + axis.text.x = element_blank(), + legend.title = element_text(size = 10), + legend.text = element_text(size = 10)) +ggsave(pp(freightWorkDir,'/freight-avg-vmt-by-category.png'),p,width=4,height=3,units='in') + + + +################ *************************** +################ validation CALTrans +################ *************************** + +##### PREPARING NETWORK AND MATCH IT WITH POSTMILE AND TRUCK AADTT DATA +#"primary","secondary","tertiary" +network <- readCsv(normalizePath(paste(freightDir,"/validation/beam/network.csv",sep=""))) +network_cleaned <- network[ + linkModes %in% c("car;bike", "car;walk;bike") & attributeOrigType %in% c("motorway","trunk","primary", "secondary")][ + ,-c("numberOfLanes", "attributeOrigId", "fromNodeId", "toNodeId", "toLocationX", "toLocationY")] +counties <- data.table::data.table( + COUNTY = c("Alameda", "Contra Costa", "Marin", "Napa", "Santa Clara", + "San Francisco", "San Mateo", "Solano", "Sonoma"), + CNTY=c("ALA", "CC", "MRN", "NAP", "SCL", "SF", "SM", "SOL", "SON") +) +linkStats <- readCsv(normalizePath(paste(freightDir,"/validation/beam/0.linkstats.new.csv.gz",sep=""))) + +#data.table::fwrite(network_cleaned, pp(freightDir,"/validation/network_cleaned.csv"), quote=F) + +# truck_aadtt_2017 <- readCsv(normalizePath(paste(freightDir,"/validation/2017_truck_aadtt.csv",sep=""))) +# truck_aadtt_2017_sfbay <- assignPostMilesGeometries(truck_aadtt_2017[counties, on=c("CNTY"="CNTY")], +# pp(freightDir, "/validation/ds1901_shp/ds1901.shp")) +truck_aadtt_2017_sfbay <- data.table::fread( + normalizePath(paste(freightDir,"/validation/2017_truck_aadtt_geocoded.csv",sep="")), + header=T, + sep=",") + +truck_aadtt_with_linkId <- assignLinkIdToTruckAADTT(network_cleaned, 26910, truck_aadtt_2017_sfbay, 500, 2) +ata.table::fwrite(truck_aadtt_with_linkId, pp(freightDir,"/validation/truck_aadtt_with_linkId.csv"), quote=F) +truck_aadtt_with_linkData <- merge(data.table::as.data.table(truck_aadtt_with_linkId), network_cleaned, by="linkId") + + +##### PREPARING BEAM/LINKSTAT DATA +#linkstats_noFreight <- readCsv(normalizePath(paste(freightDir,"/validation/beam/0.linkstats.nofreight.csv.gz",sep=""))) +#linkstats_wFreight <- readCsv(normalizePath(paste(freightDir,"/validation/beam/0.linkstats.withfreight.csv.gz",sep=""))) +#totVolume <- sum(linkstats_wFreight$volume) - sum(linkstats_noFreight$volume) +#totAADTT <- sum(truck_aadtt_2017_bay_area$TRUCK_AADT) +lsWFreight <- linkstats_wFreight[,.(volumeWithFreight=sum(volume),lengthInMeter=first(length)),by=.(link)] +lsNoFreight <- linkstats_noFreight[,.(volumeNoFreight=sum(volume)),by=.(link)] +linkStats <- lsWFreight[lsNoFreight, on=c("link")] +linkStats$truck_volume <- linkStats$volumeWithFreight - linkStats$volumeNoFreight +linkStats$vehicle_volume <- linkStats$volumeWithFreight +linkStats$truck_share <- linkStats$truck_volume/linkStats$vehicle_volume +linkStats <- linkStats[,-c("volumeWithFreight","volumeNoFreight")] +linkStats[is.na(truck_share)]$truck_share <- 0.0 +linkStats[is.infinite(truck_share)]$truck_share <- 0.0 + + +##### MERGING +truck_aadtt_with_linkStats <- merge(truck_aadtt_with_linkData, linkStats, by.x="linkId", by.y="link") + +sum(truck_aadtt_with_linkStats$TRUCK_AADT) +sum(truck_aadtt_with_linkStats$truck_volume) + +LinkStatsWithLocation <- linkStats[network_cleaned, on=c("link"="linkId")] + +LinkStats_as_sf <- st_transform(st_as_sf( + LinkStatsWithLocation, + coords = c("fromLocationX", "fromLocationY"), + crs = 26910, + agr = "constant"), 4326) + + +LinkStats_withTaz <- data.table::as.data.table(st_intersection(LinkStats_as_sf, st_buffer(sfBayTAZs, 0))) +LinkStats_withTaz <- LinkStats_withTaz[counties, on="county"] + +countyStats <- LinkStats_withTaz[,.(truck_volume=sum(truck_volume),vehicle_volume=sum(vehicle_volume)),by=.(county)][counties, on="county"] +unique_counties <- unique(countyStats$cnty) +SF_truck_aadtt_2017 <- truck_aadtt_2017[CNTY %in% unique_counties] +dist_truck_aadtt_2017 <- SF_truck_aadtt_2017[,.(TRUCK_AADT=mean(TRUCK_AADT),VEHICLE_AADT=mean(VEHICLE_AADT)),by=.(DIST)] -to_plot <- rbind(b2b_pt,b2c_pt)[,.(VMT=mean(primaryFuel+secondaryFuel)*2.77778e-7),by=.(label)] +county_truck_aadtt_2017 <- SF_truck_aadtt_2017[,.(TRUCK_AADT=sum(TRUCK_AADT),VEHICLE_AADT=sum(VEHICLE_AADT)),by=.(CNTY)] -all_pt_x <- data.table::as.data.table(rbind(b2b_pt,b2c_pt)[,c("time","vehicle","departureTime","arrivalTime","label")]) -all_pt_1 <- all_pt_x[,-c("arrivalTime")][order(time),`:=`(IDX = 1:.N),by=vehicle] -all_pt_2 <- all_pt_x[,-c("departureTime")][order(time),`:=`(IDX = 1+1:.N),by=vehicle] +truckBEAM_truckAADTT <- countyStats[county_truck_aadtt_2017, on=c("cnty"="CNTY")] -all_pt <- all_pt_1[all_pt_2, on=c("vehicle", "IDX", "label")][!is.na(arrivalTime)&!is.na(departureTime)] -all_pt[,`:=`(stopDuration = departureTime - arrivalTime)] -all_pt[,.(mean(stopDuration)),by=.(label)] +truckBEAM_truckAADTT %>% ggplot(aes(county)) + + geom_bar(truck_volume, ) -all_pt_1 <- all_pt[order(time),`:=`(IDX = 1:.N),by=vehicle] -all_pt_2 <- all_pt[order(time),`:=`(IDX = 1:.N),by=vehicle] -all_pt[,,by=.(vehicle)] +sfBayTAZs <- st_read(pp(freightDir, "/validation/TAZs/Transportation_Analysis_Zones.shp")) -all_pt_x[vehicle=="freight-vehicle-freightVehicle-b2b-1640"] +################ *************************** +################ validation HPMS +################ *************************** +sf_hpms <- st_read(pp(freightDir, "/validation/sf_hpms_inventory_clipped.geojson")) +Volume_beam <- sum(linkStats$TruckVolume) +Volume_hpms <- sum(sf_hpms$AADT_Combi+sf_hpms$AADT_Singl) +VMT_beam <- sum(linkStats$TruckVolume * linkStats$length/1609) +VMT_hpms <- (sum((sf_hpms$AADT_Combi+sf_hpms$AADT_Singl) * as.numeric(st_length(sf_hpms))/1609)) +Volume_beam/Volume_hpms +VMT_beam/VMT_hpms +freight_pt[,.(VMT=sum(length)/1609.0),by=.(vehicleType)] +freight_pt[,.(tourMT=sum(length)/1609.0),by=.(vehicle,vehicleType)][,.(avgTourMT=mean(tourMT)),by=.(vehicleType)] +write.csv( + freight_pt, + file = pp(freightWorkDir, "/freight.pt.0.events.csv"), + row.names=F, + quote=T) diff --git a/src/main/java/beam/analysis/physsim/PhyssimCalcLinkStats.java b/src/main/java/beam/analysis/physsim/PhyssimCalcLinkStats.java index 117ac09d524..89408d65a7a 100755 --- a/src/main/java/beam/analysis/physsim/PhyssimCalcLinkStats.java +++ b/src/main/java/beam/analysis/physsim/PhyssimCalcLinkStats.java @@ -1,10 +1,11 @@ package beam.analysis.physsim; +import beam.agentsim.agents.vehicles.BeamVehicle; import beam.analysis.plots.GraphUtils; +import beam.physsim.analysis.LinkStatsWithVehicleCategory; import beam.sim.BeamConfigChangesObservable; import beam.sim.BeamConfigChangesObserver; import beam.sim.config.BeamConfig; -import beam.utils.BeamCalcLinkStats; import beam.utils.VolumesAnalyzerFixed; import org.jfree.chart.*; import org.jfree.chart.plot.CategoryPlot; @@ -12,6 +13,7 @@ import org.jfree.data.category.CategoryDataset; import org.jfree.data.general.DatasetUtilities; import org.matsim.analysis.VolumesAnalyzer; +import org.matsim.api.core.v01.Id; import org.matsim.api.core.v01.network.Link; import org.matsim.api.core.v01.network.Network; import org.matsim.core.api.experimental.events.EventsManager; @@ -21,7 +23,7 @@ import org.matsim.core.utils.misc.Time; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import scala.Tuple2; + import java.awt.*; import java.io.IOException; import java.util.*; @@ -56,14 +58,18 @@ public class PhyssimCalcLinkStats implements BeamConfigChangesObserver { private BeamConfig beamConfig; private final Network network; private final OutputDirectoryHierarchy controllerIO; - private final BeamCalcLinkStats linkStats; private VolumesAnalyzer volumes; + private TravelTimeCalculatorConfigGroup ttcConfigGroup; + private Map, BeamVehicle> vehicleMap; public PhyssimCalcLinkStats(Network network, OutputDirectoryHierarchy controlerIO, BeamConfig beamConfig, - TravelTimeCalculatorConfigGroup ttcConfigGroup, BeamConfigChangesObservable beamConfigChangesObservable) { + TravelTimeCalculatorConfigGroup ttcConfigGroup, BeamConfigChangesObservable beamConfigChangesObservable, + Map, BeamVehicle> vehicleMap) { this.network = network; this.controllerIO = controlerIO; this.beamConfig = beamConfig; + this.ttcConfigGroup = ttcConfigGroup; + this.vehicleMap = vehicleMap; if (isNotTestMode()) { binSize = this.beamConfig.beam().physsim().linkStatsBinSize(); @@ -76,15 +82,15 @@ public PhyssimCalcLinkStats(Network network, OutputDirectoryHierarchy controlerI } beamConfigChangesObservable.addObserver(this); - linkStats = new BeamCalcLinkStats(network, ttcConfigGroup); } public void notifyIterationEnds(int iteration, TravelTime travelTime) { - linkStats.addData(volumes, travelTime); processData(iteration, travelTime); if (this.controllerIO != null) { if (isNotTestMode() && writeLinkStats(iteration)) { - linkStats.writeFile(this.controllerIO.getIterationFilename(iteration, "linkstats_unmodified.csv.gz")); + String filePath = this.controllerIO.getIterationFilename(iteration, "linkstats_unmodified.csv.gz"); + LinkStatsWithVehicleCategory linkStats = new LinkStatsWithVehicleCategory(network, ttcConfigGroup); + linkStats.writeLinkStatsWithTruckVolumes(volumes, travelTime, filePath); } if (beamConfig.beam().outputs().writeGraphs()) { CategoryDataset dataset = buildAndGetGraphCategoryDataset(); @@ -125,7 +131,7 @@ private void processData(int iteration, TravelTime travelTime) { double averageSpeedToFreeSpeedRatio = averageSpeed / freeSpeed; - double relativeSpeed = Math.max((Math.round(averageSpeedToFreeSpeedRatio * 50.0) / 10),minSpeed); + double relativeSpeed = Math.max((Math.round(averageSpeedToFreeSpeedRatio * 50.0) / 10), minSpeed); Map hoursDataMap = relativeSpeedFrequenciesPerBin.get(relativeSpeed); @@ -175,13 +181,13 @@ private double[][] buildModesFrequencyDataset() { Optional optionalMaxRelativeSpeedsCategories = relativeSpeedsCategoriesList.stream().max(Comparator.naturalOrder()); - if(optionalMaxRelativeSpeedsCategories.isPresent()) { + if (optionalMaxRelativeSpeedsCategories.isPresent()) { int maxRelativeSpeedsCategories = optionalMaxRelativeSpeedsCategories.get().intValue(); - dataset = new double[maxRelativeSpeedsCategories+1][noOfBins]; + dataset = new double[maxRelativeSpeedsCategories + 1][noOfBins]; for (int i = 0; i <= maxRelativeSpeedsCategories; i++) { - Map relativeSpeedBins = relativeSpeedFrequenciesPerBin.getOrDefault((double)i, new HashMap<>()); + Map relativeSpeedBins = relativeSpeedFrequenciesPerBin.getOrDefault((double) i, new HashMap<>()); double[] relativeSpeedFrequencyPerHour = new double[noOfBins]; int index = 0; @@ -218,7 +224,7 @@ private void createModesFrequencyGraph(CategoryDataset dataset, int iterationNum int max = Collections.max(relativeSpeedsCategoriesList).intValue(); - for (int i = 0; i <= max ; i++) { + for (int i = 0; i <= max; i++) { legendItems.add(new LegendItem(String.valueOf(i), getColor(i))); plot.getRenderer().setSeriesPaint(i, getColor(i)); } @@ -250,16 +256,11 @@ private Color getRandomColor() { } public void notifyIterationStarts(EventsManager eventsManager, TravelTimeCalculatorConfigGroup travelTimeCalculatorConfigGroup) { - this.linkStats.reset(); - volumes = new VolumesAnalyzerFixed(3600, travelTimeCalculatorConfigGroup.getMaxTime() - 1, network); + volumes = new VolumesAnalyzerFixed(3600, travelTimeCalculatorConfigGroup.getMaxTime() - 1, network, vehicleMap); eventsManager.addHandler(volumes); this.relativeSpeedFrequenciesPerBin.clear(); } - public void clean(){ - this.linkStats.reset(); - } - @Override public void update(BeamConfigChangesObservable observable, BeamConfig updatedBeamConfig) { this.beamConfig = updatedBeamConfig; diff --git a/src/main/java/beam/physsim/jdeqsim/AgentSimToPhysSimPlanConverter.java b/src/main/java/beam/physsim/jdeqsim/AgentSimToPhysSimPlanConverter.java index 4932055e182..c02dc705ab2 100755 --- a/src/main/java/beam/physsim/jdeqsim/AgentSimToPhysSimPlanConverter.java +++ b/src/main/java/beam/physsim/jdeqsim/AgentSimToPhysSimPlanConverter.java @@ -10,6 +10,7 @@ import beam.analysis.physsim.PhyssimNetworkComparisonEuclideanVsLengthAttribute; import beam.analysis.physsim.PhyssimNetworkLinkLengthDistribution; import beam.calibration.impl.example.CountsObjectiveFunction; +import beam.physsim.analysis.LinkStatsWithVehicleCategory; import beam.physsim.cchRoutingAssignment.OsmInfoHolder; import beam.physsim.cchRoutingAssignment.RoutingFrameworkTravelTimeCalculator; import beam.physsim.cchRoutingAssignment.RoutingFrameworkWrapperImpl; @@ -315,9 +316,9 @@ private void writeTravelTime(TravelTime travelTimeForR5, VolumesAnalyzer volumes int endTimeInSeconds = (int) Time.parseTime(beamConfig.beam().agentsim().endTime()); cfg.setMaxTime(endTimeInSeconds); Network network = agentSimScenario.getNetwork(); - BeamCalcLinkStats linkStats = new BeamCalcLinkStats(network, cfg); - linkStats.addData(volumesAnalyzer, travelTimeForR5); - linkStats.writeFile(controlerIO.getIterationFilename(iterationEndsEvent.getIteration(), "linkstats.csv.gz")); + LinkStatsWithVehicleCategory linkStats = new LinkStatsWithVehicleCategory(network, cfg); + String filePath = controlerIO.getIterationFilename(iterationEndsEvent.getIteration(), "linkstats.csv.gz"); + linkStats.writeLinkStatsWithTruckVolumes(volumesAnalyzer, travelTimeForR5, filePath); } private boolean shouldWritePlans(int iterationNumber) { diff --git a/src/main/java/beam/physsim/jdeqsim/cacc/roadcapacityadjustmentfunctions/Hao2018CaccRoadCapacityAdjustmentFunction.java b/src/main/java/beam/physsim/jdeqsim/cacc/roadcapacityadjustmentfunctions/Hao2018CaccRoadCapacityAdjustmentFunction.java index ea61cb5453b..6996a6e24da 100644 --- a/src/main/java/beam/physsim/jdeqsim/cacc/roadcapacityadjustmentfunctions/Hao2018CaccRoadCapacityAdjustmentFunction.java +++ b/src/main/java/beam/physsim/jdeqsim/cacc/roadcapacityadjustmentfunctions/Hao2018CaccRoadCapacityAdjustmentFunction.java @@ -17,6 +17,7 @@ import java.util.HashMap; import java.util.Map; import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; /* @@ -36,8 +37,8 @@ public class Hao2018CaccRoadCapacityAdjustmentFunction implements RoadCapacityAd private final boolean writeGraphs; private final OutputDirectoryHierarchy controllerIO; private final MultiValuedMap caccCapacityIncrease = new HashSetValuedHashMap<>(); - private final Map caccLinkCapacityIncrease = new HashMap<>(); - private final Map allLinksCapacityIncrease = new HashMap<>(); + private final Map caccLinkCapacityIncrease = new ConcurrentHashMap<>(); + private final Map allLinksCapacityIncrease = new ConcurrentHashMap<>(); private final Optional csvWriter; private int numberOfMixedVehicleTypeEncountersOnCACCCategoryRoads = 0; diff --git a/src/main/java/beam/utils/VolumesAnalyzerFixed.java b/src/main/java/beam/utils/VolumesAnalyzerFixed.java index f4fbb0eb96c..663c4faf7ef 100644 --- a/src/main/java/beam/utils/VolumesAnalyzerFixed.java +++ b/src/main/java/beam/utils/VolumesAnalyzerFixed.java @@ -1,12 +1,15 @@ package beam.utils; +import beam.agentsim.agents.vehicles.BeamVehicle; import org.matsim.analysis.VolumesAnalyzer; import org.matsim.api.core.v01.Id; +import org.matsim.api.core.v01.events.VehicleEntersTrafficEvent; import org.matsim.api.core.v01.network.Link; import org.matsim.api.core.v01.network.Network; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Map; import java.util.concurrent.TimeUnit; public class VolumesAnalyzerFixed extends VolumesAnalyzer { @@ -17,14 +20,37 @@ public class VolumesAnalyzerFixed extends VolumesAnalyzer { private final int timeBinSizeInSeconds; private final int maxSlotIndex; private final int maxHour; + private Map, BeamVehicle> vehicleMap; - public VolumesAnalyzerFixed(int timeBinSizeInSeconds, int maxTime, Network network) { + public VolumesAnalyzerFixed(int timeBinSizeInSeconds, int maxTime, Network network, Map, BeamVehicle> vehicleMap) { super(timeBinSizeInSeconds, maxTime, network); this.timeBinSizeInSeconds = timeBinSizeInSeconds; + this.vehicleMap = vehicleMap; this.maxSlotIndex = (maxTime / timeBinSizeInSeconds) + 1; this.maxHour = (int) TimeUnit.SECONDS.toHours(maxTime + 1); } + @Override + public void handleEvent(VehicleEntersTrafficEvent event) { + if (vehicleMap != null) { + BeamVehicle vehicle = vehicleMap.get(event.getVehicleId()); + String category = vehicleCategory(vehicle); + + VehicleEntersTrafficEvent eventWithMode = new VehicleEntersTrafficEvent(event.getTime(), event.getPersonId(), + event.getLinkId(), event.getVehicleId(), category, event.getRelativePositionOnLink()); + super.handleEvent(eventWithMode); + } else { + super.handleEvent(event); + } + } + + private String vehicleCategory(BeamVehicle vehicle) { + if (vehicle == null) { + return null; + } + return vehicle.beamVehicleType().vehicleCategory().toString(); + } + @Override public double[] getVolumesPerHourForLink(final Id linkId) { if (SECONDS_PER_HOUR % timeBinSizeInSeconds != 0) { @@ -49,4 +75,23 @@ public double[] getVolumesPerHourForLink(final Id linkId) { return volumes; } + public double[] getVolumesPerHourForLink(final Id linkId, String mode) { + if (SECONDS_PER_HOUR % timeBinSizeInSeconds != 0) + log.error("Volumes per hour and per link probably not correct! TimeBinSize: " + timeBinSizeInSeconds); + + double[] volumes = new double[maxHour]; + + int[] volumesForLink = this.getVolumesForLink(linkId, mode); + if (volumesForLink == null) return volumes; + + final int slotsPerHour = SECONDS_PER_HOUR / timeBinSizeInSeconds; + int slotIndex = 0; + for (int hour = 0; hour < maxHour; hour++) { + for (int i = 0; i < slotsPerHour; i++) { + volumes[hour] += volumesForLink[Math.min(maxSlotIndex, slotIndex++)]; + } + } + return volumes; + } + } diff --git a/src/main/python/freight/events_processing.py b/src/main/python/freight/events_processing.py new file mode 100644 index 00000000000..2cb7b09c5ce --- /dev/null +++ b/src/main/python/freight/events_processing.py @@ -0,0 +1,27 @@ +import pandas as pd +import os + +work_directory = '~/Data/FREIGHT/validation/beam/' +filename = '0.events.new.csv' +full_filename = os.path.expanduser(work_directory + filename) +compression = None +if full_filename.endswith(".gz"): + compression = 'gzip' +data = pd.read_csv(full_filename, sep=",", index_col=None, header=0, compression=compression) +print(data.type.unique()) +data_filtered = data.loc[data.type.isin(["PathTraversal", "actstart", "actend"])] +print(data_filtered.type.unique()) +data_filtered = data_filtered.loc[data_filtered.vehicle.str.startswith("freight", na=True)] +print(data_filtered.type.unique()) +data_filtered2 = data_filtered.loc[data_filtered.actType.isin(["Warehouse", "Unloading", "Loading"]) | data_filtered.actType.isnull()] +print(data_filtered.type.unique()) +# data_filtered2 = data_filtered[ +# ["time","type","vehicleType","vehicle","secondaryFuelLevel", +# "primaryFuelLevel","driver","mode","seatingCapacity","startX", +# "startY", "endX", "endY", "capacity", "arrivalTime", "departureTime", +# "secondaryFuel", "secondaryFuelType", "primaryFuelType", +# "numPassengers", "length", "primaryFuel", "actType", "fuel", "person", +# "locationY", "locationX", "duration", "chargingPointType", "parkingType", "parkingTaz"] +# ] +data_filtered2.to_csv(work_directory + "filtered." + filename) +print("END") diff --git a/src/main/python/freight/frism_to_beam_freight_plans.py b/src/main/python/freight/frism_to_beam_freight_plans.py new file mode 100644 index 00000000000..98eb27648dd --- /dev/null +++ b/src/main/python/freight/frism_to_beam_freight_plans.py @@ -0,0 +1,124 @@ +import pandas as pd +import os +from pathlib import Path + +def read_csv_file(filename): + compression = None + if filename.endswith(".gz"): + compression = 'gzip' + return pd.read_csv(filename, sep=",", index_col=None, header=0, compression=compression) + + +def add_prefix(prefix, column, row, toNum = True): + if toNum: + old = str(int(row[column])) + else: + old = str(row[column]) + new = f"{prefix}{old}" + return new + + +directory_input = os.path.expanduser('~/Data/FREIGHT/Outputs(All SF)_0322_2022') +directory_output = os.path.expanduser('~/Data/FREIGHT/Outputs(All SF)_0322_2022_merged') +Path(directory_output).mkdir(parents=True, exist_ok=True) +carriers = None +payload_plans = None +tours = None + +for filename in os.listdir(directory_input): + filepath = f'{directory_input}/{filename}' + parts = filename.split('_', 2) + unique_id = parts[0].lower() + county = parts[1].lower() + filetype = parts[2].lower() + + if filetype.startswith('carrier_'): + df = pd.read_csv(filepath) + df['carrierId'] = df.apply(lambda row: add_prefix(f'{unique_id}-{county}-', 'carrierId', row), axis=1) + df['vehicleId'] = df.apply(lambda row: add_prefix(f'{unique_id}-{county}-', 'vehicleId', row), axis=1) + df['vehicleTypeId'] = df.apply(lambda row: add_prefix('FREIGHT-', 'vehicleTypeId', row), axis=1) + df['tourId'] = df.apply(lambda row: add_prefix(f'{unique_id}-{county}-', 'tourId', row), axis=1) + if carriers is None: + carriers = df + else: + carriers = pd.concat([carriers, df]) + elif filetype.startswith('freight_tours_'): + df = pd.read_csv(filepath) + df['tour_id'] = df.apply(lambda row: add_prefix(f'{unique_id}-{county}-', 'tour_id', row), axis=1) + if tours is None: + tours = df + else: + tours = pd.concat([tours, df]) + elif filetype.startswith('payload_'): + df = pd.read_csv(filepath) + df['tourId'] = df.apply(lambda row: add_prefix(f'{unique_id}-{county}-', 'tourId', row), axis=1) + df['payloadId'] = df.apply(lambda row: add_prefix(f'{unique_id}-{county}-', 'payloadId', row, False), axis=1) + if payload_plans is None: + payload_plans = df + else: + payload_plans = pd.concat([payload_plans, df]) + else: + print(f'SKIPPING {filetype}') + + +# In[9]: + + +# carrierId,tourId,vehicleId,vehicleTypeId,warehouseZone,warehouseX,warehouseY,MESOZONE,BoundaryZONE +# carrierId,tourId,vehicleId,vehicleTypeId,warehouseZone,warehouseX,warehouseY,MESOZONE,BoundaryZONE +carriers_renames = { + 'depot_zone': 'warehouseZone', + 'depot_zone_x': 'warehouseX', + 'depot_zone_y': 'warehouseY' +} +carriers_drop = ['x', 'y'] +carriers.rename(columns=carriers_renames, inplace=True) +carriers.drop(carriers_drop, axis=1, inplace=True) +carriers['warehouseZone'] = carriers['warehouseZone'].astype(int) +carriers.to_csv(f'{directory_output}/freight-merged-carriers.csv', index=False) + + +# In[10]: + + +# tourId,departureTimeInSec,departureLocationZone,maxTourDurationInSec,departureLocationX,departureLocationY +# tourId,departureTimeInSec,departureLocationZone,maxTourDurationInSec,departureLocationX,departureLocationY +tours_renames = { + 'tour_id': 'tourId', + 'departureLocation_zone': 'departureLocationZone', + 'departureLocation_x': 'departureLocationX', + 'departureLocation_y': 'departureLocationY' +} +tours.rename(columns=tours_renames, inplace=True) +tours['departureTimeInSec'] = tours['departureTimeInSec'].astype(int) +tours['maxTourDurationInSec'] = tours['maxTourDurationInSec'].astype(int) +tours['departureLocationZone'] = tours['departureLocationZone'].astype(int) +tours.to_csv(f'{directory_output}/freight-merged-tours.csv', index=False) + + +# In[11]: + + +# payloadId,sequenceRank,tourId,payloadType,weightInKg,requestType,locationZone,estimatedTimeOfArrivalInSec,arrivalTimeWindowInSecLower,arrivalTimeWindowInSecUpper,operationDurationInSec,locationX,locationY +# payloadId,sequenceRank,tourId,payloadType,weightInKg,requestType,locationZone,estimatedTimeOfArrivalInSec,arrivalTimeWindowInSecLower,arrivalTimeWindowInSecUpper,operationDurationInSec,locationX,locationY +payload_plans_renames = { + 'arrivalTimeWindowInSec_lower': 'arrivalTimeWindowInSecLower', + 'arrivalTimeWindowInSec_upper': 'arrivalTimeWindowInSecUpper', + 'locationZone_x': 'locationX', + 'locationZone_y': 'locationY' +} +payload_plans_drop = ['weightInlb', 'cummulativeWeightInlb'] +payload_plans['weightInKg'] = abs(payload_plans['weightInlb'].astype(int)) * 0.45359237 +payload_plans.rename(columns=payload_plans_renames, inplace=True) +payload_plans.drop(payload_plans_drop, axis=1, inplace=True) +payload_plans['sequenceRank'] = payload_plans['sequenceRank'].astype(int) +payload_plans['payloadType'] = payload_plans['payloadType'].astype(int) +payload_plans['requestType'] = payload_plans['requestType'].astype(int) +payload_plans['estimatedTimeOfArrivalInSec'] = payload_plans['estimatedTimeOfArrivalInSec'].astype(int) +payload_plans['arrivalTimeWindowInSecLower'] = payload_plans['arrivalTimeWindowInSecLower'].astype(int) +payload_plans['arrivalTimeWindowInSecUpper'] = payload_plans['arrivalTimeWindowInSecUpper'].astype(int) +payload_plans['operationDurationInSec'] = payload_plans['operationDurationInSec'].astype(int) +payload_plans['locationZone'] = payload_plans['locationZone'].astype(int) +payload_plans.to_csv(f'{directory_output}/freight-merged-payload-plans.csv', index=False) + +print("END") \ No newline at end of file diff --git a/src/main/python/gemini/debug_parking_mnl.py b/src/main/python/gemini/debug_parking_mnl.py new file mode 100644 index 00000000000..82a58c5435c --- /dev/null +++ b/src/main/python/gemini/debug_parking_mnl.py @@ -0,0 +1,28 @@ +import os + +mnl_lines = ["mnlStatus,requestId,parkingZoneId,chargingPointType,parkingType,costInDollars\n"] +mnl_search_lines = ["parkingZoneId,geoId,parkingType,chargingPointType,pricingModel,reservedFor,stallsAvailable," + "maxStalls,vehicleId,parkingDuration,activityType,valueOfTime,requestId,isEV,rideHailFastChargingOnly," + "validChargingCapability,hasAvailability,validParkingType,isValidTime,isValidVehicleManager\n"] +mnl_param_lines = ["parkingZoneId,geoId,parkingType,chargingPointType,pricingModel,reservedFor,stallsAvailable," + "maxStalls,vehicleId,parkingDuration,activityType,valueOfTime,requestId,costInDollars,RangeAnxietyCost," + "WalkingEgressCost,ParkingTicketCost,HomeActivityPrefersResidentialParking\n"] +output_log_file = os.path.expanduser('~/Data/GEMINI/2022Mars-Calibration/beamLog.out.txt') +with open(output_log_file) as infile: + for line in infile: + if "SAMPLED: " in line: + mnl_lines.append("sampled,"+line.split("SAMPLED: ")[1]) + elif "CHOSEN: " in line: + mnl_lines.append("chosen,"+line.split("CHOSEN: ")[1]) + elif "SEARCH: " in line: + mnl_search_lines.append(line.split("SEARCH: ")[1]) + elif "PARAM: " in line: + mnl_param_lines.append(line.split("PARAM: ")[1]) + +with open(os.path.expanduser('~/Data/GEMINI/2022Mars-Calibration/mnl-beamLog.csv'), 'w') as f: + f.writelines(mnl_lines) +with open(os.path.expanduser('~/Data/GEMINI/2022Mars-Calibration/mnl-search-beamLog.csv'), 'w') as f: + f.writelines(mnl_search_lines) +with open(os.path.expanduser('~/Data/GEMINI/2022Mars-Calibration/mnl-param-beamLog.csv'), 'w') as f: + f.writelines(mnl_param_lines) +print("END") diff --git a/src/main/python/gemini/nrel_to_beam_charging_infrastruture.py b/src/main/python/gemini/nrel_to_beam_charging_infrastruture.py new file mode 100644 index 00000000000..c9621cf5bb8 --- /dev/null +++ b/src/main/python/gemini/nrel_to_beam_charging_infrastruture.py @@ -0,0 +1,110 @@ +import pandas as pd +import os +import random +from tqdm import tqdm + +def read_csv_file(filename): + compression = None + if filename.endswith(".gz"): + compression = 'gzip' + return pd.read_csv(filename, sep=",", index_col=None, header=0, compression=compression) + +nrel_file_input = os.path.expanduser('~/Data/GEMINI/2022Feb/siting/init1-7_2022_Feb_03_wgs84.csv') +smart_file_input = os.path.expanduser("~/Data/GEMINI/stations/taz-parking-sparse-fast-limited-l2-150-lowtech-b.csv") +nrel_file_converted_input = os.path.expanduser(nrel_file_input.split(".")[0] + "_converted.csv") +smart_file_updated_input = os.path.expanduser(smart_file_input.split(".")[0] + "_updated.csv") +smart_file_with_fees_input = os.path.expanduser(nrel_file_input.split(".")[0] + "_withFees.csv.gz") + + +def convert_nrel_data(nrel_file, nrel_file_converted): + if not os.path.exists(nrel_file_converted): + data = read_csv_file(nrel_file) + data2 = data[["subSpace", "pType", "chrgType", "field_1", "household_id", "X", "Y", "housingTypes", "propertytype", "propertysubtype", "county"]] + data2 = data2.rename(columns={ + "chrgType": "chargingPointType", + "pType": "parkingType", + "subSpace": "taz", + "X": "locationX", + "Y": "locationY", + "housingTypes": "housingType", + "propertytype": "propertyType", + "propertysubtype": "propertySubType", + "county": "county" + }) + data2["parkingZoneId"] = "" + data2["reservedFor"] = "Any" + data2["pricingModel"] = "Block" + data2["feeInCents"] = 0 + data2["numStalls"] = 1 + data2.loc[data2["household_id"].notna(), ['reservedFor']] = data2.loc[data2["household_id"].notna()].apply( + lambda row1: "household(" + str(int(row1["household_id"])) + ")", axis=1) + data2.loc[data2["field_1"].notna(), ['parkingZoneId']] = data2.loc[data2["field_1"].notna()].apply( + lambda row2: "PEV-" + str(int(row2["taz"])) + "-" + str(int(row2["field_1"])), axis=1) + nrel_data = data2.drop(columns=['household_id', 'field_1']) + nrel_data.to_csv(nrel_file_converted, index=False) + print("Reading nrel infrastructure done!") + return nrel_data + else: + return read_csv_file(nrel_file_converted) + + +# Reading fees +def reading_sf_bay_fees(smart_file, smart_file_updated): + if not os.path.exists(smart_file_updated): + smart_data = read_csv_file(smart_file) + smart_data["chargingPointType"] = "NoCharger" + smart_data.loc[(smart_data["chargingType"] == "WorkLevel2(7.2|AC)") & (smart_data["parkingType"] == "Public"), ['chargingPointType']] = "publiclevel2(7.2|AC)" + smart_data.loc[(smart_data["chargingType"] == "WorkLevel2(7.2|AC)") & (smart_data["parkingType"] == "Workplace"), ['chargingPointType']] = "worklevel2(7.2|AC)" + smart_data.loc[smart_data["chargingType"] == "Custom(150.0|DC)", ['chargingPointType']] = "publicfc(150.0|DC)" + smart_data.loc[smart_data["chargingType"] == "HomeLevel2(7.2|AC)", ['chargingPointType']] = "homelevel2(7.2|AC)" + smart_data.loc[smart_data["chargingType"] == "HomeLevel1(1.8|AC)", ['chargingPointType']] = "homelevel1(1.8|AC)" + smart_data = smart_data.drop(columns=["chargingType"]) + smart_data = smart_data.rename(columns={"ReservedFor": "reservedFor"}) + smart_data.to_csv(smart_file_updated, index=False) + print("Reading Fees done!") + return smart_data + else: + return read_csv_file(smart_file_updated) + + +def assign_fees_to_infrastructure(nrel_data, fees_data, smart_file_with_fees): + df_dict = nrel_data.to_dict('records') + for row in tqdm(df_dict): + charging_type_arg = row["chargingPointType"] + if "fc" in charging_type_arg: + charging_type_arg = "publicfc(150.0|DC)" + filtered = fees_data.loc[(fees_data["taz"] == row["taz"]) & + (fees_data["parkingType"] == row["parkingType"]) & + (fees_data["chargingPointType"] == charging_type_arg)] + if len(filtered.index) == 0: + filtered = fees_data.loc[(fees_data["parkingType"] == row["parkingType"]) & + (fees_data["chargingPointType"] == charging_type_arg)] + pd.options.mode.chained_assignment = None + filtered.loc[:, "numStalls"] = filtered.loc[:, "numStalls"].astype('int') + tot_stalls = filtered["numStalls"].sum() + cumulated = 0.0 + memorized_fee = 0.0 + rd_prob = random.uniform(0, 1) + for row2 in filtered.itertuples(): + memorized_fee = row2.feeInCents + cumulated = cumulated + float(row2.numStalls) / float(tot_stalls) + if cumulated >= rd_prob: + break + if "(150.0|DC)" in row["chargingPointType"]: + memorized_fee = memorized_fee * 1.6 + elif "(250.0|DC)" in row["chargingPointType"]: + memorized_fee = memorized_fee * 2.2 + elif "(400.0|DC)" in row["chargingPointType"]: + memorized_fee = memorized_fee * 3.1 + row["feeInCents"] = memorized_fee + output = pd.DataFrame.from_dict(df_dict) + output.reset_index(drop=True, inplace=True) + output.to_csv(smart_file_with_fees, index=False) + + +nrel_data_output = convert_nrel_data(nrel_file_input, nrel_file_converted_input) +print("convert_nrel_data done!") +fees_data_output = reading_sf_bay_fees(smart_file_input, smart_file_updated_input) +print("reading_sf_bay_fees done!") +assign_fees_to_infrastructure(nrel_data_output, fees_data_output, smart_file_with_fees_input) +print("END") diff --git a/src/main/python/gemini/unlimited_charging_infrastructure.py b/src/main/python/gemini/unlimited_charging_infrastructure.py index d0936fe6dd6..f03ececc4a4 100644 --- a/src/main/python/gemini/unlimited_charging_infrastructure.py +++ b/src/main/python/gemini/unlimited_charging_infrastructure.py @@ -1,7 +1,7 @@ #ADD MANY 50DC 150DC 250DC 350DC headerfile = "taz,parkingType,pricingModel,chargingPointType,numStalls,feeInCents,reservedFor" -with open('gemini_taz_unlimited_parking_stalls.csv', mode='w') as csv_writer: +with open('sfbay_taz_unlimited_parking_stalls.csv', mode='w') as csv_writer: csv_writer.write(headerfile+"\n") for x in range(1, 1455): @@ -10,28 +10,29 @@ csv_writer.write(f"{x},Public,Block,NoCharger,9999999,0,Any" + "\n") -with open('gemini_taz_unlimited_charging_point.csv', mode='w') as csv_writer: +with open('sfbay_taz_unlimited_charging_point.csv', mode='w') as csv_writer: csv_writer.write(headerfile+"\n") for x in range(1, 1455): - csv_writer.write(f"{x},Residential,Block,HomeLevel1(1.8|AC),9999999,50,Any" + "\n") - csv_writer.write(f"{x},Residential,Block,HomeLevel2(7.2|AC),9999999,200,Any" + "\n") + csv_writer.write(f"{x},Residential,Block,HomeLevel1(1.8|AC),9999999,0.45,Any" + "\n") + csv_writer.write(f"{x},Residential,Block,HomeLevel2(7.2|AC),9999999,1.8,Any" + "\n") - csv_writer.write(f"{x},Workplace,Block,WorkLevel2(7.2|AC),9999999,200,Any" + "\n") + csv_writer.write(f"{x},Workplace,Block,WorkLevel2(7.2|AC),9999999,14.4,Any" + "\n") - csv_writer.write(f"{x},Public,Block,PublicLevel2(7.2|AC),9999999,200,Any" + "\n") - csv_writer.write(f"{x},Public,Block,PublicFC(50|DC),9999999,1600,Any" + "\n") - csv_writer.write(f"{x},Public,Block,PublicFC(150|DC),9999999,4800,Any" + "\n") - csv_writer.write(f"{x},Public,Block,PublicXFC(250|DC),9999999,9500,Any" + "\n") - csv_writer.write(f"{x},Public,Block,PublicXFC(400|DC),9999999,15200,Any" + "\n") + csv_writer.write(f"{x},Public,Block,PublicLevel2(7.2|AC),9999999,19.3,Any" + "\n") + csv_writer.write(f"{x},Public,Block,PublicFC(150|DC),9999999,490.98,Any" + "\n") + csv_writer.write(f"{x},Public,Block,PublicFC(200|DC),9999999,654.64,Any" + "\n") + csv_writer.write(f"{x},Public,Block,PublicXFC(300|DC),9999999,981.96,Any" + "\n") + csv_writer.write(f"{x},Public,Block,PublicXFC(400|DC),9999999,1309.28,Any" + "\n") -with open('gemini_taz_unlimited_depots.csv', mode='w') as csv_writer: +with open('sfbay_taz_unlimited_depots.csv', mode='w') as csv_writer: csv_writer.write(headerfile+"\n") for x in range(1, 1455): - csv_writer.write(f"{x},Public,FlatFee,DepotFC(150.0|DC),9999999,4800,ridehail(GlobalRHM)" + "\n") - csv_writer.write(f"{x},Public,FlatFee,DepotXFC(250.0|DC),9999999,9500,ridehail(GlobalRHM)" + "\n") - csv_writer.write(f"{x},Public,FlatFee,DepotXFC(400.0|DC),9999999,15200,ridehail(GlobalRHM)" + "\n") + csv_writer.write(f"{x},Public,FlatFee,DepotFC(150.0|DC),9999999,490.98,ridehail(GlobalRHM)" + "\n") + csv_writer.write(f"{x},Public,FlatFee,DepotXFC(200.0|DC),9999999,654.64,ridehail(GlobalRHM)" + "\n") + csv_writer.write(f"{x},Public,FlatFee,DepotXFC(300.0|DC),9999999,981.96,ridehail(GlobalRHM)" + "\n") + csv_writer.write(f"{x},Public,FlatFee,DepotXFC(400.0|DC),9999999,1309.28,ridehail(GlobalRHM)" + "\n") diff --git a/src/main/resources/beam-template.conf b/src/main/resources/beam-template.conf index c7acb17f322..1b5b890d40d 100755 --- a/src/main/resources/beam-template.conf +++ b/src/main/resources/beam-template.conf @@ -31,6 +31,7 @@ beam.agentsim.lastIteration = "int | 0" beam.agentsim.endTime = "30:00:00" beam.agentsim.scheduleMonitorTask.initialDelay = 1 beam.agentsim.scheduleMonitorTask.interval = 30 +beam.agentsim.snapLocationAndRemoveInvalidInputs = "boolean | false" beam.agentsim.agents.bodyType = "BODY-TYPE-DEFAULT" diff --git a/src/main/scala/beam/agentsim/agents/freight/FreightEntities.scala b/src/main/scala/beam/agentsim/agents/freight/FreightEntities.scala index 050460bbdd2..779e6ea5507 100644 --- a/src/main/scala/beam/agentsim/agents/freight/FreightEntities.scala +++ b/src/main/scala/beam/agentsim/agents/freight/FreightEntities.scala @@ -25,8 +25,6 @@ object FreightRequestType extends Enum[FreightRequestType] { case class FreightTour( tourId: Id[FreightTour], departureTimeInSec: Int, - warehouseLocationTaz: Option[Id[TAZ]], - warehouseLocationUTM: Coord, maxTourDurationInSec: Int ) @@ -51,5 +49,7 @@ case class FreightCarrier( tourMap: Map[Id[BeamVehicle], IndexedSeq[FreightTour]], payloadPlans: Map[Id[PayloadPlan], PayloadPlan], fleet: Map[Id[BeamVehicle], BeamVehicle], - plansPerTour: Map[Id[FreightTour], IndexedSeq[PayloadPlan]] + plansPerTour: Map[Id[FreightTour], IndexedSeq[PayloadPlan]], + warehouseLocationTaz: Option[Id[TAZ]], + warehouseLocationUTM: Coord ) diff --git a/src/main/scala/beam/agentsim/agents/freight/FreightReplanner.scala b/src/main/scala/beam/agentsim/agents/freight/FreightReplanner.scala index d97375be9d1..a4c0a1dd064 100644 --- a/src/main/scala/beam/agentsim/agents/freight/FreightReplanner.scala +++ b/src/main/scala/beam/agentsim/agents/freight/FreightReplanner.scala @@ -63,7 +63,7 @@ class FreightReplanner( ): Iterable[Plan] = { routes.groupBy(_.vehicle.id).map { case (vehicleIdStr, routes) => val vehicleId = Id.createVehicleId(vehicleIdStr) - val person = population.get(freightReader.createPersonId(vehicleId)) + val person = population.get(freightReader.createPersonId(freightCarrier.carrierId, vehicleId)) val toursAndPlans = routes.zipWithIndex.map { case (route, i) => convertToFreightTourWithPayloadPlans( s"freight-tour-${route.vehicle.id}-$i".createId, @@ -73,7 +73,7 @@ class FreightReplanner( } val tours = toursAndPlans.map(_._1) val plansPerTour = toursAndPlans.map { case (tour, plans) => tour.tourId -> plans }.toMap - freightReader.createPersonPlan(tours, plansPerTour, person) + freightReader.createPersonPlan(freightCarrier, tours, plansPerTour, person) } } @@ -82,13 +82,7 @@ class FreightReplanner( route: Route, payloadPlans: Map[Id[PayloadPlan], PayloadPlan] ): (FreightTour, IndexedSeq[PayloadPlan]) = { - val tour = FreightTour( - tourId, - route.startTime, - None, - route.startLocation, - route.duration * 2 - ) + val tour = FreightTour(tourId, route.startTime, route.duration * 2) val plans = route.activities.zipWithIndex.map { case (activity, i) => val requestType: FreightRequestType = activity.service match { @@ -126,8 +120,8 @@ class FreightReplanner( private implicit def toLocation(coord: Coord): Location = Location(coord.getX, coord.getY) private implicit def toCoord(location: Location): Coord = new Coord(location.x, location.y) - private def getVehicleHouseholdLocation(vehicle: BeamVehicle): Location = { - val householdIdStr = freightReader.createHouseholdId(vehicle.id).toString + private def getVehicleHouseholdLocation(carrierId: Id[FreightCarrier]): Location = { + val householdIdStr = freightReader.createHouseholdId(carrierId).toString val x = beamServices.matsimServices.getScenario.getHouseholds.getHouseholdAttributes .getAttribute(householdIdStr, "homecoordx") .asInstanceOf[Double] @@ -178,10 +172,10 @@ class FreightReplanner( } } - def toJspritVehicle(beamVehicle: BeamVehicle, departureTime: Int) = { + def toJspritVehicle(carrierId: Id[FreightCarrier], beamVehicle: BeamVehicle, departureTime: Int) = { Vehicle( beamVehicle.id.toString, - getVehicleHouseholdLocation(beamVehicle), + getVehicleHouseholdLocation(carrierId), beamVehicle.beamVehicleType.payloadCapacityInKg.get, departureTime ) @@ -198,7 +192,7 @@ class FreightReplanner( freightCarrier.fleet.values .map(beamVehicle => { val departure = randomTimeAround(departureTime) - toJspritVehicle(beamVehicle, departure) + toJspritVehicle(freightCarrier.carrierId, beamVehicle, departure) }) .toIndexedSeq val services = freightCarrier.payloadPlans.values.map(toService).toIndexedSeq @@ -219,7 +213,7 @@ class FreightReplanner( beamVehicle = freightCarrier.fleet(vehicleId) tour <- tours services = freightCarrier.plansPerTour(tour.tourId).map(toService) - vehicles = IndexedSeq(toJspritVehicle(beamVehicle, tour.departureTimeInSec)) + vehicles = IndexedSeq(toJspritVehicle(freightCarrier.carrierId, beamVehicle, tour.departureTimeInSec)) } yield JspritWrapper.solve(Problem(vehicles, services, Some(calculateCost))) Monoid.combineAll(tourSolutions) diff --git a/src/main/scala/beam/agentsim/agents/freight/input/FreightReader.scala b/src/main/scala/beam/agentsim/agents/freight/input/FreightReader.scala index 2e40615c72c..ec96d6d07b7 100644 --- a/src/main/scala/beam/agentsim/agents/freight/input/FreightReader.scala +++ b/src/main/scala/beam/agentsim/agents/freight/input/FreightReader.scala @@ -12,6 +12,7 @@ import beam.sim.BeamServices import beam.sim.common.GeoUtils import beam.sim.config.BeamConfig import beam.sim.config.BeamConfig.Beam.Agentsim.Agents.Freight +import beam.utils.SnapCoordinateUtils.SnapLocationHelper import com.conveyal.r5.streets.StreetLayer import org.matsim.api.core.v01.population._ import org.matsim.api.core.v01.{Coord, Id} @@ -30,9 +31,9 @@ trait FreightReader { def readPayloadPlans(): Map[Id[PayloadPlan], PayloadPlan] - def createPersonId(vehicleId: Id[BeamVehicle]): Id[Person] + def createPersonId(carrierId: Id[FreightCarrier], vehicleId: Id[BeamVehicle]): Id[Person] - def createHouseholdId(vehicleId: Id[BeamVehicle]): Id[Household] + def createHouseholdId(carrierId: Id[FreightCarrier]): Id[Household] def readFreightCarriers( allTours: Map[Id[FreightTour], FreightTour], @@ -49,13 +50,14 @@ trait FreightReader { } def createPersonPlan( + carrier: FreightCarrier, tours: IndexedSeq[FreightTour], plansPerTour: Map[Id[FreightTour], IndexedSeq[PayloadPlan]], person: Person ): Plan = { val allToursPlanElements = tours.flatMap { tour => val tourInitialActivity = - createFreightActivity("Home", tour.warehouseLocationUTM, tour.departureTimeInSec, None) + createFreightActivity("Warehouse", carrier.warehouseLocationUTM, tour.departureTimeInSec, None) val firstLeg: Leg = createFreightLeg(tour.departureTimeInSec) val plans: IndexedSeq[PayloadPlan] = plansPerTour.get(tour.tourId) match { @@ -82,7 +84,7 @@ trait FreightReader { elements } - val finalActivity = createFreightActivity("Home", tours.head.warehouseLocationUTM, -1, None) + val finalActivity = createFreightActivity("Warehouse", carrier.warehouseLocationUTM, -1, None) val allPlanElements: IndexedSeq[PlanElement] = allToursPlanElements :+ finalActivity val currentPlan = PopulationUtils.createPlan(person) @@ -96,26 +98,22 @@ trait FreightReader { def generatePopulation( carriers: IndexedSeq[FreightCarrier], - personFactory: PopulationFactory, + populationFactory: PopulationFactory, householdsFactory: HouseholdsFactory - ): IndexedSeq[(Household, Plan)] = { + ): IndexedSeq[(FreightCarrier, Household, Plan, Id[Person], Id[BeamVehicle])] = { carriers.flatMap { carrier => + val freightHouseholdId = createHouseholdId(carrier.carrierId) + val household = householdsFactory.createHousehold(freightHouseholdId) + household.setIncome(new IncomeImpl(0, Income.IncomePeriod.year)) carrier.tourMap.map { case (vehicleId, tours) => - val personId = createPersonId(vehicleId) - val person = personFactory.createPerson(personId) - - val currentPlan: Plan = createPersonPlan(tours, carrier.plansPerTour, person) - + val personId = createPersonId(carrier.carrierId, vehicleId) + val person = populationFactory.createPerson(personId) + val currentPlan: Plan = createPersonPlan(carrier, tours, carrier.plansPerTour, person) person.addPlan(currentPlan) person.setSelectedPlan(currentPlan) - - val freightHouseholdId = createHouseholdId(vehicleId) - val household: Household = householdsFactory.createHousehold(freightHouseholdId) - household.setIncome(new IncomeImpl(44444, Income.IncomePeriod.year)) household.getMemberIds.add(personId) household.getVehicleIds.add(vehicleId) - - (household, currentPlan) + (carrier, household, currentPlan, personId, vehicleId) } } } @@ -185,10 +183,16 @@ object FreightReader { beamConfig: BeamConfig, geoUtils: GeoUtils, streetLayer: StreetLayer, - tazMap: TAZTreeMap + tazMap: TAZTreeMap, + outputDirMaybe: Option[String] ): FreightReader = { val rand: Random = new Random(beamConfig.matsim.modules.global.randomSeed) val config = beamConfig.beam.agentsim.agents.freight + val snapLocationHelper: SnapLocationHelper = SnapLocationHelper( + geoUtils, + streetLayer, + beamConfig.beam.routing.r5.linkRadiusMeters + ) beamConfig.beam.agentsim.agents.freight.reader match { case "Generic" => new GenericFreightReader( @@ -196,16 +200,23 @@ object FreightReader { geoUtils, rand, tazMap, - Some(ClosestUTMPointOnMap(streetLayer, beamConfig.beam.routing.r5.linkRadiusMeters)) + beamConfig.beam.agentsim.snapLocationAndRemoveInvalidInputs, + snapLocationHelper, + outputDirMaybe ) case s => throw new RuntimeException(s"Unknown freight reader $s") } } - def apply(beamConfig: BeamConfig, geoUtils: GeoUtils, streetLayer: StreetLayer): FreightReader = { + def apply( + beamConfig: BeamConfig, + geoUtils: GeoUtils, + streetLayer: StreetLayer, + outputDirMaybe: Option[String] + ): FreightReader = { val tazMap = TAZTreeMap.getTazTreeMap(beamConfig.beam.agentsim.taz.filePath) - apply(beamConfig, geoUtils, streetLayer, tazMap) + apply(beamConfig, geoUtils, streetLayer, tazMap, outputDirMaybe) } def apply(beamServices: BeamServices): FreightReader = @@ -213,21 +224,7 @@ object FreightReader { beamServices.beamConfig, beamServices.geo, beamServices.beamScenario.transportNetwork.streetLayer, - beamServices.beamScenario.tazTreeMap + beamServices.beamScenario.tazTreeMap, + None ) - - case class ClosestUTMPointOnMap(streetLayer: StreetLayer, r5LinkRadiusMeters: Double) { - - def find(wsgCoord: Coord, geoUtils: GeoUtils): Option[Coord] = { - //val wsgCoord = geoUtils.utm2Wgs(utmCoord) - val theSplit = geoUtils.getR5Split(streetLayer, wsgCoord, r5LinkRadiusMeters) - if (theSplit == null) { - None - } else { - val wgsPointOnMap = geoUtils.splitToCoord(theSplit) - val utmCoord = geoUtils.wgs2Utm(wgsPointOnMap) - Some(utmCoord) - } - } - } } diff --git a/src/main/scala/beam/agentsim/agents/freight/input/GenericFreightReader.scala b/src/main/scala/beam/agentsim/agents/freight/input/GenericFreightReader.scala index 25cf0f5f509..5c09fc33db1 100644 --- a/src/main/scala/beam/agentsim/agents/freight/input/GenericFreightReader.scala +++ b/src/main/scala/beam/agentsim/agents/freight/input/GenericFreightReader.scala @@ -1,19 +1,21 @@ package beam.agentsim.agents.freight.input import beam.agentsim.agents.freight._ -import beam.agentsim.agents.freight.input.FreightReader.ClosestUTMPointOnMap import beam.agentsim.agents.vehicles.{BeamVehicle, BeamVehicleType} import beam.agentsim.infrastructure.taz.{TAZ, TAZTreeMap} import beam.sim.common.GeoUtils import beam.sim.config.BeamConfig.Beam.Agentsim.Agents.Freight +import beam.utils.SnapCoordinateUtils +import beam.utils.SnapCoordinateUtils._ import beam.utils.csv.GenericCsvReader import beam.utils.matsim_conversion.MatsimPlanConversion.IdOps import com.typesafe.scalalogging.LazyLogging +import org.apache.commons.lang3.StringUtils.isBlank import org.matsim.api.core.v01.population._ import org.matsim.api.core.v01.{Coord, Id} import org.matsim.households.Household -import org.apache.commons.lang3.StringUtils.isBlank +import scala.collection.mutable.ListBuffer import scala.util.Random /** @@ -24,7 +26,9 @@ class GenericFreightReader( val geoUtils: GeoUtils, rnd: Random, tazTree: TAZTreeMap, - closestUTMPointOnMapMaybe: Option[ClosestUTMPointOnMap] + val snapLocationAndRemoveInvalidInputs: Boolean, + val snapLocationHelper: SnapLocationHelper, + val outputDirMaybe: Option[String] = None ) extends LazyLogging with FreightReader { @@ -40,6 +44,8 @@ class GenericFreightReader( @Override def readFreightTours(): Map[Id[FreightTour], FreightTour] = { + val errors: ListBuffer[ErrorInfo] = ListBuffer() + val maybeTours = GenericCsvReader .readAsSeq[Option[FreightTour]](config.toursFilePath) { row => def get(key: String): String = getRowValue(config.toursFilePath, row, key) @@ -47,27 +53,42 @@ class GenericFreightReader( val tourId: Id[FreightTour] = get("tourId").createId[FreightTour] val departureTimeInSec = get("departureTimeInSec").toInt val maxTourDurationInSec = get("maxTourDurationInSec").toInt + val departureLocationX = row.get("departureLocationX") + val departureLocationY = row.get("departureLocationY") + extractCoordOrTaz( - row.get("departureLocationX"), - row.get("departureLocationY"), - row.get("departureLocationZone") + departureLocationX, + departureLocationY, + row.get("departureLocationZone"), + snapLocationAndRemoveInvalidInputs ) match { - case (departureLocationZoneMaybe, Some(departureLocationUTMOnMap)) => + case (_, Right(_)) => Some( FreightTour( tourId, departureTimeInSec, - departureLocationZoneMaybe, - departureLocationUTMOnMap, maxTourDurationInSec ) ) - case _ => - logger.error(f"Following freight tour row discarded because departure location is not reachable: $row") + case (_, Left(error)) => + errors.append( + ErrorInfo( + tourId.toString, + Category.FreightTour, + error, + departureLocationX.toDouble, + departureLocationY.toDouble + ) + ) None } } + outputDirMaybe.foreach { path => + if (errors.isEmpty) logger.info("No 'snap location' error to report for freight tours.") + else SnapCoordinateUtils.writeToCsv(s"$path/${CsvFile.FreightTours}", errors) + } + maybeTours.flatten .groupBy(_.tourId) .mapValues(_.head) @@ -75,6 +96,8 @@ class GenericFreightReader( @Override def readPayloadPlans(): Map[Id[PayloadPlan], PayloadPlan] = { + val errors: ListBuffer[ErrorInfo] = ListBuffer() + val maybePlans = GenericCsvReader .readAsSeq[Option[PayloadPlan]](config.plansFilePath) { row => def get(key: String): String = getRowValue(config.plansFilePath, row, key) @@ -94,11 +117,16 @@ class GenericFreightReader( } else { requestType.toString } - extractCoordOrTaz(row.get("locationX"), row.get("locationY"), row.get("locationZone")) match { - case (locationZoneMaybe, Some(locationUTMOnMap)) => + + val payloadId = get("payloadId").createId[PayloadPlan] + val locationX = row.get("locationX") + val locationY = row.get("locationY") + + extractCoordOrTaz(locationX, locationY, row.get("locationZone"), snapLocationAndRemoveInvalidInputs) match { + case (locationZoneMaybe, Right(coord)) => Some( PayloadPlan( - get("payloadId").createId, + payloadId, get("sequenceRank").toDouble.round.toInt, get("tourId").createId, get("payloadType").createId[PayloadType], @@ -106,19 +134,32 @@ class GenericFreightReader( requestType, activityType, locationZoneMaybe, - locationUTMOnMap, + coord, get("estimatedTimeOfArrivalInSec").toDouble.toInt, get("arrivalTimeWindowInSecLower").toDouble.toInt, get("arrivalTimeWindowInSecUpper").toDouble.toInt, operationDurationInSec ) ) - case _ => - logger.error(f"Following freight plan row discarded because zone location is not reachable: $row") + case (_, Left(error)) => + errors.append( + ErrorInfo( + payloadId.toString, + Category.FreightPayloadPlan, + error, + locationX.toDouble, + locationY.toDouble + ) + ) None } } + outputDirMaybe.foreach { path => + if (errors.isEmpty) logger.info("No 'snap location' error to report for freight payload plans.") + else SnapCoordinateUtils.writeToCsv(s"$path/${CsvFile.FreightPayloadPlans}", errors) + } + maybePlans.flatten .groupBy(_.payloadId) .mapValues(_.head) @@ -130,11 +171,12 @@ class GenericFreightReader( allPlans: Map[Id[PayloadPlan], PayloadPlan], vehicleTypes: Map[Id[BeamVehicleType], BeamVehicleType] ): IndexedSeq[FreightCarrier] = { - val existingTours: Set[Id[FreightTour]] = allTours.keySet.intersect(allPlans.map(_._2.tourId).toSet) val plans: Map[Id[PayloadPlan], PayloadPlan] = allPlans.filter { case (_, plan) => existingTours.contains(plan.tourId) } + val tourIdToPlans: Map[Id[FreightTour], IndexedSeq[PayloadPlan]] = + plans.values.toIndexedSeq.groupBy(_.tourId).map { case (tourId, plans) => tourId -> plans.sortBy(_.sequenceRank) } val tours: Map[Id[FreightTour], FreightTour] = allTours.filter { case (_, tour) => existingTours.contains(tour.tourId) } @@ -175,6 +217,7 @@ class GenericFreightReader( def createCarrier(carrierId: Id[FreightCarrier], carrierRows: IndexedSeq[FreightCarrierRow]) = { val warehouseLocationUTM: Coord = carrierRows.head.warehouseLocationUTM + val warehouseLocationZone: Option[Id[TAZ]] = carrierRows.head.warehouseLocationZone val vehicles: scala.IndexedSeq[BeamVehicle] = createCarrierVehicles(carrierId, carrierRows, warehouseLocationUTM) val vehicleMap: Map[Id[BeamVehicle], BeamVehicle] = vehicles.map(vehicle => vehicle.id -> vehicle).toMap @@ -183,20 +226,32 @@ class GenericFreightReader( .mapValues { rows => rows //setting the tour warehouse location to be the carrier warehouse location - .map(row => tours(row.tourId).copy(warehouseLocationUTM = warehouseLocationUTM)) + .map(row => tours(row.tourId)) .sortBy(_.departureTimeInSec) } val carrierTourIds = tourMap.values.flatten.map(_.tourId).toSet val plansPerTour: Map[Id[FreightTour], IndexedSeq[PayloadPlan]] = - plans.values.groupBy(_.tourId).filterKeys(carrierTourIds).mapValues(_.toIndexedSeq.sortBy(_.sequenceRank)) + carrierTourIds.collect { + case tourId if tourIdToPlans.contains(tourId) => tourId -> tourIdToPlans(tourId) + }.toMap val carrierPlanIds: Set[Id[PayloadPlan]] = plansPerTour.values.flatten.map(_.payloadId).toSet - val payloadMap = plans.filterKeys(carrierPlanIds) + val payloadMap = carrierPlanIds.map(planId => planId -> plans(planId)).toMap - FreightCarrier(carrierId, tourMap, payloadMap, vehicleMap, plansPerTour) + FreightCarrier( + carrierId, + tourMap, + payloadMap, + vehicleMap, + plansPerTour, + warehouseLocationZone, + warehouseLocationUTM + ) } + val errors: ListBuffer[ErrorInfo] = ListBuffer() + val maybeCarrierRows = GenericCsvReader.readAsSeq[Option[FreightCarrierRow]](config.carriersFilePath) { row => def get(key: String): String = getRowValue(config.carriersFilePath, row, key) //carrierId,tourId,vehicleId,vehicleTypeId,warehouseZone,warehouseX,warehouseY @@ -208,20 +263,41 @@ class GenericFreightReader( logger.error(f"Following freight carrier row discarded because tour $tourId was filtered out: $row") None } else { - extractCoordOrTaz(row.get("warehouseX"), row.get("warehouseY"), row.get("warehouseZone")) match { - case (warehouseZoneMaybe, Some(warehouseUTMOnMap)) => - Some(FreightCarrierRow(carrierId, tourId, vehicleId, vehicleTypeId, warehouseZoneMaybe, warehouseUTMOnMap)) - case (_, warehouseXYMaybe) => - logger.error( - f"Following freight carrier row discarded because warehouse location ($warehouseXYMaybe) is not reachable: $row" + val warehouseX = row.get("warehouseX") + val warehouseY = row.get("warehouseY") + + extractCoordOrTaz( + row.get("warehouseX"), + row.get("warehouseY"), + row.get("warehouseZone"), + snapLocationAndRemoveInvalidInputs + ) match { + case (warehouseZoneMaybe, Right(coord)) => + Some(FreightCarrierRow(carrierId, tourId, vehicleId, vehicleTypeId, warehouseZoneMaybe, coord)) + case (_, Left(error)) => + errors.append( + ErrorInfo( + carrierId.toString, + Category.FreightCarrier, + error, + warehouseX.toDouble, + warehouseY.toDouble + ) ) None } } } + outputDirMaybe.foreach { path => + if (errors.isEmpty) logger.info("No 'snap location' error to report for freight carriers.") + else SnapCoordinateUtils.writeToCsv(s"$path/${CsvFile.FreightCarriers}", errors) + } + + val removedCarrierIds = errors.map(_.id) val carriersWithFleet = maybeCarrierRows.flatten .groupBy(_.carrierId) + .filterNot { case (carrierId, _) => removedCarrierIds.contains(carrierId.toString) } .map { case (carrierId, carrierRows) => createCarrier(carrierId, carrierRows) } @@ -235,38 +311,41 @@ class GenericFreightReader( case None => throw new IllegalArgumentException(s"Cannot find taz with id $tazId") } - private def getDistributedTazLocation(taz: TAZ): Coord = - convertedLocation(TAZTreeMap.randomLocationInTAZ(taz, rnd)) - - private def getLocationFromCoord(location: Coord): Option[Coord] = { - closestUTMPointOnMapMaybe.map(_.find(location, geoUtils)).getOrElse(Some(geoUtils.wgs2Utm(location))) - } - - private def extractCoordOrTaz(strX: String, strY: String, strZone: String): (Option[Id[TAZ]], Option[Coord]) = { + private def extractCoordOrTaz( + strX: String, + strY: String, + strZone: String, + snapLocationAndRemoveInvalidInputs: Boolean + ): (Option[Id[TAZ]], SnapCoordinateResult) = { if (isBlank(strX) || isBlank(strY)) { val taz = getTaz(strZone) - (Some(taz.tazId), Some(getDistributedTazLocation(taz))) + val coord = + if (snapLocationAndRemoveInvalidInputs) TAZTreeMap.randomLocationInTAZ(taz, rnd, snapLocationHelper) + else TAZTreeMap.randomLocationInTAZ(taz, rnd) + + (Some(taz.tazId), Right(coord)) } else { - (None, getLocationFromCoord(location(strX.toDouble, strY.toDouble))) + val wasInWgs = config.convertWgs2Utm + val loc = location(strX.toDouble, strY.toDouble) + val coord = + if (snapLocationAndRemoveInvalidInputs) snapLocationHelper.computeResult(loc, wasInWgs) + else Right(loc) + + (None, coord) } } @Override - def createPersonId(vehicleId: Id[BeamVehicle]): Id[Person] = { - if (vehicleId.toString.startsWith(freightIdPrefix)) { - Id.createPersonId(s"$vehicleId-agent") - } else { - Id.createPersonId(s"$freightIdPrefix-$vehicleId-agent") - } + def createPersonId(carrierId: Id[FreightCarrier], vehicleId: Id[BeamVehicle]): Id[Person] = { + val updatedCarrierId = carrierId.toString.replace(freightIdPrefix + "-", "") + val updatedVehicleId = vehicleId.toString.replace(freightIdPrefix + "-", "") + Id.createPersonId(s"$freightIdPrefix-$updatedCarrierId-$updatedVehicleId-agent") } @Override - def createHouseholdId(vehicleId: Id[BeamVehicle]): Id[Household] = { - if (vehicleId.toString.startsWith(freightIdPrefix)) { - s"$vehicleId-household".createId - } else { - s"$freightIdPrefix-$vehicleId-household".createId - } + def createHouseholdId(carrierId: Id[FreightCarrier]): Id[Household] = { + val updatedCarrierId = carrierId.toString.replace(freightIdPrefix + "-", "") + s"$freightIdPrefix-$updatedCarrierId-household".createId } } diff --git a/src/main/scala/beam/agentsim/agents/household/HouseholdActor.scala b/src/main/scala/beam/agentsim/agents/household/HouseholdActor.scala index a8aebf4c673..a254d417868 100755 --- a/src/main/scala/beam/agentsim/agents/household/HouseholdActor.scala +++ b/src/main/scala/beam/agentsim/agents/household/HouseholdActor.scala @@ -11,7 +11,6 @@ import beam.agentsim.agents.modalbehaviors.ChoosesMode.{CavTripLegsRequest, CavT import beam.agentsim.agents.modalbehaviors.DrivesVehicle.VehicleOrToken import beam.agentsim.agents.modalbehaviors.ModeChoiceCalculator import beam.agentsim.agents.planning.BeamPlan -import beam.agentsim.agents.planning.BeamPlan.atHome import beam.agentsim.agents.ridehail.RideHailAgent.{ ModifyPassengerSchedule, ModifyPassengerScheduleAck, @@ -19,7 +18,7 @@ import beam.agentsim.agents.ridehail.RideHailAgent.{ } import beam.agentsim.agents.ridehail.RideHailManager.RoutingResponses import beam.agentsim.agents.vehicles.EnergyEconomyAttributes.Powertrain -import beam.agentsim.agents.vehicles.VehicleCategory.{Bike, Car, VehicleCategory} +import beam.agentsim.agents.vehicles.VehicleCategory.{VehicleCategory, _} import beam.agentsim.agents.vehicles._ import beam.agentsim.events.SpaceTime import beam.agentsim.infrastructure.{ParkingInquiry, ParkingInquiryResponse} @@ -183,11 +182,16 @@ object HouseholdActor { private var members: Map[Id[Person], PersonIdWithActorRef] = Map() + private val isFreightCarrier: Boolean = household.getId.toString.toLowerCase.contains("freight") + // Data need to execute CAV dispatch private val cavPlans: mutable.ListBuffer[CAVSchedule] = mutable.ListBuffer() private var cavPassengerSchedules: Map[BeamVehicle, PassengerSchedule] = Map() private var personAndActivityToCav: Map[(Id[Person], Activity), BeamVehicle] = Map() private var personAndActivityToLegs: Map[(Id[Person], Activity), List[BeamLeg]] = Map() + private val householdVehicleCategories = List(Car, Bike) + + private var whoDrivesThisVehicle: Map[Id[BeamVehicle], Id[Person]] = Map() private val realDistribution: UniformRealDistribution = new UniformRealDistribution() realDistribution.reseedRandomGenerator(beamScenario.beamConfig.matsim.modules.global.randomSeed) @@ -195,40 +199,47 @@ object HouseholdActor { override def loggedReceive: Receive = { case TriggerWithId(InitializeTrigger(tick), triggerId) => + val vehiclesByCategory = + vehicles.filter(_._2.beamVehicleType.automationLevel <= 3).groupBy(_._2.beamVehicleType.vehicleCategory) + var vehiclesByAllCategories = vehiclesByCategory + if (!isFreightCarrier) { + //We should create a vehicle manager for cars and bikes for all households in case they are generated during the simulation + vehiclesByAllCategories = householdVehicleCategories + .map(cat => cat -> Map[Id[BeamVehicle], BeamVehicle]()) + .toMap ++ vehiclesByCategory + } val homeCoordFromPlans = household.members - .flatMap(person => - person.getSelectedPlan.getPlanElements.asScala.flatMap { - case act: Activity if atHome(act) => Some(act.getCoord) - case _ => None + .flatMap { person => + if (isFreightCarrier) { + val vehicleIdFromPlans = Id.create( + beamServices.matsimServices.getScenario.getPopulation.getPersonAttributes + .getAttribute(person.getId.toString, "vehicle") + .toString, + classOf[BeamVehicle] + ) + whoDrivesThisVehicle = whoDrivesThisVehicle + (vehicleIdFromPlans -> person.getId) } - ) + person.getSelectedPlan.getPlanElements.asScala.headOption.flatMap { + case act: Activity if isFreightCarrier && act.getType == "Warehouse" => Some(act.getCoord) + case act: Activity if !isFreightCarrier && act.getType == "Home" => Some(act.getCoord) + case _ => None + } + } .headOption .getOrElse(fallbackHomeCoord) - val vehiclesByCategory = - vehicles.filter(_._2.beamVehicleType.automationLevel <= 3).groupBy(_._2.beamVehicleType.vehicleCategory) - - //We should create a vehicle manager for cars and bikes for all households in case they are generated during the simulation - - val vehiclesByAllCategories = (List(Car, Bike) ++ vehiclesByCategory.keys).distinct - .map(cat => cat -> vehiclesByCategory.getOrElse(cat, Map[Id[BeamVehicle], BeamVehicle]())) - .toMap - val fleetManagers = vehiclesByAllCategories.map { case (category, vehicleMap) => - val emergencyGenerator = new EmergencyHouseholdVehicleGenerator( - household, - homeCoordFromPlans, - beamScenario, - vehiclesAdjustment, - category - ) + val fleetManagers = vehiclesByAllCategories.map { case (category, vehiclesInCategory) => + val emergencyGenerator = + new EmergencyHouseholdVehicleGenerator(household, beamScenario, vehiclesAdjustment, category) val fleetManager = context.actorOf( Props( new HouseholdFleetManager( parkingManager, - vehicleMap, + vehiclesInCategory, homeCoordFromPlans, Some(emergencyGenerator), + whoDrivesThisVehicle, beamServices.beamConfig.beam.debug ) ), @@ -540,7 +551,6 @@ object HouseholdActor { class EmergencyHouseholdVehicleGenerator( household: Household, - homeCoordFromPlans: Coord, beamScenario: BeamScenario, vehiclesAdjustment: VehiclesAdjustment, defaultCategory: VehicleCategory @@ -551,57 +561,69 @@ object HouseholdActor { private val generateEmergencyHousehold = beamScenario.beamConfig.beam.agentsim.agents.vehicles.generateEmergencyHouseholdVehicleWhenPlansRequireIt - def createVehicle(personId: Id[Person], vehicleIndex: Int, category: VehicleCategory): Option[BeamVehicle] = { - val vehicleTypeMaybe = - if (generateEmergencyHousehold && defaultCategory == category) { - category match { - case VehicleCategory.Car => - vehiclesAdjustment - .sampleVehicleTypesForHousehold( - 1, - VehicleCategory.Car, - household.getIncome.getIncome, - household.getMemberIds.size(), - householdPopulation = null, - homeCoordFromPlans, - realDistribution - ) - .headOption - .orElse { - beamScenario.vehicleTypes.get( - Id.create( - beamScenario.beamConfig.beam.agentsim.agents.vehicles.dummySharedCar.vehicleTypeId, - classOf[BeamVehicleType] - ) - ) - } - case VehicleCategory.Bike => - beamScenario.vehicleTypes - .get( + def sampleVehicleTypeForEmergencyUse( + personId: Id[Person], + category: VehicleCategory, + whenWhere: SpaceTime + ): Option[BeamVehicleType] = { + if (generateEmergencyHousehold && defaultCategory == category) { + category match { + case VehicleCategory.Car => + vehiclesAdjustment + .sampleVehicleTypesForHousehold( + 1, + VehicleCategory.Car, + household.getIncome.getIncome, + household.getMemberIds.size(), + householdPopulation = null, + whenWhere.loc, + realDistribution + ) + .headOption + .orElse { + beamScenario.vehicleTypes.get( Id.create( - beamScenario.beamConfig.beam.agentsim.agents.vehicles.dummySharedBike.vehicleTypeId, + beamScenario.beamConfig.beam.agentsim.agents.vehicles.dummySharedCar.vehicleTypeId, classOf[BeamVehicleType] ) ) - case _ => - logger.warn( - s"Person $personId is requiring a vehicle that belongs to category $category that is neither Car nor Bike" + } + case VehicleCategory.Bike => + beamScenario.vehicleTypes + .get( + Id.create( + beamScenario.beamConfig.beam.agentsim.agents.vehicles.dummySharedBike.vehicleTypeId, + classOf[BeamVehicleType] + ) ) - None - } - } else None - vehicleTypeMaybe map { vehicleType => - val vehicle = new BeamVehicle( - Id.createVehicleId(personId.toString + "-emergency-" + vehicleIndex), - new Powertrain(vehicleType.primaryFuelConsumptionInJoulePerMeter), - vehicleType - ) - beamScenario.privateVehicles.put(vehicle.id, vehicle) - vehicle.initializeFuelLevelsFromUniformDistribution( - beamScenario.beamConfig.beam.agentsim.agents.vehicles.meanPrivateVehicleStartingSOC - ) - vehicle - } + case _ => + logger.warn( + s"Person $personId is requiring a vehicle that belongs to category $category that is neither Car nor Bike" + ) + None + } + } else None + } + + def createAndAddVehicle( + vehicleType: BeamVehicleType, + personId: Id[Person], + vehicleIndex: Int, + whenWhere: SpaceTime, + manager: ActorRef + ): BeamVehicle = { + val vehicle = new BeamVehicle( + Id.createVehicleId(personId.toString + "-emergency-" + vehicleIndex), + new Powertrain(vehicleType.primaryFuelConsumptionInJoulePerMeter), + vehicleType + ) + vehicle.initializeFuelLevelsFromUniformDistribution( + beamScenario.beamConfig.beam.agentsim.agents.vehicles.meanPrivateVehicleStartingSOC + ) + beamScenario.privateVehicles.put(vehicle.id, vehicle) + vehicle.setManager(Some(manager)) + vehicle.spaceTime = whenWhere + vehicle } } } diff --git a/src/main/scala/beam/agentsim/agents/household/HouseholdFleetManager.scala b/src/main/scala/beam/agentsim/agents/household/HouseholdFleetManager.scala index bf92ad135e1..98aa94799d1 100644 --- a/src/main/scala/beam/agentsim/agents/household/HouseholdFleetManager.scala +++ b/src/main/scala/beam/agentsim/agents/household/HouseholdFleetManager.scala @@ -10,7 +10,6 @@ import beam.agentsim.agents.InitializeTrigger import beam.agentsim.agents.household.HouseholdActor._ import beam.agentsim.agents.household.HouseholdFleetManager.ResolvedParkingResponses import beam.agentsim.agents.modalbehaviors.DrivesVehicle.ActualVehicle -import beam.agentsim.agents.planning.BeamPlan.atHome import beam.agentsim.agents.vehicles.BeamVehicle import beam.agentsim.events.SpaceTime import beam.agentsim.infrastructure.{ParkingInquiry, ParkingInquiryResponse} @@ -20,6 +19,7 @@ import beam.agentsim.scheduler.Trigger.TriggerWithId import beam.sim.config.BeamConfig.Beam.Debug import beam.utils.logging.pattern.ask import beam.utils.logging.{ExponentialLazyLogging, LoggingMessageActor} +import org.matsim.api.core.v01.population.Person import org.matsim.api.core.v01.{Coord, Id} import java.util.concurrent.TimeUnit @@ -30,6 +30,7 @@ class HouseholdFleetManager( vehicles: Map[Id[BeamVehicle], BeamVehicle], homeCoord: Coord, maybeEmergencyHouseholdVehicleGenerator: Option[EmergencyHouseholdVehicleGenerator], + whoDrivesThisVehicle: Map[Id[BeamVehicle], Id[Person]], // so far only freight module is using this collection implicit val debug: Debug ) extends LoggingMessageActor with ExponentialLazyLogging { @@ -105,71 +106,31 @@ class HouseholdFleetManager( case GetVehicleTypes(triggerId) => sender() ! VehicleTypesResponse(vehicles.values.map(_.beamVehicleType).toSet, triggerId) - case MobilityStatusInquiry(personId, whenWhere, originActivity, requireVehicleCategoryAvailable, triggerId) => - { - for { - neededVehicleCategory <- requireVehicleCategoryAvailable - emergencyHouseholdVehicleGenerator <- maybeEmergencyHouseholdVehicleGenerator - vehicle <- emergencyHouseholdVehicleGenerator.createVehicle(personId, nextVehicleIndex, neededVehicleCategory) - } yield { - val vehicleCreatedOutOfThinAir: Boolean = if (availableVehicles.isEmpty) { - logger.warn( - s"No vehicles available for category ${neededVehicleCategory} available for person ${personId.toString}, creating a new vehicle with id ${vehicle.id.toString}" - ) - emergencyHouseholdVehicleGenerator.createVehicle(personId, nextVehicleIndex, neededVehicleCategory) match { - case Some(vehicle) => - nextVehicleIndex += 1 - vehicle.setManager(Some(self)) - vehicle.spaceTime = whenWhere - val mobilityRequester = sender() - vehiclesInternal(vehicle.id) = vehicle - - // Pipe my car through the parking manager - // and complete initialization only when I got them all. - val responseFuture = parkingManager ? ParkingInquiry.init( - whenWhere, - "wherever", - triggerId = triggerId - ) - responseFuture.collect { case ParkingInquiryResponse(stall, _, otherTriggerId) => - vehicle.useParkingStall(stall) - logger.debug("Vehicle {} is now taken, which was just created", vehicle.id) - vehicle.becomeDriver(mobilityRequester) - MobilityStatusResponse(Vector(ActualVehicle(vehicle)), otherTriggerId) - } pipeTo mobilityRequester - true - case _ => - false - } - } else false - - if (!vehicleCreatedOutOfThinAir) { - availableVehicles = availableVehicles match { - case firstVehicle :: rest => - logger.debug("Vehicle {} is now taken", firstVehicle.id) - firstVehicle.becomeDriver(sender) - sender() ! MobilityStatusResponse(Vector(ActualVehicle(firstVehicle)), triggerId) - rest - case _ => - logger.error(s"THE LIST OF VEHICLES SHOULDN'T BE EMPTY") - availableVehicles - } - } - } - }.getOrElse { - availableVehicles = availableVehicles match { - //in case of replanning because of TRANSIT failure WALK_TRANSIT is used - //but we may want to introduce maxWalkingDistance and check that the agent is close enough to the vehicle - case firstVehicle :: rest if atHome(originActivity) => - logger.debug("Vehicle {} is now taken", firstVehicle.id) - firstVehicle.becomeDriver(sender) - sender() ! MobilityStatusResponse(Vector(ActualVehicle(firstVehicle)), triggerId) - rest - case _ => - logger.debug(s"Not returning vehicle because no default is defined or agent is not at home") - sender() ! MobilityStatusResponse(Vector(), triggerId) - availableVehicles - } + case inquiry @ MobilityStatusInquiry(personId, _, _, requireVehicleCategoryAvailable, triggerId) => + val availableVehicleMaybe: Option[BeamVehicle] = requireVehicleCategoryAvailable match { + case Some(_) if personId.toString.contains("freight") => + whoDrivesThisVehicle + .filter(_._2 == personId) + .flatMap { case (vehicleId, _) => availableVehicles.find(_.id == vehicleId) } + .headOption + case Some(requireVehicleCategory) => + availableVehicles.find(_.beamVehicleType.vehicleCategory == requireVehicleCategory) + case _ => availableVehicles.headOption + } + + availableVehicleMaybe match { + case Some(availableVehicle) => + logger.debug("Vehicle {} is now taken", availableVehicle.id) + availableVehicle.becomeDriver(sender) + sender() ! MobilityStatusResponse(Vector(ActualVehicle(availableVehicle)), triggerId) + availableVehicles = availableVehicles.filter(_ != availableVehicle) + case None if createAnEmergencyVehicle(inquiry).nonEmpty => + logger.debug(s"An emergency vehicle has been created!") + case _ => + if (availableVehicles.isEmpty) + logger.error(s"THE LIST OF VEHICLES SHOULD NOT BE EMPTY") + logger.debug(s"Not returning vehicle because no default for is defined") + sender() ! MobilityStatusResponse(Vector(), triggerId) } case Finish => @@ -182,6 +143,50 @@ class HouseholdFleetManager( case x => logger.warn(s"No handler for $x") } + + /** + * @param inquiry + * @return + */ + private def createAnEmergencyVehicle(inquiry: MobilityStatusInquiry): Option[BeamVehicle] = { + for { + category <- inquiry.requireVehicleCategoryAvailable + emergency <- maybeEmergencyHouseholdVehicleGenerator + vehicleType <- emergency.sampleVehicleTypeForEmergencyUse(inquiry.personId, category, inquiry.whereWhen) + } yield { + val vehicle = emergency.createAndAddVehicle( + vehicleType, + inquiry.personId, + nextVehicleIndex, + inquiry.whereWhen, + self + ) + // Create a vehicle out of thin air + nextVehicleIndex += 1 + val mobilityRequester = sender() + vehiclesInternal(vehicle.id) = vehicle + + // Pipe my car through the parking manager + // and complete initialization only when I got them all. + val responseFuture = parkingManager ? ParkingInquiry.init( + inquiry.whereWhen, + "wherever", + triggerId = inquiry.triggerId + ) + logger.warn( + s"No vehicles available for category ${category} available for " + + s"person ${inquiry.personId.toString}, creating a new vehicle with id ${vehicle.id.toString}" + ) + + responseFuture.collect { case ParkingInquiryResponse(stall, _, otherTriggerId) => + vehicle.useParkingStall(stall) + logger.debug("Vehicle {} is now taken, which was just created", vehicle.id) + vehicle.becomeDriver(mobilityRequester) + MobilityStatusResponse(Vector(ActualVehicle(vehicle)), otherTriggerId) + } pipeTo mobilityRequester + vehicle + } + } } object HouseholdFleetManager { diff --git a/src/main/scala/beam/agentsim/agents/modalbehaviors/ChoosesMode.scala b/src/main/scala/beam/agentsim/agents/modalbehaviors/ChoosesMode.scala index 325ef3bad41..94a64da73f7 100755 --- a/src/main/scala/beam/agentsim/agents/modalbehaviors/ChoosesMode.scala +++ b/src/main/scala/beam/agentsim/agents/modalbehaviors/ChoosesMode.scala @@ -11,8 +11,8 @@ import beam.agentsim.agents.modalbehaviors.DrivesVehicle.{ActualVehicle, Token, import beam.agentsim.agents.planning.Strategy.ModeChoiceStrategy import beam.agentsim.agents.ridehail.{RideHailInquiry, RideHailRequest, RideHailResponse} import beam.agentsim.agents.vehicles.AccessErrorCodes.RideHailNotRequestedError -import beam.agentsim.agents.vehicles.VehicleCategory.VehicleCategory import beam.agentsim.agents.vehicles.EnergyEconomyAttributes.Powertrain +import beam.agentsim.agents.vehicles.VehicleCategory.VehicleCategory import beam.agentsim.agents.vehicles.VehicleProtocol.StreetVehicle import beam.agentsim.agents.vehicles._ import beam.agentsim.events.{ModeChoiceEvent, SpaceTime} diff --git a/src/main/scala/beam/agentsim/agents/parking/ChoosesParking.scala b/src/main/scala/beam/agentsim/agents/parking/ChoosesParking.scala index 77a1b5e072e..b043b848de8 100755 --- a/src/main/scala/beam/agentsim/agents/parking/ChoosesParking.scala +++ b/src/main/scala/beam/agentsim/agents/parking/ChoosesParking.scala @@ -11,12 +11,12 @@ import beam.agentsim.agents.freight.input.FreightReader.FREIGHT_REQUEST_TYPE import beam.agentsim.agents.modalbehaviors.DrivesVehicle.StartLegTrigger import beam.agentsim.agents.parking.ChoosesParking._ import beam.agentsim.agents.vehicles.VehicleProtocol.StreetVehicle -import beam.agentsim.agents.vehicles.{BeamVehicle, PassengerSchedule, VehicleCategory, VehicleManager} +import beam.agentsim.agents.vehicles.{BeamVehicle, PassengerSchedule, VehicleManager} import beam.agentsim.events.{LeavingParkingEvent, ParkingEvent, SpaceTime} import beam.agentsim.infrastructure.ChargingNetworkManager._ +import beam.agentsim.infrastructure.ParkingInquiry.ParkingSearchMode import beam.agentsim.infrastructure.charging.{ChargingPointType, ElectricCurrentType} import beam.agentsim.infrastructure.parking.PricingModel -import beam.agentsim.infrastructure.ParkingInquiry.ParkingSearchMode import beam.agentsim.infrastructure.taz.TAZTreeMap import beam.agentsim.infrastructure.{ParkingInquiry, ParkingInquiryResponse, ParkingStall} import beam.agentsim.scheduler.BeamAgentScheduler.{CompletionNotice, ScheduleTrigger} diff --git a/src/main/scala/beam/agentsim/infrastructure/ChargingNetwork.scala b/src/main/scala/beam/agentsim/infrastructure/ChargingNetwork.scala index bca09474253..52b876c3a04 100644 --- a/src/main/scala/beam/agentsim/infrastructure/ChargingNetwork.scala +++ b/src/main/scala/beam/agentsim/infrastructure/ChargingNetwork.scala @@ -26,11 +26,13 @@ import scala.util.Random */ class ChargingNetwork[GEO: GeoLevel](val chargingZones: Map[Id[ParkingZoneId], ParkingZone[GEO]]) extends ParkingNetwork[GEO](chargingZones) { - import ChargingNetwork._ override protected val searchFunctions: Option[InfrastructureFunctions[GEO]] = None + protected val beamVehicleIdToChargingVehicleMap: mutable.HashMap[Id[BeamVehicle], ChargingVehicle] = + mutable.HashMap.empty + protected val chargingZoneKeyToChargingStationMap: Map[Id[ParkingZoneId], ChargingStation] = chargingZones.map { case (zoneId, zone) => zoneId -> ChargingStation(zone) } @@ -68,7 +70,7 @@ class ChargingNetwork[GEO: GeoLevel](val chargingZones: Map[Id[ParkingZoneId], P * @return charging vehicle */ def lookupVehicle(vehicleId: Id[BeamVehicle]): Option[ChargingVehicle] = - chargingZoneKeyToChargingStationMap.values.view.flatMap(_.lookupVehicle(vehicleId)).headOption + beamVehicleIdToChargingVehicleMap.get(vehicleId) /** * clear charging vehicle map @@ -87,8 +89,8 @@ class ChargingNetwork[GEO: GeoLevel](val chargingZones: Map[Id[ParkingZoneId], P activityType: String, theSender: ActorRef ): Option[ChargingVehicle] = lookupStation(request.stall.parkingZoneId) - .map( - _.connect( + .map { chargingStation => + val chargingVehicle = chargingStation.connect( request.tick, request.vehicle, request.stall, @@ -98,7 +100,9 @@ class ChargingNetwork[GEO: GeoLevel](val chargingZones: Map[Id[ParkingZoneId], P request.shiftDuration, theSender ) - ) + beamVehicleIdToChargingVehicleMap.put(chargingVehicle.vehicle.id, chargingVehicle) + chargingVehicle + } /** * @param vehicleId vehicle to end charge @@ -121,6 +125,7 @@ class ChargingNetwork[GEO: GeoLevel](val chargingZones: Map[Id[ParkingZoneId], P */ def disconnectVehicle(vehicleId: Id[BeamVehicle], tick: Int): Option[ChargingVehicle] = { lookupVehicle(vehicleId) map { chargingVehicle => + beamVehicleIdToChargingVehicleMap.remove(vehicleId) chargingVehicle.chargingStation.disconnect(chargingVehicle.vehicle.id, tick) } getOrElse { logger.debug(s"Vehicle $vehicleId is already disconnected") diff --git a/src/main/scala/beam/agentsim/infrastructure/parking/ParkingZoneSearch.scala b/src/main/scala/beam/agentsim/infrastructure/parking/ParkingZoneSearch.scala index 7749313afe3..d8738f5f720 100644 --- a/src/main/scala/beam/agentsim/infrastructure/parking/ParkingZoneSearch.scala +++ b/src/main/scala/beam/agentsim/infrastructure/parking/ParkingZoneSearch.scala @@ -289,7 +289,7 @@ object ParkingZoneSearch { override def lookupParkingZonesInNextSearchAreaUnlessThresholdReached( zoneQuadTree: QuadTree[GEO] ): Option[List[GEO]] = { - if (thisInnerDistance > maxDistance) None + if (thisInnerDistance >= maxDistance) None else { val result = zoneQuadTree .getElliptical(originUTM.getX, originUTM.getY, destinationUTM.getX, destinationUTM.getY, thisInnerDistance) diff --git a/src/main/scala/beam/agentsim/infrastructure/taz/TAZTreeMap.scala b/src/main/scala/beam/agentsim/infrastructure/taz/TAZTreeMap.scala index 98fcdedfecb..679ea79d2eb 100755 --- a/src/main/scala/beam/agentsim/infrastructure/taz/TAZTreeMap.scala +++ b/src/main/scala/beam/agentsim/infrastructure/taz/TAZTreeMap.scala @@ -1,9 +1,6 @@ package beam.agentsim.infrastructure.taz -import java.io._ -import java.util -import scala.collection.JavaConverters._ -import scala.collection.mutable +import beam.utils.SnapCoordinateUtils.SnapLocationHelper import beam.utils.matsim_conversion.ShapeUtils import beam.utils.matsim_conversion.ShapeUtils.{HasQuadBounds, QuadTreeBounds} import com.vividsolutions.jts.geom.Geometry @@ -13,8 +10,12 @@ import org.matsim.core.utils.gis.ShapeFileReader import org.opengis.feature.simple.SimpleFeature import org.slf4j.LoggerFactory +import java.io._ +import java.util import scala.annotation.tailrec +import scala.collection.JavaConverters._ import scala.collection.concurrent.TrieMap +import scala.collection.mutable /** * TAZTreeMap manages a quadTree to find the closest TAZ to any coordinate. @@ -197,7 +198,7 @@ object TAZTreeMap { def randomLocationInTAZ( taz: TAZ, - rand: scala.util.Random = new scala.util.Random(System.currentTimeMillis()) + rand: scala.util.Random ): Coord = { val radius = Math.sqrt(taz.areaInSquareMeters / Math.PI) / 2 val a = 2 * Math.PI * rand.nextDouble() @@ -207,6 +208,35 @@ object TAZTreeMap { new Coord(taz.coord.getX + x, taz.coord.getY + y) } + def randomLocationInTAZ( + taz: TAZ, + rand: scala.util.Random, + snapLocationHelper: SnapLocationHelper + ): Coord = { + val tazId = taz.tazId.toString + val max = 10000 + var counter = 0 + var split: Coord = null + while (split == null && counter < max) { + snapLocationHelper.computeResult(randomLocationInTAZ(taz, rand)) match { + case Right(splitCoord) => + split = splitCoord + case _ => + } + counter += 1 + } + + if (split == null) { + val loc = randomLocationInTAZ(taz, rand) + logger.warn( + s"Could not found valid location within taz $tazId even in $max attempts. Creating one anyway $loc." + ) + split = loc + } + + split + } + /** * performs a concentric ring search from the present location to find elements up to the SearchMaxRadius * @param quadTree tree to search diff --git a/src/main/scala/beam/physsim/analysis/LinkStatsWithVehicleCategory.scala b/src/main/scala/beam/physsim/analysis/LinkStatsWithVehicleCategory.scala new file mode 100644 index 00000000000..9e146b40ea6 --- /dev/null +++ b/src/main/scala/beam/physsim/analysis/LinkStatsWithVehicleCategory.scala @@ -0,0 +1,95 @@ +package beam.physsim.analysis + +import beam.utils.BeamCalcLinkStats +import beam.utils.BeamCalcLinkStats.LinkData +import beam.utils.csv.CsvWriter +import org.matsim.analysis.VolumesAnalyzer +import org.matsim.api.core.v01.Id +import org.matsim.api.core.v01.network.{Link, Network} +import org.matsim.core.config.groups.TravelTimeCalculatorConfigGroup +import org.matsim.core.router.util.TravelTime + +import scala.collection.JavaConverters._ +import scala.util.Try + +/** + * @author Dmitry Openkov + */ +class LinkStatsWithVehicleCategory( + network: Network, + ttConfigGroup: TravelTimeCalculatorConfigGroup +) { + + def calculateLinkData( + volumesAnalyzer: VolumesAnalyzer, + travelTimeForR5: TravelTime, + categories: IndexedSeq[String] + ): (Map[Id[Link], LinkData], Map[String, Map[Id[Link], LinkData]], Int) = { + val calc = new BeamCalcLinkStats(network, ttConfigGroup) + calc.reset() + calc.addData(volumesAnalyzer, travelTimeForR5) + val totalLinkData = calc.getLinkData.asScala.toMap + val linkData = categories.map { category => + calc.reset() + calc.addData(volumesAnalyzer, travelTimeForR5, category) + category -> calc.getLinkData.asScala.toMap + }.toMap + (totalLinkData, linkData, calc.getNofHours) + } + + def writeToFile( + totalLinkData: Map[Id[Link], BeamCalcLinkStats.LinkData], + linkData: Map[String, Map[Id[Link], BeamCalcLinkStats.LinkData]], + nofHours: Int, + aggregation: Seq[(Seq[String], String)], + filePath: String + ): Try[Unit] = { + val header = Seq("link", "from", "to", "hour", "length", "freespeed", "capacity", "stat", "volume") ++ + aggregation.map(_._2) :+ "traveltime" + val csvWriter = new CsvWriter(filePath, header) + val rows = totalLinkData.view.flatMap { case (linkId, data) => + val link = network.getLinks.get(linkId) + for { + hour <- 0 until nofHours + } yield { + val categories = aggregation.flatMap(_._1).distinct + val categoryToVolume = categories.map { category => + category -> linkData(category).get(linkId).map(_.getSumVolume(hour)).getOrElse(0.0) + }.toMap + val aggregatedVolumes = aggregation.map { case (categoryGroup, _) => + categoryGroup.map(category => categoryToVolume(category)).sum + } + val categoryVolumeSum = categoryToVolume.values.reduceOption(_ + _).getOrElse(0.0) + val otherVolume = totalLinkData.get(linkId).map(_.getSumVolume(hour)).getOrElse(0.0) - categoryVolumeSum + Seq( + linkId, + link.getFromNode.getId, + link.getToNode.getId, + hour, + link.getLength, + link.getFreespeed, + link.getCapacity, + "AVG", + otherVolume + ) ++ aggregatedVolumes ++ Seq(data.calculateAverageTravelTime(hour)) + } + + } + csvWriter.writeAllAndClose(rows) + } + + def writeLinkStatsWithTruckVolumes( + volumesAnalyzer: VolumesAnalyzer, + travelTimeForR5: TravelTime, + filePath: String + ): Try[(Map[Id[Link], LinkData], Map[String, Map[Id[Link], LinkData]], Int)] = { + val categoryMapping = IndexedSeq( + Seq("LightDutyTruck", "HeavyDutyTruck") -> "TruckVolume", + Seq("HeavyDutyTruck") -> "HDTruckVolume" + ) + val categories = categoryMapping.flatMap(_._1).distinct + val (totalLinkData, linkData, nofHours) = calculateLinkData(volumesAnalyzer, travelTimeForR5, categories) + writeToFile(totalLinkData, linkData, nofHours, categoryMapping, filePath) + .map(_ => (totalLinkData, linkData, nofHours)) + } +} diff --git a/src/main/scala/beam/physsim/jdeqsim/JDEQSimRunner.scala b/src/main/scala/beam/physsim/jdeqsim/JDEQSimRunner.scala index 7a594f4df57..8a8db0352eb 100644 --- a/src/main/scala/beam/physsim/jdeqsim/JDEQSimRunner.scala +++ b/src/main/scala/beam/physsim/jdeqsim/JDEQSimRunner.scala @@ -63,7 +63,8 @@ class JDEQSimRunner( controlerIO, beamServices.beamConfig, jdeqSimScenario.getConfig.travelTimeCalculator, - beamConfigChangesObservable + beamConfigChangesObservable, + beamServices.beamScenario.privateVehicles.view.toMap.asJava ) linkStatsGraph.notifyIterationStarts(jdeqsimEvents, jdeqSimScenario.getConfig.travelTimeCalculator) diff --git a/src/main/scala/beam/replanning/SupplementaryTripGenerator.scala b/src/main/scala/beam/replanning/SupplementaryTripGenerator.scala index 92d1a6d03f5..8db4efb2eef 100755 --- a/src/main/scala/beam/replanning/SupplementaryTripGenerator.scala +++ b/src/main/scala/beam/replanning/SupplementaryTripGenerator.scala @@ -8,6 +8,7 @@ import beam.router.Modes.BeamMode import beam.router.Modes.BeamMode.{CAR, CAV, RIDE_HAIL, RIDE_HAIL_POOLED, WALK, WALK_TRANSIT} import beam.sim.BeamServices import beam.sim.population.AttributesOfIndividual +import beam.utils.SnapCoordinateUtils.SnapLocationHelper import org.matsim.api.core.v01.population.{Activity, Person, Plan} import org.matsim.api.core.v01.{Coord, Id} import org.matsim.core.population.PopulationUtils @@ -22,9 +23,10 @@ class SupplementaryTripGenerator( val attributesOfIndividual: AttributesOfIndividual, val destinationChoiceModel: DestinationChoiceModel, val beamServices: BeamServices, - val personId: Id[Person] + val personId: Id[Person], + val snapLocationHelper: SnapLocationHelper ) { - val r: Random.type = scala.util.Random + val rnd: Random = new scala.util.Random(System.currentTimeMillis()) val personSpecificSeed: Long = personId.hashCode().toLong val travelTimeBufferInSec: Int = 30 * 60 @@ -186,8 +188,8 @@ class SupplementaryTripGenerator( false -> noTrip ) - tripMNL.sampleAlternative(tripChoice, r) match { - case Some(mnlSample) if mnlSample.alternativeType => destinationMNL.sampleAlternative(modeChoice, r) + tripMNL.sampleAlternative(tripChoice, rnd) match { + case Some(mnlSample) if mnlSample.alternativeType => destinationMNL.sampleAlternative(modeChoice, rnd) case _ => None } } @@ -199,7 +201,7 @@ class SupplementaryTripGenerator( val newActivity = PopulationUtils.createActivityFromCoord( newActivityType, - TAZTreeMap.randomLocationInTAZ(chosenAlternative.taz) + TAZTreeMap.randomLocationInTAZ(chosenAlternative.taz, rnd, snapLocationHelper) ) val activityBeforeNewActivity = PopulationUtils.createActivityFromCoord(prevActivity.getType, prevActivity.getCoord) @@ -244,7 +246,8 @@ class SupplementaryTripGenerator( Map[SupplementaryTripAlternative, Map[SupplementaryTripAlternative, Map[DestinationParameters, Double]]]() } else { TAZs.map { taz => - val destinationCoord: Coord = TAZTreeMap.randomLocationInTAZ(taz) + val destinationCoord: Coord = + TAZTreeMap.randomLocationInTAZ(taz, rnd, snapLocationHelper) val additionalActivity = PopulationUtils.createActivityFromCoord(newActivityType, destinationCoord) additionalActivity.setStartTime(startTime) additionalActivity.setEndTime(endTime) @@ -379,7 +382,7 @@ class SupplementaryTripGenerator( } else { None } chosenStartIndex match { case Some(index) => - val startTime = math.max((r.nextDouble() + index) * 3600, altStart + travelTimeBufferInSec) + val startTime = math.max((rnd.nextDouble() + index) * 3600, altStart + travelTimeBufferInSec) ( actType, startTime.toInt, @@ -412,7 +415,7 @@ class SupplementaryTripGenerator( keyToProb: Map[A, Double] ): Option[A] = { val totalProb = keyToProb.values.sum - val randomDraw = r.nextDouble() + val randomDraw = rnd.nextDouble() val probs = keyToProb.values.scanLeft(0.0)(_ + _ / totalProb).drop(1) keyToProb.keys.zip(probs).dropWhile { _._2 <= randomDraw }.headOption match { case Some(result) => Some(result._1) diff --git a/src/main/scala/beam/router/LinkTravelTimeContainer.scala b/src/main/scala/beam/router/LinkTravelTimeContainer.scala index 1fda41de1d3..caf58b1131f 100755 --- a/src/main/scala/beam/router/LinkTravelTimeContainer.scala +++ b/src/main/scala/beam/router/LinkTravelTimeContainer.scala @@ -1,8 +1,9 @@ package beam.router -import beam.utils.FileUtils.using -import beam.utils.{FileUtils, TravelTimeCalculatorHelper} +import beam.utils.csv.GenericCsvReader +import beam.utils.{MathUtils, TravelTimeCalculatorHelper} import com.typesafe.scalalogging.LazyLogging +import org.apache.commons.lang3.StringUtils.isBlank import org.matsim.api.core.v01.network.Link import org.matsim.api.core.v01.population.Person import org.matsim.core.router.util.TravelTime @@ -10,6 +11,7 @@ import org.matsim.vehicles.Vehicle import scala.collection.JavaConverters._ import scala.collection.mutable +import scala.util.Try class LinkTravelTimeContainer(fileName: String, timeBinSizeInSeconds: Int, maxHour: Int) extends TravelTime @@ -23,34 +25,35 @@ class LinkTravelTimeContainer(fileName: String, timeBinSizeInSeconds: Int, maxHo val linkTravelTimeMap: mutable.HashMap[String, Array[Double]] = mutable.HashMap() logger.info(s"Stats fileName [$fileName] is being loaded") - using(FileUtils.readerFromFile(fileName)) { bufferedReader => - var line: String = null - while ({ - line = bufferedReader.readLine - line != null - }) { - val linkStats = line.split(",") - if (linkStats.length == 10 && "avg".equalsIgnoreCase(linkStats(7))) { - val linkId = linkStats(0) - val hour = linkStats(3).toDouble.toInt - val travelTime = linkStats(9).toDouble - linkTravelTimeMap.get(linkId) match { - case Some(travelTimePerHourArr) => - travelTimePerHourArr.update(hour, travelTime) - case None => - val travelTimePerHourArr = Array.ofDim[Double](maxHour) - travelTimePerHourArr.update(hour, travelTime) - linkTravelTimeMap.put(linkId, travelTimePerHourArr) - } - } + val (iterator, closable) = + GenericCsvReader.readAs(fileName, mapper, (row: (String, Int, Double)) => !isBlank(row._1)) + try { + iterator.foreach { case (linkId, hour, travelTime) => + val travelTimePerHourArr = linkTravelTimeMap.getOrElseUpdate(linkId, Array.ofDim[Double](maxHour)) + travelTimePerHourArr.update(hour, travelTime) } + } finally { + Try(closable.close()) } + val end = System.currentTimeMillis() logger.info("LinkTravelTimeMap is initialized in {} ms", end - start) linkTravelTimeMap } + def mapper(row: java.util.Map[String, String]): (String, Int, Double) = { + val linkId = row.get("link") + val stat = row.get("stat") + if (isBlank(linkId) || isBlank(stat) || !stat.equalsIgnoreCase("avg")) { + ("", 0, 0.0) + } else { + val hour = MathUtils.doubleToInt(row.get("hour").toDouble) + val travelTime = row.get("traveltime").toDouble + (linkId, hour, travelTime) + } + } + def getLinkTravelTime(link: Link, time: Double, person: Person, vehicle: Vehicle): Double = { travelTimeCalculator.getLinkTravelTime(link, time, person, vehicle) } diff --git a/src/main/scala/beam/router/r5/CarWeightCalculator.scala b/src/main/scala/beam/router/r5/CarWeightCalculator.scala index a97b137706c..e786ff31571 100644 --- a/src/main/scala/beam/router/r5/CarWeightCalculator.scala +++ b/src/main/scala/beam/router/r5/CarWeightCalculator.scala @@ -5,6 +5,7 @@ import org.matsim.core.router.util.TravelTime import java.util.concurrent.ThreadLocalRandom import java.util.concurrent.atomic.AtomicInteger +import scala.util.Try class CarWeightCalculator(workerParams: R5Parameters, travelTimeNoiseFraction: Double = 0d) { private val networkHelper = workerParams.networkHelper @@ -32,7 +33,8 @@ class CarWeightCalculator(workerParams: R5Parameters, travelTimeNoiseFraction: D travelTime: TravelTime, vehicleType: Option[BeamVehicleType], time: Double, - shouldAddNoise: Boolean + shouldAddNoise: Boolean, + heavyGoodsVehicle: Boolean = false ): Double = { val link = networkHelper.getLinkUnsafe(linkId) assert(link != null) @@ -54,6 +56,13 @@ class CarWeightCalculator(workerParams: R5Parameters, travelTimeNoiseFraction: D physSimTravelTime * travelTimeNoises(idx) } val linkTravelTime = Math.max(physSimTravelTimeWithNoise, minTravelTime) - Math.min(linkTravelTime, maxTravelTime) + val result = Math.min(linkTravelTime, maxTravelTime) + + if (heavyGoodsVehicle) { + // TODO this is only prototype + val isLinkHgv = Try(link.getAttributes.getAttribute("hgv")).map(_.asInstanceOf[Boolean]).getOrElse(false) + if (isLinkHgv) result / 10 else result * 10 + } else + result } } diff --git a/src/main/scala/beam/router/r5/R5Wrapper.scala b/src/main/scala/beam/router/r5/R5Wrapper.scala index c43457b5f25..91c97075149 100644 --- a/src/main/scala/beam/router/r5/R5Wrapper.scala +++ b/src/main/scala/beam/router/r5/R5Wrapper.scala @@ -1,7 +1,7 @@ package beam.router.r5 import beam.agentsim.agents.choice.mode.DrivingCost -import beam.agentsim.agents.vehicles.BeamVehicleType +import beam.agentsim.agents.vehicles.{BeamVehicleType, VehicleCategory} import beam.agentsim.agents.vehicles.VehicleProtocol.StreetVehicle import beam.agentsim.events.SpaceTime import beam.router.BeamRouter.{RoutingRequest, RoutingResponse, _} @@ -1135,7 +1135,14 @@ class R5Wrapper(workerParams: R5Parameters, travelTime: TravelTime, travelTimeNo val maxSpeed: Double = vehicleType.maxVelocity.getOrElse(profileRequest.getSpeedForMode(streetMode)) val minTravelTime = edge.getLengthM / maxSpeed if (streetMode == StreetMode.CAR) { - carWeightCalculator.calcTravelTime(linkId, travelTime, Some(vehicleType), time, shouldAddNoise) + carWeightCalculator.calcTravelTime( + linkId, + travelTime, + Some(vehicleType), + time, + shouldAddNoise, + vehicleType.vehicleCategory == VehicleCategory.HeavyDutyTruck + ) } else if (streetMode == StreetMode.BICYCLE && shouldApplyBicycleScaleFactor) { val scaleFactor = bikeLanesAdjustment.scaleFactor(vehicleType, linkId) minTravelTime * scaleFactor diff --git a/src/main/scala/beam/sim/BeamHelper.scala b/src/main/scala/beam/sim/BeamHelper.scala index e5f0e5cd9d1..f28e45b1cf4 100755 --- a/src/main/scala/beam/sim/BeamHelper.scala +++ b/src/main/scala/beam/sim/BeamHelper.scala @@ -31,6 +31,7 @@ import beam.sim.modules.{BeamAgentModule, UtilsModule} import beam.sim.population.PopulationScaling import beam.sim.termination.TerminationCriterionProvider import beam.utils.BeamVehicleUtils.{readBeamVehicleTypeFile, readFuelTypeFile, readVehiclesFile} +import beam.utils.SnapCoordinateUtils.SnapLocationHelper import beam.utils._ import beam.utils.csv.readers import beam.utils.plan.sampling.AvailableModeUtils @@ -38,7 +39,13 @@ import beam.utils.scenario.generic.GenericScenarioSource import beam.utils.scenario.matsim.BeamScenarioSource import beam.utils.scenario.urbansim.censusblock.{ScenarioAdjuster, UrbansimReaderV2} import beam.utils.scenario.urbansim.{CsvScenarioReader, ParquetScenarioReader, UrbanSimScenarioSource} -import beam.utils.scenario.{BeamScenarioLoader, InputType, PreviousRunPlanMerger, UrbanSimScenarioLoader} +import beam.utils.scenario.{ + BeamScenarioLoader, + InputType, + PreviousRunPlanMerger, + ScenarioLoaderHelper, + UrbanSimScenarioLoader +} import com.conveyal.r5.streets.StreetLayer import com.conveyal.r5.transit.TransportNetwork import com.fasterxml.jackson.databind.ObjectMapper @@ -50,7 +57,7 @@ import com.typesafe.config.{ConfigFactory, Config => TypesafeConfig} import com.typesafe.scalalogging.LazyLogging import kamon.Kamon import org.matsim.api.core.v01.network.Link -import org.matsim.api.core.v01.population.{Activity, Plan, Population} +import org.matsim.api.core.v01.population.{Activity, Population} import org.matsim.api.core.v01.{Id, Scenario} import org.matsim.core.api.experimental.events.EventsManager import org.matsim.core.config.groups.TravelTimeCalculatorConfigGroup @@ -61,7 +68,7 @@ import org.matsim.core.events.ParallelEventsManagerImpl import org.matsim.core.scenario.{MutableScenario, ScenarioBuilder, ScenarioByInstanceModule, ScenarioUtils} import org.matsim.core.trafficmonitoring.TravelTimeCalculator import org.matsim.core.utils.collections.QuadTree -import org.matsim.households.{Household, Households} +import org.matsim.households.Households import org.matsim.utils.objectattributes.AttributeConverter import org.matsim.vehicles.Vehicle @@ -269,7 +276,7 @@ trait BeamHelper extends LazyLogging { ) } - def loadScenario(beamConfig: BeamConfig): BeamScenario = { + def loadScenario(beamConfig: BeamConfig, outputDirMaybe: Option[String] = None): BeamScenario = { val vehicleTypes = maybeScaleTransit(beamConfig, readBeamVehicleTypeFile(beamConfig)) val vehicleCsvReader = new VehicleCsvReader(beamConfig) val baseFilePath = Paths.get(beamConfig.beam.agentsim.agents.vehicles.vehicleTypesFilePath).getParent @@ -300,7 +307,7 @@ trait BeamHelper extends LazyLogging { val linkToTAZMapping: Map[Link, TAZ] = LinkLevelOperations.getLinkToTazMapping(networkCoordinator.network, tazMap) val (freightCarriers, fixedActivitiesDurationsFromFreight) = - readFreights(beamConfig, networkCoordinator.transportNetwork.streetLayer, vehicleTypes) + readFreights(beamConfig, networkCoordinator.transportNetwork.streetLayer, vehicleTypes, outputDirMaybe) val fixedActivitiesDurationsFromConfig: Map[String, Double] = { val maybeFixedDurationsList = beamConfig.beam.agentsim.agents.activities.activityTypeToFixedDurationMap @@ -344,13 +351,14 @@ trait BeamHelper extends LazyLogging { def readFreights( beamConfig: BeamConfig, streetLayer: StreetLayer, - vehicleTypes: Map[Id[BeamVehicleType], BeamVehicleType] + vehicleTypes: Map[Id[BeamVehicleType], BeamVehicleType], + outputDirMaybe: Option[String] ): (IndexedSeq[FreightCarrier], Map[String, Double]) = { val freightConfig = beamConfig.beam.agentsim.agents.freight if (freightConfig.enabled) { val geoUtils = new GeoUtilsImpl(beamConfig) - val freightReader = FreightReader(beamConfig, geoUtils, streetLayer) + val freightReader = FreightReader(beamConfig, geoUtils, streetLayer, outputDirMaybe) val tours = freightReader.readFreightTours() val plans = freightReader.readPayloadPlans() val carriers = freightReader.readFreightCarriers(tours, plans, vehicleTypes) @@ -643,32 +651,17 @@ trait BeamHelper extends LazyLogging { (beamExecutionConfig, scenario, beamScenario, services, plansMerged) } - def fixDanglingPersons(result: MutableScenario): Unit = { - val peopleViaHousehold = result.getHouseholds.getHouseholds - .values() - .asScala - .flatMap { x => - x.getMemberIds.asScala - } - .toSet - val danglingPeople = result.getPopulation.getPersons - .values() - .asScala - .filter(person => !peopleViaHousehold.contains(person.getId)) - if (danglingPeople.nonEmpty) { - logger.error(s"There are ${danglingPeople.size} persons not connected to household, removing them") - danglingPeople.foreach { p => - result.getPopulation.removePerson(p.getId) - } - } - } - protected def buildScenarioFromMatsimConfig( matsimConfig: MatsimConfig, beamScenario: BeamScenario ): MutableScenario = { + val snapLocationHelper = SnapLocationHelper( + new GeoUtilsImpl(beamScenario.beamConfig), + beamScenario.transportNetwork.streetLayer, + beamScenario.beamConfig.beam.routing.r5.linkRadiusMeters + ) val result = ScenarioUtils.loadScenario(matsimConfig).asInstanceOf[MutableScenario] - fixDanglingPersons(result) + ScenarioLoaderHelper.validateScenario(result, snapLocationHelper) result.setNetwork(beamScenario.network) result } @@ -706,6 +699,33 @@ trait BeamHelper extends LazyLogging { PopulationScaling.samplePopulation(scenario, beamScenario, beamServices.beamConfig, beamServices, outputDir) } + if (beamScenario.beamConfig.beam.agentsim.snapLocationAndRemoveInvalidInputs) { + logger.info(s""" + |The parameter `beam.agentsim.snapLocationAndRemoveInvalidInputs` is enabled. + |This may take some time to finish depending on the size of population/households.""".stripMargin) + + val beforeHouseholdsCount = scenario.getHouseholds.getHouseholds.size() + val beforePopulationCount = scenario.getPopulation.getPersons.size() + + val snapLocationHelper = SnapLocationHelper( + new GeoUtilsImpl(beamScenario.beamConfig), + beamScenario.transportNetwork.streetLayer, + beamScenario.beamConfig.beam.routing.r5.linkRadiusMeters + ) + + ScenarioLoaderHelper.validateScenario(scenario, snapLocationHelper, Some(outputDir)) + + val afterHouseholdsCount = scenario.getHouseholds.getHouseholds.size() + val afterPopulationCount = scenario.getPopulation.getPersons.size() + + logger.info( + s""" + |After snapping locations and validating scenario: + |Number of households: $afterHouseholdsCount. Removed: ${beforeHouseholdsCount - afterHouseholdsCount}. + |Number of persons: $afterPopulationCount. Removed: ${beforePopulationCount - afterPopulationCount}.""".stripMargin + ) + } + // write static metrics, such as population size, vehicles fleet size, etc. // necessary to be called after population sampling BeamStaticMetricsWriter.writeSimulationParameters( @@ -721,7 +741,8 @@ trait BeamHelper extends LazyLogging { beamServices.beamConfig, beamServices.geo, beamServices.beamScenario.transportNetwork.streetLayer, - beamServices.beamScenario.tazTreeMap + beamServices.beamScenario.tazTreeMap, + Some(outputDir) ) generatePopulationForPayloadPlans( beamScenario, @@ -800,15 +821,14 @@ trait BeamHelper extends LazyLogging { matsimConfig: MatsimConfig ): (MutableScenario, BeamScenario, Boolean) = { val scenarioConfig = beamConfig.beam.exchange.scenario - val src = scenarioConfig.source.toLowerCase - val fileFormat = scenarioConfig.fileFormat + val outputDirOpt = Option(matsimConfig.controler().getOutputDirectory) val (scenario, beamScenario, plansMerged) = ProfilingUtils.timed(s"Load scenario using $src/$fileFormat", x => logger.info(x)) { if (src == "urbansim" || src == "urbansim_v2" || src == "generic") { - val beamScenario = loadScenario(beamConfig) + val beamScenario = loadScenario(beamConfig, outputDirOpt) val emptyScenario = ScenarioBuilder(matsimConfig, beamScenario.network).build val geoUtils = new GeoUtilsImpl(beamConfig) val (scenario, plansMerged) = { @@ -877,24 +897,24 @@ trait BeamHelper extends LazyLogging { } else if (src == "beam") { fileFormat match { case "csv" => - val beamScenario = loadScenario(beamConfig) + val beamScenario = loadScenario(beamConfig, outputDirOpt) val scenario = { val source = new BeamScenarioSource( beamConfig, rdr = readers.BeamCsvScenarioReader ) val scenarioBuilder = ScenarioBuilder(matsimConfig, beamScenario.network) - new BeamScenarioLoader(scenarioBuilder, beamScenario, source, new GeoUtilsImpl(beamConfig)) - .loadScenario() + new BeamScenarioLoader( + scenarioBuilder, + beamScenario, + source, + new GeoUtilsImpl(beamConfig) + ).loadScenario() }.asInstanceOf[MutableScenario] (scenario, beamScenario, false) case "xml" => - val beamScenario = loadScenario(beamConfig) - val scenario = { - val result = ScenarioUtils.loadScenario(matsimConfig).asInstanceOf[MutableScenario] - fixDanglingPersons(result) - result - } + val beamScenario = loadScenario(beamConfig, outputDirOpt) + val scenario = ScenarioUtils.loadScenario(matsimConfig).asInstanceOf[MutableScenario] (scenario, beamScenario, false) case unknown => throw new IllegalArgumentException(s"Beam does not support [$unknown] file type") @@ -916,30 +936,29 @@ trait BeamHelper extends LazyLogging { beamScenario.freightCarriers .flatMap(_.fleet) .foreach { case (id, vehicle) => beamScenario.privateVehicles.put(id, vehicle) } - - val plans: IndexedSeq[(Household, Plan)] = converter.generatePopulation( - beamScenario.freightCarriers, - population.getFactory, - households.getFactory - ) - val allowedModes = Seq(BeamMode.CAR.value) - plans.foreach { case (household, plan) => - households.getHouseholds.put(household.getId, household) - population.addPerson(plan.getPerson) - AvailableModeUtils.setAvailableModesForPerson_v2( - beamScenario, - plan.getPerson, - household, - population, - allowedModes + converter + .generatePopulation( + beamScenario.freightCarriers, + population.getFactory, + households.getFactory ) - val freightVehicle = beamScenario.privateVehicles(household.getVehicleIds.get(0)) - households.getHouseholdAttributes - .putAttribute(household.getId.toString, "homecoordx", freightVehicle.spaceTime.loc.getX) - households.getHouseholdAttributes - .putAttribute(household.getId.toString, "homecoordy", freightVehicle.spaceTime.loc.getY) - } + .foreach { case (carrier, household, plan, personId, vehicleId) => + households.getHouseholdAttributes + .putAttribute(household.getId.toString, "homecoordx", carrier.warehouseLocationUTM.getX) + households.getHouseholdAttributes + .putAttribute(household.getId.toString, "homecoordy", carrier.warehouseLocationUTM.getY) + population.getPersonAttributes.putAttribute(personId.toString, "vehicle", vehicleId.toString) + households.getHouseholds.put(household.getId, household) + population.addPerson(plan.getPerson) + AvailableModeUtils.setAvailableModesForPerson_v2( + beamScenario, + plan.getPerson, + household, + population, + allowedModes + ) + } } def setupBeamWithConfig( @@ -966,6 +985,7 @@ trait BeamHelper extends LazyLogging { Kamon.init(config.withFallback(ConfigFactory.load())) } + logger.info("Agentsim random seed for population scaling is set to {}.", beamConfig.beam.agentsim.randomSeed) logger.info("Starting beam on branch {} at commit {}.", BashUtils.getBranch, BashUtils.getCommitHash) logger.info( diff --git a/src/main/scala/beam/sim/BeamMobsim.scala b/src/main/scala/beam/sim/BeamMobsim.scala index f023de5c1a5..6659ff22c0a 100755 --- a/src/main/scala/beam/sim/BeamMobsim.scala +++ b/src/main/scala/beam/sim/BeamMobsim.scala @@ -27,6 +27,7 @@ import beam.sim.metrics.{Metrics, MetricsSupport, SimulationMetricCollector} import beam.sim.monitoring.ErrorListener import beam.sim.population.AttributesOfIndividual import beam.sim.vehiclesharing.Fleets +import beam.utils.SnapCoordinateUtils.SnapLocationHelper import beam.utils._ import beam.utils.csv.writers.PlansCsvWriter import beam.utils.logging.{LoggingMessageActor, MessageLogger} @@ -70,6 +71,12 @@ class BeamMobsim @Inject() ( import beamServices._ val physsimConfig = beamConfig.beam.physsim + val snapLocationHelper = SnapLocationHelper( + geo, + beamScenario.transportNetwork.streetLayer, + beamConfig.beam.routing.r5.linkRadiusMeters + ) + override def run(): Unit = { logger.info("Starting Iteration") startMeasuringIteration() @@ -224,7 +231,8 @@ class BeamMobsim @Inject() ( person.getCustomAttributes.get("beam-attributes").asInstanceOf[AttributesOfIndividual], destinationChoiceModel, beamServices, - person.getId + person.getId, + snapLocationHelper ) val newPlan = supplementaryTripGenerator.generateNewPlans(person.getSelectedPlan, destinationChoiceModel, modesAvailable) diff --git a/src/main/scala/beam/sim/common/GeoUtils.scala b/src/main/scala/beam/sim/common/GeoUtils.scala index 7c480e4dbaf..ddb56a0e49c 100755 --- a/src/main/scala/beam/sim/common/GeoUtils.scala +++ b/src/main/scala/beam/sim/common/GeoUtils.scala @@ -27,6 +27,7 @@ case class EdgeWithCoord(edgeIndex: Int, wgsCoord: Coordinate) trait GeoUtils extends ExponentialLazyLogging { def localCRS: String + val defaultMaxRadiusForMapSearch = 10000 private lazy val notExponentialLogger = Logger(LoggerFactory.getLogger(getClass.getName)) lazy val utm2Wgs: GeotoolsTransformation = @@ -64,7 +65,7 @@ trait GeoUtils extends ExponentialLazyLogging { def getNearestR5EdgeToUTMCoord( streetLayer: StreetLayer, coordUTM: Coord, - maxRadius: Double + maxRadius: Double = defaultMaxRadiusForMapSearch ): Int = { getNearestR5Edge(streetLayer, utm2Wgs(coordUTM), maxRadius) } @@ -72,7 +73,7 @@ trait GeoUtils extends ExponentialLazyLogging { def getNearestR5Edge( streetLayer: StreetLayer, coordWGS: Coord, - maxRadius: Double + maxRadius: Double = defaultMaxRadiusForMapSearch ): Int = { val theSplit = getR5Split(streetLayer, coordWGS, maxRadius, StreetMode.WALK) if (theSplit == null) { @@ -105,7 +106,7 @@ trait GeoUtils extends ExponentialLazyLogging { def snapToR5Edge( streetLayer: StreetLayer, coordWGS: Coord, - maxRadius: Double, + maxRadius: Double = defaultMaxRadiusForMapSearch, streetMode: StreetMode = StreetMode.WALK ): Coord = { val theSplit = getR5Split(streetLayer, coordWGS, maxRadius, streetMode) @@ -119,7 +120,7 @@ trait GeoUtils extends ExponentialLazyLogging { def getR5Split( streetLayer: StreetLayer, coord: Coord, - maxRadius: Double, + maxRadius: Double = defaultMaxRadiusForMapSearch, streetMode: StreetMode = StreetMode.WALK ): Split = { var radius = 10.0 diff --git a/src/main/scala/beam/sim/config/BeamConfig.scala b/src/main/scala/beam/sim/config/BeamConfig.scala index 57f17ef5f07..8aa7a57c6e1 100644 --- a/src/main/scala/beam/sim/config/BeamConfig.scala +++ b/src/main/scala/beam/sim/config/BeamConfig.scala @@ -2,6 +2,8 @@ package beam.sim.config +import scala.util.Random + case class BeamConfig( beam: BeamConfig.Beam, matsim: BeamConfig.Matsim @@ -37,6 +39,7 @@ object BeamConfig { object Beam { case class Agentsim( + randomSeed: scala.Int, agentSampleSizeAsFractionOfPopulation: scala.Double, agents: BeamConfig.Beam.Agentsim.Agents, chargingNetworkManager: BeamConfig.Beam.Agentsim.ChargingNetworkManager, @@ -55,7 +58,8 @@ object BeamConfig { thresholdForWalkingInMeters: scala.Int, timeBinSize: scala.Int, toll: BeamConfig.Beam.Agentsim.Toll, - tuning: BeamConfig.Beam.Agentsim.Tuning + tuning: BeamConfig.Beam.Agentsim.Tuning, + snapLocationAndRemoveInvalidInputs: scala.Boolean ) object Agentsim { @@ -2159,6 +2163,10 @@ object BeamConfig { def apply(c: com.typesafe.config.Config): BeamConfig.Beam.Agentsim = { BeamConfig.Beam.Agentsim( + randomSeed = + if (c.hasPathOrNull("randomSeed")) + c.getInt("randomSeed") + else new Random().nextInt(), agentSampleSizeAsFractionOfPopulation = if (c.hasPathOrNull("agentSampleSizeAsFractionOfPopulation")) c.getDouble("agentSampleSizeAsFractionOfPopulation") @@ -2211,7 +2219,11 @@ object BeamConfig { tuning = BeamConfig.Beam.Agentsim.Tuning( if (c.hasPathOrNull("tuning")) c.getConfig("tuning") else com.typesafe.config.ConfigFactory.parseString("tuning{}") - ) + ), + snapLocationAndRemoveInvalidInputs = + if (c.hasPathOrNull("snapLocationAndRemoveInvalidInputs")) + c.getBoolean("snapLocationAndRemoveInvalidInputs") + else false ) } } diff --git a/src/main/scala/beam/sim/population/PopulationScaling.scala b/src/main/scala/beam/sim/population/PopulationScaling.scala index b8fa57900ec..a04f2fad126 100644 --- a/src/main/scala/beam/sim/population/PopulationScaling.scala +++ b/src/main/scala/beam/sim/population/PopulationScaling.scala @@ -126,7 +126,7 @@ class PopulationScaling extends LazyLogging { val numAgents = math.round( beamConfig.beam.agentsim.agentSampleSizeAsFractionOfPopulation * scenario.getPopulation.getPersons.size() ) - val rand = new Random(beamServices.beamConfig.matsim.modules.global.randomSeed) + val rand = new Random(beamServices.beamConfig.beam.agentsim.randomSeed) val notSelectedHouseholdIds = mutable.Set[Id[Household]]() val notSelectedVehicleIds = mutable.Set[Id[Vehicle]]() val notSelectedPersonIds = mutable.Set[Id[Person]]() @@ -378,14 +378,5 @@ object PopulationScaling { val populationAdjustment = PopulationAdjustment.getPopulationAdjustment(beamServices) populationAdjustment.update(scenario) - - // write static metrics, such as population size, vehicles fleet size, etc. - // necessary to be called after population sampling - BeamStaticMetricsWriter.writeSimulationParameters( - scenario, - beamScenario, - beamServices, - beamConfig - ) } } diff --git a/src/main/scala/beam/utils/SnapCoordinateUtils.scala b/src/main/scala/beam/utils/SnapCoordinateUtils.scala new file mode 100644 index 00000000000..15d00410d84 --- /dev/null +++ b/src/main/scala/beam/utils/SnapCoordinateUtils.scala @@ -0,0 +1,78 @@ +package beam.utils + +import beam.sim.common.GeoUtils +import beam.utils.csv.CsvWriter +import com.conveyal.r5.streets.StreetLayer +import com.typesafe.scalalogging.LazyLogging +import enumeratum._ +import org.matsim.api.core.v01.Coord + +import scala.collection.concurrent.TrieMap + +object SnapCoordinateUtils extends LazyLogging { + + sealed abstract class Error(override val entryName: String) extends EnumEntry + + object Error extends Enum[Error] { + val values = findValues + + case object OutOfBoundingBoxError extends Error("OutOfBoundingBox") + case object R5SplitNullError extends Error("R5SplitNull") + } + + sealed abstract class Category(override val entryName: String) extends EnumEntry + + object Category extends Enum[Category] { + val values = findValues + + case object ScenarioPerson extends Category("Person") + case object ScenarioHousehold extends Category("Household") + case object FreightTour extends Category("Tour") + case object FreightPayloadPlan extends Category("PayloadPlan") + case object FreightCarrier extends Category("Carrier") + } + + object CsvFile { + val Plans = "snapLocationPlanErrors.csv" + val Households = "snapLocationHouseholdErrors.csv" + val FreightTours = "snapLocationFreightTourErrors.csv" + val FreightPayloadPlans = "snapLocationFreightPayloadPlanErrors.csv" + val FreightCarriers = "snapLocationFreightCarrierErrors.csv" + } + + final case class ErrorInfo(id: String, category: Category, error: Error, planX: Double, planY: Double) + + type SnapCoordinateResult = Either[Error, Coord] + + final case class SnapLocationHelper(geo: GeoUtils, streetLayer: StreetLayer, maxRadius: Double) { + private val store: TrieMap[Coord, Option[Coord]] = TrieMap.empty + + def find(planCoord: Coord, isWgs: Boolean = false): Option[Coord] = { + val coord = if (isWgs) planCoord else geo.utm2Wgs(planCoord) + store.get(coord).flatten + } + + def computeResult(planCoord: Coord, isWgs: Boolean = false): SnapCoordinateResult = { + val coord = if (isWgs) planCoord else geo.utm2Wgs(planCoord) + if (streetLayer.envelope.contains(coord.getX, coord.getY)) { + val snapCoordOpt = store.getOrElseUpdate( + coord, + Option(geo.getR5Split(streetLayer, coord, maxRadius)).map { split => + val updatedPlanCoord = geo.splitToCoord(split) + geo.wgs2Utm(updatedPlanCoord) + } + ) + snapCoordOpt.fold[SnapCoordinateResult](Left(Error.R5SplitNullError))(coord => Right(coord)) + } else Left(Error.OutOfBoundingBoxError) + } + } + + def writeToCsv(path: String, errors: Seq[ErrorInfo]): Unit = { + new CsvWriter(path, "id", "category", "error", "x", "y") + .writeAllAndClose( + errors.map(error => List(error.id, error.category.entryName, error.error.entryName, error.planX, error.planY)) + ) + logger.info("See location error info at {}.", path) + } + +} diff --git a/src/main/scala/beam/utils/scenario/BeamScenarioLoader.scala b/src/main/scala/beam/utils/scenario/BeamScenarioLoader.scala index d7c2161e653..32e155314cc 100644 --- a/src/main/scala/beam/utils/scenario/BeamScenarioLoader.scala +++ b/src/main/scala/beam/utils/scenario/BeamScenarioLoader.scala @@ -198,22 +198,29 @@ class BeamScenarioLoader( case (_, listOfElementsGroupedByPlan) if listOfElementsGroupedByPlan.nonEmpty => val person = population.getPersons.get(Id.createPersonId(personId.id)) - val currentPlan = PopulationUtils.createPlan(person) - currentPlan.setScore(listOfElementsGroupedByPlan.head.planScore) - person.addPlan(currentPlan) - - val personWithoutSelectedPlan = person.getSelectedPlan == null - val isCurrentPlanIndexSelected = listOfElementsGroupedByPlan.head.planSelected - val isLastPlanIteration = person.getPlans.size() == listOfElementsGroupedByPerson.size - if (personWithoutSelectedPlan && (isCurrentPlanIndexSelected || isLastPlanIteration)) { - person.setSelectedPlan(currentPlan) - } + if (person == null) { + logger.warn( + "Could not find person {} while adding plans (maybe it doesn't belong to any household!?)", + personId.id + ) + } else { + val currentPlan = PopulationUtils.createPlan(person) + currentPlan.setScore(listOfElementsGroupedByPlan.head.planScore) + person.addPlan(currentPlan) + + val personWithoutSelectedPlan = person.getSelectedPlan == null + val isCurrentPlanIndexSelected = listOfElementsGroupedByPlan.head.planSelected + val isLastPlanIteration = person.getPlans.size() == listOfElementsGroupedByPerson.size + if (personWithoutSelectedPlan && (isCurrentPlanIndexSelected || isLastPlanIteration)) { + person.setSelectedPlan(currentPlan) + } - listOfElementsGroupedByPlan.foreach { planElement => - if (planElement.planElementType == PlanElement.Leg) { - buildAndAddLegToPlan(currentPlan, planElement) - } else if (planElement.planElementType == PlanElement.Activity) { - buildAndAddActivityToPlan(currentPlan, planElement) + listOfElementsGroupedByPlan.foreach { planElement => + if (planElement.planElementType == PlanElement.Leg) { + buildAndAddLegToPlan(currentPlan, planElement) + } else if (planElement.planElementType == PlanElement.Activity) { + buildAndAddActivityToPlan(currentPlan, planElement) + } } } } diff --git a/src/main/scala/beam/utils/scenario/ScenarioLoaderHelper.scala b/src/main/scala/beam/utils/scenario/ScenarioLoaderHelper.scala new file mode 100644 index 00000000000..3399aad497f --- /dev/null +++ b/src/main/scala/beam/utils/scenario/ScenarioLoaderHelper.scala @@ -0,0 +1,154 @@ +package beam.utils.scenario + +import beam.utils.SnapCoordinateUtils +import beam.utils.SnapCoordinateUtils.{Category, CsvFile, Error, ErrorInfo, SnapLocationHelper} +import com.typesafe.scalalogging.LazyLogging +import org.matsim.api.core.v01.population.{Activity, Leg, Person} +import org.matsim.api.core.v01.{Coord, Id} +import org.matsim.core.scenario.MutableScenario +import org.matsim.households.{Household, HouseholdsFactoryImpl} + +import scala.collection.mutable.ListBuffer +import scala.jdk.CollectionConverters.{collectionAsScalaIterableConverter, seqAsJavaListConverter} + +object ScenarioLoaderHelper extends LazyLogging { + + import org.matsim.api.core.v01.population.PlanElement + + private def createHouseholdWithGivenMembers( + household: Household, + members: List[Id[Person]] + ): Household = { + val householdResult = new HouseholdsFactoryImpl().createHousehold(household.getId) + householdResult.setIncome(household.getIncome) + householdResult.setVehicleIds(household.getVehicleIds) + householdResult.setMemberIds(members.asJava) + householdResult + } + + private def updatePlanElementCoord( + personId: Id[Person], + elements: Vector[PlanElement], + snapLocationHelper: SnapLocationHelper + ): Vector[ErrorInfo] = { + val errors = elements.foldLeft[Vector[ErrorInfo]](Vector.empty) { (errors, element) => + element match { + case _: Leg => errors + case a: Activity => + val planCoord = a.getCoord + snapLocationHelper.computeResult(planCoord) match { + case Right(_) => + // note: we don't want to update coord in-place here since we might end up removing plan from the person + errors + case Left(error) => + errors :+ ErrorInfo( + personId.toString, + Category.ScenarioPerson, + error, + planCoord.getX, + planCoord.getY + ) + } + } + } + + if (errors.isEmpty) { + elements.foreach { + case a: Activity => + val planCoord = a.getCoord + snapLocationHelper.find(planCoord) match { + case Some(coord) => + a.setCoord(coord) + case None => + logger.error("UNEXPECTED: there should be mapped split location for {} coord.", planCoord) + } + case _ => + } + } + + errors + } + + def validateScenario( + scenario: MutableScenario, + snapLocationHelper: SnapLocationHelper, + outputDirMaybe: Option[String] = None + ): Unit = { + + val planErrors: ListBuffer[ErrorInfo] = ListBuffer() + + val people: List[Person] = scenario.getPopulation.getPersons.values().asScala.toList + people.par.foreach { person => + val plans = person.getPlans.asScala.toList + plans.foreach { plan => + val elements: Vector[PlanElement] = plan.getPlanElements.asScala.toVector + val errors: Vector[ErrorInfo] = updatePlanElementCoord(person.getId, elements, snapLocationHelper) + if (errors.nonEmpty) { + planErrors.appendAll(errors) + person.removePlan(plan) + } + } + val planCount = person.getPlans.size() + if (planCount == 0) { + scenario.getPopulation.removePerson(person.getId) + } + } + + outputDirMaybe.foreach { path => + if (planErrors.isEmpty) logger.info("No 'snap location' error to report for scenario plans.") + else SnapCoordinateUtils.writeToCsv(s"$path/${CsvFile.Plans}", planErrors) + } + + val validPeople: Set[Id[Person]] = scenario.getPopulation.getPersons.values().asScala.map(_.getId).toSet + + val households: List[Household] = scenario.getHouseholds.getHouseholds.values().asScala.toList + households.par.foreach { household => + val members = household.getMemberIds.asScala.toSet + val validMembers = validPeople.intersect(members) + + if (validMembers.isEmpty) { + scenario.getHouseholds.getHouseholdAttributes.removeAllAttributes(household.getId.toString) + scenario.getHouseholds.getHouseholds.remove(household.getId) + } else if (validMembers != members) { + val updatedHousehold = createHouseholdWithGivenMembers(household, validMembers.toList) + scenario.getHouseholds.getHouseholds.replace(household.getId, updatedHousehold) + } + } + + val householdErrors: ListBuffer[ErrorInfo] = ListBuffer() + + val householdsWithMembers: List[Household] = scenario.getHouseholds.getHouseholds.values().asScala.toList + householdsWithMembers.par.foreach { household => + val householdId = household.getId.toString + val attr = scenario.getHouseholds.getHouseholdAttributes + val locationX = attr.getAttribute(householdId, "homecoordx").asInstanceOf[Double] + val locationY = attr.getAttribute(householdId, "homecoordy").asInstanceOf[Double] + val planCoord = new Coord(locationX, locationY) + + snapLocationHelper.computeResult(planCoord) match { + case Right(splitCoord) => + attr.putAttribute(householdId, "homecoordx", splitCoord.getX) + attr.putAttribute(householdId, "homecoordy", splitCoord.getY) + case Left(error) => + household.getMemberIds.asScala.toList.foreach(personId => scenario.getPopulation.getPersons.remove(personId)) + scenario.getHouseholds.getHouseholds.remove(household.getId) + householdErrors.append( + ErrorInfo( + householdId, + Category.ScenarioHousehold, + error, + planCoord.getX, + planCoord.getY + ) + ) + } + } + + outputDirMaybe.foreach { path => + if (householdErrors.isEmpty) logger.info("No 'snap location' error to report for scenario households.") + else SnapCoordinateUtils.writeToCsv(s"$path/${CsvFile.Households}", householdErrors) + } + + } + +} diff --git a/src/main/scala/beam/utils/scenario/UrbanSimScenarioLoader.scala b/src/main/scala/beam/utils/scenario/UrbanSimScenarioLoader.scala index 725bf5e1fda..4f3173a3aaf 100644 --- a/src/main/scala/beam/utils/scenario/UrbanSimScenarioLoader.scala +++ b/src/main/scala/beam/utils/scenario/UrbanSimScenarioLoader.scala @@ -45,62 +45,24 @@ class UrbanSimScenarioLoader( def loadScenario(): (Scenario, Boolean) = { clear() - val wereCoordinatesInWGS = beamScenario.beamConfig.beam.exchange.scenario.convertWgs2Utm - val plansF = Future { val plans = scenarioSource.getPlans logger.info(s"Read ${plans.size} plans") - val activities = plans.view.filter { p => - p.activityType.exists(actType => actType.toLowerCase == "home") - } - - val personIdsWithinRange = - activities - .filter { act => - val actCoord = new Coord(act.activityLocationX.get, act.activityLocationY.get) - val wgsCoord = if (wereCoordinatesInWGS) geo.utm2Wgs(actCoord) else actCoord - beamScenario.transportNetwork.streetLayer.envelope.contains(wgsCoord.getX, wgsCoord.getY) - } - .map { act => - act.personId - } - .toSet - val planWithinRange = plans.filter(p => personIdsWithinRange.contains(p.personId)) - val filteredCnt = plans.size - planWithinRange.size - if (filteredCnt > 0) { - logger.info(s"Filtered out $filteredCnt plans. Total number of plans: ${planWithinRange.size}") - } - planWithinRange + plans } + val personsF = Future { val persons: Iterable[PersonInfo] = scenarioSource.getPersons logger.info(s"Read ${persons.size} persons") persons } + val householdsF = Future { val households = scenarioSource.getHousehold logger.info(s"Read ${households.size} households") - val householdIdsWithinBoundingBox = households.view - .filter { hh => - val coord = new Coord(hh.locationX, hh.locationY) - val wgsCoord = if (wereCoordinatesInWGS) geo.utm2Wgs(coord) else coord - beamScenario.transportNetwork.streetLayer.envelope.contains(wgsCoord.getX, wgsCoord.getY) - } - .map { hh => - hh.householdId - } - .toSet - - val householdsInsideBoundingBox = - households.filter(household => householdIdsWithinBoundingBox.contains(household.householdId)) - val filteredCnt = households.size - householdsInsideBoundingBox.size - if (filteredCnt > 0) { - logger.info( - s"Filtered out $filteredCnt households. Total number of households: ${householdsInsideBoundingBox.size}" - ) - } - householdsInsideBoundingBox + households } + val inputPlans = Await.result(plansF, 1800.seconds) logger.info(s"Reading plans done.") val persons = Await.result(personsF, 1800.seconds) diff --git a/src/test/java/beam/analysis/physsim/PhyssimCalcLinkStatsTest.java b/src/test/java/beam/analysis/physsim/PhyssimCalcLinkStatsTest.java index 8e9cf361c58..7f2c5b73c4f 100755 --- a/src/test/java/beam/analysis/physsim/PhyssimCalcLinkStatsTest.java +++ b/src/test/java/beam/analysis/physsim/PhyssimCalcLinkStatsTest.java @@ -47,7 +47,7 @@ public static void createDummySimWithXML() { eventsManager.addHandler(travelTimeCalculator); BeamConfig beamConfig = BeamConfig.apply(TestConfigUtils.testConfig("test/input/equil-square/equil-0.001k.conf").resolve().withValue("beam.physsim.quick_fix_minCarSpeedInMetersPerSecond", ConfigValueFactory.fromAnyRef(0.0))); - physsimCalcLinkStats = new PhyssimCalcLinkStats(network, null, beamConfig, defaultTravelTimeCalculator, new BeamConfigChangesObservable(beamConfig, Option.empty()) ); + physsimCalcLinkStats = new PhyssimCalcLinkStats(network, null, beamConfig, defaultTravelTimeCalculator, new BeamConfigChangesObservable(beamConfig, Option.empty()), null); //physsimCalcLinkStats = new PhyssimCalcLinkStats(network, null, null); diff --git a/src/test/scala/beam/agentsim/agents/freight/FreightReplannerSpec.scala b/src/test/scala/beam/agentsim/agents/freight/FreightReplannerSpec.scala index ff18111ad48..98a52939999 100644 --- a/src/test/scala/beam/agentsim/agents/freight/FreightReplannerSpec.scala +++ b/src/test/scala/beam/agentsim/agents/freight/FreightReplannerSpec.scala @@ -122,7 +122,7 @@ class FreightReplannerSpec extends AnyWordSpecLike with Matchers with BeamHelper beamServices.beamScenario.freightCarriers.find(_.carrierId == "freight-carrier-1".createId[FreightCarrier]).get replanner.replan(carrier) val person = beamServices.matsimServices.getScenario.getPopulation.getPersons - .get(Id.createPersonId("freight-vehicle-1-agent")) + .get(Id.createPersonId("freight-carrier-1-vehicle-1-agent")) val plan = person.getSelectedPlan plan.getPlanElements should have size 9 plan.getPlanElements.get(0) shouldBe a[Activity] diff --git a/src/test/scala/beam/agentsim/agents/freight/input/GenericFreightReaderSpec.scala b/src/test/scala/beam/agentsim/agents/freight/input/GenericFreightReaderSpec.scala index a093b00b2c7..eae38e6d8aa 100644 --- a/src/test/scala/beam/agentsim/agents/freight/input/GenericFreightReaderSpec.scala +++ b/src/test/scala/beam/agentsim/agents/freight/input/GenericFreightReaderSpec.scala @@ -5,6 +5,7 @@ import beam.agentsim.infrastructure.taz.TAZTreeMap import beam.sim.common.GeoUtils import beam.sim.config.BeamConfig.Beam.Agentsim.Agents.Freight import beam.utils.BeamVehicleUtils +import beam.utils.SnapCoordinateUtils.SnapLocationHelper import beam.utils.matsim_conversion.MatsimPlanConversion.IdOps import org.matsim.api.core.v01.population.{Activity, Person, Plan, PopulationFactory} import org.matsim.api.core.v01.{Coord, Id} @@ -45,7 +46,17 @@ class GenericFreightReaderSpec extends AnyWordSpecLike with Matchers { val rnd = new Random(2333L) - private val reader = new GenericFreightReader(freightConfig, geoUtils, rnd, tazMap, None) + val snapLocationHelper = Mockito.mock(classOf[SnapLocationHelper]) + + private val reader = + new GenericFreightReader( + freightConfig, + geoUtils, + rnd, + tazMap, + snapLocationAndRemoveInvalidInputs = false, + snapLocationHelper + ) "PayloadPlansConverter" should { "read Payload Plans" in { @@ -70,7 +81,6 @@ class GenericFreightReaderSpec extends AnyWordSpecLike with Matchers { val tour3 = tours("tour-3".createId) tour3.tourId should be("tour-3".createId) tour3.departureTimeInSec should be(15000) - tour3.warehouseLocationUTM should be(new Coord(170308.4, 2964.6)) tour3.maxTourDurationInSec should be(36000) } @@ -88,7 +98,6 @@ class GenericFreightReaderSpec extends AnyWordSpecLike with Matchers { carrier1.tourMap(Id.createVehicleId("freight-vehicle-2")).head should have( 'tourId ("tour-1".createId[FreightTour]), 'departureTimeInSec (1000), - 'warehouseLocationUTM (new Coord(169637.3661199976, 3030.52756066406)), 'maxTourDurationInSec (36000) ) carrier1.plansPerTour should have size 3 @@ -136,7 +145,7 @@ class GenericFreightReaderSpec extends AnyWordSpecLike with Matchers { ) personPlans should have size 3 - val plan1 = personPlans(Id.createPersonId("freight-vehicle-1-agent")) + val plan1 = personPlans(Id.createPersonId("freight-carrier-1-vehicle-1-agent")) plan1.getPlanElements should have size 15 plan1.getPlanElements.get(2).asInstanceOf[Activity].getCoord should be( new Coord(169567.3017564815, 836.6518909569604) @@ -144,7 +153,7 @@ class GenericFreightReaderSpec extends AnyWordSpecLike with Matchers { plan1.getPlanElements.get(12).asInstanceOf[Activity].getCoord should be( new Coord(169576.80444138843, 3380.0075111142937) ) - val plan4 = personPlans(Id.createPersonId("freight-vehicle-3-agent")) + val plan4 = personPlans(Id.createPersonId("freight-carrier-2-vehicle-3-agent")) plan4.getPlanElements should have size 5 plan4.getPlanElements.get(2).asInstanceOf[Activity].getCoord should be( new Coord(169900.11498160253, 3510.2356380579545) @@ -154,12 +163,26 @@ class GenericFreightReaderSpec extends AnyWordSpecLike with Matchers { } private def readCarriers: IndexedSeq[FreightCarrier] = { - val converter = new GenericFreightReader(freightConfig, geoUtils, new Random(4324L), tazMap, None) + val converter = new GenericFreightReader( + freightConfig, + geoUtils, + new Random(4324L), + tazMap, + snapLocationAndRemoveInvalidInputs = false, + snapLocationHelper + ) val payloadPlans: Map[Id[PayloadPlan], PayloadPlan] = converter.readPayloadPlans() val tours = converter.readFreightTours() val vehicleTypes = BeamVehicleUtils.readBeamVehicleTypeFile("test/input/beamville/vehicleTypes.csv") val freightCarriers: IndexedSeq[FreightCarrier] = - new GenericFreightReader(freightConfig, geoUtils, new Random(73737L), tazMap, None).readFreightCarriers( + new GenericFreightReader( + freightConfig, + geoUtils, + new Random(73737L), + tazMap, + snapLocationAndRemoveInvalidInputs = false, + snapLocationHelper + ).readFreightCarriers( tours, payloadPlans, vehicleTypes diff --git a/src/test/scala/beam/analysis/physsim/BeamCalcLinkStatsSpec.scala b/src/test/scala/beam/analysis/physsim/BeamCalcLinkStatsSpec.scala index 8c837094b7c..e4a5cdeb9e0 100644 --- a/src/test/scala/beam/analysis/physsim/BeamCalcLinkStatsSpec.scala +++ b/src/test/scala/beam/analysis/physsim/BeamCalcLinkStatsSpec.scala @@ -55,7 +55,7 @@ class BeamCalcLinkStatsSpec extends AnyWordSpecLike with Matchers with BeforeAnd beamCalcLinkStats = new BeamCalcLinkStats(network, ttccg) beamCalcLinkStats.reset() - val volumes = new VolumesAnalyzerFixed(3600, ttccg.getMaxTime() - 1, network) + val volumes = new VolumesAnalyzerFixed(3600, ttccg.getMaxTime() - 1, network, null) events.addHandler(volumes) val reader = new MatsimEventsReader(events) diff --git a/src/test/scala/beam/physsim/analysis/LinkStatsWithVehicleCategorySpec.scala b/src/test/scala/beam/physsim/analysis/LinkStatsWithVehicleCategorySpec.scala new file mode 100644 index 00000000000..c219329f718 --- /dev/null +++ b/src/test/scala/beam/physsim/analysis/LinkStatsWithVehicleCategorySpec.scala @@ -0,0 +1,145 @@ +package beam.physsim.analysis + +import beam.agentsim.agents.vehicles.VehicleCategory.VehicleCategory +import beam.agentsim.agents.vehicles.{BeamVehicle, BeamVehicleType, VehicleCategory} +import beam.router.LinkTravelTimeContainer +import beam.utils.VolumesAnalyzerFixed +import beam.utils.matsim_conversion.MatsimPlanConversion.IdOps +import org.matsim.api.core.v01.Id +import org.matsim.api.core.v01.network.{Link, Network} +import org.matsim.api.core.v01.population.Person +import org.matsim.core.config.ConfigUtils +import org.matsim.core.config.groups.TravelTimeCalculatorConfigGroup +import org.matsim.core.events.{EventsUtils, MatsimEventsReader} +import org.matsim.core.network.io.MatsimNetworkReader +import org.matsim.core.scenario.ScenarioUtils +import org.matsim.core.trafficmonitoring.TravelTimeCalculator +import org.matsim.vehicles.Vehicle +import org.mockito.Mockito.{mock, when} +import org.scalatest.OptionValues._ +import org.scalatest.TryValues._ +import org.scalatest.matchers.should.Matchers +import org.scalatest.wordspec.AnyWordSpecLike + +import scala.collection.JavaConverters._ + +/** + * @author Dmitry Openkov + */ +class LinkStatsWithVehicleCategorySpec extends AnyWordSpecLike with Matchers { + private val EVENTS_FILE_PATH = "test/input/equil-square/test-data/physSimEvents-relative-speeds.xml" + private val NETWORK_FILE_PATH = "test/input/equil-square/test-data/physSimNetwork-relative-speeds.xml" + private val LINKSTATS_CSV_PATH = "output/test/linkstats-by-category.csv" + + val ( + network: Network, + ttConfigGroup: TravelTimeCalculatorConfigGroup, + travelTimeCalculator: TravelTimeCalculator, + volumeAnalyzer: VolumesAnalyzerFixed + ) = readPhysSimEvents + + "LinkStatsWithVehicleCategory" when { + val linkStats = new LinkStatsWithVehicleCategory(network, ttConfigGroup) + val (totalLinkData, linkData, nofHours) = linkStats.calculateLinkData( + volumeAnalyzer, + travelTimeCalculator.getLinkTravelTimes, + IndexedSeq("MediumDutyPassenger", "Car") + ) + "calculates volume" should { + "return correct volume for each vehicle category" in { + nofHours shouldBe 68 + withClue(s"According to event file $EVENTS_FILE_PATH, bused 1,6 went through link 248 2 times:") { + val linkDataOption = linkData("MediumDutyPassenger").get(Id.createLinkId(248)) + linkDataOption.value.getSumVolume(9) shouldBe 0.0 + linkDataOption.value.getSumVolume(10) shouldBe 1.0 + linkDataOption.value.getSumVolume(11) shouldBe 1.0 + linkDataOption.value.getSumVolume(12) shouldBe 0.0 + } + withClue(s"According to event file $EVENTS_FILE_PATH, cars 1-9 went through link 248 9 times:") { + val linkDataOption = linkData("Car").get(Id.createLinkId(248)) + linkDataOption.value.getSumVolume(5) shouldBe 0.0 + linkDataOption.value.getSumVolume(6) shouldBe 0.0 + linkDataOption.value.getSumVolume(7) shouldBe 1.0 + linkDataOption.value.getSumVolume(20) shouldBe 1.0 + linkDataOption.value.getSumVolume(21) shouldBe 2.0 + linkDataOption.value.getSumVolume(23) shouldBe 1.0 + linkDataOption.value.getSumVolume(33) shouldBe 1.0 + linkDataOption.value.getSumVolume(40) shouldBe 1.0 + linkDataOption.value.getSumVolume(51) shouldBe 1.0 + linkDataOption.value.getSumVolume(53) shouldBe 1.0 + linkDataOption.value.getSumVolume(54) shouldBe 0.0 + } + } + "return correct total volume" in { + withClue(s"According to event file $EVENTS_FILE_PATH, vehicles went through link 112 12 times:") { + val linkDataOption = totalLinkData.get(Id.createLinkId(112)) + linkDataOption.value.getSumVolume(5) shouldBe 0.0 + linkDataOption.value.getSumVolume(6) shouldBe 8.0 + linkDataOption.value.getSumVolume(7) shouldBe 4.0 + linkDataOption.value.getSumVolume(8) shouldBe 0.0 + } + } + } + "saves linkstats to csv" should { + "save it that WarmStart could read it" in { + val result = linkStats.writeLinkStatsWithTruckVolumes( + volumeAnalyzer, + travelTimeCalculator.getLinkTravelTimes, + LINKSTATS_CSV_PATH + ) + val (_, _, nofHours) = result.success.value + nofHours shouldBe 68 + val container = new LinkTravelTimeContainer(LINKSTATS_CSV_PATH, 3600, nofHours) + val link = mock(classOf[Link]) + when(link.getId).thenReturn(Id.createLinkId(4)) + val person = mock(classOf[Person]) + val vehicle = mock(classOf[Vehicle]) + val travelTime17 = container.getLinkTravelTime(link, 17 * 3600, person, vehicle) + travelTime17 shouldBe 0.4 +- 0.01 + val travelTime18 = container.getLinkTravelTime(link, 18 * 3600, person, vehicle) + travelTime18 shouldBe 44478.89 +- 0.01 + } + } + } + + private def readPhysSimEvents + : (Network, TravelTimeCalculatorConfigGroup, TravelTimeCalculator, VolumesAnalyzerFixed) = { + val config = ConfigUtils.createConfig + config.travelTimeCalculator().setMaxTime(245162) + val scenario = ScenarioUtils.createScenario(ConfigUtils.createConfig) + val network = scenario.getNetwork + val matsimNetworkReader = new MatsimNetworkReader(network) + matsimNetworkReader.readFile(NETWORK_FILE_PATH) + + val ttConfigGroup: TravelTimeCalculatorConfigGroup = config.travelTimeCalculator + val travelTimeCalculator = new TravelTimeCalculator(network, ttConfigGroup) + val eventsManager = EventsUtils.createEventsManager + eventsManager.addHandler(travelTimeCalculator) + val volumeAnalyzer = + new VolumesAnalyzerFixed(3600, ttConfigGroup.getMaxTime - 1, network, createVehicleMap().asJava) + eventsManager.addHandler(volumeAnalyzer) + + val matsimEventsReader = new MatsimEventsReader(eventsManager) + matsimEventsReader.readFile(EVENTS_FILE_PATH) + (network, ttConfigGroup, travelTimeCalculator, volumeAnalyzer) + } + + def createVehicleMap(): Map[Id[BeamVehicle], BeamVehicle] = { + Map( + createVehicleWithId("bus:B2-EAST-1", VehicleCategory.MediumDutyPassenger), + createVehicleWithId("bus:B2-EAST-6", VehicleCategory.MediumDutyPassenger) + ) ++ (1 to 9).map(i => createVehicleWithId(i.toString, VehicleCategory.Car)) + } + + private def createVehicleWithId( + vehicleIdStr: String, + vehicleCategory: VehicleCategory + ): (Id[BeamVehicle], BeamVehicle) = { + val vehicleId = vehicleIdStr.createId[BeamVehicle] + val vehicleType = mock(classOf[BeamVehicleType]) + when(vehicleType.vehicleCategory).thenReturn(vehicleCategory) + val vehicle = mock(classOf[BeamVehicle]) + when(vehicle.beamVehicleType).thenReturn(vehicleType) + (vehicleId, vehicle) + } +} diff --git a/src/test/scala/beam/router/LinkTravelTimeContainerSpec.scala b/src/test/scala/beam/router/LinkTravelTimeContainerSpec.scala new file mode 100644 index 00000000000..f6bf048858f --- /dev/null +++ b/src/test/scala/beam/router/LinkTravelTimeContainerSpec.scala @@ -0,0 +1,28 @@ +package beam.router + +import org.matsim.api.core.v01.Id +import org.matsim.api.core.v01.network.Link +import org.matsim.api.core.v01.population.Person +import org.matsim.vehicles.Vehicle +import org.scalatest.matchers.should.Matchers +import org.scalatest.wordspec.AnyWordSpecLike +import org.mockito.Mockito.{mock, when} + +/** + * @author Dmitry Openkov + */ +class LinkTravelTimeContainerSpec extends AnyWordSpecLike with Matchers { + "LinkTravelTimeContainer" when { + "provided with a valid linkstats file" should { + "read travel time correctly" in { + val container = new LinkTravelTimeContainer("test/test-resources/beam/router/0.linkstats.csv.gz", 3600, 30) + val link = mock(classOf[Link]) + when(link.getId).thenReturn(Id.createLinkId(233)) + val person = mock(classOf[Person]) + val vehicle = mock(classOf[Vehicle]) + val travelTime = container.getLinkTravelTime(link, 4000, person, vehicle) + travelTime shouldBe 0.355 +- 0.0001 + } + } + } +} diff --git a/src/test/scala/beam/sim/population/PopulationSamplingSpec.scala b/src/test/scala/beam/sim/population/PopulationSamplingSpec.scala new file mode 100644 index 00000000000..b040893a92d --- /dev/null +++ b/src/test/scala/beam/sim/population/PopulationSamplingSpec.scala @@ -0,0 +1,157 @@ +package beam.sim.population + +import beam.integration.IntegrationSpecCommon +import beam.sim.config.{BeamConfig, MatSimBeamConfigBuilder} +import beam.sim.{BeamHelper, BeamScenario, BeamServices} +import beam.utils.FileUtils +import com.typesafe.config.ConfigValueFactory +import org.matsim.core.scenario.{MutableScenario, ScenarioUtils} +import org.scalatest.matchers.must.Matchers +import org.scalatest.matchers.should.Matchers.convertToAnyShouldWrapper +import org.scalatest.wordspec.AnyWordSpecLike + +import java.nio.file.Files + +class PopulationSamplingSpec extends AnyWordSpecLike with Matchers with BeamHelper with IntegrationSpecCommon { + + private def getObjectsForPopulationSampling( + seed: Integer = 0 + ): (MutableScenario, BeamScenario, BeamConfig, BeamServices, String) = { + if (seed.equals(0)) { + return prepareObjectsForPopulationSampling(getConfigWithRandomSeed) + } + prepareObjectsForPopulationSampling(getConfigWithSeed(seed)) + } + + private def getConfigWithRandomSeed: BeamConfig = { + BeamConfig( + baseConfig + .withValue( + "beam.agentsim.agentSampleSizeAsFractionOfPopulation", + ConfigValueFactory.fromAnyRef(0.5) + ) + .resolve() + ) + } + + private def getConfigWithSeed(seed: Integer): BeamConfig = { + BeamConfig( + baseConfig + .withValue( + "beam.agentsim.agentSampleSizeAsFractionOfPopulation", + ConfigValueFactory.fromAnyRef(0.5) + ) + .withValue("beam.agentsim.randomSeed", ConfigValueFactory.fromAnyRef(seed)) + .resolve() + ) + } + + private def prepareObjectsForPopulationSampling( + config: BeamConfig + ): (MutableScenario, BeamScenario, BeamConfig, BeamServices, String) = { + val beamScenario = loadScenario(config) + val configBuilder = new MatSimBeamConfigBuilder(baseConfig) + val matsimConfig = configBuilder.buildMatSimConf() + matsimConfig.planCalcScore().setMemorizingExperiencedPlans(true) + FileUtils.setConfigOutputFile(config, matsimConfig) + + val scenario = ScenarioUtils.loadScenario(matsimConfig).asInstanceOf[MutableScenario] + scenario.setNetwork(beamScenario.network) + + val injector = org.matsim.core.controler.Injector.createInjector( + scenario.getConfig, + module(baseConfig, config, scenario, beamScenario) + ) + + val beamServices: BeamServices = injector.getInstance(classOf[BeamServices]) + val temporaryOutputDirectory = Files.createTempDirectory("dummyOutputDirectory").toString + (scenario, beamScenario, config, beamServices, temporaryOutputDirectory) + } + + "PopulationSampling" must { + "sample the same agents for the same randomSeed value" in { + val samplingDataOne = getObjectsForPopulationSampling(1441) + val samplingDataTwo = getObjectsForPopulationSampling(1441) + val agentsBeforeSamplingOneSize = samplingDataOne._1.getPopulation.getPersons.keySet().size() + PopulationScaling.samplePopulation( + samplingDataOne._1, + samplingDataOne._2, + samplingDataOne._3, + samplingDataOne._4, + samplingDataOne._5 + ) + + agentsBeforeSamplingOneSize should be <= 50 + val agentsAfterSamplingOne = + samplingDataOne._1.getPopulation.getPersons.keySet().stream().map(a => a.leftSide).toArray + + PopulationScaling.samplePopulation( + samplingDataTwo._1, + samplingDataTwo._2, + samplingDataTwo._3, + samplingDataTwo._4, + samplingDataTwo._5 + ) + val agentsAfterSamplingTwo = + samplingDataTwo._1.getPopulation.getPersons.keySet().stream().map(a => a.leftSide).toArray + + agentsAfterSamplingOne shouldBe agentsAfterSamplingTwo + } + + "sample different agents if randomSeed is not set" in { + val samplingDataOne = getObjectsForPopulationSampling() + val samplingDataTwo = getObjectsForPopulationSampling() + val agentsBeforeSamplingOneSize = samplingDataOne._1.getPopulation.getPersons.keySet().size() + PopulationScaling.samplePopulation( + samplingDataOne._1, + samplingDataOne._2, + samplingDataOne._3, + samplingDataOne._4, + samplingDataOne._5 + ) + PopulationScaling.samplePopulation( + samplingDataTwo._1, + samplingDataTwo._2, + samplingDataTwo._3, + samplingDataTwo._4, + samplingDataTwo._5 + ) + + agentsBeforeSamplingOneSize should be <= 50 + val agentsAfterSamplingOne = + samplingDataOne._1.getPopulation.getPersons.keySet().stream().map(a => a.leftSide).toArray + val agentsAfterSamplingTwo = + samplingDataTwo._1.getPopulation.getPersons.keySet().stream().map(a => a.leftSide).toArray + + agentsAfterSamplingOne should not equal agentsAfterSamplingTwo + } + + "sample different agents for different randomSeeds" in { + val samplingDataOne = getObjectsForPopulationSampling(1441) + val samplingDataTwo = getObjectsForPopulationSampling(23) + val agentsBeforeSamplingOneSize = samplingDataOne._1.getPopulation.getPersons.keySet().size() + PopulationScaling.samplePopulation( + samplingDataOne._1, + samplingDataOne._2, + samplingDataOne._3, + samplingDataOne._4, + samplingDataOne._5 + ) + PopulationScaling.samplePopulation( + samplingDataTwo._1, + samplingDataTwo._2, + samplingDataTwo._3, + samplingDataTwo._4, + samplingDataTwo._5 + ) + + agentsBeforeSamplingOneSize should be <= 50 + val agentsAfterSamplingOne = + samplingDataOne._1.getPopulation.getPersons.keySet().stream().map(a => a.leftSide).toArray + val agentsAfterSamplingTwo = + samplingDataTwo._1.getPopulation.getPersons.keySet().stream().map(a => a.leftSide).toArray + + agentsAfterSamplingOne should not equal agentsAfterSamplingTwo + } + } +} diff --git a/src/test/scala/beam/utils/SnapCoordinateSpec.scala b/src/test/scala/beam/utils/SnapCoordinateSpec.scala new file mode 100644 index 00000000000..c6b1c690909 --- /dev/null +++ b/src/test/scala/beam/utils/SnapCoordinateSpec.scala @@ -0,0 +1,449 @@ +package beam.utils + +import beam.agentsim.agents.freight.{FreightCarrier, FreightTour, PayloadPlan} +import beam.sim.BeamHelper +import beam.sim.common.GeoUtilsImpl +import beam.sim.config.{BeamConfig, MatSimBeamConfigBuilder} +import beam.utils.SnapCoordinateUtils.{Category, CsvFile, Error, ErrorInfo, SnapCoordinateResult, SnapLocationHelper} +import beam.utils.TestConfigUtils.testConfig +import beam.utils.scenario.ScenarioLoaderHelper +import com.typesafe.config.{ConfigFactory, Config => TypesafeConfig} +import org.matsim.api.core.v01.Coord +import org.matsim.api.core.v01.population.{Activity, Population} +import org.matsim.core.config.Config +import org.matsim.households.Households +import org.scalatest.matchers.should.Matchers +import org.scalatest.wordspec.AnyWordSpec + +import scala.io.Source +import scala.jdk.CollectionConverters.collectionAsScalaIterableConverter + +class SnapCoordinateSpec extends AnyWordSpec with Matchers with BeamHelper { + + val pwd: String = System.getenv("PWD") + + def readErrorCsv(path: String): List[ErrorInfo] = { + val src = Source.fromFile(path) + val errors = src + .getLines() + .drop(1) + .map(_.split(",")) + .map { row => + ErrorInfo(row(0), Category.withName(row(1)), Error.withName(row(2)), row(3).toDouble, row(4).toDouble) + } + .toList + src.close() + errors + } + + def intersection(population: Population, path: String): Set[String] = { + val invalidPersonIds = readErrorCsv(path).map(_.id).toSet + val validPersonIds = population.getPersons.keySet().asScala.map(_.toString).toSet + validPersonIds.intersect(invalidPersonIds) + } + + def intersection(households: Households, path: String): Set[String] = { + val invalidHouseholdIds = readErrorCsv(path).map(_.id).toSet + val validHouseholdIds = households.getHouseholds.keySet().asScala.map(_.toString).toSet + validHouseholdIds.intersect(invalidHouseholdIds) + } + + def intersection(freightTours: Array[FreightTour], path: String): Set[String] = { + val invalidIds = readErrorCsv(path).map(_.id).toSet + val validIds = freightTours.map(_.tourId.toString).toSet + validIds.intersect(invalidIds) + } + + def intersection(freightPayloadPlans: Array[PayloadPlan], path: String): Set[String] = { + val invalidIds = readErrorCsv(path).map(_.id).toSet + val validIds = freightPayloadPlans.map(_.payloadId.toString).toSet + validIds.intersect(invalidIds) + } + + def intersection(freightCarriers: Array[FreightCarrier], path: String): Set[String] = { + val invalidIds = readErrorCsv(path).map(_.id).toSet + val validIds = freightCarriers.map(_.carrierId.toString).toSet + validIds.intersect(invalidIds) + } + + "scenario plan" should { + "contain all valid locations" in { + lazy val config: TypesafeConfig = ConfigFactory + .parseString(""" + |beam.routing.r5.linkRadiusMeters = 350 + |""".stripMargin) + .withFallback(testConfig("test/input/beamville/beam.conf")) + .resolve() + + val configBuilder = new MatSimBeamConfigBuilder(config) + val matsimConfig: Config = configBuilder.buildMatSimConf() + matsimConfig.planCalcScore().setMemorizingExperiencedPlans(true) + val beamConfig: BeamConfig = BeamConfig(config) + + val outputDir = FileUtils.setConfigOutputFile(beamConfig, matsimConfig) + + val (scenario, beamScenario, _) = buildBeamServicesAndScenario( + beamConfig, + matsimConfig + ) + + val snapLocationHelper: SnapLocationHelper = SnapLocationHelper( + new GeoUtilsImpl(beamConfig), + beamScenario.transportNetwork.streetLayer, + beamConfig.beam.routing.r5.linkRadiusMeters + ) + ScenarioLoaderHelper.validateScenario(scenario, snapLocationHelper, Some(outputDir)) + + val dummyCoord = new Coord() + val population: Seq[SnapCoordinateResult] = scenario.getPopulation.getPersons + .values() + .asScala + .flatMap { person => + person.getPlans.asScala.flatMap { plan => + plan.getPlanElements.asScala.map { + case e: Activity => snapLocationHelper.computeResult(e.getCoord) + case _ => Right(dummyCoord) + } + } + } + .toList + + val households: Seq[SnapCoordinateResult] = scenario.getHouseholds.getHouseholds + .values() + .asScala + .map { household => + val locationX = scenario.getHouseholds.getHouseholdAttributes + .getAttribute(household.getId.toString, "homecoordx") + .asInstanceOf[Double] + val locationY = scenario.getHouseholds.getHouseholdAttributes + .getAttribute(household.getId.toString, "homecoordy") + .asInstanceOf[Double] + val coord = new Coord(locationX, locationY) + snapLocationHelper.computeResult(coord) + } + .toList + + (population ++ households).forall(_.isRight) shouldBe true + } + + "remove invalid persons and households [case1 xml input]" in { + lazy val config: TypesafeConfig = ConfigFactory + .parseString(s""" + |beam.agentsim.agents.plans { + | inputPlansFilePath = "$pwd/test/test-resources/beam/input/snap-location/scenario/case1/xml/population.xml" + | inputPersonAttributesFilePath = "$pwd/test/test-resources/beam/input/snap-location/scenario/case1/xml/populationAttributes.xml" + |} + |beam.agentsim.agents.households { + | inputFilePath = "$pwd/test/test-resources/beam/input/snap-location/scenario/case1/xml/households.xml" + | inputHouseholdAttributesFilePath = "$pwd/test/test-resources/beam/input/snap-location/scenario/case1/xml/householdAttributes.xml" + |} + |beam.routing.r5.linkRadiusMeters = 350 + |""".stripMargin) + .withFallback(testConfig("test/input/beamville/beam.conf")) + .resolve() + + val configBuilder = new MatSimBeamConfigBuilder(config) + val matsimConfig: Config = configBuilder.buildMatSimConf() + matsimConfig.planCalcScore().setMemorizingExperiencedPlans(true) + val beamConfig: BeamConfig = BeamConfig(config) + + val outputDir = FileUtils.setConfigOutputFile(beamConfig, matsimConfig) + + val (scenario, beamScenario, _) = buildBeamServicesAndScenario( + beamConfig, + matsimConfig + ) + + val snapLocationHelper: SnapLocationHelper = SnapLocationHelper( + new GeoUtilsImpl(beamConfig), + beamScenario.transportNetwork.streetLayer, + beamConfig.beam.routing.r5.linkRadiusMeters + ) + ScenarioLoaderHelper.validateScenario(scenario, snapLocationHelper, Some(outputDir)) + + val households = scenario.getHouseholds.getHouseholds.values().asScala.toList + + scenario.getPopulation.getPersons.size() shouldBe 2 + intersection(scenario.getPopulation, path = s"$outputDir/${CsvFile.Plans}") shouldBe Set.empty + + households.foreach { household => + household.getMemberIds.size() shouldBe 1 + } + } + + "remove invalid persons and households [case2 xml input]" in { + lazy val config: TypesafeConfig = ConfigFactory + .parseString(s""" + |beam.agentsim.agents.plans { + | inputPlansFilePath = "$pwd/test/test-resources/beam/input/snap-location/scenario/case2/xml/population.xml" + | inputPersonAttributesFilePath = "$pwd/test/test-resources/beam/input/snap-location/scenario/case2/xml/populationAttributes.xml" + |} + |beam.agentsim.agents.households { + | inputFilePath = "$pwd/test/test-resources/beam/input/snap-location/scenario/case2/xml/households.xml" + | inputHouseholdAttributesFilePath = "$pwd/test/test-resources/beam/input/snap-location/scenario/case2/xml/householdAttributes.xml" + |} + |beam.routing.r5.linkRadiusMeters = 350 + |""".stripMargin) + .withFallback(testConfig("test/input/beamville/beam.conf")) + .resolve() + + val configBuilder = new MatSimBeamConfigBuilder(config) + val matsimConfig: Config = configBuilder.buildMatSimConf() + matsimConfig.planCalcScore().setMemorizingExperiencedPlans(true) + val beamConfig: BeamConfig = BeamConfig(config) + + val outputDir = FileUtils.setConfigOutputFile(beamConfig, matsimConfig) + + val (scenario, beamScenario, _) = buildBeamServicesAndScenario( + beamConfig, + matsimConfig + ) + + val snapLocationHelper: SnapLocationHelper = SnapLocationHelper( + new GeoUtilsImpl(beamConfig), + beamScenario.transportNetwork.streetLayer, + beamConfig.beam.routing.r5.linkRadiusMeters + ) + ScenarioLoaderHelper.validateScenario(scenario, snapLocationHelper, Some(outputDir)) + + scenario.getPopulation.getPersons.size() shouldBe 1 + intersection(scenario.getPopulation, path = s"$outputDir/${CsvFile.Plans}") shouldBe Set.empty + + scenario.getHouseholds.getHouseholds.size() shouldBe 1 + intersection(scenario.getHouseholds, path = s"$outputDir/${CsvFile.Households}") shouldBe Set.empty + } + + "remove invalid persons and households [case1 csv input]" in { + lazy val config: TypesafeConfig = ConfigFactory + .parseString(s""" + |beam.agentsim.agents.plans.inputPlansFilePath = "$pwd/test/test-resources/beam/input/snap-location/scenario/case1/csv/plans.csv" + |beam.agentsim.agents.households.inputFilePath = "$pwd/test/test-resources/beam/input/snap-location/scenario/case1/csv/households.csv" + |beam.routing.r5.linkRadiusMeters = 350 + |""".stripMargin) + .withFallback(testConfig("test/input/beamville/beam-csv.conf")) + .resolve() + + val configBuilder = new MatSimBeamConfigBuilder(config) + val matsimConfig: Config = configBuilder.buildMatSimConf() + matsimConfig.planCalcScore().setMemorizingExperiencedPlans(true) + val beamConfig: BeamConfig = BeamConfig(config) + + val outputDir = FileUtils.setConfigOutputFile(beamConfig, matsimConfig) + + val (scenario, beamScenario, _) = buildBeamServicesAndScenario( + beamConfig, + matsimConfig + ) + + val snapLocationHelper: SnapLocationHelper = SnapLocationHelper( + new GeoUtilsImpl(beamConfig), + beamScenario.transportNetwork.streetLayer, + beamConfig.beam.routing.r5.linkRadiusMeters + ) + ScenarioLoaderHelper.validateScenario(scenario, snapLocationHelper, Some(outputDir)) + + val households = scenario.getHouseholds.getHouseholds.values().asScala.toList + + scenario.getPopulation.getPersons.size() shouldBe 2 + intersection(scenario.getPopulation, path = s"$outputDir/${CsvFile.Plans}") shouldBe Set.empty + + households.foreach { household => + household.getMemberIds.size() shouldBe 1 + } + } + + "remove invalid persons and households [case2 csv input]" in { + lazy val config: TypesafeConfig = ConfigFactory + .parseString(s""" + |beam.agentsim.agents.plans.inputPlansFilePath = "$pwd/test/test-resources/beam/input/snap-location/scenario/case2/csv/plans.csv" + |beam.agentsim.agents.households.inputFilePath = "$pwd/test/test-resources/beam/input/snap-location/scenario/case2/csv/households.csv" + |beam.routing.r5.linkRadiusMeters = 350 + |""".stripMargin) + .withFallback(testConfig("test/input/beamville/beam-csv.conf")) + .resolve() + + val configBuilder = new MatSimBeamConfigBuilder(config) + val matsimConfig: Config = configBuilder.buildMatSimConf() + matsimConfig.planCalcScore().setMemorizingExperiencedPlans(true) + val beamConfig: BeamConfig = BeamConfig(config) + + val outputDir = FileUtils.setConfigOutputFile(beamConfig, matsimConfig) + + val (scenario, beamScenario, _) = buildBeamServicesAndScenario( + beamConfig, + matsimConfig + ) + + val snapLocationHelper: SnapLocationHelper = SnapLocationHelper( + new GeoUtilsImpl(beamConfig), + beamScenario.transportNetwork.streetLayer, + beamConfig.beam.routing.r5.linkRadiusMeters + ) + ScenarioLoaderHelper.validateScenario(scenario, snapLocationHelper, Some(outputDir)) + + scenario.getPopulation.getPersons.size() shouldBe 1 + intersection(scenario.getPopulation, path = s"$outputDir/${CsvFile.Plans}") shouldBe Set.empty + + scenario.getHouseholds.getHouseholds.size() shouldBe 1 + intersection(scenario.getHouseholds, path = s"$outputDir/${CsvFile.Households}") shouldBe Set.empty + } + + "remove invalid persons and households [case1 urbansimv2 input]" in { + lazy val config: TypesafeConfig = ConfigFactory + .parseString(s""" + |beam.exchange.scenario.folder = "$pwd/test/test-resources/beam/input/snap-location/scenario/case1/urbansim_v2" + |beam.routing.r5.linkRadiusMeters = 350 + |""".stripMargin) + .withFallback(testConfig("test/input/beamville/beam-urbansimv2.conf")) + .resolve() + + val configBuilder = new MatSimBeamConfigBuilder(config) + val matsimConfig: Config = configBuilder.buildMatSimConf() + matsimConfig.planCalcScore().setMemorizingExperiencedPlans(true) + val beamConfig: BeamConfig = BeamConfig(config) + + val outputDir = FileUtils.setConfigOutputFile(beamConfig, matsimConfig) + + val (scenario, beamScenario, _) = buildBeamServicesAndScenario( + beamConfig, + matsimConfig + ) + + val snapLocationHelper: SnapLocationHelper = SnapLocationHelper( + new GeoUtilsImpl(beamConfig), + beamScenario.transportNetwork.streetLayer, + beamConfig.beam.routing.r5.linkRadiusMeters + ) + ScenarioLoaderHelper.validateScenario(scenario, snapLocationHelper, Some(outputDir)) + + val households = scenario.getHouseholds.getHouseholds.values().asScala.toList + + scenario.getPopulation.getPersons.size() shouldBe 2 + intersection(scenario.getPopulation, path = s"$outputDir/${CsvFile.Plans}") shouldBe Set.empty + + households.foreach { household => + household.getMemberIds.size() shouldBe 1 + } + } + + "remove invalid persons and households [case2 urbansimv2 input]" in { + lazy val config: TypesafeConfig = ConfigFactory + .parseString(s""" + |beam.exchange.scenario.folder = "$pwd/test/test-resources/beam/input/snap-location/scenario/case2/urbansim_v2" + |beam.routing.r5.linkRadiusMeters = 350 + |""".stripMargin) + .withFallback(testConfig("test/input/beamville/beam-urbansimv2.conf")) + .resolve() + + val configBuilder = new MatSimBeamConfigBuilder(config) + val matsimConfig: Config = configBuilder.buildMatSimConf() + matsimConfig.planCalcScore().setMemorizingExperiencedPlans(true) + val beamConfig: BeamConfig = BeamConfig(config) + + val outputDir = FileUtils.setConfigOutputFile(beamConfig, matsimConfig) + + val (scenario, beamScenario, _) = buildBeamServicesAndScenario( + beamConfig, + matsimConfig + ) + + val snapLocationHelper: SnapLocationHelper = SnapLocationHelper( + new GeoUtilsImpl(beamConfig), + beamScenario.transportNetwork.streetLayer, + beamConfig.beam.routing.r5.linkRadiusMeters + ) + ScenarioLoaderHelper.validateScenario(scenario, snapLocationHelper, Some(outputDir)) + + scenario.getPopulation.getPersons.size() shouldBe 1 + intersection(scenario.getPopulation, path = s"$outputDir/${CsvFile.Plans}") shouldBe Set.empty + + scenario.getHouseholds.getHouseholds.size() shouldBe 1 + intersection(scenario.getHouseholds, path = s"$outputDir/${CsvFile.Households}") shouldBe Set.empty + } + + "remove invalid freight tours" in { + lazy val config: TypesafeConfig = ConfigFactory + .parseString(s""" + |beam.agentsim.agents.freight.toursFilePath = "$pwd/test/test-resources/beam/input/snap-location/freight/freight-tours.csv" + |beam.routing.r5.linkRadiusMeters = 350 + |beam.agentsim.snapLocationAndRemoveInvalidInputs = true + |""".stripMargin) + .withFallback(testConfig("test/input/beamville/beam-freight.conf")) + .resolve() + + val configBuilder = new MatSimBeamConfigBuilder(config) + val matsimConfig: Config = configBuilder.buildMatSimConf() + matsimConfig.planCalcScore().setMemorizingExperiencedPlans(true) + val beamConfig: BeamConfig = BeamConfig(config) + + val outputDir = FileUtils.setConfigOutputFile(beamConfig, matsimConfig) + + val (_, beamScenario, _) = buildBeamServicesAndScenario( + beamConfig, + matsimConfig + ) + + intersection( + beamScenario.freightCarriers.flatMap(_.tourMap.values).flatten.toArray, + path = s"$outputDir/${CsvFile.FreightTours}" + ) shouldBe Set.empty + } + + "remove invalid freight payload plans" in { + lazy val config: TypesafeConfig = ConfigFactory + .parseString(s""" + |beam.agentsim.agents.freight.plansFilePath = "$pwd/test/test-resources/beam/input/snap-location/freight/payload-plans.csv" + |beam.routing.r5.linkRadiusMeters = 350 + |beam.agentsim.snapLocationAndRemoveInvalidInputs = true + |""".stripMargin) + .withFallback(testConfig("test/input/beamville/beam-freight.conf")) + .resolve() + + val configBuilder = new MatSimBeamConfigBuilder(config) + val matsimConfig: Config = configBuilder.buildMatSimConf() + matsimConfig.planCalcScore().setMemorizingExperiencedPlans(true) + val beamConfig: BeamConfig = BeamConfig(config) + + val outputDir = FileUtils.setConfigOutputFile(beamConfig, matsimConfig) + + val (_, beamScenario, _) = buildBeamServicesAndScenario( + beamConfig, + matsimConfig + ) + + intersection( + beamScenario.freightCarriers.flatMap(_.payloadPlans.values).toArray, + path = s"$outputDir/${CsvFile.FreightPayloadPlans}" + ) shouldBe Set.empty + } + + "remove invalid freight carriers" in { + lazy val config: TypesafeConfig = ConfigFactory + .parseString(s""" + |beam.agentsim.agents.freight.carriersFilePath = "$pwd/test/test-resources/beam/input/snap-location/freight/freight-carriers.csv" + |beam.routing.r5.linkRadiusMeters = 350 + |beam.agentsim.snapLocationAndRemoveInvalidInputs = true + |""".stripMargin) + .withFallback(testConfig("test/input/beamville/beam-freight.conf")) + .resolve() + + val configBuilder = new MatSimBeamConfigBuilder(config) + val matsimConfig: Config = configBuilder.buildMatSimConf() + matsimConfig.planCalcScore().setMemorizingExperiencedPlans(true) + val beamConfig: BeamConfig = BeamConfig(config) + + val outputDir = FileUtils.setConfigOutputFile(beamConfig, matsimConfig) + + val (_, beamScenario, _) = buildBeamServicesAndScenario( + beamConfig, + matsimConfig + ) + + intersection( + beamScenario.freightCarriers.toArray, + path = s"$outputDir/${CsvFile.FreightCarriers}" + ) shouldBe Set.empty + } + } + +} diff --git a/test/test-resources/beam/input/snap-location/freight/freight-carriers.csv b/test/test-resources/beam/input/snap-location/freight/freight-carriers.csv new file mode 100644 index 00000000000..d87ec9d3c7d --- /dev/null +++ b/test/test-resources/beam/input/snap-location/freight/freight-carriers.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:403e9ccbedb831269fd57e1a5edd48e068cdf33750bf1bd02867246d7a8c058e +size 232 diff --git a/test/test-resources/beam/input/snap-location/freight/freight-tours.csv b/test/test-resources/beam/input/snap-location/freight/freight-tours.csv new file mode 100644 index 00000000000..3595f6b82ff --- /dev/null +++ b/test/test-resources/beam/input/snap-location/freight/freight-tours.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a05f0f2b94046948b249cd3bf5d683b3bbb34951d40fc1be70a28ba6ef2256d0 +size 248 diff --git a/test/test-resources/beam/input/snap-location/freight/payload-plans.csv b/test/test-resources/beam/input/snap-location/freight/payload-plans.csv new file mode 100644 index 00000000000..9c6e9b76a46 --- /dev/null +++ b/test/test-resources/beam/input/snap-location/freight/payload-plans.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ca6c183d0add95c0d68a7b2d26bb6c62bab40d723b13271ef697c38060ca92e +size 939 diff --git a/test/test-resources/beam/input/snap-location/scenario/case1/csv/households.csv b/test/test-resources/beam/input/snap-location/scenario/case1/csv/households.csv new file mode 100644 index 00000000000..433dae022fe --- /dev/null +++ b/test/test-resources/beam/input/snap-location/scenario/case1/csv/households.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:40a521bee13f8eccc5d13843cb4125f75e91da7425ed4f2bbe1e385f4a0b0312 +size 95 diff --git a/test/test-resources/beam/input/snap-location/scenario/case1/csv/plans.csv b/test/test-resources/beam/input/snap-location/scenario/case1/csv/plans.csv new file mode 100644 index 00000000000..6e7de309326 --- /dev/null +++ b/test/test-resources/beam/input/snap-location/scenario/case1/csv/plans.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:10f892985fdd48bf3cdae8fe1d10f62352b4ce8032f61244fb8a9c0693672c9d +size 1314 diff --git a/test/test-resources/beam/input/snap-location/scenario/case1/note b/test/test-resources/beam/input/snap-location/scenario/case1/note new file mode 100644 index 00000000000..bba4e27fe6b --- /dev/null +++ b/test/test-resources/beam/input/snap-location/scenario/case1/note @@ -0,0 +1,8 @@ +OutOfBoundingBox +Person 1 x 1627138.4 y 1117.0 + +R5SplitNull +Person 1 x 166045.2 y 2705.4 +Person 3 x 166675.4 y 3068.543 +Person 4 x 170108.4 y 1960.535 + diff --git a/test/test-resources/beam/input/snap-location/scenario/case1/urbansim_v2/blocks.csv.gz b/test/test-resources/beam/input/snap-location/scenario/case1/urbansim_v2/blocks.csv.gz new file mode 100644 index 00000000000..8eccf4c87c2 --- /dev/null +++ b/test/test-resources/beam/input/snap-location/scenario/case1/urbansim_v2/blocks.csv.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:567708718a96701b3977ddb90c21917b6d5dcbab79ad54d302cc8b91dfdb6cca +size 249 diff --git a/test/test-resources/beam/input/snap-location/scenario/case1/urbansim_v2/households.csv.gz b/test/test-resources/beam/input/snap-location/scenario/case1/urbansim_v2/households.csv.gz new file mode 100644 index 00000000000..d051b989b01 --- /dev/null +++ b/test/test-resources/beam/input/snap-location/scenario/case1/urbansim_v2/households.csv.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fee1ebbd211c0f9d9b53f9294a053dd930e5d20edcaf84534cf5395e6c02d1b4 +size 156 diff --git a/test/test-resources/beam/input/snap-location/scenario/case1/urbansim_v2/persons.csv.gz b/test/test-resources/beam/input/snap-location/scenario/case1/urbansim_v2/persons.csv.gz new file mode 100644 index 00000000000..a7a76074200 --- /dev/null +++ b/test/test-resources/beam/input/snap-location/scenario/case1/urbansim_v2/persons.csv.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8a4f7fc7716d6f202f51e60baa5da09d7ac44480c9ec1e71db6a3a7805f662c1 +size 217 diff --git a/test/test-resources/beam/input/snap-location/scenario/case1/urbansim_v2/plans.csv.gz b/test/test-resources/beam/input/snap-location/scenario/case1/urbansim_v2/plans.csv.gz new file mode 100644 index 00000000000..7673b1ab478 --- /dev/null +++ b/test/test-resources/beam/input/snap-location/scenario/case1/urbansim_v2/plans.csv.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be7f1b757d480628e044501adbe99f1a48245467d6c767fa440a7d233971a978 +size 615 diff --git a/test/test-resources/beam/input/snap-location/scenario/case1/xml/householdAttributes.xml b/test/test-resources/beam/input/snap-location/scenario/case1/xml/householdAttributes.xml new file mode 100755 index 00000000000..15563c7198b --- /dev/null +++ b/test/test-resources/beam/input/snap-location/scenario/case1/xml/householdAttributes.xml @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b239a0087a0fea5d6d629dc8b90177fa818fbf9bd4eadb650bfc1c2f13f3f253 +size 705 diff --git a/test/test-resources/beam/input/snap-location/scenario/case1/xml/households.xml b/test/test-resources/beam/input/snap-location/scenario/case1/xml/households.xml new file mode 100755 index 00000000000..2211510e3d0 --- /dev/null +++ b/test/test-resources/beam/input/snap-location/scenario/case1/xml/households.xml @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1749a97d76b2735b8736104ffee1c1c887be650224cf39e4a62426ce17cf3a18 +size 875 diff --git a/test/test-resources/beam/input/snap-location/scenario/case1/xml/population.xml b/test/test-resources/beam/input/snap-location/scenario/case1/xml/population.xml new file mode 100755 index 00000000000..996d9d151bd --- /dev/null +++ b/test/test-resources/beam/input/snap-location/scenario/case1/xml/population.xml @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53c0ae571973fe243adda66f3d68ae6dc1dce8962187d17575373bc89e03efbd +size 2850 diff --git a/test/test-resources/beam/input/snap-location/scenario/case1/xml/populationAttributes.xml b/test/test-resources/beam/input/snap-location/scenario/case1/xml/populationAttributes.xml new file mode 100755 index 00000000000..c2d270cac35 --- /dev/null +++ b/test/test-resources/beam/input/snap-location/scenario/case1/xml/populationAttributes.xml @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:adb3199cd0c3d15ee9244af92c81479522908329ca44893bd7cb958554856f37 +size 998 diff --git a/test/test-resources/beam/input/snap-location/scenario/case2/csv/households.csv b/test/test-resources/beam/input/snap-location/scenario/case2/csv/households.csv new file mode 100644 index 00000000000..829dcbf2848 --- /dev/null +++ b/test/test-resources/beam/input/snap-location/scenario/case2/csv/households.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:621b455d74f0323748f7e481f890cd5cbd3080262b681749efb4492e093a2b41 +size 96 diff --git a/test/test-resources/beam/input/snap-location/scenario/case2/csv/plans.csv b/test/test-resources/beam/input/snap-location/scenario/case2/csv/plans.csv new file mode 100644 index 00000000000..a7aae41899f --- /dev/null +++ b/test/test-resources/beam/input/snap-location/scenario/case2/csv/plans.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:456f1efc4fc8933ac32f832b51a0fa5b670716be7450c4cd7fb6e7d7b9244c28 +size 1313 diff --git a/test/test-resources/beam/input/snap-location/scenario/case2/note b/test/test-resources/beam/input/snap-location/scenario/case2/note new file mode 100644 index 00000000000..41b8909e821 --- /dev/null +++ b/test/test-resources/beam/input/snap-location/scenario/case2/note @@ -0,0 +1,7 @@ +OutOfBoundingBox +Household 1 x 1627138.4 y 1117.0 + +R5SplitNull +Person 1 x 166045.2 y 2705.4 +Person 3 x 166675.4 y 3068.543 +Person 4 x 170108.4 y 1960.535 diff --git a/test/test-resources/beam/input/snap-location/scenario/case2/urbansim_v2/blocks.csv.gz b/test/test-resources/beam/input/snap-location/scenario/case2/urbansim_v2/blocks.csv.gz new file mode 100644 index 00000000000..a77ffc2555e --- /dev/null +++ b/test/test-resources/beam/input/snap-location/scenario/case2/urbansim_v2/blocks.csv.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f4b46994dcb316c264c77feed485283bee13cf25fea4bf80c3b49b075e82eedd +size 250 diff --git a/test/test-resources/beam/input/snap-location/scenario/case2/urbansim_v2/households.csv.gz b/test/test-resources/beam/input/snap-location/scenario/case2/urbansim_v2/households.csv.gz new file mode 100644 index 00000000000..d051b989b01 --- /dev/null +++ b/test/test-resources/beam/input/snap-location/scenario/case2/urbansim_v2/households.csv.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fee1ebbd211c0f9d9b53f9294a053dd930e5d20edcaf84534cf5395e6c02d1b4 +size 156 diff --git a/test/test-resources/beam/input/snap-location/scenario/case2/urbansim_v2/persons.csv.gz b/test/test-resources/beam/input/snap-location/scenario/case2/urbansim_v2/persons.csv.gz new file mode 100644 index 00000000000..0427fc031f3 --- /dev/null +++ b/test/test-resources/beam/input/snap-location/scenario/case2/urbansim_v2/persons.csv.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b8c1661871669f2c20b4f22272184285ab64a5ea213aaf248d1a81ca9ec9a1c5 +size 217 diff --git a/test/test-resources/beam/input/snap-location/scenario/case2/urbansim_v2/plans.csv.gz b/test/test-resources/beam/input/snap-location/scenario/case2/urbansim_v2/plans.csv.gz new file mode 100644 index 00000000000..28e5c4f98c5 --- /dev/null +++ b/test/test-resources/beam/input/snap-location/scenario/case2/urbansim_v2/plans.csv.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a7d3af7a4245e2e843b0c77666a2d04a4a2f26024f4ad76b62f6b8102ffc618c +size 613 diff --git a/test/test-resources/beam/input/snap-location/scenario/case2/xml/householdAttributes.xml b/test/test-resources/beam/input/snap-location/scenario/case2/xml/householdAttributes.xml new file mode 100755 index 00000000000..8115f49478f --- /dev/null +++ b/test/test-resources/beam/input/snap-location/scenario/case2/xml/householdAttributes.xml @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56c400740feb094463a5baabf1a66d719ef6a8059bd07ac930f390786b039f3e +size 706 diff --git a/test/test-resources/beam/input/snap-location/scenario/case2/xml/households.xml b/test/test-resources/beam/input/snap-location/scenario/case2/xml/households.xml new file mode 100755 index 00000000000..2211510e3d0 --- /dev/null +++ b/test/test-resources/beam/input/snap-location/scenario/case2/xml/households.xml @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1749a97d76b2735b8736104ffee1c1c887be650224cf39e4a62426ce17cf3a18 +size 875 diff --git a/test/test-resources/beam/input/snap-location/scenario/case2/xml/population.xml b/test/test-resources/beam/input/snap-location/scenario/case2/xml/population.xml new file mode 100755 index 00000000000..e810d12ebdc --- /dev/null +++ b/test/test-resources/beam/input/snap-location/scenario/case2/xml/population.xml @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9cfa90074dc01de642da79b6896271d74d3d36967731dabdea2a006f345568c +size 2849 diff --git a/test/test-resources/beam/input/snap-location/scenario/case2/xml/populationAttributes.xml b/test/test-resources/beam/input/snap-location/scenario/case2/xml/populationAttributes.xml new file mode 100755 index 00000000000..c2d270cac35 --- /dev/null +++ b/test/test-resources/beam/input/snap-location/scenario/case2/xml/populationAttributes.xml @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:adb3199cd0c3d15ee9244af92c81479522908329ca44893bd7cb958554856f37 +size 998 diff --git a/test/test-resources/beam/router/0.linkstats.csv.gz b/test/test-resources/beam/router/0.linkstats.csv.gz index 5ee3fca9a7e..9238d49d25a 100644 --- a/test/test-resources/beam/router/0.linkstats.csv.gz +++ b/test/test-resources/beam/router/0.linkstats.csv.gz @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:298c5a2730f3b0c9d9d654b86e6283cf9d37c7b7ba555a8ceb934de71bdd9341 -size 40571 +oid sha256:df8f992e0640a656a48b91f9f06456bb3e07573c2c7b46e6a8e0b9131ab77d82 +size 31764