Merge remote-tracking branch 'origin/develop' into do/#3441-mode-choi…

…ce-strategy

# Conflicts:
#	src/main/R/freight/freight-processing.R
#	src/main/scala/beam/agentsim/agents/freight/input/FreightReader.scala
#	src/main/scala/beam/agentsim/agents/household/HouseholdActor.scala
#	src/main/scala/beam/agentsim/agents/household/HouseholdFleetManager.scala

aws/src/main/python/updateDependencies/lambda_function.py

@@ -37,15 +37,19 @@
   - sudo apt-get install gcc-8 g++-8 -y
   - sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-8 60 --slave /usr/bin/g++ g++ /usr/bin/g++-8
   - sudo apt install jq -y
+  - sudo add-apt-repository ppa:git-core/ppa -y
+  - sudo apt-get update
+  - sudo apt-get install git -y
   - echo "-------------------Finished updating Beam dependencies----------------------"
   - cd /home/ubuntu/git/beam
   - echo "send notification ..."
   - /home/ubuntu/git/glip.sh -i "http://icons.iconarchive.com/icons/uiconstock/socialmedia/32/AWS-icon.png" -a "Updating Dependencies" -b "Beam automated deployment image update started on $(ec2metadata --instance-id)."
   - echo "git checkout ..."
-  - sudo git reset origin/HEAD
+  - sudo git reset --hard origin/HEAD
   - sudo git checkout -- .
   - sudo git clean -df
   - sudo git checkout develop
+  - sudo git reset --hard origin/develop
   - sudo git pull
   - sudo git fetch
   - sudo git fetch --prune
@@ -101,6 +105,7 @@ def init_ec2(region):
 
 def deploy(script, instance_type, region_prefix, shutdown_behaviour, instance_name, en_vars):
     res = ec2.run_instances(ImageId=en_vars[region_prefix + 'IMAGE_ID'],
+
                             InstanceType=instance_type,
                             UserData=script,
                             KeyName=en_vars[region_prefix + 'KEY_NAME'],
@@ -114,7 +119,7 @@ def deploy(script, instance_type, region_prefix, shutdown_behaviour, instance_na
                                 'Tags': [ {
                                     'Key': 'Name',
                                     'Value': instance_name
-                                } ]
+                                 }]
                             } ])
     return res['Instances'][0]['InstanceId']
 

build.gradle

@@ -136,7 +136,7 @@ configurations.all {
 
 dependencies {
 
-    implementation(group: 'com.github.LBNL-UCB-STI', name: 'beam-utilities', version: 'v0.2.15') {
+    implementation(group: 'com.github.LBNL-UCB-STI', name: 'beam-utilities', version: 'v0.2.17') {
         exclude group: 'com.github.LBNL-UCB-STI', module: 'r5'
         exclude group: 'org.matsim', module: 'matsim'
     }

src/main/R/common/helpers.R

@@ -11,18 +11,18 @@ mergefiles <- function(directory){
   for (file in list.files(directory)){
     # if the merged dataset doesn't exist, create it
     filepath <- paste(directory, file, sep="/")
-
+    
     if (!exists("dataset")){
       dataset <- readCsv(filepath)
     }
-
+    
     # if the merged dataset does exist, append to it
     if (exists("dataset")){
       temp_dataset <-readCsv(filepath)
       dataset<-rbind(dataset, temp_dataset)
       rm(temp_dataset)
     }
-
+    
   }
   return(dataset)
 }
@@ -66,3 +66,141 @@ sankeyDiagram <- function(source, target, value, title) {
 }
 
 
+clusteringFreightBy <- function(data,cols,dataCbg,numClusters,labelData) {
+  data[,hour:=as.integer(arrivalTime/3600)%%24]
+  data_asSf <- st_as_sf(data,coords=cols,crs=4326,agr="constant")
+  data_withCBG_asSf <- st_intersection(data_asSf,dataCbg)
+  data_withCBG_asSf$X <- st_coordinates(data_withCBG_asSf$geometry)[,1]
+  data_withCBG_asSf$Y <- st_coordinates(data_withCBG_asSf$geometry)[,2]
+  data_withCBG <- data.table::as.data.table(data_withCBG_asSf)
+  data_withCBG[,cluster:=kmeans(data_withCBG[,.(X,Y)],numClusters)$cluster]
+  result <- data_withCBG[,.(count=.N,x2=mean(X),y2=mean(Y)),by=.(hour,cluster)]
+  result$label <- labelData
+  result
+}
+
+assignPostMilesGeometries <- function(TRUCK_DATA, POSTMILES_SHP) {
+  getElementWithoutFlag <- function(list_elements, flag) {
+    i <- 1
+    while(list_elements[i] == '') {
+      i <- i+1
+    }
+    return(list_elements[i])
+  }
+  TRUCK_DATA$X <- 0.0
+  TRUCK_DATA$Y <- 0.0
+  ### GEOCODING ###
+  #TRUCK_DATA$lon <- 0.0
+  #TRUCK_DATA$lat <- 0.0
+  #TRUCK_DATA$geoAddress <- ""
+  #nrow(TRUCK_DATA)
+  postmiles <- st_read(POSTMILES_SHP)
+  for(i in 1:nrow(TRUCK_DATA)) {
+    county <- TRUCK_DATA$COUNTY[i]
+    cnty <- TRUCK_DATA$CNTY[i]
+    dist <- TRUCK_DATA$DIST[i]
+    rte <- TRUCK_DATA$RTE[i]
+    leg <- TRUCK_DATA$LEG[i]
+    pm <- TRUCK_DATA$POST_MILE[i]
+    filteredPM <- postmiles %>% filter(County == cnty, 
+                                       District == dist, 
+                                       Route == rte, 
+                                       startsWith(as.character(PMc), as.character(pm)))
+    if(nrow(filteredPM) == 0) {
+      filteredPM <- postmiles %>% filter(County == cnty, District == dist, Route == rte)
+    }
+    if(nrow(filteredPM) > 1) {
+      pm_numeric <- 0
+      if(!is.na(as.numeric(pm))) {
+        pm_numeric <- as.numeric(pm)
+      } else {
+        element <- getElementWithoutFlag(unlist(strsplit(pm, "R")), "R")
+        element <- getElementWithoutFlag(unlist(strsplit(element, "L")), "L")
+        element <- getElementWithoutFlag(unlist(strsplit(element, "T")), "T")
+        element <- getElementWithoutFlag(unlist(strsplit(element, "M")), "M")
+        pm_numeric <- as.numeric(element)
+      }
+      filteredPM <- filteredPM %>% rowwise() %>% mutate(diff = abs(as.numeric(PM) - pm_numeric))
+      filteredPM <- filteredPM %>% filter(diff == min(filteredPM$diff))
+    }
+    if(nrow(filteredPM) > 0) {
+      TRUCK_DATA$X[i] <- st_coordinates(filteredPM$geometry[1])[1]
+      TRUCK_DATA$Y[i] <- st_coordinates(filteredPM$geometry[1])[2]
+    }
+    # address <- paste(county, " County, California, USA", sep="")
+    # descriptions <- unlist(strsplit(TRUCK_DATA$DESCRIPTION[i],","))
+    # if(length(descriptions) > 1) {
+    #   address <- paste(str_trim(descriptions[2]), ", ", str_trim(descriptions[1]), ", ", address, sep="")
+    # } else {
+    #   address <- paste(str_trim(descriptions), ", ", address, sep="")
+    # }
+    # address <- ", CA"
+    # address <- paste(TRUCK_DATA$DESCRIPTION[i], ", ", address, sep="")
+    # address <- gsub("RTE.", "route", address)
+    # address <- gsub("JCT.", "", address)
+    # print(address)
+    # result <- geocode(address, output = "latlona", source = "google")
+    # TRUCK_DATA$lon[i] <- as.numeric(result[1])
+    # TRUCK_DATA$lat[i] <- as.numeric(result[2])
+    # TRUCK_DATA$geoAddress[i] <- as.character(result[3])
+    # print(TRUCK_DATA[i])
+  }
+  data.table::fwrite(TRUCK_DATA, normalizePath(pp(freightDir,"/validation/2017_truck_aadtt_geocoded.csv")), quote=T)
+  print("END OF assignPostMilesGeometries")
+  return(TRUCK_DATA)
+}
+
+assignLinkIdToTruckAADTT <- function(NETWORK_CLEANED, NETWORK_CRS, TRUCK_AADTT, MAX_DISTANCE_IN_METER, EXPANSION_FACTOR) {
+  network_sf <- st_transform(st_as_sf(
+    NETWORK_CLEANED,
+    coords = c("fromLocationX", "fromLocationY"),
+    crs = NETWORK_CRS,
+    agr = "constant"), 4326)
+
+  truck_aadtt_sf <- st_as_sf(
+    TRUCK_AADTT,
+    coords = c("X", "Y"),
+    crs = 4326,
+    agr = "constant")
+
+  network_sf$tempDistance <- NA
+  truck_aadtt_sf$linkId <- NA
+  expansionFactor <- EXPANSION_FACTOR
+  maxDistanceToSearch <- MAX_DISTANCE_IN_METER
+  counter1 <- 0
+  counter2 <- 0
+  for (row in 1:nrow(truck_aadtt_sf)) {
+    if(row %% 100 == 0) {
+      print(paste(row, " entries have been processed so far!"))
+    }
+    current_pm <- truck_aadtt_sf[row,]
+    currentDist <- 20
+    current_pm_links <- st_is_within_distance(current_pm, network_sf, dist = currentDist)
+    list_of_links <- current_pm_links[1][[1]]
+    while(length(list_of_links) < 1 & currentDist < maxDistanceToSearch) {
+      currentDist <- currentDist * 2
+      current_pm_links <- st_is_within_distance(current_pm, network_sf, dist = currentDist)
+      list_of_links = current_pm_links[1][[1]]
+    }
+    selected_links <- network_sf[list_of_links,]
+    if (length(list_of_links) == 0){
+      counter1 <- counter1 + 1
+      print("no links!!!")
+    } else if (length(list_of_links) == 1){
+      truck_aadtt_sf$linkId[row] <- network_sf[list_of_links,]$linkId
+      network_sf <- network_sf[-list_of_links,]
+      counter2 <- counter2 + 1
+    } else {
+      selected_pm <- truck_aadtt_sf[row,]
+      selected_network <- network_sf[selected_links,]
+      selected_network$tempDistance <- st_distance(selected_network, selected_pm)
+      selected_network_filtered <- slice(selected_network, which.min(tempDistance))
+      truck_aadtt_sf$linkId[row] <- selected_network_filtered$linkId
+      network_sf <- network_sf %>% filter(linkId != selected_network_filtered$linkId)
+      counter2 <- counter2 + 1
+    }
+  }
+  print(paste("# unmatched links ", counter1))
+  print(paste("# matched links ", counter2))
+  return(truck_aadtt_sf)
+}