Merge branch 'develop' of github.com:usdot-jpo-ode/jpo-conflictvisual…

…izer into decoder

api/jpo-conflictvisualizer-api/src/main/java/us/dot/its/jpo/ode/api/accessors/map/ProcessedMapRepository.java

@@ -11,7 +11,7 @@
 import us.dot.its.jpo.ode.api.models.IntersectionReferenceData;
 
 public interface ProcessedMapRepository extends DataLoader<ProcessedMap<LineString>>{
-    Query getQuery(Integer intersectionID, Long startTime, Long endTime,boolean latest);
+    Query getQuery(Integer intersectionID, Long startTime, Long endTime, boolean latest, boolean compact);
 
     long getQueryResultCount(Query query);
 

api/jpo-conflictvisualizer-api/src/main/java/us/dot/its/jpo/ode/api/accessors/map/ProcessedMapRepositoryImpl.java

@@ -57,7 +57,7 @@ public class ProcessedMapRepositoryImpl implements ProcessedMapRepository {
     private ObjectMapper mapper = DateJsonMapper.getInstance();
     private Logger logger = LoggerFactory.getLogger(ProcessedMapRepositoryImpl.class);
 
-    public Query getQuery(Integer intersectionID, Long startTime, Long endTime, boolean latest) {
+    public Query getQuery(Integer intersectionID, Long startTime, Long endTime, boolean latest, boolean compact) {
         Query query = new Query();
 
         if (intersectionID != null) {
@@ -81,8 +81,13 @@ public Query getQuery(Integer intersectionID, Long startTime, Long endTime, bool
             query.limit(props.getMaximumResponseSize());
         }
 
+        if (compact){
+            query.fields().exclude("recordGeneratedAt", "properties.validationMessages");
+        }else{
+            query.fields().exclude("recordGeneratedAt");
+        }
+
         query.addCriteria(Criteria.where("properties.timeStamp").gte(startTimeString).lte(endTimeString));
-        query.fields().exclude("recordGeneratedAt");
         return query;
     }
 

api/jpo-conflictvisualizer-api/src/main/java/us/dot/its/jpo/ode/api/accessors/spat/ProcessedSpatRepository.java

@@ -9,7 +9,7 @@
 import org.springframework.data.mongodb.core.query.Query;
 
 public interface ProcessedSpatRepository extends DataLoader<ProcessedSpat>{
-    Query getQuery(Integer intersectionID, Long startTime, Long endTime);
+    Query getQuery(Integer intersectionID, Long startTime, Long endTime, boolean latest, boolean compact);
 
     long getQueryResultCount(Query query);
 

api/jpo-conflictvisualizer-api/src/main/java/us/dot/its/jpo/ode/api/accessors/spat/ProcessedSpatRepositoryImpl.java

@@ -6,6 +6,7 @@
 import java.util.ArrayList;
 import java.util.List;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.data.domain.Sort;
 import org.springframework.data.mongodb.core.MongoTemplate;
 import org.springframework.data.mongodb.core.query.Criteria;
 import org.springframework.data.mongodb.core.query.Query;
@@ -40,7 +41,7 @@ public class ProcessedSpatRepositoryImpl implements ProcessedSpatRepository {
     private DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd-HH");
     private ObjectMapper mapper = DateJsonMapper.getInstance();
 
-    public Query getQuery(Integer intersectionID, Long startTime, Long endTime) {
+    public Query getQuery(Integer intersectionID, Long startTime, Long endTime, boolean latest, boolean compact) {
         Query query = new Query();
 
         if (intersectionID != null) {
@@ -56,9 +57,21 @@ public Query getQuery(Integer intersectionID, Long startTime, Long endTime) {
         if (endTime != null) {
             endTimeString = Instant.ofEpochMilli(endTime).toString();
         }
-	    query.limit(props.getMaximumResponseSize());
-        query.addCriteria(Criteria.where("odeReceivedAt").gte(startTimeString).lte(endTimeString));
-        query.fields().exclude("recordGeneratedAt");
+
+        if (latest) {
+            query.with(Sort.by(Sort.Direction.DESC, "utcTimeStamp"));
+            query.limit(1);
+        }else{
+            query.limit(props.getMaximumResponseSize());
+        }
+
+        if (compact){
+            query.fields().exclude("recordGeneratedAt", "validationMessages");
+        }else{
+            query.fields().exclude("recordGeneratedAt");
+        }
+
+        query.addCriteria(Criteria.where("utcTimeStamp").gte(startTimeString).lte(endTimeString));
         return query;
     }
 
@@ -72,26 +85,20 @@ public List<ProcessedSpat> findProcessedSpats(Query query) {
     }
 
     public List<IDCount> getSpatBroadcastRates(int intersectionID, Long startTime, Long endTime){
-        Query query = getQuery(intersectionID, startTime, endTime);
+        Query query = getQuery(intersectionID, startTime, endTime, false, true);
 
         query.fields().include("utcTimeStamp");
         List<Map> times = mongoTemplate.find(query, Map.class, collectionName);
 
-        //List<ZonedDateTime> spats =  findProcessedSpats(query);
-        System.out.println("Retreived Spat List" + times.size());
         Map<String, IDCount> results = new HashMap<>();
 
         for(Map doc: times){
-            //ZonedDateTime time = spat.getUtcTimeStamp();
-            System.out.println(doc);
             ZonedDateTime time = mapper.convertValue(doc.get("utcTimeStamp"), ZonedDateTime.class);
             String key = time.format(formatter);
 
-            //String key = map.getProperties().getTimeStamp().substring(0,10) + map.getProperties().getTimeStamp().substring(11,13);
             if(results.containsKey(key)){
                 IDCount count = results.get(key);
                 count.setCount(count.getCount() +1);
-                //results.put(key, count);
             }
             else{
                 IDCount count = new IDCount();
@@ -101,46 +108,12 @@ public List<IDCount> getSpatBroadcastRates(int intersectionID, Long startTime, L
             }
         }
 
-        System.out.println("Finished Message Parsing");
-
-        //AggregationResults<IDCount> result = mongoTemplate.aggregate(aggregation, collectionName, IDCount.class);
-        //List<IDCount> results = result.getMappedResults();
-        //results = new ArrayList<IDCount>(results);
-
         List<IDCount> outputCounts = new ArrayList<>(results.values());
         for (IDCount r : outputCounts) {
             r.setCount((double) r.getCount() / 3600.0);
         }
         return outputCounts;
-        // String startTimeString = Instant.ofEpochMilli(0).toString();
-        // String endTimeString = Instant.now().toString();
-
-        // if (startTime != null) {
-        //     startTimeString = Instant.ofEpochMilli(startTime).toString();
-        // }
-        // if (endTime != null) {
-        //     endTimeString = Instant.ofEpochMilli(endTime).toString();
-        // }
-
-        // AggregationOptions options = AggregationOptions.builder().allowDiskUse(true).build();
-
-        // Aggregation aggregation = Aggregation.newAggregation(
-        //     Aggregation.match(Criteria.where("intersectionId").is(intersectionID)),
-        //     Aggregation.match(Criteria.where("utcTimeStamp").gte(startTimeString).lte(endTimeString)),
-        //     Aggregation.project("utcTimeStamp"),
-        //     Aggregation.project()
-        //         .and(DateOperators.DateFromString.fromStringOf("utcTimeStamp")).as("date"),
-        //     Aggregation.project()
-        //         .and(DateOperators.DateToString.dateOf("date").toString("%Y-%m-%d-%H")).as("dateStr"),
-        //     Aggregation.group("dateStr").count().as("count"),
-        //     Aggregation.sort(Sort.Direction.ASC, "_id")
-        // ).withOptions(options);
-
-        // AggregationResults<IDCount> result = mongoTemplate.aggregate(aggregation, collectionName, IDCount.class);
-        // List<IDCount> results = result.getMappedResults();
-        // for (IDCount r: results){
-        //     r.setCount((float)r.getCount() / 3600.0);    
-        // }
+
 
     }
 
@@ -187,5 +160,4 @@ public List<IDCount> getSpatBroadcastRateDistribution(int intersectionID, Long s
     public void add(ProcessedSpat item) {
         mongoTemplate.save(item, collectionName);
     }
-
 }

api/jpo-conflictvisualizer-api/src/main/java/us/dot/its/jpo/ode/api/controllers/MapController.java

@@ -46,13 +46,14 @@ public ResponseEntity<List<ProcessedMap<LineString>>> findMaps(
             @RequestParam(name = "intersection_id", required = false) Integer intersectionID,
             @RequestParam(name = "start_time_utc_millis", required = false) Long startTime,
             @RequestParam(name = "end_time_utc_millis", required = false) Long endTime,
-            @RequestParam(name= "latest", required = false, defaultValue = "false") boolean latest,
-            @RequestParam(name = "test", required = false, defaultValue = "false") boolean testData) {
+            @RequestParam(name = "latest", required = false, defaultValue = "false") boolean latest,
+            @RequestParam(name = "test", required = false, defaultValue = "false") boolean testData,
+            @RequestParam(name = "compact", required = false, defaultValue = "false") boolean compact) {
 
         if (testData) {
             return ResponseEntity.ok(MockMapGenerator.getProcessedMaps());
         } else {
-            Query query = processedMapRepo.getQuery(intersectionID, startTime, endTime, latest);
+            Query query = processedMapRepo.getQuery(intersectionID, startTime, endTime, latest, compact);
             long count = processedMapRepo.getQueryResultCount(query);
 
             logger.info("Returning ProcessedMap Response with Size: " + count);
@@ -73,7 +74,7 @@ public ResponseEntity<Long> countMaps(
         if (testData) {
             return ResponseEntity.ok(5L);
         } else {
-            Query query = processedMapRepo.getQuery(intersectionID, startTime, endTime, false);
+            Query query = processedMapRepo.getQuery(intersectionID, startTime, endTime, false, true);
             long count = processedMapRepo.getQueryResultCount(query);
 
             logger.info("Found: " + count + "Processed Map Messages");

api/jpo-conflictvisualizer-api/src/main/java/us/dot/its/jpo/ode/api/controllers/SpatController.java

@@ -46,12 +46,14 @@ public ResponseEntity<List<ProcessedSpat>> findSpats(
             @RequestParam(name = "intersection_id", required = false) Integer intersectionID,
             @RequestParam(name = "start_time_utc_millis", required = false) Long startTime,
             @RequestParam(name = "end_time_utc_millis", required = false) Long endTime,
+            @RequestParam(name = "latest", required = false, defaultValue = "false") boolean latest,
+            @RequestParam(name = "compact", required = false, defaultValue = "false") boolean compact,
             @RequestParam(name = "test", required = false, defaultValue = "false") boolean testData) {
 
         if (testData) {
             return ResponseEntity.ok(MockSpatGenerator.getProcessedSpats());
         } else {
-            Query query = processedSpatRepo.getQuery(intersectionID, startTime, endTime);
+            Query query = processedSpatRepo.getQuery(intersectionID, startTime, endTime, latest, compact);
             long count = processedSpatRepo.getQueryResultCount(query);
             logger.info("Returning Processed Spat Response with Size: " + count);
             return ResponseEntity.ok(processedSpatRepo.findProcessedSpats(query));
@@ -70,7 +72,7 @@ public ResponseEntity<Long> countSpats(
         if (testData) {
             return ResponseEntity.ok(80L);
         } else {
-            Query query = processedSpatRepo.getQuery(intersectionID, startTime, endTime);
+            Query query = processedSpatRepo.getQuery(intersectionID, startTime, endTime,false, true);
             long count = processedSpatRepo.getQueryResultCount(query);
             logger.info("Found: " + count + "Processed Spat Messages");
             return ResponseEntity.ok(count);

api/jpo-conflictvisualizer-api/src/test/java/us/dot/its/jpo/ode/api/MapTest.java

@@ -43,10 +43,10 @@ public void testProcessedMap() {
 
     List<ProcessedMap<LineString>> list = MockMapGenerator.getProcessedMaps();
 
-    Query query = processedMapRepo.getQuery(null, null, null, false);
+    Query query = processedMapRepo.getQuery(null, null, null, false, false);
     when(processedMapRepo.findProcessedMaps(query)).thenReturn(list);
 
-    ResponseEntity<List<ProcessedMap<LineString>>> result = controller.findMaps(null, null, null, false, false);
+    ResponseEntity<List<ProcessedMap<LineString>>> result = controller.findMaps(null, null, null, false, false, false);
     assertThat(result.getStatusCode()).isEqualTo(HttpStatus.OK);
     assertThat(result.getBody()).isEqualTo(list);
   }

api/jpo-conflictvisualizer-api/src/test/java/us/dot/its/jpo/ode/api/MockKeyCloakAuth.java

@@ -5,7 +5,6 @@
 import java.util.Collection;
 import java.util.Set;
 
-
 import org.springframework.security.core.Authentication;
 import org.springframework.security.core.GrantedAuthority;
 import org.springframework.security.core.authority.SimpleGrantedAuthority;

api/jpo-conflictvisualizer-api/src/test/java/us/dot/its/jpo/ode/api/SpatTest.java

@@ -41,10 +41,10 @@ public void testProcessedSpat() {
 
     List<ProcessedSpat> list = MockSpatGenerator.getProcessedSpats();
 
-    Query query = processedSpatRepo.getQuery(null, null, null);
+    Query query = processedSpatRepo.getQuery(null, null, null, false,  false);
     when(processedSpatRepo.findProcessedSpats(query)).thenReturn(list);
 
-    ResponseEntity<List<ProcessedSpat>> result = controller.findSpats(null, null, null, false);
+    ResponseEntity<List<ProcessedSpat>> result = controller.findSpats(null, null, null, false,false,false);
     assertThat(result.getStatusCode()).isEqualTo(HttpStatus.OK);
     // assertThat(result.getHeaders().getContentType()).isEqualTo(MediaType.APPLICATION_JSON);
     assertThat(result.getBody()).isEqualTo(list);

api/jpo-conflictvisualizer-api/src/test/java/us/dot/its/jpo/ode/api/accessorTests/map/ProcessedMapRepositoryImplTest.java

@@ -52,7 +52,7 @@ public void testGetQuery() {
 
         boolean latest = true;
 
-        Query query = repository.getQuery(intersectionID, startTime, endTime, latest);
+        Query query = repository.getQuery(intersectionID, startTime, endTime, latest, false);
 
 
         // Assert IntersectionID

api/jpo-conflictvisualizer-api/src/test/java/us/dot/its/jpo/ode/api/accessorTests/spat/ProcessedSpatRepositoryImplTest.java

@@ -51,7 +51,7 @@ void setUp() {
     @Test
     public void testGetQuery() {
 
-        Query query = repository.getQuery(intersectionID, startTime, endTime);
+        Query query = repository.getQuery(intersectionID, startTime, endTime, false, false);
 
 
 
@@ -60,7 +60,7 @@ public void testGetQuery() {
 
 
         // Assert Start and End Time
-        Document queryTimeDocument = (Document)query.getQueryObject().get("odeReceivedAt");
+        Document queryTimeDocument = (Document)query.getQueryObject().get("utcTimeStamp");
         assertThat(queryTimeDocument.getString("$gte")).isEqualTo(Instant.ofEpochMilli(startTime).toString());
         assertThat(queryTimeDocument.getString("$lte")).isEqualTo(Instant.ofEpochMilli(endTime).toString());
 

gui/src/apis/mm-api.ts

@@ -16,18 +16,24 @@ class MessageMonitorApi {
     roadRegulatorId,
     startTime,
     endTime,
+    latest,
+    compact,
   }: {
     token: string;
     intersectionId: number;
     roadRegulatorId: number;
     startTime?: Date;
     endTime?: Date;
+    latest?: boolean;
+    compact?: boolean;
   }): Promise<ProcessedSpat[]> {
     const queryParams: Record<string, string> = {};
     queryParams["intersection_id"] = intersectionId.toString();
     queryParams["road_regulator_id"] = roadRegulatorId.toString();
     if (startTime) queryParams["start_time_utc_millis"] = startTime.getTime().toString();
     if (endTime) queryParams["end_time_utc_millis"] = endTime.getTime().toString();
+    if (latest) queryParams["latest"] = latest.toString();
+    if (compact) queryParams["compact"] = compact.toString();
 
     var response = await authApiHelper.invokeApi({
       path: "/spat/json",

gui/src/components/map/map-component.tsx

@@ -268,6 +268,7 @@ const generateQueryParams = (
         endDate: new Date(Date.now() + endOffset),
         eventDate: new Date(Date.now()),
         vehicleId: undefined,
+        default: true,
       };
   }
 };
@@ -304,6 +305,7 @@ const MapTab = (props: MyProps) => {
     vehicleId?: string;
     intersectionId?: number;
     roadRegulatorId?: number;
+    default?: boolean;
   }>({
     ...generateQueryParams(props.sourceData, props.sourceDataType),
     intersectionId: props.intersectionId,
@@ -845,7 +847,23 @@ const MapTab = (props: MyProps) => {
           odeReceivedAt: getTimestamp(bsm.metadata.odeReceivedAt),
         },
       }));
-    } else if (importedMessageData == undefined) {
+    }
+    else if (queryParams.default == true) {
+      const latestSpats = await MessageMonitorApi.getSpatMessages({
+        token: session?.accessToken,
+        intersectionId: queryParams.intersectionId,
+        roadRegulatorId: queryParams.roadRegulatorId,
+        latest: true,
+      });
+      if (latestSpats && latestSpats.length > 0) {
+        setQueryParams({
+          ...generateQueryParams({ timestamp: getTimestamp(latestSpats.at(-1)?.utcTimeStamp) }, "timestamp"),
+          intersectionId: queryParams.intersectionId,
+          roadRegulatorId: queryParams.roadRegulatorId,
+        });
+      }
+    }
+    else if (importedMessageData == undefined) {
       // ######################### Retrieve MAP Data #########################
       const rawMapPromise = MessageMonitorApi.getMapMessages({
         token: session?.accessToken,
@@ -890,11 +908,6 @@ const MapTab = (props: MyProps) => {
         queryParams.startDate,
         queryParams.endDate
       );
-      toast.promise(surroundingEventsPromise, {
-        loading: `Loading Event Data`,
-        success: `Successfully got Event Data`,
-        error: `Failed to get Event data. Please see console`,
-      });
       surroundingEventsPromise.then((events) => setSurroundingEvents(events));
 
       // ######################### BSM Events By Minute #########################
@@ -920,11 +933,6 @@ const MapTab = (props: MyProps) => {
         startTime: queryParams.startDate,
         endTime: queryParams.endDate,
       });
-      toast.promise(surroundingNotificationsPromise, {
-        loading: `Loading Notification Data`,
-        success: `Successfully got Notification Data`,
-        error: `Failed to get Notification data. Please see console`,
-      });
       surroundingNotificationsPromise.then((notifications) => setSurroundingNotifications(notifications));
     } else {
       rawMap = importedMessageData.mapData;

gui/src/pages/index.tsx

@@ -93,9 +93,10 @@ const Page = () => {
       >
         <Container maxWidth={false}>
           <Grid container spacing={3} alignItems="flex-start">
+            { /*
             <Box mt={3} mr={-3}>
               <MessageCountWidget accessToken={session?.accessToken} intersectionId={intersectionId} />
-            </Box>
+      </Box> 
             <Grid item xs={12} md={3} style={{ minWidth: 324 }}>
               <MessageCountWeekChart
               accessToken={session?.accessToken}
@@ -114,6 +115,7 @@ const Page = () => {
               barColor="#10B981"
               disclaimer=""/>
             </Grid>
+            
             <Grid item xs={12} md={3} style={{ minWidth: 324 }}>
               <MessageCountWeekChart
               accessToken={session?.accessToken}
@@ -123,6 +125,7 @@ const Page = () => {
               barColor="#900bae"
               disclaimer="duplicate messages not included"/>
             </Grid>
+            */}
             <ConnectionOfTravelAssessmentCard assessment={connectionOfTravelAssessment} />
             <StopLineStopAssessmentCard assessment={stopLineStopAssessment} />
             <SignalStateEventAssessmentCard assessment={signalStateEventAssessment} />