Merge pull request usdot-jpo-ode#108 from usdot-jpo-ode/missing-bsms

Fixing missing BSMs

gui/package.json

@@ -33,7 +33,6 @@
     "@types/react-dom": "^18.0.10",
     "@zalter/identity-js": "1.2.1",
     "color": "^4.2.3",
-    "color-convert": "^2.0.1",
     "cookie": "^0.5.0",
     "date-fns": "2.29.3",
     "dayjs": "^1.11.7",

gui/src/components/intersection-selector/intersection-selection-map.tsx

@@ -3,6 +3,7 @@ import Map, { Layer, MapRef, Marker, Popup, Source, SymbolLayer } from "react-ma
 
 import { Container, Col } from "reactstrap";
 import getConfig from "next/config";
+import mbStyle from "../../intersectionMapStyle.json";
 
 const { publicRuntimeConfig } = getConfig();
 
@@ -146,7 +147,7 @@ const IntersectionMap = (props: Props) => {
         <Map
           {...viewState}
           ref={myRef}
-          mapStyle={publicRuntimeConfig.MAPBOX_STYLE_URL!}
+          mapStyle={mbStyle as mapboxgl.Style}
           mapboxAccessToken={MAPBOX_API_TOKEN}
           attributionControl={true}
           customAttribution={['<a href="https://www.cotrip.com/" target="_blank">© CDOT</a>']}

gui/src/components/map/map-component.tsx

@@ -175,6 +175,7 @@ const markerLayer: LayerProps = {
   },
 };
 
+// TS in milliseconds
 export const getTimestamp = (dt: any): number => {
   try {
     const dtFromString = Date.parse(dt as any as string);
@@ -193,6 +194,44 @@ export const getTimestamp = (dt: any): number => {
   }
 };
 
+// algorithm:
+// 1. Convert odeReceivedAt to a unix timestamp, in milliseconds
+// 2. Convert the secMarkMs to seconds
+// 3. Pull out the second and minute from the odeReceivedAt timestamp
+// case 1: odeReceivedAtSecondOfMinute < secMarkSecondOfMinute
+//   - decrement the minute by 1
+//   - set the second to secMarkSecond
+//   # Logic: Assume clocks are relatively in sync, so if the secMark is in the future, it must be in the next minute
+// case 2: Math.abs((odeReceivedAtSecondOfMinute - 60) - secMarkSecondOfMinute) < 30
+//   - increment the minute by 1
+//   - set the second to secMarkSecond
+//   # Logic: We know the odeReceivedAt is
+export const getTimestampWithSecMark = (odeReceivedAt: any, secMarkMs: number): number => {
+  const dt: number = getTimestamp(odeReceivedAt);
+  const odeReceivedAtDate = new Date(dt);
+  const odeReceivedAtSecondOfMinute = odeReceivedAtDate.getUTCSeconds();
+  const secMarkSecondOfMinute = Math.floor(secMarkMs / 1000);
+
+  const diff = odeReceivedAtSecondOfMinute - secMarkSecondOfMinute; // delay, in seconds
+
+  let adjustedDate = new Date(dt);
+
+  if (diff < 0) {
+    adjustedDate.setUTCMinutes(adjustedDate.getUTCMinutes() - 1);
+  }
+
+  adjustedDate.setUTCSeconds(secMarkSecondOfMinute);
+  return adjustedDate.getTime();
+};
+// test cases
+// odeReceivedAtSecondOfMinute = 10, secMarkSecondOfMinute = 15 -> odeReceivedAt is 55 seconds ahead. Subtract 1 minute and set seconds to secMarkSecondOfMinute
+// odeReceivedAtSecondOfMinute = 10, secMarkSecondOfMinute = 5 -> odeReceivedAt is 5 seconds behind. Set seconds to secMarkSecondOfMinute
+// odeReceivedAtSecondOfMinute = 5, secMarkSecondOfMinute = 59 -> odeReceivedAt is 6 seconds behind. Set seconds to secMarkSecondOfMinute
+// odeReceivedAtSecondOfMinute = 59, secMarkSecondOfMinute = 55 -> odeReceivedAt is 4 seconds ahead. Add 1 minute and set seconds to secMarkSecondOfMinute
+
+// const diff_normal = odeReceivedAtSecondOfMinute - secMarkSecondOfMinute // delay, in seconds
+// const diff_ode = (odeReceivedAtSecondOfMinute + 60) - (secMarkSecondOfMinute) // delay, in seconds
+
 const generateQueryParams = (
   source:
     | MessageMonitor.Notification
@@ -694,7 +733,8 @@ const MapTab = forwardRef<MAP_REFERENCE_TYPE | undefined, MapProps>(
             type: "Feature",
             properties: {
               ...bsm.payload.data.coreData,
-              odeReceivedAt: new Date(bsm.metadata.odeReceivedAt as string).getTime() / 1000,
+              odeReceivedAt:
+                getTimestampWithSecMark(bsm.metadata.odeReceivedAt, bsm.payload.data.coreData.secMark) / 1000,
             },
             geometry: {
               type: "Point",
@@ -885,7 +925,8 @@ const MapTab = forwardRef<MAP_REFERENCE_TYPE | undefined, MapProps>(
           ...bsm,
           metadata: {
             ...bsm.metadata,
-            odeReceivedAt: getTimestamp(bsm.metadata.odeReceivedAt),
+            odeReceivedAt:
+              getTimestampWithSecMark(bsm.metadata.odeReceivedAt, bsm.payload.data.coreData.secMark) / 1000,
           },
         }));
       } else if (queryParams.default == true) {
@@ -1077,7 +1118,7 @@ const MapTab = forwardRef<MAP_REFERENCE_TYPE | undefined, MapProps>(
       currentSpatData: ProcessedSpat[],
       currentBsmData: BsmFeatureCollection
     ) => {
-      if (props.sourceDataType == "exact") {
+      if (props.sourceDataType == "exact" && currentMapData.length == 0) {
         const uniqueIds = new Set(currentBsmData.features.map((bsm) => bsm.properties?.id).sort());
         // generate equally spaced unique colors for each uniqueId
         const colors = generateColorDictionary(uniqueIds);
@@ -1092,8 +1133,8 @@ const MapTab = forwardRef<MAP_REFERENCE_TYPE | undefined, MapProps>(
           paint: { ...prevValue.paint, "circle-color": bsmLayerStyle },
         }));
         setBsmData(currentBsmData);
-      }
-      if (currentMapData.length == 0) {
+        return;
+      } else if (currentMapData.length == 0) {
         console.log("Did not attempt to render map, no map messages available:", currentMapData);
         return;
       }
@@ -1121,6 +1162,21 @@ const MapTab = forwardRef<MAP_REFERENCE_TYPE | undefined, MapProps>(
 
       setSpatSignalGroups(spatSignalGroupsLocal);
 
+      const uniqueIds = new Set(currentBsmData.features.map((bsm) => bsm.properties?.id).sort());
+      // generate equally spaced unique colors for each uniqueId
+      const colors = generateColorDictionary(uniqueIds);
+      setMapLegendColors((prevValue) => ({
+        ...prevValue,
+        bsmColors: colors,
+      }));
+      // add color to each feature
+      const bsmLayerStyle = generateMapboxStyleExpression(colors);
+      setBsmLayerStyle((prevValue) => ({
+        ...prevValue,
+        paint: { ...prevValue.paint, "circle-color": bsmLayerStyle },
+      }));
+      setBsmData(currentBsmData);
+
       // ######################### Message Data #########################
       rawData["map"] = currentMapData;
       rawData["spat"] = currentSpatData;
@@ -1464,7 +1520,6 @@ const MapTab = forwardRef<MAP_REFERENCE_TYPE | undefined, MapProps>(
       const filteredStartTime = startTime + sliderValue / 10 - timeWindowSeconds;
       const filteredEndTime = startTime + sliderValue / 10;
 
-      console.log("Filtered Time Interval:", filteredStartTime, filteredEndTime, sliderValue, timeWindowSeconds);
       setRenderTimeInterval([filteredStartTime, filteredEndTime]);
     }, [sliderValue, queryParams, timeWindowSeconds]);
 

gui/src/components/map/popup.tsx

@@ -5,6 +5,7 @@ import { Container, Col } from "reactstrap";
 
 import { Paper, Box, Typography } from "@mui/material";
 import { CustomTable } from "./custom-table";
+import { format } from "date-fns";
 
 export const getSelectedLayerPopupContent = (feature: any) => {
   switch (feature?.layer?.id) {
@@ -18,7 +19,7 @@ export const getSelectedLayerPopupContent = (feature: any) => {
             data={[
               ["Id", bsm.id],
               ["Message Count", bsm.msgCnt],
-              ["Time", bsm.secMark / 1000],
+              ["Time", format(bsm.odeReceivedAt * 1000, "MM/dd/yyyy HH:mm:ss.SSS")],
               ["Speed", bsm.speed],
               ["Heading", bsm.heading],
             ]}

gui/src/components/map/side-panel.tsx

@@ -227,7 +227,7 @@ export const SidePanel = (props: SidePanelProps) => {
                         headers={["Time", "Vehicle ID", "Speed", "Heading"]}
                         data={
                           bsms?.features.map((bsm) => [
-                            bsm.properties.secMark / 1000,
+                            format(bsm.properties.odeReceivedAt * 1000, "MM/dd/yyyy HH:mm:ss.SSS"),
                             bsm.properties.id,
                             bsm.properties.speed,
                             bsm.properties.heading,

gui/src/components/map/utilities/colors.ts

@@ -1,5 +1,4 @@
 import { sha256 } from "js-sha256";
-import { hsl, rgb } from "color-convert";
 import * as Color from "color";
 
 const DISALLOWED_COLORS = new Set(
@@ -18,7 +17,6 @@ const DISALLOWED_COLORS = new Set(
     ]) // Extract the hue
   ).sort((a, b) => a - b) // ascending order
 );
-console.log("Color dictionary DISALLOWED_COLORS:", DISALLOWED_COLORS);
 
 // pre-allocate color space around disallowed colors
 const OUTPUT_COLOR_SPACE_LENGTH = 360;
@@ -38,20 +36,16 @@ for (let i = 0; i < OUTPUT_COLOR_SPACE_LENGTH; i += COLOR_SPACE_INCREMENT) {
 
 const OUTPUT_COLOR_SPACE_LENGTH_BEFORE_SCALING = 360 - totalAvoidanceSpace;
 
-console.log("Color dictionary Total avoidance space:", totalAvoidanceSpace);
-
 // Generate the new color space
 let prevColor = 0;
 for (let i = 0; i < OUTPUT_COLOR_SPACE_LENGTH; i += COLOR_SPACE_INCREMENT) {
   let newColor =
     prevColor + (COLOR_SPACE_INCREMENT * OUTPUT_COLOR_SPACE_LENGTH_BEFORE_SCALING) / OUTPUT_COLOR_SPACE_LENGTH;
   for (const disallowedHue of DISALLOWED_COLORS.values()) {
-    // console.log(disallowedHue, newColor, Math.abs(disallowedHue - newColor) < COLOR_SPACE_AVOID_WIDTH);
     if (Math.abs(disallowedHue - newColor) < COLOR_SPACE_AVOID_WIDTH) {
       newColor = disallowedHue + COLOR_SPACE_AVOID_WIDTH;
     }
   }
-  //   console.log("Color dictionary New color:", i, Math.round(i * 10) / 10, newColor, prevColor, newColor - prevColor);
   COLOR_SPACE_MAP.set(Math.round(i * 10) / 10, newColor);
   prevColor = newColor;
 }
@@ -75,21 +69,12 @@ export function generateColorDictionary(inputSet: Set<string>): { [key: string]:
 
     // Map the hue to the shrunken color space
     const keyInMap = Math.round(hue * 10) / 10;
-    console.log("Color dictionary Key:", hue, keyInMap, COLOR_SPACE_MAP.get(keyInMap), DISALLOWED_COLORS);
     hue = COLOR_SPACE_MAP.get(keyInMap) || hue;
 
     const color = Color.hsl(hue, 100, 50); // Use the hue to generate a color
-    console.log("Color dictionary Color:", hue, 100, 65, color, color.hex());
     colorDictionary[key] = color.hex();
   });
 
-  //   console.log(
-  //     "Color dictionary: ",
-  //     inputSet,
-  //     colorDictionary,
-  //     numbers.map((n) => ({ k: n, v: COLOR_SPACE_MAP.get(n) }))
-  //   );
-
   return colorDictionary;
 }
 
@@ -102,5 +87,3 @@ export function generateMapboxStyleExpression(colors: { [key: string]: string })
   layerStyle.push("#000000"); // other
   return layerStyle;
 }
-
-// console.log(generateColorDictionary(new Set(["0139C942", "13906CFE", "37BDF36C", "5C6FA267"])));

gui/src/pages/decoder.tsx

@@ -22,8 +22,6 @@ const DecoderPage = () => {
 
   const [currentBsms, setCurrentBsms] = useState([] as OdeBsmData[]);
 
-  console.log("Data", data);
-
   useEffect(() => {
     const freshData = [] as DecoderDataEntry[];
     for (let i = 0; i < 3; i++) {
@@ -64,7 +62,6 @@ const DecoderPage = () => {
         if (type == "BSM") {
           setSelectedBsms((prevBsms) => [...prevBsms, id]);
         }
-        console.log("Response", response);
         setData((prevData) => {
           return {
             ...prevData,
@@ -227,7 +224,6 @@ const DecoderPage = () => {
       .filter((v) => v.type === "BSM" && v.status === "COMPLETED" && selectedBsms.includes(v.id))
       .map((v) => v.decodedResponse?.bsm);
     setCurrentBsms(newBsmData.filter((v) => v !== undefined) as OdeBsmData[]);
-    console.log("Current BSMs", newBsmData.length);
   }, [data, selectedBsms]);
 
   return (
@@ -314,7 +310,6 @@ const DecoderPage = () => {
             selectedBsms={selectedBsms}
             setSelectedBsms={setSelectedBsms}
             centerMapOnLocation={(lat: number, long: number) => {
-              console.log("Centering map on location", lat, long, mapRef.current);
               if (mapRef.current) {
                 mapRef.current?.centerMapOnPoint({ latitude: lat, longitude: long });
               }