diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 79fd809d6..2f10f264d 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -25,6 +25,8 @@ "redhat.vscode-commons", "ms-vscode.cpptools", "ms-vscode.cmake-tools", + "ms-vscode.makefile-tools", + "Oracle.oracle-java", "ms-python.python" ] } diff --git a/README.md b/README.md index a53071c19..64fb1c03e 100644 --- a/README.md +++ b/README.md @@ -289,6 +289,8 @@ git submodule deinit -f . && git submodule update --recursive --init - Make a copy of [sample.env](./sample.env) rename it as `.env` **_and_**; - Make a copy of [jpo-utils/sample.env](jpo-utils/sample.env), rename it as `.env` (keep this one in the `jpo-utils/` directory), and fill in the variables as described in the [jpo-utils README](jpo-utils/README.md) - **The .env files will contain private keys, do not add them to version control.** + - Log-based alerts are enabled by default in the `sample.env`. If you don't want log messages notifying you when no TIMs were ingested in a specific period of time, + you will want to update your `.env` file to set `ODE_TIM_INGEST_MONITORING_ENABLED=false`. See [TimIngestWatcher](jpo-ode-svcs/src/main/java/us/dot/its/jpo/ode/traveler/TimIngestWatcher.java) to see the log-based monitoring provided. **Make:** diff --git a/asn1_codec b/asn1_codec index 6ce8588e7..b6c05cca8 160000 --- a/asn1_codec +++ b/asn1_codec @@ -1 +1 @@ -Subproject commit 6ce8588e77cde969fa419f1cc960c4eb05829ff1 +Subproject commit b6c05cca889bbab42bcc6b4a73c336c0b09ac930 diff --git a/docker-compose.yml b/docker-compose.yml index e5f5e809a..1fbe96c1a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -41,6 +41,8 @@ services: DATA_SIGNING_ENABLED_RSU: ${DATA_SIGNING_ENABLED_RSU} DATA_SIGNING_ENABLED_SDW: ${DATA_SIGNING_ENABLED_SDW} DEFAULT_SNMP_PROTOCOL: ${DEFAULT_SNMP_PROTOCOL} + ODE_TIM_INGEST_MONITORING_ENABLED: ${ODE_TIM_INGEST_MONITORING_ENABLED} + ODE_TIM_INGEST_MONITORING_INTERVAL: ${ODE_TIM_INGEST_MONITORING_INTERVAL} depends_on: kafka: condition: service_healthy diff --git a/jpo-cvdp b/jpo-cvdp index 23b593eee..4c3662ab4 160000 --- a/jpo-cvdp +++ b/jpo-cvdp @@ -1 +1 @@ -Subproject commit 23b593eeebacd82b7f334ac926709d424c7bcc28 +Subproject commit 4c3662ab416d75c88779459f8789afaffab31695 diff --git a/jpo-ode-core/mvnw b/jpo-ode-core/mvnw old mode 100644 new mode 100755 diff --git a/jpo-ode-core/mvnw.cmd b/jpo-ode-core/mvnw.cmd old mode 100644 new mode 100755 diff --git a/jpo-ode-svcs/mvnw b/jpo-ode-svcs/mvnw old mode 100644 new mode 100755 diff --git a/jpo-ode-svcs/mvnw.cmd b/jpo-ode-svcs/mvnw.cmd old mode 100644 new mode 100755 diff --git a/jpo-ode-svcs/run.bat b/jpo-ode-svcs/run.bat old mode 100644 new mode 100755 diff --git a/jpo-ode-svcs/run.sh b/jpo-ode-svcs/run.sh old mode 100644 new mode 100755 diff --git a/jpo-ode-svcs/src/main/java/us/dot/its/jpo/ode/ConfigEnvironmentVariables.java b/jpo-ode-svcs/src/main/java/us/dot/its/jpo/ode/ConfigEnvironmentVariables.java new file mode 100644 index 000000000..9992afeb6 --- /dev/null +++ b/jpo-ode-svcs/src/main/java/us/dot/its/jpo/ode/ConfigEnvironmentVariables.java @@ -0,0 +1,6 @@ +package us.dot.its.jpo.ode; + +public class ConfigEnvironmentVariables { + public static final String ODE_TIM_INGEST_MONITORING_ENABLED = "ODE_TIM_INGEST_MONITORING_ENABLED"; + public static final String ODE_TIM_INGEST_MONITORING_INTERVAL = "ODE_TIM_INGEST_MONITORING_INTERVAL"; // in seconds +} diff --git a/jpo-ode-svcs/src/main/java/us/dot/its/jpo/ode/traveler/TimDepositController.java b/jpo-ode-svcs/src/main/java/us/dot/its/jpo/ode/traveler/TimDepositController.java index 56dea7112..8afcadad0 100644 --- a/jpo-ode-svcs/src/main/java/us/dot/its/jpo/ode/traveler/TimDepositController.java +++ b/jpo-ode-svcs/src/main/java/us/dot/its/jpo/ode/traveler/TimDepositController.java @@ -19,6 +19,8 @@ import java.text.SimpleDateFormat; import java.time.format.DateTimeParseException; import java.util.Date; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; import org.json.JSONObject; import org.slf4j.Logger; @@ -34,6 +36,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode; +import us.dot.its.jpo.ode.ConfigEnvironmentVariables; import us.dot.its.jpo.ode.OdeProperties; import us.dot.its.jpo.ode.context.AppContext; import us.dot.its.jpo.ode.model.OdeMsgMetadata.GeneratedBy; @@ -62,6 +65,7 @@ public class TimDepositController { private static final Logger logger = LoggerFactory.getLogger(TimDepositController.class); + private static final TimIngestTracker INGEST_MONITOR = TimIngestTracker.getInstance(); private static final String ERRSTR = "error"; private static final String WARNING = "warning"; @@ -105,6 +109,27 @@ public TimDepositController(OdeProperties odeProperties) { ? Boolean.parseBoolean(System.getenv("DATA_SIGNING_ENABLED_SDW")) : true; + // start the TIM ingest monitoring service if enabled + Boolean timIngestMonitoringEnabled = Boolean.valueOf(odeProperties.getProperty(ConfigEnvironmentVariables.ODE_TIM_INGEST_MONITORING_ENABLED)); + if (timIngestMonitoringEnabled) { + logger.info("TIM ingest monitoring enabled."); + + ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); + // 3600 seconds, or one hour, was determined to be a sane default for the monitoring interval if monitoring is enabled + // but there was no interval set in the .env file + String interval = odeProperties.getProperty(ConfigEnvironmentVariables.ODE_TIM_INGEST_MONITORING_INTERVAL); + // getProperty(name, default) method will not use the default value if the value is set to an empty string, so we are using getProperty(name) + // and then checking if it is null or empty to protect against the case where the value is set to an empty string in the .env file so that we can + // use Long.valueOf() without risk of a NumberFormatException. + if (interval == null || interval.isEmpty()) { + interval = "3600"; + } + Long monitoringInterval = Long.valueOf(interval); + + scheduledExecutorService.scheduleAtFixedRate(new TimIngestWatcher(monitoringInterval), monitoringInterval, monitoringInterval, java.util.concurrent.TimeUnit.SECONDS); + } else { + logger.info("TIM ingest monitoring disabled."); + } } /** @@ -261,6 +286,8 @@ public synchronized ResponseEntity depositTim(String jsonString, Request return ResponseEntity.status(HttpStatus.BAD_REQUEST).body(JsonUtils.jsonKeyValue(ERRSTR, errMsg)); } + INGEST_MONITOR.incrementTotalMessagesReceived(); + return ResponseEntity.status(HttpStatus.OK).body(JsonUtils.jsonKeyValue(SUCCESS, "true")); } diff --git a/jpo-ode-svcs/src/main/java/us/dot/its/jpo/ode/traveler/TimIngestTracker.java b/jpo-ode-svcs/src/main/java/us/dot/its/jpo/ode/traveler/TimIngestTracker.java new file mode 100644 index 000000000..940e200d1 --- /dev/null +++ b/jpo-ode-svcs/src/main/java/us/dot/its/jpo/ode/traveler/TimIngestTracker.java @@ -0,0 +1,30 @@ +package us.dot.its.jpo.ode.traveler; + +public class TimIngestTracker { + + private long totalMessagesReceived; + + private TimIngestTracker() { + totalMessagesReceived = 0; + } + + public static TimIngestTracker getInstance() { + return TimIngestMonitorHolder.INSTANCE; + } + + private static class TimIngestMonitorHolder { + private static final TimIngestTracker INSTANCE = new TimIngestTracker(); + } + + public long getTotalMessagesReceived() { + return totalMessagesReceived; + } + + public void incrementTotalMessagesReceived() { + totalMessagesReceived++; + } + + public void resetTotalMessagesReceived() { + totalMessagesReceived = 0; + } + } diff --git a/jpo-ode-svcs/src/main/java/us/dot/its/jpo/ode/traveler/TimIngestWatcher.java b/jpo-ode-svcs/src/main/java/us/dot/its/jpo/ode/traveler/TimIngestWatcher.java new file mode 100644 index 000000000..5de3fa94f --- /dev/null +++ b/jpo-ode-svcs/src/main/java/us/dot/its/jpo/ode/traveler/TimIngestWatcher.java @@ -0,0 +1,29 @@ +package us.dot.its.jpo.ode.traveler; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class TimIngestWatcher implements Runnable { + + private static final Logger logger = LoggerFactory.getLogger(TimIngestWatcher.class.getName()); + private final long interval; + + public TimIngestWatcher(long interval) { + this.interval = interval; + } + + @Override + public void run() { + TimIngestTracker tracker = TimIngestTracker.getInstance(); + long ingested = tracker.getTotalMessagesReceived(); + + if (ingested == 0) { + logger.warn("ODE has not received TIM deposits in {} seconds.", interval); + } else { + logger.debug("ODE has received {} TIM deposits in the last {} seconds.", ingested, interval); + } + + // After checking the number of TIMs ingested in the last interval, reset the counter + tracker.resetTotalMessagesReceived(); + } +} \ No newline at end of file diff --git a/jpo-ode-svcs/src/main/java/us/dot/its/jpo/ode/udp/controller/UdpServicesController.java b/jpo-ode-svcs/src/main/java/us/dot/its/jpo/ode/udp/controller/UdpServicesController.java index c27352a9b..6cd9229bc 100644 --- a/jpo-ode-svcs/src/main/java/us/dot/its/jpo/ode/udp/controller/UdpServicesController.java +++ b/jpo-ode-svcs/src/main/java/us/dot/its/jpo/ode/udp/controller/UdpServicesController.java @@ -8,12 +8,12 @@ import us.dot.its.jpo.ode.OdeProperties; import us.dot.its.jpo.ode.udp.bsm.BsmReceiver; import us.dot.its.jpo.ode.udp.generic.GenericReceiver; -import us.dot.its.jpo.ode.udp.tim.TimReceiver; -import us.dot.its.jpo.ode.udp.ssm.SsmReceiver; -import us.dot.its.jpo.ode.udp.srm.SrmReceiver; -import us.dot.its.jpo.ode.udp.spat.SpatReceiver; import us.dot.its.jpo.ode.udp.map.MapReceiver; import us.dot.its.jpo.ode.udp.psm.PsmReceiver; +import us.dot.its.jpo.ode.udp.spat.SpatReceiver; +import us.dot.its.jpo.ode.udp.srm.SrmReceiver; +import us.dot.its.jpo.ode.udp.ssm.SsmReceiver; +import us.dot.its.jpo.ode.udp.tim.TimReceiver; /** * Centralized UDP service dispatcher. diff --git a/jpo-ode-svcs/src/main/java/us/dot/its/jpo/ode/udp/tim/TimReceiver.java b/jpo-ode-svcs/src/main/java/us/dot/its/jpo/ode/udp/tim/TimReceiver.java index cff647a00..666d34e1f 100644 --- a/jpo-ode-svcs/src/main/java/us/dot/its/jpo/ode/udp/tim/TimReceiver.java +++ b/jpo-ode-svcs/src/main/java/us/dot/its/jpo/ode/udp/tim/TimReceiver.java @@ -1,18 +1,19 @@ package us.dot.its.jpo.ode.udp.tim; import java.net.DatagramPacket; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; -import us.dot.its.jpo.ode.coder.StringPublisher; import us.dot.its.jpo.ode.OdeProperties; +import us.dot.its.jpo.ode.coder.StringPublisher; import us.dot.its.jpo.ode.udp.AbstractUdpReceiverPublisher; import us.dot.its.jpo.ode.udp.UdpHexDecoder; public class TimReceiver extends AbstractUdpReceiverPublisher { private static Logger logger = LoggerFactory.getLogger(TimReceiver.class); - + private StringPublisher timPublisher; @Autowired diff --git a/jpo-ode-svcs/src/test/java/us/dot/its/jpo/ode/traveler/TimDepositControllerTest.java b/jpo-ode-svcs/src/test/java/us/dot/its/jpo/ode/traveler/TimDepositControllerTest.java index 911a77919..744b0a3fd 100644 --- a/jpo-ode-svcs/src/test/java/us/dot/its/jpo/ode/traveler/TimDepositControllerTest.java +++ b/jpo-ode-svcs/src/test/java/us/dot/its/jpo/ode/traveler/TimDepositControllerTest.java @@ -15,9 +15,8 @@ ******************************************************************************/ package us.dot.its.jpo.ode.traveler; -import static org.junit.Assert.assertEquals; - import org.apache.commons.io.IOUtils; +import static org.junit.Assert.assertEquals; import org.junit.jupiter.api.Test; import org.springframework.http.ResponseEntity; @@ -38,6 +37,7 @@ import us.dot.its.jpo.ode.util.XmlUtils.XmlUtilsException; import us.dot.its.jpo.ode.wrapper.MessageProducer; + public class TimDepositControllerTest { @Tested @@ -167,6 +167,15 @@ public void testDepositingTimWithExtraProperties(@Capturing TimTransmogrifier ca assertEquals("{\"success\":\"true\"}", actualResponse.getBody()); } + @Test + public void testSuccessfulTimIngestIsTracked(@Capturing TimTransmogrifier capturingTimTransmogrifier, @Capturing XmlUtils capturingXmlUtils) { + String timToSubmit = "{\"request\":{\"rsus\":[],\"snmp\":{},\"randomProp1\":true,\"randomProp2\":\"hello world\"},\"tim\":{\"msgCnt\":\"13\",\"timeStamp\":\"2017-03-13T01:07:11-05:00\",\"randomProp3\":123,\"randomProp4\":{\"nestedProp1\":\"foo\",\"nestedProp2\":\"bar\"}}}"; + long priorIngestCount = TimIngestTracker.getInstance().getTotalMessagesReceived(); + ResponseEntity actualResponse = testTimDepositController.postTim(timToSubmit); + assertEquals("{\"success\":\"true\"}", actualResponse.getBody()); + assertEquals(priorIngestCount + 1, TimIngestTracker.getInstance().getTotalMessagesReceived()); + } + @Test public void testSuccessfulRsuMessageReturnsSuccessMessagePost(@Capturing TimTransmogrifier capturingTimTransmogrifier, @Capturing XmlUtils capturingXmlUtils) { String timToSubmit = "{\"request\": {\"rsus\": [{\"latitude\": 30.123456, \"longitude\": -100.12345, \"rsuId\": 123, \"route\": \"myroute\", \"milepost\": 10, \"rsuTarget\": \"172.0.0.1\", \"rsuRetries\": 3, \"rsuTimeout\": 5000, \"rsuIndex\": 7, \"rsuUsername\": \"myusername\", \"rsuPassword\": \"mypassword\"}], \"snmp\": {\"rsuid\": \"83\", \"msgid\": 31, \"mode\": 1, \"channel\": 183, \"interval\": 2000, \"deliverystart\": \"2024-05-13T14:30:00Z\", \"deliverystop\": \"2024-05-13T22:30:00Z\", \"enable\": 1, \"status\": 4}}, \"tim\": {\"msgCnt\": \"1\", \"timeStamp\": \"2024-05-10T19:01:22Z\", \"packetID\": \"123451234512345123\", \"urlB\": \"null\", \"dataframes\": [{\"startDateTime\": \"2024-05-13T20:30:05.014Z\", \"durationTime\": \"30\", \"sspTimRights\": \"1\", \"frameType\": \"advisory\", \"msgId\": {\"roadSignID\": {\"mutcdCode\": \"warning\", \"viewAngle\": \"1111111111111111\", \"position\": {\"latitude\": 30.123456, \"longitude\": -100.12345}}}, \"priority\": \"5\", \"sspLocationRights\": \"1\", \"regions\": [{\"name\": \"I_myroute_RSU_172.0.0.1\", \"anchorPosition\": {\"latitude\": 30.123456, \"longitude\": -100.12345}, \"laneWidth\": \"50\", \"directionality\": \"3\", \"closedPath\": \"false\", \"description\": \"path\", \"path\": {\"scale\": 0, \"nodes\": [{\"delta\": \"node-LL\", \"nodeLat\": 0.0, \"nodeLong\": 0.0}, {\"delta\": \"node-LL\", \"nodeLat\": 0.0, \"nodeLong\": 0.0}], \"type\": \"ll\"}, \"direction\": \"0000000000010000\"}], \"sspMsgTypes\": \"1\", \"sspMsgContent\": \"1\", \"content\": \"workZone\", \"items\": [\"771\"], \"url\": \"null\"}]}}"; diff --git a/jpo-ode-svcs/src/test/java/us/dot/its/jpo/ode/traveler/TimIngestTrackerTest.java b/jpo-ode-svcs/src/test/java/us/dot/its/jpo/ode/traveler/TimIngestTrackerTest.java new file mode 100644 index 000000000..f9ae23735 --- /dev/null +++ b/jpo-ode-svcs/src/test/java/us/dot/its/jpo/ode/traveler/TimIngestTrackerTest.java @@ -0,0 +1,25 @@ +package us.dot.its.jpo.ode.traveler; + +import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import org.junit.jupiter.api.Test; + +public class TimIngestTrackerTest { + + @Test + public void testCanIncrementTotalMessagesReceived() { + TimIngestTracker testTimIngestTracker = TimIngestTracker.getInstance(); + long priorCount = testTimIngestTracker.getTotalMessagesReceived(); + testTimIngestTracker.incrementTotalMessagesReceived(); + assertTrue(testTimIngestTracker.getTotalMessagesReceived() > priorCount); + } + + @Test + public void testCanResetTotalMessagesReceived() { + TimIngestTracker testTimIngestTracker = TimIngestTracker.getInstance(); + testTimIngestTracker.incrementTotalMessagesReceived(); + assertTrue(testTimIngestTracker.getTotalMessagesReceived()> 0); + testTimIngestTracker.resetTotalMessagesReceived(); + assertEquals(0, testTimIngestTracker.getTotalMessagesReceived()); + } +} diff --git a/jpo-ode-svcs/src/test/java/us/dot/its/jpo/ode/traveler/TimIngestWatcherTest.java b/jpo-ode-svcs/src/test/java/us/dot/its/jpo/ode/traveler/TimIngestWatcherTest.java new file mode 100644 index 000000000..51947277d --- /dev/null +++ b/jpo-ode-svcs/src/test/java/us/dot/its/jpo/ode/traveler/TimIngestWatcherTest.java @@ -0,0 +1,18 @@ +package us.dot.its.jpo.ode.traveler; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import org.junit.jupiter.api.Test; + +public class TimIngestWatcherTest { + + @Test + public void testRun() { + TimIngestWatcher watcher = new TimIngestWatcher(0); + watcher.run(); + + // we can't easily test that the run method wrote the correct log message, but we can test that it reset the total messages received after running + TimIngestTracker testTimIngestTracker = TimIngestTracker.getInstance(); + assertEquals(0, testTimIngestTracker.getTotalMessagesReceived()); + } + +} \ No newline at end of file diff --git a/jpo-sdw-depositor b/jpo-sdw-depositor index 0180d3024..ca109a47a 160000 --- a/jpo-sdw-depositor +++ b/jpo-sdw-depositor @@ -1 +1 @@ -Subproject commit 0180d30242fdfe7c178f3e060267b8690220c473 +Subproject commit ca109a47a5e20776c5ddb5df98dfdce402023c9a diff --git a/jpo-security-svcs b/jpo-security-svcs index eaa5a3ab4..95533c2c4 160000 --- a/jpo-security-svcs +++ b/jpo-security-svcs @@ -1 +1 @@ -Subproject commit eaa5a3ab49be37c5457677840b94d674e4be5e45 +Subproject commit 95533c2c49dd2f3327cbe4afdd658a9d3362ea7d diff --git a/jpo-utils b/jpo-utils index 99c36526a..5beea2a7a 160000 --- a/jpo-utils +++ b/jpo-utils @@ -1 +1 @@ -Subproject commit 99c36526af8c349e6ec92440cc017fffbf2c5652 +Subproject commit 5beea2a7a1dc1b25d868b3d8ef79f6292b07f819 diff --git a/sample.env b/sample.env index 0a2020704..625ceff5f 100644 --- a/sample.env +++ b/sample.env @@ -103,4 +103,10 @@ AEM_LOG_TO_FILE=false AEM_LOG_LEVEL=INFO ADM_LOG_TO_FILE=false ADM_LOG_TO_CONSOLE=true -ADM_LOG_LEVEL=INFO \ No newline at end of file +ADM_LOG_LEVEL=INFO + +# ODE Monitoring +ODE_TIM_INGEST_MONITORING_ENABLED=true +# The interval is measured in seconds. 60 seconds is a sane default for local monitoring, but you may want to increase +# the interval to 3600 (1 hour) or more for production monitoring to reduce the noise in logs for healthy systems +ODE_TIM_INGEST_MONITORING_INTERVAL=60 \ No newline at end of file