434 add validation to observations

.github/workflows/build-docker-image.yml

@@ -32,6 +32,9 @@ on:
 jobs:
   # this workflow contains a single job called "build"
   build:
+    # Do not build if it is an RC build
+    if: ${{ ! contains(github.ref_name, "RC") }}
+
     # the type of runner that the job will run on
     runs-on: ubuntu-latest
     strategy:

src/mtconnect/agent.cpp

@@ -56,9 +56,8 @@
 #include "mtconnect/sink/rest_sink/file_cache.hpp"
 #include "mtconnect/sink/rest_sink/session.hpp"
 #include "mtconnect/utilities.hpp"
-#include "mtconnect/version.h"
-
 #include "mtconnect/validation/observations.hpp"
+#include "mtconnect/version.h"
 
 using namespace std;
 
@@ -171,13 +170,14 @@ namespace mtconnect {
 
     if (!m_observationsInitialized)
     {
-      if (m_intSchemaVersion < SCHEMA_VERSION(2, 5) && IsOptionSet(m_options, mtconnect::configuration::Validation))
+      if (m_intSchemaVersion < SCHEMA_VERSION(2, 5) &&
+          IsOptionSet(m_options, mtconnect::configuration::Validation))
       {
         m_validation = false;
         for (auto &printer : m_printers)
           printer.second->setValidation(false);
       }
-      
+
       for (auto device : m_deviceIndex)
         initializeDataItems(device);
 
@@ -201,7 +201,6 @@ namespace mtconnect {
       }
 
       m_observationsInitialized = true;
-
     }
   }
 

src/mtconnect/pipeline/validator.hpp

@@ -27,37 +27,41 @@
 
 namespace mtconnect::pipeline {
   using namespace entity;
-
-  /// @brief Map a token list to data items or asset types
+
+  /// @brief Validate obsrvations based on Controlled Vocabularies
+  ///
+  /// - Does not validate data sets and tables
+  /// - Validates all events, not samples or conditions
   class AGENT_LIB_API Validator : public Transform
   {
   public:
     Validator(const Validator &) = default;
     Validator(PipelineContextPtr context)
-      : Transform("Validator"),
-        m_contract(context->m_contract.get())
+      : Transform("Validator"), m_contract(context->m_contract.get())
     {
       m_guard = TypeGuard<observation::Event>(RUN) || TypeGuard<observation::Observation>(SKIP);
     }
-
+
+    /// @brief validate the Event
+    /// @param entity The Event entity
+    /// @returns modified entity with quality and deprecated properties
     EntityPtr operator()(entity::EntityPtr &&entity) override
     {
       using namespace observation;
       using namespace mtconnect::validation::observations;
       auto evt = std::dynamic_pointer_cast<Event>(entity);
-      
+
       auto di = evt->getDataItem();
-      auto &type = di->getType();
-      auto &value = evt->getValue<std::string>();
-
-      if (value == "UNAVAILABLE")
+      if (evt->isUnavailable() || di->isDataSet())
       {
         evt->setProperty("quality", std::string("VALID"));
       }
       else
       {
+        auto &value = evt->getValue<std::string>();
+
         // Optimize
-        auto vocab = ControlledVocabularies.find(type);
+        auto vocab = ControlledVocabularies.find(evt->getName());
         if (vocab != ControlledVocabularies.end())
         {
           auto &lits = vocab->second;
@@ -67,7 +71,12 @@ namespace mtconnect::pipeline {
             if (lit != lits.end())
             {
               evt->setProperty("quality", std::string("VALID"));
-              // Check for deprecated
+
+              // Check if deprecated
+              if (lit->second > 0 && m_contract->getSchemaVersion() > lit->second)
+              {
+                evt->setProperty("deprecated", true);
+              }
             }
             else
             {
@@ -76,13 +85,14 @@ namespace mtconnect::pipeline {
               auto &id = di->getId();
               if (m_logOnce.count(id) < 1)
               {
-                LOG(warning) << id << ": Invalid value for '" << type << "' " <<  evt->getValue<std::string>();
+                LOG(warning) << "DataItem '" << id << "': Invalid value for '" << evt->getName()
+                             << "': '" << evt->getValue<std::string>() << '\'';
                 m_logOnce.insert(id);
               }
               else
               {
-                LOG(trace) << id << ": Invalid value for '" << type << "' " <<  evt->getValue<std::string>();
-
+                LOG(trace) << "DataItem '" << id << "': Invalid value for '" << evt->getName()
+                           << "': '" << evt->getValue<std::string>() << '\'';
               }
             }
           }
@@ -96,7 +106,7 @@ namespace mtconnect::pipeline {
           evt->setProperty("quality", std::string("UNVERIFIABLE"));
         }
       }
-      
+
       return next(std::move(evt));
     }
 

src/mtconnect/printer/json_printer.cpp

@@ -60,8 +60,7 @@ namespace mtconnect::printer {
   template <typename T>
   inline void header(AutoJsonObject<T> &obj, const string &version, const string &hostname,
                      const uint64_t instanceId, const unsigned int bufferSize,
-                     const string &schemaVersion, const string modelChangeTime,
-                     bool validation)
+                     const string &schemaVersion, const string modelChangeTime, bool validation)
   {
     obj.AddPairs("version", version, "creationTime", getCurrentTime(GMT), "testIndicator", false,
                  "instanceId", instanceId, "sender", hostname, "schemaVersion", schemaVersion);
@@ -72,18 +71,17 @@ namespace mtconnect::printer {
       obj.AddPairs("bufferSize", bufferSize);
     if (validation)
       obj.AddPairs("validation", true);
-
   }
 
   template <typename T>
   inline void probeAssetHeader(AutoJsonObject<T> &obj, const string &version,
                                const string &hostname, const uint64_t instanceId,
                                const unsigned int bufferSize, const unsigned int assetBufferSize,
                                const unsigned int assetCount, const string &schemaVersion,
-                               const string modelChangeTime,
-                               const bool validation)
+                               const string modelChangeTime, const bool validation)
   {
-    header(obj, version, hostname, instanceId, bufferSize, schemaVersion, modelChangeTime, validation);
+    header(obj, version, hostname, instanceId, bufferSize, schemaVersion, modelChangeTime,
+           validation);
     obj.AddPairs("assetBufferSize", assetBufferSize, "assetCount", assetCount);
   }
 
@@ -92,10 +90,10 @@ namespace mtconnect::printer {
                            const uint64_t instanceId, const unsigned int bufferSize,
                            const uint64_t nextSequence, const uint64_t firstSequence,
                            const uint64_t lastSequence, const string &schemaVersion,
-                           const string modelChangeTime,
-                           const bool validation)
+                           const string modelChangeTime, const bool validation)
   {
-    header(obj, version, hostname, instanceId, bufferSize, schemaVersion, modelChangeTime, validation);
+    header(obj, version, hostname, instanceId, bufferSize, schemaVersion, modelChangeTime,
+           validation);
     obj.AddPairs("nextSequence", nextSequence, "lastSequence", lastSequence, "firstSequence",
                  firstSequence);
   }
@@ -213,8 +211,7 @@ namespace mtconnect::printer {
       {
         AutoJsonObject obj(writer, "Header");
         probeAssetHeader(obj, m_version, m_senderName, instanceId, 0, bufferSize, assetCount,
-                         *m_schemaVersion, m_modelChangeTime,
-                         m_validation);
+                         *m_schemaVersion, m_modelChangeTime, m_validation);
       }
       {
         obj.Key("Assets");
@@ -412,8 +409,7 @@ namespace mtconnect::printer {
       {
         AutoJsonObject obj(writer, "Header");
         streamHeader(obj, m_version, m_senderName, instanceId, bufferSize, nextSeq, firstSeq,
-                     lastSeq, *m_schemaVersion, m_modelChangeTime,
-                     m_validation);
+                     lastSeq, *m_schemaVersion, m_modelChangeTime, m_validation);
       }
 
       {

src/mtconnect/printer/printer.hpp

@@ -49,7 +49,9 @@ namespace mtconnect {
     public:
       /// @brief construct a printer
       /// @param pretty `true` if content should be pretty printed
-      Printer(bool pretty = false, bool validation = false) : m_pretty(pretty), m_validation(validation) {}
+      Printer(bool pretty = false, bool validation = false)
+        : m_pretty(pretty), m_validation(validation)
+      {}
       virtual ~Printer() = default;
 
       /// @brief Generate an MTConnect Error document
@@ -145,19 +147,18 @@ namespace mtconnect {
           const_cast<Printer *>(this)->m_schemaVersion.emplace(ver);
         }
       }
-      
+
       /// @brief Get validation header flag state
       /// @returns validation state
       bool getValidation() const { return m_validation; }
-      
+
       /// @brief sets validation state
       /// @param validation the validation state
       void setValidation(bool v) { m_validation = v; }
-
 
     protected:
-      bool m_pretty; //< Turns pretty printing on
-      bool m_validation; //< Sets validation flag in header
+      bool m_pretty;      //< Turns pretty printing on
+      bool m_validation;  //< Sets validation flag in header
       std::string m_modelChangeTime;
       std::optional<std::string> m_schemaVersion;
       std::string m_senderName {"localhost"};

src/mtconnect/printer/xml_printer.cpp

@@ -100,7 +100,10 @@ namespace mtconnect::printer {
     xmlBufferPtr m_buf;
   };
 
-  XmlPrinter::XmlPrinter(bool pretty, bool validation) : Printer(pretty, validation) { NAMED_SCOPE("xml.printer"); }
+  XmlPrinter::XmlPrinter(bool pretty, bool validation) : Printer(pretty, validation)
+  {
+    NAMED_SCOPE("xml.printer");
+  }
 
   void XmlPrinter::addDevicesNamespace(const std::string &urn, const std::string &location,
                                        const std::string &prefix)
@@ -636,7 +639,7 @@ namespace mtconnect::printer {
 
     addAttribute(writer, "sender", m_senderName);
     addAttribute(writer, "instanceId", to_string(instanceId));
-    
+
     if (m_validation)
       addAttribute(writer, "validation", "true"s);
 

src/mtconnect/source/adapter/adapter_pipeline.cpp

@@ -139,7 +139,7 @@ namespace mtconnect {
       // Convert values
       if (IsOptionSet(m_options, configuration::ConversionRequired))
         next = next->bind(make_shared<ConvertSample>());
-            
+
       // Validate Values
       if (IsOptionSet(m_options, configuration::Validation))
         next = next->bind(make_shared<Validator>(m_context));

src/mtconnect/source/loopback_source.cpp

@@ -38,7 +38,7 @@ namespace mtconnect::source {
   void LoopbackPipeline::build(const ConfigOptions &options)
   {
     m_options = options;
-    
+
     clear();
     TransformPtr next = m_start;