Merge pull request #436 from mtconnect/434_add_validation_to_observat…

…ions

.github/workflows/build-docker-image.yml

@@ -32,6 +32,9 @@ on:
 jobs:
  # this workflow contains a single job called "build"
  build:
+ # Do not build if it is an RC build
+ if: ${{ ! contains(github.ref_name, "RC") }}
+
  # the type of runner that the job will run on
  runs-on: ubuntu-latest
  strategy:

src/mtconnect/agent.cpp

@@ -56,9 +56,8 @@
 #include "mtconnect/sink/rest_sink/file_cache.hpp"
 #include "mtconnect/sink/rest_sink/session.hpp"
 #include "mtconnect/utilities.hpp"
-#include "mtconnect/version.h"
-
 #include "mtconnect/validation/observations.hpp"
+#include "mtconnect/version.h"
 
 using namespace std;
 
@@ -171,13 +170,14 @@ namespace mtconnect {
 
  if (!m_observationsInitialized)
  {
- if (m_intSchemaVersion < SCHEMA_VERSION(2, 5) && IsOptionSet(m_options, mtconnect::configuration::Validation))
+ if (m_intSchemaVersion < SCHEMA_VERSION(2, 5) &&
+ IsOptionSet(m_options, mtconnect::configuration::Validation))
  {
  m_validation = false;
  for (auto &printer : m_printers)
  printer.second->setValidation(false);
  }
- 
+
  for (auto device : m_deviceIndex)
  initializeDataItems(device);
 
@@ -201,7 +201,6 @@ namespace mtconnect {
  }
 
  m_observationsInitialized = true;
-
  }
  }
 

src/mtconnect/pipeline/validator.hpp

@@ -27,37 +27,41 @@
 
 namespace mtconnect::pipeline {
  using namespace entity;
-
- /// @brief Map a token list to data items or asset types
+
+ /// @brief Validate obsrvations based on Controlled Vocabularies
+ ///
+ /// - Does not validate data sets and tables
+ /// - Validates all events, not samples or conditions
  class AGENT_LIB_API Validator : public Transform
  {
  public:
  Validator(const Validator &) = default;
  Validator(PipelineContextPtr context)
- : Transform("Validator"),
- m_contract(context->m_contract.get())
+ : Transform("Validator"), m_contract(context->m_contract.get())
  {
  m_guard = TypeGuard<observation::Event>(RUN) || TypeGuard<observation::Observation>(SKIP);
  }
-
+
+ /// @brief validate the Event
+ /// @param entity The Event entity
+ /// @returns modified entity with quality and deprecated properties
  EntityPtr operator()(entity::EntityPtr &&entity) override
  {
  using namespace observation;
  using namespace mtconnect::validation::observations;
  auto evt = std::dynamic_pointer_cast<Event>(entity);
- 
+
  auto di = evt->getDataItem();
- auto &type = di->getType();
- auto &value = evt->getValue<std::string>();
-
- if (value == "UNAVAILABLE")
+ if (evt->isUnavailable() || di->isDataSet())
  {
  evt->setProperty("quality", std::string("VALID"));
  }
  else
  {
+ auto &value = evt->getValue<std::string>();
+
  // Optimize
- auto vocab = ControlledVocabularies.find(type);
+ auto vocab = ControlledVocabularies.find(evt->getName());
  if (vocab != ControlledVocabularies.end())
  {
  auto &lits = vocab->second;
@@ -67,7 +71,12 @@ namespace mtconnect::pipeline {
  if (lit != lits.end())
  {
  evt->setProperty("quality", std::string("VALID"));
- // Check for deprecated
+
+ // Check if deprecated
+ if (lit->second > 0 && m_contract->getSchemaVersion() > lit->second)
+ {
+ evt->setProperty("deprecated", true);
+ }
  }
  else
  {
@@ -76,13 +85,14 @@ namespace mtconnect::pipeline {
  auto &id = di->getId();
  if (m_logOnce.count(id) < 1)
  {
- LOG(warning) << id << ": Invalid value for '" << type << "' " << evt->getValue<std::string>();
+ LOG(warning) << "DataItem '" << id << "': Invalid value for '" << evt->getName()
+ << "': '" << evt->getValue<std::string>() << '\'';
  m_logOnce.insert(id);
  }
  else
  {
- LOG(trace) << id << ": Invalid value for '" << type << "' " << evt->getValue<std::string>();
-
+ LOG(trace) << "DataItem '" << id << "': Invalid value for '" << evt->getName()
+ << "': '" << evt->getValue<std::string>() << '\'';
  }
  }
  }
@@ -96,7 +106,7 @@ namespace mtconnect::pipeline {
  evt->setProperty("quality", std::string("UNVERIFIABLE"));
  }
  }
- 
+
  return next(std::move(evt));
  }
 

src/mtconnect/printer/json_printer.cpp

@@ -60,8 +60,7 @@ namespace mtconnect::printer {
  template <typename T>
  inline void header(AutoJsonObject<T> &obj, const string &version, const string &hostname,
  const uint64_t instanceId, const unsigned int bufferSize,
- const string &schemaVersion, const string modelChangeTime,
- bool validation)
+ const string &schemaVersion, const string modelChangeTime, bool validation)
  {
  obj.AddPairs("version", version, "creationTime", getCurrentTime(GMT), "testIndicator", false,
  "instanceId", instanceId, "sender", hostname, "schemaVersion", schemaVersion);
@@ -72,18 +71,17 @@ namespace mtconnect::printer {
  obj.AddPairs("bufferSize", bufferSize);
  if (validation)
  obj.AddPairs("validation", true);
-
  }
 
  template <typename T>
  inline void probeAssetHeader(AutoJsonObject<T> &obj, const string &version,
  const string &hostname, const uint64_t instanceId,
  const unsigned int bufferSize, const unsigned int assetBufferSize,
  const unsigned int assetCount, const string &schemaVersion,
- const string modelChangeTime,
- const bool validation)
+ const string modelChangeTime, const bool validation)
  {
- header(obj, version, hostname, instanceId, bufferSize, schemaVersion, modelChangeTime, validation);
+ header(obj, version, hostname, instanceId, bufferSize, schemaVersion, modelChangeTime,
+ validation);
  obj.AddPairs("assetBufferSize", assetBufferSize, "assetCount", assetCount);
  }
 
@@ -92,10 +90,10 @@ namespace mtconnect::printer {
  const uint64_t instanceId, const unsigned int bufferSize,
  const uint64_t nextSequence, const uint64_t firstSequence,
  const uint64_t lastSequence, const string &schemaVersion,
- const string modelChangeTime,
- const bool validation)
+ const string modelChangeTime, const bool validation)
  {
- header(obj, version, hostname, instanceId, bufferSize, schemaVersion, modelChangeTime, validation);
+ header(obj, version, hostname, instanceId, bufferSize, schemaVersion, modelChangeTime,
+ validation);
  obj.AddPairs("nextSequence", nextSequence, "lastSequence", lastSequence, "firstSequence",
  firstSequence);
  }
@@ -213,8 +211,7 @@ namespace mtconnect::printer {
  {
  AutoJsonObject obj(writer, "Header");
  probeAssetHeader(obj, m_version, m_senderName, instanceId, 0, bufferSize, assetCount,
- *m_schemaVersion, m_modelChangeTime,
- m_validation);
+ *m_schemaVersion, m_modelChangeTime, m_validation);
  }
  {
  obj.Key("Assets");
@@ -412,8 +409,7 @@ namespace mtconnect::printer {
  {
  AutoJsonObject obj(writer, "Header");
  streamHeader(obj, m_version, m_senderName, instanceId, bufferSize, nextSeq, firstSeq,
- lastSeq, *m_schemaVersion, m_modelChangeTime,
- m_validation);
+ lastSeq, *m_schemaVersion, m_modelChangeTime, m_validation);
  }
 
  {

src/mtconnect/printer/printer.hpp

@@ -49,7 +49,9 @@ namespace mtconnect {
  public:
  /// @brief construct a printer
  /// @param pretty `true` if content should be pretty printed
- Printer(bool pretty = false, bool validation = false) : m_pretty(pretty), m_validation(validation) {}
+ Printer(bool pretty = false, bool validation = false)
+ : m_pretty(pretty), m_validation(validation)
+ {}
  virtual ~Printer() = default;
 
  /// @brief Generate an MTConnect Error document
@@ -145,19 +147,18 @@ namespace mtconnect {
  const_cast<Printer *>(this)->m_schemaVersion.emplace(ver);
  }
  }
- 
+
  /// @brief Get validation header flag state
  /// @returns validation state
  bool getValidation() const { return m_validation; }
- 
+
  /// @brief sets validation state
  /// @param validation the validation state
  void setValidation(bool v) { m_validation = v; }
-
 
  protected:
- bool m_pretty; //< Turns pretty printing on
- bool m_validation; //< Sets validation flag in header
+ bool m_pretty;  //< Turns pretty printing on
+ bool m_validation;  //< Sets validation flag in header
  std::string m_modelChangeTime;
  std::optional<std::string> m_schemaVersion;
  std::string m_senderName {"localhost"};

src/mtconnect/printer/xml_printer.cpp

@@ -100,7 +100,10 @@ namespace mtconnect::printer {
  xmlBufferPtr m_buf;
  };
 
- XmlPrinter::XmlPrinter(bool pretty, bool validation) : Printer(pretty, validation) { NAMED_SCOPE("xml.printer"); }
+ XmlPrinter::XmlPrinter(bool pretty, bool validation) : Printer(pretty, validation)
+ {
+ NAMED_SCOPE("xml.printer");
+ }
 
  void XmlPrinter::addDevicesNamespace(const std::string &urn, const std::string &location,
  const std::string &prefix)
@@ -636,7 +639,7 @@ namespace mtconnect::printer {
 
  addAttribute(writer, "sender", m_senderName);
  addAttribute(writer, "instanceId", to_string(instanceId));
- 
+
  if (m_validation)
  addAttribute(writer, "validation", "true"s);
 

src/mtconnect/source/adapter/adapter_pipeline.cpp

@@ -139,7 +139,7 @@ namespace mtconnect {
  // Convert values
  if (IsOptionSet(m_options, configuration::ConversionRequired))
  next = next->bind(make_shared<ConvertSample>());
- 
+
  // Validate Values
  if (IsOptionSet(m_options, configuration::Validation))
  next = next->bind(make_shared<Validator>(m_context));

src/mtconnect/source/loopback_source.cpp

@@ -38,7 +38,7 @@ namespace mtconnect::source {
  void LoopbackPipeline::build(const ConfigOptions &options)
  {
  m_options = options;
- 
+
  clear();
  TransformPtr next = m_start;