feat: add the XPath/RegEx Data Extractor and Expression Evaluation fo…

…r HTTP Probe (megaease#192)

* feat: add the XPath/RegEx Data Extractor and Expression Evaluation for HTTP probe

* lint warning

* 1)remove time_format, 2) support the xpatch in place

* refactoring the unit test

* add a test case mix the extract funciton and variable in evaluation expression

conf/conf.go

@@ -54,7 +54,7 @@ func Get() *Conf {
 // Schedule is the schedule.
 type Schedule int
 
-//
+// Schedule enum
 const (
  Hourly Schedule = iota
  Daily

conf/log.go

@@ -178,7 +178,7 @@ func (l *Log) GetWriter() io.Writer {
  return (io.Writer)(l.Writer)
 }
 
-//Rotate rotate the log file
+// Rotate rotate the log file
 func (l *Log) Rotate() {
  if l.Writer == nil || l.IsStdout == true {
  return

eval/eval.go

@@ -0,0 +1,236 @@
+/*
+ * Copyright (c) 2022, MegaEase
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package eval
+
+import (
+ "fmt"
+ "time"
+
+ "github.com/Knetic/govaluate"
+ log "github.com/sirupsen/logrus"
+)
+
+// Variable is the variable type
+type Variable struct {
+ Name string `yaml:"name"`
+ Type VarType `yaml:"type"`
+ Query string `yaml:"query"`
+ Value interface{}
+}
+
+// NewVariable is the function to create a variable
+func NewVariable(name string, t VarType, query string) *Variable {
+ return &Variable{
+ Name: name,
+ Type: t,
+ Query: query,
+ Value: nil,
+ }
+}
+
+// Evaluator is the structure of evaluator
+type Evaluator struct {
+ Variables []Variable `yaml:"variables"`
+ DocType DocType `yaml:"doc"`
+ Expression string `yaml:"expression"`
+ Document string `yaml:"-"`
+ Extractor Extractor `yaml:"-"`
+ EvalFuncs map[string]govaluate.ExpressionFunction `yaml:"-"`
+}
+
+// NewEvaluator is the function to create a evaluator
+func NewEvaluator(doc string, t DocType, exp string) *Evaluator {
+ e := &Evaluator{
+ Variables: make([]Variable, 0),
+ DocType: t,
+ Expression: exp,
+ Document: doc,
+ }
+ e.Config()
+ return e
+}
+
+// Config is the function to config the evaluator
+func (e *Evaluator) Config() error {
+ e.configExtractor()
+ e.configEvalFunctions()
+ return nil
+}
+
+func (e *Evaluator) configExtractor() {
+ switch e.DocType {
+ case HTML:
+ e.Extractor = NewHTMLExtractor(e.Document)
+ case XML:
+ e.Extractor = NewXMLExtractor(e.Document)
+ case JSON:
+ e.Extractor = NewJSONExtractor(e.Document)
+ case TEXT:
+ e.Extractor = NewRegexExtractor(e.Document)
+ default:
+ e.Extractor = nil
+ log.Errorf("Unsupported document type: %s", e.DocType)
+ }
+}
+
+func (e *Evaluator) configEvalFunctions() {
+
+ extract := func(t VarType, query string, failed interface{}) (interface{}, error) {
+ v := Variable{
+ Type: t,
+ Query: query,
+ }
+ if err := e.ExtractValue(&v); err != nil {
+ return failed, err
+ }
+ return v.Value, nil
+ }
+
+ e.EvalFuncs = map[string]govaluate.ExpressionFunction{
+
+ // Extract value by XPath/Regex Expression
+ "x_str": func(args ...interface{}) (interface{}, error) {
+ return extract(String, args[0].(string), "")
+ },
+ "x_float": func(args ...interface{}) (interface{}, error) {
+ return extract(Float, args[0].(string), 0.0)
+ },
+ "x_int": func(args ...interface{}) (interface{}, error) {
+ v, e := extract(Int, args[0].(string), 0)
+ return float64(v.(int)), e
+ },
+ "x_bool": func(args ...interface{}) (interface{}, error) {
+ return extract(Bool, args[0].(string), false)
+ },
+ "x_time": func(args ...interface{}) (interface{}, error) {
+ v := Variable{
+ Type: Time,
+ Query: args[0].(string),
+ }
+
+ if err := e.ExtractValue(&v); err != nil {
+ return (time.Time{}), err
+ }
+ return (float64)(v.Value.(int64)), nil
+ },
+ "x_duration": func(args ...interface{}) (interface{}, error) {
+ v, e := extract(Duration, args[0].(string), 0)
+ return (float64)(v.(time.Duration)), e
+ },
+
+ // Functional functions
+ "strlen": func(args ...interface{}) (interface{}, error) {
+ length := len(args[0].(string))
+ return (float64)(length), nil
+ },
+ "now": func(args ...interface{}) (interface{}, error) {
+ return (float64)(time.Now().Unix()), nil
+ },
+ "duration": func(args ...interface{}) (interface{}, error) {
+ str := args[0].(string)
+ d, err := time.ParseDuration(str)
+ if err != nil {
+ return nil, err
+ }
+ return (float64)(d), nil
+ },
+ }
+}
+
+// SetDocument is the function to set the document
+func (e *Evaluator) SetDocument(t DocType, doc string) {
+ if e.DocType != t {
+ e.DocType = t
+ e.Document = doc
+ e.configExtractor()
+ } else {
+ e.Document = doc
+ e.Extractor.SetDocument(doc)
+ }
+}
+
+// AddVariable is the function to add a variable
+func (e *Evaluator) AddVariable(v *Variable) {
+ e.Variables = append(e.Variables, *v)
+}
+
+// CleanVariable is the function to clean the variable
+func (e *Evaluator) CleanVariable() {
+ e.Variables = make([]Variable, 0)
+}
+
+// Evaluate is the function to evaluate the expression
+func (e *Evaluator) Evaluate() (bool, error) {
+
+ if err := e.Extract(); err != nil {
+ return false, err
+ }
+
+ expression, err := govaluate.NewEvaluableExpressionWithFunctions(e.Expression, e.EvalFuncs)
+ if err != nil {
+ return false, err
+ }
+
+ variables := make(map[string]interface{})
+ for _, v := range e.Variables {
+ variables[v.Name] = v.Value
+ }
+
+ result, err := expression.Evaluate(variables)
+ if err != nil {
+ return false, err
+ }
+ switch result.(type) {
+ case bool:
+ return result.(bool), nil
+ case float64:
+ return result.(float64) != 0, nil
+ case string:
+ return result.(string) != "", nil
+ }
+ return false, fmt.Errorf("Unsupported type: %T", result)
+}
+
+// Extract is the function to extract the value from the document
+func (e *Evaluator) Extract() error {
+ for i := 0; i < len(e.Variables); i++ {
+ if err := e.ExtractValue(&e.Variables[i]); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+// ExtractValue is the function to extract the value from the document
+func (e *Evaluator) ExtractValue(v *Variable) error {
+ if e.DocType == Unsupported || e.Extractor == nil {
+ return fmt.Errorf("Unsupported document type: %s", e.DocType)
+ }
+ e.Extractor.SetQuery(v.Query)
+ e.Extractor.SetVarType(v.Type)
+ value, err := e.Extractor.Extract()
+ if err != nil {
+ return err
+ }
+ if v.Type == Time {
+ v.Value = value.(time.Time).Local().Unix()
+ } else {
+ v.Value = value
+ }
+ return nil
+}