Custom data (#500)

* add dynamic object * add custom data store * update custom data API * update API for mesh * add client command for custom data * fix mesh layout typo * rename for adding test cases * add unit tests * add more testcases * improve coverage * add documentation for custom data * Apply suggestions from code review Co-authored-by: Samu Tamminen <[email protected]> * Apply suggestions from code review Co-authored-by: Samu Tamminen <[email protected]> * update according to comment Co-authored-by: Samu Tamminen <[email protected]>
easegress-io · Feb 11, 2022 · e2a96b8 · e2a96b8
1 parent a99ec2d
commit e2a96b8
Show file tree

Hide file tree

Showing 18 changed files with 1,897 additions and 299 deletions.
diff --git a/cmd/client/command/common.go b/cmd/client/command/common.go
@@ -63,8 +63,10 @@ const (
  wasmCodeURL = apiURL + "/wasm/code"
  wasmDataURL = apiURL + "/wasm/data/%s/%s"
 
- customDataKindURL = apiURL + "/customdata/%s"
- customDataURL = apiURL + "/customdata/%s/%s"
+ customDataKindURL = apiURL + "/customdatakinds"
+ customDataKindItemURL = apiURL + "/customdatakinds/%s"
+ customDataURL = apiURL + "/customdata/%s"
+ customDataItemURL = apiURL + "/customdata/%s/%s"
 
  // MeshTenantsURL is the mesh tenant prefix.
  MeshTenantsURL = apiURL + "/mesh/tenants"

diff --git a/cmd/client/command/customdata.go b/cmd/client/command/customdata.go
@@ -24,6 +24,118 @@ import (
  "github.com/spf13/cobra"
 )
 
+// CustomDataKindCmd defines custom data kind command.
+func CustomDataKindCmd() *cobra.Command {
+ cmd := &cobra.Command{
+ Use: "custom-data-kind",
+ Short: "View and change custom data kind",
+ }
+
+ cmd.AddCommand(listCustomDataKindCmd())
+ cmd.AddCommand(getCustomDataKindCmd())
+ cmd.AddCommand(createCustomDataKindCmd())
+ cmd.AddCommand(updateCustomDataKindCmd())
+ cmd.AddCommand(deleteCustomDataKindCmd())
+
+ return cmd
+}
+
+func listCustomDataKindCmd() *cobra.Command {
+ cmd := &cobra.Command{
+ Use: "list",
+ Short: "List all custom data kinds",
+ Example: "egctl custom-data-kind list",
+
+ Run: func(cmd *cobra.Command, args []string) {
+ handleRequest(http.MethodGet, makeURL(customDataKindURL), nil, cmd)
+ },
+ }
+
+ return cmd
+}
+
+func getCustomDataKindCmd() *cobra.Command {
+ cmd := &cobra.Command{
+ Use: "get",
+ Short: "Get a custom data kind",
+ Example: "egctl custom-data-kind get <kind>",
+ Args: func(cmd *cobra.Command, args []string) error {
+ if len(args) != 1 {
+ return errors.New("requires custom data kind to be retrieved")
+ }
+ return nil
+ },
+
+ Run: func(cmd *cobra.Command, args []string) {
+ handleRequest(http.MethodGet, makeURL(customDataKindItemURL, args[0]), nil, cmd)
+ },
+ }
+
+ return cmd
+}
+
+func createCustomDataKindCmd() *cobra.Command {
+ var specFile string
+ cmd := &cobra.Command{
+ Use: "create",
+ Short: "Create a custom data kind from a yaml file or stdin",
+ Example: "egctl custom-data-kind create -f <kind file>",
+ Run: func(cmd *cobra.Command, args []string) {
+ visitor := buildYAMLVisitor(specFile, cmd)
+ visitor.Visit(func(yamlDoc []byte) error {
+ handleRequest(http.MethodPost, makeURL(customDataKindURL), yamlDoc, cmd)
+ return nil
+ })
+ visitor.Close()
+ },
+ }
+
+ cmd.Flags().StringVarP(&specFile, "file", "f", "", "A yaml file specifying the change request.")
+
+ return cmd
+}
+
+func updateCustomDataKindCmd() *cobra.Command {
+ var specFile string
+ cmd := &cobra.Command{
+ Use: "update",
+ Short: "Update a custom data from a yaml file or stdin",
+ Example: "egctl custom-data-kind update -f <kind file>",
+ Run: func(cmd *cobra.Command, args []string) {
+ visitor := buildYAMLVisitor(specFile, cmd)
+ visitor.Visit(func(yamlDoc []byte) error {
+ handleRequest(http.MethodPut, makeURL(customDataKindURL), yamlDoc, cmd)
+ return nil
+ })
+ visitor.Close()
+ },
+ }
+
+ cmd.Flags().StringVarP(&specFile, "file", "f", "", "A yaml file specifying the change request.")
+
+ return cmd
+}
+
+func deleteCustomDataKindCmd() *cobra.Command {
+ cmd := &cobra.Command{
+ Use: "delete",
+ Short: "Delete a custom data kind",
+ Example: "egctl custom-data-kind delete <kind>",
+ Args: func(cmd *cobra.Command, args []string) error {
+ if len(args) != 1 {
+ return errors.New("requires custom data kind to be retrieved")
+ }
+ return nil
+ },
+
+ Run: func(cmd *cobra.Command, args []string) {
+ handleRequest(http.MethodDelete, makeURL(customDataKindItemURL, args[0]), nil, cmd)
+ },
+ }
+
+ return cmd
+}
+
 // CustomDataCmd defines custom data command.
 func CustomDataCmd() *cobra.Command {
  cmd := &cobra.Command{
@@ -33,15 +145,18 @@ func CustomDataCmd() *cobra.Command {
 
  cmd.AddCommand(listCustomDataCmd())
  cmd.AddCommand(getCustomDataCmd())
+ cmd.AddCommand(createCustomDataCmd())
  cmd.AddCommand(updateCustomDataCmd())
+ cmd.AddCommand(batchUpdateCustomDataCmd())
+ cmd.AddCommand(deleteCustomDataCmd())
 
  return cmd
 }
 
 func getCustomDataCmd() *cobra.Command {
  cmd := &cobra.Command{
  Use: "get",
- Short: "Get an custom data",
+ Short: "Get a custom data",
  Example: "egctl custom-data get <kind> <id>",
  Args: func(cmd *cobra.Command, args []string) error {
  if len(args) != 2 {
@@ -51,7 +166,7 @@ func getCustomDataCmd() *cobra.Command {
  },
 
  Run: func(cmd *cobra.Command, args []string) {
- handleRequest(http.MethodGet, makeURL(customDataURL, args[0], args[1]), nil, cmd)
+ handleRequest(http.MethodGet, makeURL(customDataItemURL, args[0], args[1]), nil, cmd)
  },
  }
 
@@ -70,19 +185,73 @@ func listCustomDataCmd() *cobra.Command {
  return nil
  },
  Run: func(cmd *cobra.Command, args []string) {
- handleRequest(http.MethodGet, makeURL(customDataKindURL, args[0]), nil, cmd)
+ handleRequest(http.MethodGet, makeURL(customDataURL, args[0]), nil, cmd)
  },
  }
 
  return cmd
 }
 
+func createCustomDataCmd() *cobra.Command {
+ var specFile string
+ cmd := &cobra.Command{
+ Use: "create",
+ Short: "Create a custom data from a yaml file or stdin",
+ Example: "egctl custom-data create <kind> -f <data item file>",
+ Args: func(cmd *cobra.Command, args []string) error {
+ if len(args) != 1 {
+ return errors.New("requires custom data kind to be retrieved")
+ }
+ return nil
+ },
+ Run: func(cmd *cobra.Command, args []string) {
+ visitor := buildYAMLVisitor(specFile, cmd)
+ visitor.Visit(func(yamlDoc []byte) error {
+ handleRequest(http.MethodPost, makeURL(customDataURL, args[0]), yamlDoc, cmd)
+ return nil
+ })
+ visitor.Close()
+ },
+ }
+
+ cmd.Flags().StringVarP(&specFile, "file", "f", "", "A yaml file specifying the change request.")
+
+ return cmd
+}
+
 func updateCustomDataCmd() *cobra.Command {
  var specFile string
  cmd := &cobra.Command{
  Use: "update",
+ Short: "Update a custom data from a yaml file or stdin",
+ Example: "egctl custom-data update <kind> -f <data item file>",
+ Args: func(cmd *cobra.Command, args []string) error {
+ if len(args) != 1 {
+ return errors.New("requires custom data kind to be retrieved")
+ }
+ return nil
+ },
+ Run: func(cmd *cobra.Command, args []string) {
+ visitor := buildYAMLVisitor(specFile, cmd)
+ visitor.Visit(func(yamlDoc []byte) error {
+ handleRequest(http.MethodPut, makeURL(customDataURL, args[0]), yamlDoc, cmd)
+ return nil
+ })
+ visitor.Close()
+ },
+ }
+
+ cmd.Flags().StringVarP(&specFile, "file", "f", "", "A yaml file specifying the change request.")
+
+ return cmd
+}
+
+func batchUpdateCustomDataCmd() *cobra.Command {
+ var specFile string
+ cmd := &cobra.Command{
+ Use: "batch-update",
  Short: "Batch update custom data from a yaml file or stdin",
- Example: "egctl custom-data update <kind> -f <change request file>",
+ Example: "egctl custom-data batch-update <kind> -f <change request file>",
  Args: func(cmd *cobra.Command, args []string) error {
  if len(args) != 1 {
  return errors.New("requires custom data kind to be retrieved")
@@ -92,7 +261,7 @@ func updateCustomDataCmd() *cobra.Command {
  Run: func(cmd *cobra.Command, args []string) {
  visitor := buildYAMLVisitor(specFile, cmd)
  visitor.Visit(func(yamlDoc []byte) error {
- handleRequest(http.MethodPost, makeURL(customDataKindURL, args[0]), yamlDoc, cmd)
+ handleRequest(http.MethodPost, makeURL(customDataItemURL, args[0], "items"), yamlDoc, cmd)
  return nil
  })
  visitor.Close()
@@ -103,3 +272,22 @@ func updateCustomDataCmd() *cobra.Command {
 
  return cmd
 }
+
+func deleteCustomDataCmd() *cobra.Command {
+ cmd := &cobra.Command{
+ Use: "delete",
+ Short: "Delete a custom data item",
+ Example: "egctl custom-data delete <kind> <id>",
+ Args: func(cmd *cobra.Command, args []string) error {
+ if len(args) != 2 {
+ return errors.New("requires custom data kind and id to be retrieved")
+ }
+ return nil
+ },
+ Run: func(cmd *cobra.Command, args []string) {
+ handleRequest(http.MethodDelete, makeURL(customDataItemURL, args[0], args[1]), nil, cmd)
+ },
+ }
+
+ return cmd
+}
diff --git a/cmd/client/main.go b/cmd/client/main.go
@@ -110,6 +110,7 @@ func main() {
  command.ObjectCmd(),
  command.MemberCmd(),
  command.WasmCmd(),
+ command.CustomDataKindCmd(),
  command.CustomDataCmd(),
  completionCmd,
  )

diff --git a/doc/README.md b/doc/README.md
@@ -9,6 +9,7 @@
   - [4.1.1 System Controllers](#411-system-controllers)
   - [4.1.2 Business Controllers](#412-business-controllers)
   - [4.2 Filters](#42-filters)
+  - [4.3 Custom Data](#43-custom-data)
 
 ## 1. Cookbook / How-To Guide
 
@@ -91,4 +92,6 @@ It could be created, updated, deleted by admin operation. They control various r
 - [Validator](./reference/filters.md#Validator) - The Validator filter validates requests, forwards valid ones, and rejects invalid ones. 
 - [WasmHost](./reference/filters.md#WasmHost) - The WasmHost filter implements a host environment for user-developed WebAssembly code. 
 
+### 4.3 Custom Data
 
+- [Custom Data Management](./reference/customdata.md) - Create/Read/Update/Delete custom data kinds and custom data items.
diff --git a/doc/reference/customdata.md b/doc/reference/customdata.md
@@ -0,0 +1,105 @@
+# Custom Data Management
+
+The `Custom Data` feature implements a storage for 'any' data, which can be used by other components for data persistence.
+
+Because the schema of the data being persisted varies from components, a `CustomDataKind` must be defined to distinguish the data.
+
+## CustomDataKind
+
+The YAML example below defines a CustomDataKind:
+
+```yaml
+name: kind1
+idField: name
+jsonSchema:
+ type: object
+ properties:
+ name:
+ type: string
+ required: true
+```
+
+The `name` field is required, it is the kind name of custom data items of this kind.
+The `idField` is optional, and its default value is `name`, the value of this field of a data item is used as its identifier.
+The `jsonSchema` is optional, if provided, data items of this kind will be validated against this [JSON Schema](http:https://json-schema.org/).
+
+## CustomData
+
+CustomData is a map, the keys of this map must be strings while the values can be any valid JSON values, but the keys of a nested map must be strings too.
+
+A CustomData item must contain the `idField` defined by its corresponding CustomDataKind, for example, the data items of the kind defined in the above example must contain the `name` field as their identifiers.
+
+Below is an example of a CustomData:
+
+```yaml
+name: data1
+field1: 12
+field2: abc
+field3: [1, 2, 3, 4]
+```
+
+## API
+
+* **Create a CustomDataKind**
+ * **URL**: http:https://{ip}:{port}/apis/v1/customdatakinds
+ * **Method**: POST
+ * **Body**: CustomDataKind definition is YAML.
+
+* **Update a CustomDataKind**
+ * **URL**: http:https://{ip}:{port}/apis/v1/customdatakinds
+ * **Method**: PUT
+ * **Body**: CustomDataKind definition is YAML.
+
+* **Query the definition of a CustomDataKind**
+ * **URL**: http:https://{ip}:{port}/apis/v1/customdatakinds/{kind name}
+ * **Method**: GET
+
+* **List the definition of all CustomDataKind**
+ * **URL**: http:https://{ip}:{port}/apis/v1/customdatakinds
+ * **Method**: GET
+
+* **Delete a CustomDataKind**
+ * **URL**: http:https://{ip}:{port}/apis/v1/customdatakinds/{kind name}
+ * **Method**: DELETE
+
+* **Create a CustomData**
+ * **URL**: http:https://{ip}:{port}/apis/v1/customdata/{kind name}
+ * **Method**: POST
+ * **Body**: CustomData definition is YAML.
+
+* **Update a CustomData**
+ * **URL**: http:https://{ip}:{port}/apis/v1/customdata/{kind name}
+ * **Method**: PUT
+ * **Body**: CustomData definition is YAML.
+
+* **Query the definition of a CustomData**
+ * **URL**: http:https://{ip}:{port}/apis/v1/customdata/{kind name}/{data id}
+ * **Method**: GET
+
+* **List the definition of all CustomData of a kind**
+ * **URL**: http:https://{ip}:{port}/apis/v1/customdata/{kind name}
+ * **Method**: GET
+
+* **Delete a CustomData**
+ * **URL**: http:https://{ip}:{port}/apis/v1/customdata/{kind name}/{data id}
+ * **Method**: DELETE
+
+* **Delete all CustomData of a kind**
+ * **URL**: http:https://{ip}:{port}/apis/v1/customdata/{kind name}
+ * **Method**: DELETE
+
+* **Bulk update**
+ * **URL**: http:https://{ip}:{port}/apis/v1/customdata/{kind name}/items
+ * **Method**: POST
+ * **Body**: A change request in YAML, as defined below.
+
+```yaml
+rebuild: false
+delete: [data1, data2]
+list:
+- name: data3
+ field1: 12
+- name: data4
+ field1: foo
+```
+When `rebuild` is true (default is false), all existing data items are deleted before processing the data items in `list`. `delete` is an array of data identifiers to be deleted, this array is ignored when `rebuild` is true. `list` is an array of data items to be created or updated.