From ab208ebc9f90772a9335df6ed149cdadab6f7f0a Mon Sep 17 00:00:00 2001 From: Nevill Date: Thu, 2 Sep 2021 18:40:59 +0800 Subject: [PATCH 1/5] improve egctl to support sending multi configs at once (fix #29) --- cmd/client/command/common.go | 30 ++++++++++++++++++++++-------- cmd/client/command/object.go | 10 ++++++---- cmd/client/command/wasm.go | 5 +++-- 3 files changed, 31 insertions(+), 14 deletions(-) diff --git a/cmd/client/command/common.go b/cmd/client/command/common.go index 4fbb9c7da5..7cb0d2de40 100644 --- a/cmd/client/command/common.go +++ b/cmd/client/command/common.go @@ -23,6 +23,7 @@ import ( "io/ioutil" "net/http" "os" + "strings" yamljsontool "github.com/ghodss/yaml" "github.com/spf13/cobra" @@ -41,6 +42,8 @@ type ( Code int `yaml:"code"` Message string `yaml:"message"` } + + yamlHandler func(doc, name string) ) // CommandlineGlobalFlags is the singleton of GlobalFlags. @@ -61,8 +64,9 @@ const ( statusObjectURL = apiURL + "/status/objects/%s" statusObjectsURL = apiURL + "/status/objects" - wasmCodeURL = apiURL + "/wasm/code" - wasmDataURL = apiURL + "/wasm/data/%s/%s" + wasmCodeURL = apiURL + "/wasm/code" + wasmDataURL = apiURL + "/wasm/data/%s/%s" + yamlSeparator = "---" // MeshTenantsURL is the mesh tenant prefix. MeshTenantsURL = apiURL + "/mesh/tenants" @@ -163,7 +167,7 @@ func printBody(body []byte) { fmt.Printf("%s", output) } -func readFromFileOrStdin(specFile string, cmd *cobra.Command) ([]byte, string) { +func readFromFileOrStdin(specFile string, cmd *cobra.Command, handler yamlHandler) { var buff []byte var err error if specFile != "" { @@ -182,10 +186,20 @@ func readFromFileOrStdin(specFile string, cmd *cobra.Command) ([]byte, string) { Kind string `yaml:"kind"` Name string `yaml:"name"` } - err = yaml.Unmarshal(buff, &spec) - if err != nil { - ExitWithErrorf("%s failed, invalid spec: %v", cmd.Short, err) - } - return buff, spec.Name + yamlDocs := strings.Split(string(buff), yamlSeparator) + + // make sure each yaml doc valid + for _, yamlDoc := range yamlDocs { + if len(strings.TrimSpace(yamlDoc)) == 0 { + continue + } + err = yaml.Unmarshal([]byte(yamlDoc), &spec) + if err != nil { + ExitWithErrorf("%s failed, invalid spec: %v", cmd.Short, err) + break + } + + handler(yamlDoc, spec.Name) + } } diff --git a/cmd/client/command/object.go b/cmd/client/command/object.go index e96308aac1..d185b714bd 100644 --- a/cmd/client/command/object.go +++ b/cmd/client/command/object.go @@ -61,8 +61,9 @@ func createObjectCmd() *cobra.Command { Use: "create", Short: "Create an object from a yaml file or stdin", Run: func(cmd *cobra.Command, args []string) { - buff, _ := readFromFileOrStdin(specFile, cmd) - handleRequest(http.MethodPost, makeURL(objectsURL), buff, cmd) + readFromFileOrStdin(specFile, cmd, func(doc, name string) { + handleRequest(http.MethodPost, makeURL(objectsURL), []byte(doc), cmd) + }) }, } @@ -77,8 +78,9 @@ func updateObjectCmd() *cobra.Command { Use: "update", Short: "Update an object from a yaml file or stdin", Run: func(cmd *cobra.Command, args []string) { - buff, name := readFromFileOrStdin(specFile, cmd) - handleRequest(http.MethodPut, makeURL(objectURL, name), buff, cmd) + readFromFileOrStdin(specFile, cmd, func(doc, name string) { + handleRequest(http.MethodPut, makeURL(objectURL, name), []byte(doc), cmd) + }) }, } diff --git a/cmd/client/command/wasm.go b/cmd/client/command/wasm.go index 3c79dbc255..e8af5e797d 100644 --- a/cmd/client/command/wasm.go +++ b/cmd/client/command/wasm.go @@ -85,8 +85,9 @@ func wasmApplyDataCmd() *cobra.Command { }, Run: func(cmd *cobra.Command, args []string) { - buf, _ := readFromFileOrStdin(specFile, cmd) - handleRequest(http.MethodPut, makeURL(wasmDataURL, args[0], args[1]), buf, cmd) + readFromFileOrStdin(specFile, cmd, func(doc, name string) { + handleRequest(http.MethodPut, makeURL(wasmDataURL, args[0], args[1]), []byte(doc), cmd) + }) }, } cmd.Flags().StringVarP(&specFile, "file", "f", "", "A yaml file specifying the object.") From ed49a3ab81415152c1b60030ab155697a0ba9f0e Mon Sep 17 00:00:00 2001 From: Nevill Date: Mon, 6 Sep 2021 23:19:25 +0800 Subject: [PATCH 2/5] add a visitor to do the YAML parsing --- cmd/client/command/common.go | 17 ++++++++ cmd/client/command/object.go | 14 +++++-- cmd/client/command/visitor.go | 78 +++++++++++++++++++++++++++++++++++ cmd/client/command/wasm.go | 7 +++- 4 files changed, 110 insertions(+), 6 deletions(-) create mode 100644 cmd/client/command/visitor.go diff --git a/cmd/client/command/common.go b/cmd/client/command/common.go index 7cb0d2de40..ebee06ed46 100644 --- a/cmd/client/command/common.go +++ b/cmd/client/command/common.go @@ -203,3 +203,20 @@ func readFromFileOrStdin(specFile string, cmd *cobra.Command, handler yamlHandle handler(yamlDoc, spec.Name) } } + +func buildVisitorFromFileOrStdin(specFile string, cmd *cobra.Command) Visitor { + var buff []byte + var err error + if specFile != "" { + buff, err = ioutil.ReadFile(specFile) + if err != nil { + ExitWithErrorf("%s failed: %v", cmd.Short, err) + } + } else { + buff, err = ioutil.ReadAll(os.Stdin) + if err != nil { + ExitWithErrorf("%s failed: %v", cmd.Short, err) + } + } + return NewStreamVisitor(string(buff)) +} diff --git a/cmd/client/command/object.go b/cmd/client/command/object.go index d185b714bd..4ced6f6106 100644 --- a/cmd/client/command/object.go +++ b/cmd/client/command/object.go @@ -61,8 +61,11 @@ func createObjectCmd() *cobra.Command { Use: "create", Short: "Create an object from a yaml file or stdin", Run: func(cmd *cobra.Command, args []string) { - readFromFileOrStdin(specFile, cmd, func(doc, name string) { - handleRequest(http.MethodPost, makeURL(objectsURL), []byte(doc), cmd) + visitor := buildVisitorFromFileOrStdin(specFile, cmd) + visitor.Visit(func(docs []spec) { + for _, d := range docs { + handleRequest(http.MethodPost, makeURL(objectsURL), []byte(d.doc), cmd) + } }) }, } @@ -78,8 +81,11 @@ func updateObjectCmd() *cobra.Command { Use: "update", Short: "Update an object from a yaml file or stdin", Run: func(cmd *cobra.Command, args []string) { - readFromFileOrStdin(specFile, cmd, func(doc, name string) { - handleRequest(http.MethodPut, makeURL(objectURL, name), []byte(doc), cmd) + visitor := buildVisitorFromFileOrStdin(specFile, cmd) + visitor.Visit(func(docs []spec) { + for _, d := range docs { + handleRequest(http.MethodPut, makeURL(objectURL, d.Name), []byte(d.doc), cmd) + } }) }, } diff --git a/cmd/client/command/visitor.go b/cmd/client/command/visitor.go new file mode 100644 index 0000000000..5d23c71a41 --- /dev/null +++ b/cmd/client/command/visitor.go @@ -0,0 +1,78 @@ +package command + +import ( + "bufio" + "io" + "strings" + + "k8s.io/apimachinery/pkg/util/yaml" +) + +// VisitorFunc executes visition logic +type VisitorFunc func([]spec) + +// Visitor walk through the document via VisitorFunc +type Visitor interface { + Visit(VisitorFunc) +} + +type spec struct { + Kind string + Name string + doc string +} + +type streamVisitor struct { + io.Reader +} + +// NewStreamVisitor returns a streamVisitor. +func NewStreamVisitor(src string) *streamVisitor { + return &streamVisitor{ + Reader: strings.NewReader(src), + } +} + +type yamlDecoder struct { + reader *yaml.YAMLReader + doc string +} + +func newYAMLDecoder(r io.Reader) *yamlDecoder { + return &yamlDecoder{ + reader: yaml.NewYAMLReader(bufio.NewReader(r)), + } +} + +// Decods reads a YAML document into bytes and try to yaml.Unmarshal it. +func (d *yamlDecoder) Decode(into interface{}) error { + bytes, err := d.reader.Read() + if err != nil && err != io.EOF { + return err + } + d.doc = string(bytes) + if len(bytes) != 0 { + err = yaml.Unmarshal(bytes, into) + } + return err +} + +// Visit implements Visitor over a stream. +func (v *streamVisitor) Visit(fn VisitorFunc) { + d := newYAMLDecoder(v.Reader) + var validDocs []spec + for { + var s spec + if err := d.Decode(&s); err != nil { + if err == io.EOF { + break + } else { + ExitWithErrorf("error parsing %s: %v", d.doc, err) + } + } + s.doc = d.doc + //TODO can validate spec's Kind here + validDocs = append(validDocs, s) + } + fn(validDocs) +} diff --git a/cmd/client/command/wasm.go b/cmd/client/command/wasm.go index e8af5e797d..b53b37cfaa 100644 --- a/cmd/client/command/wasm.go +++ b/cmd/client/command/wasm.go @@ -85,8 +85,11 @@ func wasmApplyDataCmd() *cobra.Command { }, Run: func(cmd *cobra.Command, args []string) { - readFromFileOrStdin(specFile, cmd, func(doc, name string) { - handleRequest(http.MethodPut, makeURL(wasmDataURL, args[0], args[1]), []byte(doc), cmd) + visitor := buildVisitorFromFileOrStdin(specFile, cmd) + visitor.Visit(func(docs []spec) { + for _, d := range docs { + handleRequest(http.MethodPut, makeURL(wasmDataURL, args[0], args[1]), []byte(d.doc), cmd) + } }) }, } From 139f53bffdaaa499255743206a3e5993e3bf9ea7 Mon Sep 17 00:00:00 2001 From: Nevill Date: Mon, 6 Sep 2021 23:37:01 +0800 Subject: [PATCH 3/5] rollback function readFromFileOrStdin --- cmd/client/command/common.go | 30 ++++++++---------------------- 1 file changed, 8 insertions(+), 22 deletions(-) diff --git a/cmd/client/command/common.go b/cmd/client/command/common.go index ebee06ed46..0f03ef3af6 100644 --- a/cmd/client/command/common.go +++ b/cmd/client/command/common.go @@ -23,7 +23,6 @@ import ( "io/ioutil" "net/http" "os" - "strings" yamljsontool "github.com/ghodss/yaml" "github.com/spf13/cobra" @@ -42,8 +41,6 @@ type ( Code int `yaml:"code"` Message string `yaml:"message"` } - - yamlHandler func(doc, name string) ) // CommandlineGlobalFlags is the singleton of GlobalFlags. @@ -64,9 +61,8 @@ const ( statusObjectURL = apiURL + "/status/objects/%s" statusObjectsURL = apiURL + "/status/objects" - wasmCodeURL = apiURL + "/wasm/code" - wasmDataURL = apiURL + "/wasm/data/%s/%s" - yamlSeparator = "---" + wasmCodeURL = apiURL + "/wasm/code" + wasmDataURL = apiURL + "/wasm/data/%s/%s" // MeshTenantsURL is the mesh tenant prefix. MeshTenantsURL = apiURL + "/mesh/tenants" @@ -167,7 +163,7 @@ func printBody(body []byte) { fmt.Printf("%s", output) } -func readFromFileOrStdin(specFile string, cmd *cobra.Command, handler yamlHandler) { +func readFromFileOrStdin(specFile string, cmd *cobra.Command) ([]byte, string) { var buff []byte var err error if specFile != "" { @@ -186,22 +182,12 @@ func readFromFileOrStdin(specFile string, cmd *cobra.Command, handler yamlHandle Kind string `yaml:"kind"` Name string `yaml:"name"` } - - yamlDocs := strings.Split(string(buff), yamlSeparator) - - // make sure each yaml doc valid - for _, yamlDoc := range yamlDocs { - if len(strings.TrimSpace(yamlDoc)) == 0 { - continue - } - err = yaml.Unmarshal([]byte(yamlDoc), &spec) - if err != nil { - ExitWithErrorf("%s failed, invalid spec: %v", cmd.Short, err) - break - } - - handler(yamlDoc, spec.Name) + err = yaml.Unmarshal(buff, &spec) + if err != nil { + ExitWithErrorf("%s failed, invalid spec: %v", cmd.Short, err) } + + return buff, spec.Name } func buildVisitorFromFileOrStdin(specFile string, cmd *cobra.Command) Visitor { From e3b448e14d22bab1fb6cb43ede78d8822d7c365e Mon Sep 17 00:00:00 2001 From: Nevill Date: Tue, 7 Sep 2021 16:34:16 +0800 Subject: [PATCH 4/5] Update comments. Co-authored-by: Bomin Zhang --- cmd/client/command/visitor.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/client/command/visitor.go b/cmd/client/command/visitor.go index 5d23c71a41..7ab56258c7 100644 --- a/cmd/client/command/visitor.go +++ b/cmd/client/command/visitor.go @@ -44,7 +44,7 @@ func newYAMLDecoder(r io.Reader) *yamlDecoder { } } -// Decods reads a YAML document into bytes and try to yaml.Unmarshal it. +// Decode reads a YAML document into bytes and tries to yaml.Unmarshal it. func (d *yamlDecoder) Decode(into interface{}) error { bytes, err := d.reader.Read() if err != nil && err != io.EOF { From 5897eba8370f9c6f649d262828e8cbc9ca84f596 Mon Sep 17 00:00:00 2001 From: Nevill Date: Tue, 7 Sep 2021 16:50:44 +0800 Subject: [PATCH 5/5] some improvements - change the signature of VisitorFunc - remove useless function readFromFileOrStdin --- cmd/client/command/common.go | 27 --------------------------- cmd/client/command/object.go | 12 ++++-------- cmd/client/command/visitor.go | 10 ++++++---- cmd/client/command/wasm.go | 6 ++---- 4 files changed, 12 insertions(+), 43 deletions(-) diff --git a/cmd/client/command/common.go b/cmd/client/command/common.go index 0f03ef3af6..f1c83610bd 100644 --- a/cmd/client/command/common.go +++ b/cmd/client/command/common.go @@ -163,33 +163,6 @@ func printBody(body []byte) { fmt.Printf("%s", output) } -func readFromFileOrStdin(specFile string, cmd *cobra.Command) ([]byte, string) { - var buff []byte - var err error - if specFile != "" { - buff, err = ioutil.ReadFile(specFile) - if err != nil { - ExitWithErrorf("%s failed: %v", cmd.Short, err) - } - } else { - buff, err = ioutil.ReadAll(os.Stdin) - if err != nil { - ExitWithErrorf("%s failed: %v", cmd.Short, err) - } - } - - var spec struct { - Kind string `yaml:"kind"` - Name string `yaml:"name"` - } - err = yaml.Unmarshal(buff, &spec) - if err != nil { - ExitWithErrorf("%s failed, invalid spec: %v", cmd.Short, err) - } - - return buff, spec.Name -} - func buildVisitorFromFileOrStdin(specFile string, cmd *cobra.Command) Visitor { var buff []byte var err error diff --git a/cmd/client/command/object.go b/cmd/client/command/object.go index 4ced6f6106..82deac4bf8 100644 --- a/cmd/client/command/object.go +++ b/cmd/client/command/object.go @@ -62,10 +62,8 @@ func createObjectCmd() *cobra.Command { Short: "Create an object from a yaml file or stdin", Run: func(cmd *cobra.Command, args []string) { visitor := buildVisitorFromFileOrStdin(specFile, cmd) - visitor.Visit(func(docs []spec) { - for _, d := range docs { - handleRequest(http.MethodPost, makeURL(objectsURL), []byte(d.doc), cmd) - } + visitor.Visit(func(s *spec) { + handleRequest(http.MethodPost, makeURL(objectsURL), []byte(s.doc), cmd) }) }, } @@ -82,10 +80,8 @@ func updateObjectCmd() *cobra.Command { Short: "Update an object from a yaml file or stdin", Run: func(cmd *cobra.Command, args []string) { visitor := buildVisitorFromFileOrStdin(specFile, cmd) - visitor.Visit(func(docs []spec) { - for _, d := range docs { - handleRequest(http.MethodPut, makeURL(objectURL, d.Name), []byte(d.doc), cmd) - } + visitor.Visit(func(s *spec) { + handleRequest(http.MethodPost, makeURL(objectsURL), []byte(s.doc), cmd) }) }, } diff --git a/cmd/client/command/visitor.go b/cmd/client/command/visitor.go index 7ab56258c7..983324ed17 100644 --- a/cmd/client/command/visitor.go +++ b/cmd/client/command/visitor.go @@ -9,7 +9,7 @@ import ( ) // VisitorFunc executes visition logic -type VisitorFunc func([]spec) +type VisitorFunc func(*spec) // Visitor walk through the document via VisitorFunc type Visitor interface { @@ -60,7 +60,7 @@ func (d *yamlDecoder) Decode(into interface{}) error { // Visit implements Visitor over a stream. func (v *streamVisitor) Visit(fn VisitorFunc) { d := newYAMLDecoder(v.Reader) - var validDocs []spec + var validSpecs []spec for { var s spec if err := d.Decode(&s); err != nil { @@ -72,7 +72,9 @@ func (v *streamVisitor) Visit(fn VisitorFunc) { } s.doc = d.doc //TODO can validate spec's Kind here - validDocs = append(validDocs, s) + validSpecs = append(validSpecs, s) + } + for _, s := range validSpecs { + fn(&s) } - fn(validDocs) } diff --git a/cmd/client/command/wasm.go b/cmd/client/command/wasm.go index b53b37cfaa..4e907798df 100644 --- a/cmd/client/command/wasm.go +++ b/cmd/client/command/wasm.go @@ -86,10 +86,8 @@ func wasmApplyDataCmd() *cobra.Command { Run: func(cmd *cobra.Command, args []string) { visitor := buildVisitorFromFileOrStdin(specFile, cmd) - visitor.Visit(func(docs []spec) { - for _, d := range docs { - handleRequest(http.MethodPut, makeURL(wasmDataURL, args[0], args[1]), []byte(d.doc), cmd) - } + visitor.Visit(func(s *spec) { + handleRequest(http.MethodPost, makeURL(objectsURL), []byte(s.doc), cmd) }) }, }