From 6e0e68275b69f337d4f73727c05b527c9895a2a4 Mon Sep 17 00:00:00 2001 From: chen Date: Mon, 30 Oct 2023 18:23:06 +0800 Subject: [PATCH 01/22] finish include directive --- cmd/client/commandv2/convert.go | 34 ++++ cmd/client/commandv2/convert/nginx/cmd.go | 53 ++++++ cmd/client/commandv2/convert/nginx/convert.go | 62 +++++++ .../commandv2/convert/nginx/convert_test.go | 111 +++++++++++++ cmd/client/commandv2/convert/nginx/env.go | 152 ++++++++++++++++++ .../commandv2/convert/nginx/test_test.go | 62 +++++++ cmd/client/general/message.go | 10 ++ cmd/client/main.go | 12 +- go.mod | 3 + go.sum | 6 + 10 files changed, 504 insertions(+), 1 deletion(-) create mode 100644 cmd/client/commandv2/convert.go create mode 100644 cmd/client/commandv2/convert/nginx/cmd.go create mode 100644 cmd/client/commandv2/convert/nginx/convert.go create mode 100644 cmd/client/commandv2/convert/nginx/convert_test.go create mode 100644 cmd/client/commandv2/convert/nginx/env.go create mode 100644 cmd/client/commandv2/convert/nginx/test_test.go diff --git a/cmd/client/commandv2/convert.go b/cmd/client/commandv2/convert.go new file mode 100644 index 0000000000..c3d55bfa19 --- /dev/null +++ b/cmd/client/commandv2/convert.go @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2017, MegaEase + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package commandv2 + +import ( + "github.com/spf13/cobra" + + "github.com/megaease/easegress/v2/cmd/client/commandv2/convert/nginx" +) + +// ConvertCmd returns convert command. +func ConvertCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "convert", + Short: "Convert other format to easegress yaml file", + } + cmd.AddCommand(nginx.Cmd()) + return cmd +} diff --git a/cmd/client/commandv2/convert/nginx/cmd.go b/cmd/client/commandv2/convert/nginx/cmd.go new file mode 100644 index 0000000000..e14a6c1cb5 --- /dev/null +++ b/cmd/client/commandv2/convert/nginx/cmd.go @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2017, MegaEase + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nginx + +import ( + "encoding/json" + "fmt" + + "github.com/megaease/easegress/v2/cmd/client/general" + crossplane "github.com/nginxinc/nginx-go-crossplane" + "github.com/spf13/cobra" +) + +// Cmd returns convert nginx.conf command. +func Cmd() *cobra.Command { + var nginxConf string + cmd := &cobra.Command{ + Use: "nginx", + Short: "Convert nginx.conf to easegress yaml file", + Args: func(cmd *cobra.Command, args []string) error { + return nil + }, + Run: func(cmd *cobra.Command, args []string) { + payload, err := crossplane.Parse(nginxConf, &crossplane.ParseOptions{}) + if err != nil { + general.ExitWithErrorf("parse nginx.conf failed: %v", err) + } + data, err := json.Marshal(payload) + if err != nil { + general.ExitWithError(err) + } + fmt.Println(string(data)) + general.Warnf("warn") + }, + } + cmd.Flags().StringVarP(&nginxConf, "file", "f", "", "nginx.conf file path") + return cmd +} diff --git a/cmd/client/commandv2/convert/nginx/convert.go b/cmd/client/commandv2/convert/nginx/convert.go new file mode 100644 index 0000000000..14fe5aa433 --- /dev/null +++ b/cmd/client/commandv2/convert/nginx/convert.go @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2017, MegaEase + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nginx + +import ( + "github.com/megaease/easegress/v2/cmd/client/general" + crossplane "github.com/nginxinc/nginx-go-crossplane" +) + +const ( + DirectiveInclude = "include" + DirectiveHTTP = "http" + DirectiveServer = "server" + DirectiveLocation = "location" +) + +// loadIncludes loads include files recursively. +// The result contains all directives in the include files and the original directives. +func loadIncludes(fileName string, directives crossplane.Directives, payload *crossplane.Payload) crossplane.Directives { + res := crossplane.Directives{} + for _, d := range directives { + if d.Directive != DirectiveInclude { + d.File = fileName + res = append(res, d) + continue + } + + if len(d.Args) != 1 { + continue + } + name := d.Args[0] + var include crossplane.Directives + for _, config := range payload.Config { + if config.File == name { + include = config.Parsed + break + } + } + if include == nil { + general.Warnf("can't find include file %s in line %d of file %s", name, d.Line, fileName) + continue + } + include = loadIncludes(name, include, payload) + res = append(res, include...) + } + return res +} diff --git a/cmd/client/commandv2/convert/nginx/convert_test.go b/cmd/client/commandv2/convert/nginx/convert_test.go new file mode 100644 index 0000000000..e5df54effe --- /dev/null +++ b/cmd/client/commandv2/convert/nginx/convert_test.go @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2017, MegaEase + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nginx + +import ( + "fmt" + "testing" + + crossplane "github.com/nginxinc/nginx-go-crossplane" + "github.com/stretchr/testify/assert" +) + +func TestLoadIncludes(t *testing.T) { + tempDir := newTempTestDir(t) + defer tempDir.Clean() + + { + conf := ` + events {} + http { + include %s; + server { + listen 80; + location / { + proxy_pass http://localhost:8888; + } + } + } + ` + conf1 := `include %s; proxy_set_header Conf-One One;` + conf2 := `proxy_set_header Conf-Two Two;` + + file2 := tempDir.Create(t, []byte(conf2)) + file1 := tempDir.Create(t, []byte(fmt.Sprintf(conf1, file2))) + file := tempDir.Create(t, []byte(fmt.Sprintf(conf, file1))) + + payload, err := crossplane.Parse(file, &crossplane.ParseOptions{}) + assert.Nil(t, err) + httpDirectives := payload.Config[0].Parsed[1].Block + httpDirectives = loadIncludes(file, httpDirectives, payload) + assert.Equal(t, 3, len(httpDirectives)) + + // first directive from conf2 + d2 := httpDirectives[0] + assert.Equal(t, "proxy_set_header", d2.Directive) + assert.Equal(t, []string{"Conf-Two", "Two"}, d2.Args) + assert.Equal(t, file2, d2.File) + // second directive from conf1 + d1 := httpDirectives[1] + assert.Equal(t, "proxy_set_header", d1.Directive) + assert.Equal(t, []string{"Conf-One", "One"}, d1.Args) + assert.Equal(t, file1, d1.File) + } + + { + // test invalid includes + conf := ` + events {} + http { + include not-exist.conf; + include %s invalid-args.conf; + include; + include %s; + server { + listen 80; + location / { + proxy_pass http://localhost:8888; + } + } + } + ` + conf1 := `include %s; proxy_set_header Conf-One One;` + conf2 := `proxy_set_header Conf-Two Two;` + + file2 := tempDir.Create(t, []byte(conf2)) + file1 := tempDir.Create(t, []byte(fmt.Sprintf(conf1, file2))) + file := tempDir.Create(t, []byte(fmt.Sprintf(conf, file1, file1))) + + payload, err := crossplane.Parse(file, &crossplane.ParseOptions{}) + assert.Nil(t, err) + httpDirectives := payload.Config[0].Parsed[1].Block + httpDirectives = loadIncludes(file, httpDirectives, payload) + assert.Equal(t, 3, len(httpDirectives)) + + // first directive from conf2 + d2 := httpDirectives[0] + assert.Equal(t, "proxy_set_header", d2.Directive) + assert.Equal(t, []string{"Conf-Two", "Two"}, d2.Args) + assert.Equal(t, file2, d2.File) + // second directive from conf1 + d1 := httpDirectives[1] + assert.Equal(t, "proxy_set_header", d1.Directive) + assert.Equal(t, []string{"Conf-One", "One"}, d1.Args) + assert.Equal(t, file1, d1.File) + } +} diff --git a/cmd/client/commandv2/convert/nginx/env.go b/cmd/client/commandv2/convert/nginx/env.go new file mode 100644 index 0000000000..5886c47e54 --- /dev/null +++ b/cmd/client/commandv2/convert/nginx/env.go @@ -0,0 +1,152 @@ +/* + * Copyright (c) 2017, MegaEase + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nginx + +import ( + "encoding/json" + + "github.com/megaease/easegress/v2/pkg/filters/proxies" + "github.com/megaease/easegress/v2/pkg/object/httpserver/routers" + crossplane "github.com/nginxinc/nginx-go-crossplane" +) + +// Env is the environment for converting. +// Server is the server environment. Used to create HTTPServer or in future GRPCServer. +// Proxy is the proxy environment. Used to create Pipeline. +type Env struct { + Server *ServerEnv + Proxy *ProxyEnv + Upstream []*Directive +} + +type Directive = crossplane.Directive + +// ServerEnv is the environment for creating server. +// Listen contains the listen address, port and protocol. +// ServerName contains the server name (hosts in easegress). +type ServerEnv struct { + Listen *Directive + ServerName *Directive + SSLClientCertificate *Directive + SSLCertificate []*Directive + SSLCertificateKey []*Directive +} + +type ProxyEnv struct { + Kind string + Pass *Directive + ProxySetHeader []*Directive +} + +type HTTPServerInfo struct { + Port uint16 + Address string + Hosts []routers.Host + + CaCert string + Certs map[string]string + Keys map[string]string +} + +type PipelineInfo struct { + Servers []proxies.Server + LoadBalance *proxies.LoadBalanceSpec + SetHeaders map[string]string +} + +func (env *Env) Clone() (*Env, error) { + data, err := json.Marshal(env) + if err != nil { + return nil, err + } + var newEnv Env + err = json.Unmarshal(data, &newEnv) + if err != nil { + return nil, err + } + return &newEnv, nil +} + +func (env *Env) MustClone() *Env { + newEnv, err := env.Clone() + if err != nil { + panic(err) + } + return newEnv +} + +func NewEnv() *Env { + return &Env{ + Server: &ServerEnv{ + SSLCertificate: make([]*Directive, 0), + SSLCertificateKey: make([]*Directive, 0), + }, + Proxy: &ProxyEnv{ + ProxySetHeader: make([]*Directive, 0), + }, + Upstream: make([]*Directive, 0), + } +} + +var nginxBackends = map[string]struct{}{ + "root": {}, + "return": {}, + "rewrite": {}, + "try_files": {}, + "error_page": {}, + "proxy_pass": {}, + "fastcgi_pass": {}, + "uwsgi_pass": {}, + "scgi_pass": {}, + "memcached_pass": {}, + "grpc_pass": {}, +} + +func (env *Env) Update(d *Directive) { + directiveMap := map[string]**Directive{ + "listen": &env.Server.Listen, + "server_name": &env.Server.ServerName, + "ssl_client_certificate": &env.Server.SSLClientCertificate, + } + if val, ok := directiveMap[d.Directive]; ok { + *val = d + return + } + + arrMap := map[string][]*Directive{ + "ssl_certificate": env.Server.SSLCertificate, + "ssl_certificate_key": env.Server.SSLCertificateKey, + "proxy_set_header": env.Proxy.ProxySetHeader, + "upstream": env.Upstream, + } + arr, ok := arrMap[d.Directive] + if ok { + arrMap[d.Directive] = append(arr, d) + return + } + + _, ok = nginxBackends[d.Directive] + if ok { + env.Proxy.Kind = d.Directive + env.Proxy.Pass = d + } +} + +func (env *Env) Validate() error { + return nil +} diff --git a/cmd/client/commandv2/convert/nginx/test_test.go b/cmd/client/commandv2/convert/nginx/test_test.go new file mode 100644 index 0000000000..028cf1df0f --- /dev/null +++ b/cmd/client/commandv2/convert/nginx/test_test.go @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2017, MegaEase + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nginx + +import ( + "encoding/json" + "os" + "testing" + + "github.com/stretchr/testify/require" +) + +type tempTestDir struct { + dir string + files []string +} + +func newTempTestDir(t *testing.T) *tempTestDir { + dir, err := os.MkdirTemp("", "test") + require.Nil(t, err) + return &tempTestDir{dir: dir} +} + +func (dir *tempTestDir) Create(t *testing.T, content []byte) string { + file, err := os.CreateTemp(dir.dir, "test") + require.Nil(t, err) + defer file.Close() + + _, err = file.Write(content) + require.Nil(t, err) + dir.files = append(dir.files, file.Name()) + return file.Name() +} + +func (dir *tempTestDir) Clean() { + for _, file := range dir.files { + os.Remove(file) + } + os.Remove(dir.dir) +} + +func printJson(t *testing.T, v interface{}) { + t.Helper() + b, err := json.MarshalIndent(v, "", " ") + require.Nil(t, err) + t.Log(string(b)) +} diff --git a/cmd/client/general/message.go b/cmd/client/general/message.go index ebcb540082..c87e5df9d2 100644 --- a/cmd/client/general/message.go +++ b/cmd/client/general/message.go @@ -32,3 +32,13 @@ func SuccessMsg(action CmdType, values ...string) string { func ErrorMsg(action CmdType, err error, values ...string) error { return fmt.Errorf("%s %s failed, %v", action, strings.Join(values, " "), err) } + +// Warnf prints the warning message. +func Warnf(format string, args ...interface{}) { + fmt.Printf("WARNING: "+format+"\n", args...) +} + +// Infof prints the info message. +func Infof(format string, args ...interface{}) { + fmt.Printf("INFO: "+format+"\n", args...) +} diff --git a/cmd/client/main.go b/cmd/client/main.go index 886ac51cf4..1115fef885 100644 --- a/cmd/client/main.go +++ b/cmd/client/main.go @@ -40,6 +40,11 @@ var basicGroup = &cobra.Group{ Title: `Basic Commands`, } +var advancedGroup = &cobra.Group{ + ID: "advanced", + Title: `Advanced Commands`, +} + var otherGroup = &cobra.Group{ ID: "other", Title: `Other Commands`, @@ -77,6 +82,11 @@ func main() { commandv2.EditCmd(), ) + addCommandWithGroup( + advancedGroup, + commandv2.ConvertCmd(), + ) + addCommandWithGroup( otherGroup, commandv2.APIsCmd(), @@ -98,7 +108,7 @@ func main() { command.CustomDataCmd(), ) - rootCmd.AddGroup(basicGroup, otherGroup, deprecatedGroup) + rootCmd.AddGroup(basicGroup, advancedGroup, otherGroup, deprecatedGroup) for _, c := range rootCmd.Commands() { general.GenerateExampleFromChild(c) diff --git a/go.mod b/go.mod index 65c18e27be..7f433f34af 100644 --- a/go.mod +++ b/go.mod @@ -103,7 +103,10 @@ require ( github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.2 // indirect github.com/hashicorp/go-version v1.6.0 // indirect github.com/huandu/xstrings v1.3.2 // indirect + github.com/jstemmer/go-junit-report v1.0.0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect + github.com/maxbrunsfeld/counterfeiter/v6 v6.6.1 // indirect + github.com/nginxinc/nginx-go-crossplane v0.4.33 // indirect github.com/onsi/ginkgo/v2 v2.9.5 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b // indirect diff --git a/go.sum b/go.sum index cd2fed6773..b55b7e04cb 100644 --- a/go.sum +++ b/go.sum @@ -514,6 +514,8 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jstemmer/go-junit-report v1.0.0 h1:8X1gzZpR+nVQLAht+L/foqOeX2l9DTZoaIPbEQHxsds= +github.com/jstemmer/go-junit-report v1.0.0/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtblin/go-ldap-client v0.0.0-20170223121919-b73f66626b33 h1:XDpFOMOZq0u0Ar4F0p/wklqQXp/AMV1pTF5T5bDoUfQ= github.com/jtblin/go-ldap-client v0.0.0-20170223121919-b73f66626b33/go.mod h1:+0BcLY5d54TVv6irFzHoiFvwAHR6T0g9B+by/UaS9T0= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= @@ -589,6 +591,8 @@ github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/ github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/maxbrunsfeld/counterfeiter/v6 v6.6.1 h1:9XE5ykDiC8eNSqIPkxx0EsV3kMX1oe4kQWRZjIgytUA= +github.com/maxbrunsfeld/counterfeiter/v6 v6.6.1/go.mod h1:qbKwBR+qQODzH2WD/s53mdgp/xVcXMlJb59GRFOp6Z4= github.com/megaease/easemesh-api v1.4.4 h1:E18mtLfj8ffuPTeN7MqZeakJgT/tJ92JNIZsY2k2GE0= github.com/megaease/easemesh-api v1.4.4/go.mod h1:GuAE5DwqK6lI/ovoRKjyPxBCSoMhj0NLp9PRejj0Hnw= github.com/megaease/grace v1.0.0 h1:b44R3j6e/iaN62F4ZUnru9nzL1VaIcxxUZjSPVtTVzI= @@ -623,6 +627,8 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nacos-group/nacos-sdk-go v1.1.0 h1:6ESrAegx2pqp3Vi8mqDi7s2Vq+I+u0oYLn646K4wx6o= github.com/nacos-group/nacos-sdk-go v1.1.0/go.mod h1:Y/9Dj0Bl04hWUO1DaL4+r+fLzv5nl9kn58Vt1OGvWdw= +github.com/nginxinc/nginx-go-crossplane v0.4.33 h1:Sr6ptE+xzQ7Vc5xs1SXNDDBWvVwKA7BTtQJZIGTS7RY= +github.com/nginxinc/nginx-go-crossplane v0.4.33/go.mod h1:UzbZnyFv0vPlt1Urbnp/mrFCzBL4tYCReFuNBpFQEfI= github.com/nrdcg/dnspod-go v0.4.0 h1:c/jn1mLZNKF3/osJ6mz3QPxTudvPArXTjpkmYj0uK6U= github.com/nrdcg/dnspod-go v0.4.0/go.mod h1:vZSoFSFeQVm2gWLMkyX61LZ8HI3BaqtHZWgPTGKr6KQ= github.com/onsi/ginkgo/v2 v2.9.5 h1:+6Hr4uxzP4XIUyAkg61dWBw8lb/gc4/X5luuxN/EC+Q= From 52cddb082468fd94b15c3ce974873b6429bbee5c Mon Sep 17 00:00:00 2001 From: chen Date: Tue, 31 Oct 2023 18:20:52 +0800 Subject: [PATCH 02/22] httpserver spec support wildcards --- pkg/object/httpserver/routers/spec.go | 21 ++++++++++++++++++++ pkg/object/httpserver/routers/spec_test.go | 23 ++++++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/pkg/object/httpserver/routers/spec.go b/pkg/object/httpserver/routers/spec.go index 1647df72bc..cf12d58993 100644 --- a/pkg/object/httpserver/routers/spec.go +++ b/pkg/object/httpserver/routers/spec.go @@ -22,6 +22,7 @@ import ( "net/http" "net/url" "regexp" + "strings" "github.com/megaease/easegress/v2/pkg/logger" "github.com/megaease/easegress/v2/pkg/util/ipfilter" @@ -38,6 +39,8 @@ type Paths []*Path type Host struct { IsRegexp bool `json:"isRegexp" jsonschema:"omitempty"` Value string `json:"value" jsonschema:"required"` + prefix string `json:"-"` + suffix string `json:"-"` re *regexp.Regexp } @@ -124,6 +127,20 @@ func (rule *Rule) Init() { for i := range rule.Hosts { h := &rule.Hosts[i] if !h.IsRegexp { + if h.Value != "" { + count := strings.Count(h.Value, "*") + if count > 1 { + logger.Errorf("invalid host %s, only one wildcard is allowed", h.Value) + continue + } + if h.Value[0] == '*' { + h.suffix = h.Value[1:] + } else if h.Value[len(h.Value)-1] == '*' { + h.prefix = h.Value[:len(h.Value)-1] + } else { + logger.Errorf("invalid host %s, only wildcard prefix or suffix is allowed", h.Value) + } + } continue } if re, err := regexp.Compile(h.Value); err != nil { @@ -154,6 +171,10 @@ func (rule *Rule) MatchHost(ctx *RouteContext) bool { } } else if host == h.Value { return true + } else if h.prefix != "" && strings.HasPrefix(host, h.prefix) { + return true + } else if h.suffix != "" && strings.HasSuffix(host, h.suffix) { + return true } } diff --git a/pkg/object/httpserver/routers/spec_test.go b/pkg/object/httpserver/routers/spec_test.go index f27dedaf20..2ceff529ef 100644 --- a/pkg/object/httpserver/routers/spec_test.go +++ b/pkg/object/httpserver/routers/spec_test.go @@ -126,6 +126,29 @@ func TestRuleMatch(t *testing.T) { rule.Init() assert.NotNil(rule) assert.False(rule.MatchHost(ctx)) + + testCases := []struct { + request string + value string + result bool + }{ + {request: "http://www.megaease.com:8080", value: "www.megaease.com", result: true}, + {request: "http://www.megaease.com:8080", value: "*.megaease.com", result: true}, + {request: "http://www.sub.megaease.com:8080", value: "*.megaease.com", result: true}, + {request: "http://www.example.megaease.com:8080", value: "*.megaease.com", result: true}, + {request: "http://www.megaease.com:8080", value: "www.megaease.*", result: true}, + {request: "http://www.megaease.cn:8080", value: "www.megaease.*", result: true}, + {request: "http://www.google.com:8080", value: "*.megaease.com", result: false}, + } + for _, tc := range testCases { + stdr, _ := http.NewRequest(http.MethodGet, tc.request, nil) + req, _ := httpprot.NewRequest(stdr) + ctx := NewContext(req) + + rule = &Rule{Hosts: []Host{{Value: tc.value}}} + rule.Init() + assert.Equal(tc.result, rule.MatchHost(ctx)) + } } func TestRuleAllowIP(t *testing.T) { From c8fa3e27806c9faf74c4e7eb59c6536a4e7e593c Mon Sep 17 00:00:00 2001 From: chen Date: Tue, 31 Oct 2023 18:38:53 +0800 Subject: [PATCH 03/22] get server info from nginx.conf --- cmd/client/commandv2/convert/nginx/cmd.go | 2 +- cmd/client/commandv2/convert/nginx/convert.go | 44 --- .../commandv2/convert/nginx/convert_test.go | 93 ------ cmd/client/commandv2/convert/nginx/env.go | 286 +++++++++++++----- .../commandv2/convert/nginx/env_test.go | 57 ++++ cmd/client/commandv2/convert/nginx/parse.go | 137 +++++++++ .../commandv2/convert/nginx/parse_test.go | 140 +++++++++ .../commandv2/convert/nginx/test_test.go | 12 +- cmd/client/commandv2/convert/nginx/utils.go | 24 ++ 9 files changed, 576 insertions(+), 219 deletions(-) create mode 100644 cmd/client/commandv2/convert/nginx/env_test.go create mode 100644 cmd/client/commandv2/convert/nginx/parse.go create mode 100644 cmd/client/commandv2/convert/nginx/parse_test.go create mode 100644 cmd/client/commandv2/convert/nginx/utils.go diff --git a/cmd/client/commandv2/convert/nginx/cmd.go b/cmd/client/commandv2/convert/nginx/cmd.go index e14a6c1cb5..4f57bd4675 100644 --- a/cmd/client/commandv2/convert/nginx/cmd.go +++ b/cmd/client/commandv2/convert/nginx/cmd.go @@ -40,12 +40,12 @@ func Cmd() *cobra.Command { if err != nil { general.ExitWithErrorf("parse nginx.conf failed: %v", err) } + parsePayload(payload) data, err := json.Marshal(payload) if err != nil { general.ExitWithError(err) } fmt.Println(string(data)) - general.Warnf("warn") }, } cmd.Flags().StringVarP(&nginxConf, "file", "f", "", "nginx.conf file path") diff --git a/cmd/client/commandv2/convert/nginx/convert.go b/cmd/client/commandv2/convert/nginx/convert.go index 14fe5aa433..0d041760e3 100644 --- a/cmd/client/commandv2/convert/nginx/convert.go +++ b/cmd/client/commandv2/convert/nginx/convert.go @@ -16,47 +16,3 @@ */ package nginx - -import ( - "github.com/megaease/easegress/v2/cmd/client/general" - crossplane "github.com/nginxinc/nginx-go-crossplane" -) - -const ( - DirectiveInclude = "include" - DirectiveHTTP = "http" - DirectiveServer = "server" - DirectiveLocation = "location" -) - -// loadIncludes loads include files recursively. -// The result contains all directives in the include files and the original directives. -func loadIncludes(fileName string, directives crossplane.Directives, payload *crossplane.Payload) crossplane.Directives { - res := crossplane.Directives{} - for _, d := range directives { - if d.Directive != DirectiveInclude { - d.File = fileName - res = append(res, d) - continue - } - - if len(d.Args) != 1 { - continue - } - name := d.Args[0] - var include crossplane.Directives - for _, config := range payload.Config { - if config.File == name { - include = config.Parsed - break - } - } - if include == nil { - general.Warnf("can't find include file %s in line %d of file %s", name, d.Line, fileName) - continue - } - include = loadIncludes(name, include, payload) - res = append(res, include...) - } - return res -} diff --git a/cmd/client/commandv2/convert/nginx/convert_test.go b/cmd/client/commandv2/convert/nginx/convert_test.go index e5df54effe..0d041760e3 100644 --- a/cmd/client/commandv2/convert/nginx/convert_test.go +++ b/cmd/client/commandv2/convert/nginx/convert_test.go @@ -16,96 +16,3 @@ */ package nginx - -import ( - "fmt" - "testing" - - crossplane "github.com/nginxinc/nginx-go-crossplane" - "github.com/stretchr/testify/assert" -) - -func TestLoadIncludes(t *testing.T) { - tempDir := newTempTestDir(t) - defer tempDir.Clean() - - { - conf := ` - events {} - http { - include %s; - server { - listen 80; - location / { - proxy_pass http://localhost:8888; - } - } - } - ` - conf1 := `include %s; proxy_set_header Conf-One One;` - conf2 := `proxy_set_header Conf-Two Two;` - - file2 := tempDir.Create(t, []byte(conf2)) - file1 := tempDir.Create(t, []byte(fmt.Sprintf(conf1, file2))) - file := tempDir.Create(t, []byte(fmt.Sprintf(conf, file1))) - - payload, err := crossplane.Parse(file, &crossplane.ParseOptions{}) - assert.Nil(t, err) - httpDirectives := payload.Config[0].Parsed[1].Block - httpDirectives = loadIncludes(file, httpDirectives, payload) - assert.Equal(t, 3, len(httpDirectives)) - - // first directive from conf2 - d2 := httpDirectives[0] - assert.Equal(t, "proxy_set_header", d2.Directive) - assert.Equal(t, []string{"Conf-Two", "Two"}, d2.Args) - assert.Equal(t, file2, d2.File) - // second directive from conf1 - d1 := httpDirectives[1] - assert.Equal(t, "proxy_set_header", d1.Directive) - assert.Equal(t, []string{"Conf-One", "One"}, d1.Args) - assert.Equal(t, file1, d1.File) - } - - { - // test invalid includes - conf := ` - events {} - http { - include not-exist.conf; - include %s invalid-args.conf; - include; - include %s; - server { - listen 80; - location / { - proxy_pass http://localhost:8888; - } - } - } - ` - conf1 := `include %s; proxy_set_header Conf-One One;` - conf2 := `proxy_set_header Conf-Two Two;` - - file2 := tempDir.Create(t, []byte(conf2)) - file1 := tempDir.Create(t, []byte(fmt.Sprintf(conf1, file2))) - file := tempDir.Create(t, []byte(fmt.Sprintf(conf, file1, file1))) - - payload, err := crossplane.Parse(file, &crossplane.ParseOptions{}) - assert.Nil(t, err) - httpDirectives := payload.Config[0].Parsed[1].Block - httpDirectives = loadIncludes(file, httpDirectives, payload) - assert.Equal(t, 3, len(httpDirectives)) - - // first directive from conf2 - d2 := httpDirectives[0] - assert.Equal(t, "proxy_set_header", d2.Directive) - assert.Equal(t, []string{"Conf-Two", "Two"}, d2.Args) - assert.Equal(t, file2, d2.File) - // second directive from conf1 - d1 := httpDirectives[1] - assert.Equal(t, "proxy_set_header", d1.Directive) - assert.Equal(t, []string{"Conf-One", "One"}, d1.Args) - assert.Equal(t, file1, d1.File) - } -} diff --git a/cmd/client/commandv2/convert/nginx/env.go b/cmd/client/commandv2/convert/nginx/env.go index 5886c47e54..4fddead703 100644 --- a/cmd/client/commandv2/convert/nginx/env.go +++ b/cmd/client/commandv2/convert/nginx/env.go @@ -19,54 +19,81 @@ package nginx import ( "encoding/json" - - "github.com/megaease/easegress/v2/pkg/filters/proxies" - "github.com/megaease/easegress/v2/pkg/object/httpserver/routers" - crossplane "github.com/nginxinc/nginx-go-crossplane" + "fmt" + "regexp" + "strconv" + "strings" ) +var nginxBackends = map[string]struct{}{ + "root": {}, + "return": {}, + "rewrite": {}, + "try_files": {}, + "error_page": {}, + "proxy_pass": {}, + "fastcgi_pass": {}, + "uwsgi_pass": {}, + "scgi_pass": {}, + "memcached_pass": {}, + "grpc_pass": {}, +} + // Env is the environment for converting. // Server is the server environment. Used to create HTTPServer or in future GRPCServer. // Proxy is the proxy environment. Used to create Pipeline. type Env struct { - Server *ServerEnv - Proxy *ProxyEnv - Upstream []*Directive -} + Server *ServerEnv `json:"server"` + Proxy *ProxyEnv `json:"proxy"` + Upstream []*Directive `json:"upstream"` -type Directive = crossplane.Directive + updateFn map[string]func(*Directive) `json:"-"` +} // ServerEnv is the environment for creating server. // Listen contains the listen address, port and protocol. // ServerName contains the server name (hosts in easegress). type ServerEnv struct { - Listen *Directive - ServerName *Directive - SSLClientCertificate *Directive - SSLCertificate []*Directive - SSLCertificateKey []*Directive + Listen *Directive `json:"listen"` + ServerName *Directive `json:"server_name"` + SSLClientCertificate *Directive `json:"ssl_client_certificate"` + SSLCertificate []*Directive `json:"ssl_certificate"` + SSLCertificateKey []*Directive `json:"ssl_certificate_key"` } type ProxyEnv struct { - Kind string - Pass *Directive - ProxySetHeader []*Directive + Kind string `json:"kind"` + Pass *Directive `json:"pass"` + ProxySetHeader []*Directive `json:"proxy_set_header"` } -type HTTPServerInfo struct { - Port uint16 - Address string - Hosts []routers.Host - - CaCert string - Certs map[string]string - Keys map[string]string +func getEnvUpdateFn(env *Env) map[string]func(*Directive) { + m := map[string]func(*Directive){ + "listen": func(d *Directive) { env.Server.Listen = d }, + "server_name": func(d *Directive) { env.Server.ServerName = d }, + "ssl_client_certificate": func(d *Directive) { env.Server.SSLClientCertificate = d }, + "ssl_certificate": func(d *Directive) { env.Server.SSLCertificate = append(env.Server.SSLCertificate, d) }, + "ssl_certificate_key": func(d *Directive) { env.Server.SSLCertificateKey = append(env.Server.SSLCertificateKey, d) }, + "proxy_set_header": func(d *Directive) { env.Proxy.ProxySetHeader = append(env.Proxy.ProxySetHeader, d) }, + "upstream": func(d *Directive) { env.Upstream = append(env.Upstream, d) }, + } + return m } -type PipelineInfo struct { - Servers []proxies.Server - LoadBalance *proxies.LoadBalanceSpec - SetHeaders map[string]string +func newEnv() *Env { + env := &Env{ + Server: &ServerEnv{ + SSLCertificate: make([]*Directive, 0), + SSLCertificateKey: make([]*Directive, 0), + }, + Proxy: &ProxyEnv{ + ProxySetHeader: make([]*Directive, 0), + }, + Upstream: make([]*Directive, 0), + } + + env.updateFn = getEnvUpdateFn(env) + return env } func (env *Env) Clone() (*Env, error) { @@ -79,6 +106,7 @@ func (env *Env) Clone() (*Env, error) { if err != nil { return nil, err } + newEnv.updateFn = getEnvUpdateFn(env) return &newEnv, nil } @@ -90,63 +118,169 @@ func (env *Env) MustClone() *Env { return newEnv } -func NewEnv() *Env { - return &Env{ - Server: &ServerEnv{ - SSLCertificate: make([]*Directive, 0), - SSLCertificateKey: make([]*Directive, 0), - }, - Proxy: &ProxyEnv{ - ProxySetHeader: make([]*Directive, 0), - }, - Upstream: make([]*Directive, 0), +func (env *Env) Update(d *Directive) { + fn, ok := env.updateFn[d.Directive] + if ok { + fn(d) + return + } + _, ok = nginxBackends[d.Directive] + if ok { + env.Proxy.Kind = d.Directive + env.Proxy.Pass = d } } -var nginxBackends = map[string]struct{}{ - "root": {}, - "return": {}, - "rewrite": {}, - "try_files": {}, - "error_page": {}, - "proxy_pass": {}, - "fastcgi_pass": {}, - "uwsgi_pass": {}, - "scgi_pass": {}, - "memcached_pass": {}, - "grpc_pass": {}, -} - -func (env *Env) Update(d *Directive) { - directiveMap := map[string]**Directive{ - "listen": &env.Server.Listen, - "server_name": &env.Server.ServerName, - "ssl_client_certificate": &env.Server.SSLClientCertificate, +func (env *Env) GetServerInfo() (*ServerInfo, error) { + info := &ServerInfo{ + Port: 80, + Certs: make(map[string]string), + Keys: make(map[string]string), } - if val, ok := directiveMap[d.Directive]; ok { - *val = d - return + + if env.Server.Listen != nil { + address, port, https, err := processListen(env.Server.Listen) + if err != nil { + return nil, err + } + info.Address = address + info.Port = port + info.HTTPS = https } - arrMap := map[string][]*Directive{ - "ssl_certificate": env.Server.SSLCertificate, - "ssl_certificate_key": env.Server.SSLCertificateKey, - "proxy_set_header": env.Proxy.ProxySetHeader, - "upstream": env.Upstream, + if env.Server.ServerName != nil { + serverName := env.Server.ServerName + hosts, err := processServerName(serverName) + if err != nil { + return nil, err + } + info.Hosts = hosts } - arr, ok := arrMap[d.Directive] - if ok { - arrMap[d.Directive] = append(arr, d) - return + + return info, nil +} + +func processServerName(d *Directive) ([]HostInfo, error) { + hosts := make([]HostInfo, 0) + for _, arg := range d.Args { + if strings.HasPrefix(arg, "~") { + _, err := regexp.Compile(arg[1:]) + if err != nil { + return nil, fmt.Errorf("%s: %v", directiveInfo(d), err) + } + hosts = append(hosts, HostInfo{ + Value: arg[1:], + IsRegexp: true, + }) + } else { + count := strings.Count(arg, "*") + if count > 1 { + return nil, fmt.Errorf("%s: host %s contains more than one wildcard", directiveInfo(d), arg) + } + if count == 1 { + if arg[0] != '*' && arg[len(arg)-1] != '*' { + return nil, fmt.Errorf("%s: host %s contains wildcard in the middle", directiveInfo(d), arg) + } + } + hosts = append(hosts, HostInfo{ + Value: arg, + IsRegexp: false, + }) + } } + return hosts, nil +} - _, ok = nginxBackends[d.Directive] - if ok { - env.Proxy.Kind = d.Directive - env.Proxy.Pass = d +func processListen(d *Directive) (address string, port int, https bool, err error) { + mustContainArgs(d, 1) + address, port, err = splitAddressPort(d.Args[0]) + if err != nil { + return "", 0, false, fmt.Errorf("%s: %v", directiveInfo(d), err) } + for _, arg := range d.Args[1:] { + if arg == "ssl" { + return address, port, true, nil + } + } + return address, port, false, nil +} + +type ServerInfo struct { + Port int + Address string + HTTPS bool + Hosts []HostInfo + + CaCert string + Certs map[string]string + Keys map[string]string +} + +type HostInfo struct { + Value string + IsRegexp bool +} + +type ProxyInfo struct { + Servers []*BackendInfo + SetHeaders map[string]string } -func (env *Env) Validate() error { - return nil +type BackendInfo struct { + Server string + Weight int +} + +// splitAddressPort splits the listen directive into address and port. +// nginx examples: +// listen 127.0.0.1:8000; +// listen 127.0.0.1; +// listen 8000; +// listen *:8000; +// listen localhost:8000; +// listen [::]:8000; +// listen [::1]; +func splitAddressPort(listen string) (string, int, error) { + if listen == "" { + return "", 0, fmt.Errorf("listen is empty") + } + if listen[0] == '[' { + end := strings.Index(listen, "]") + if end < 0 { + return "", 0, fmt.Errorf("invalid listen: %s", listen) + } + host := listen[0 : end+1] + portPart := listen[end+1:] + if portPart == "" { + return host, 80, nil + } + if portPart[0] != ':' { + return "", 0, fmt.Errorf("invalid listen: %s", listen) + } + portPart = portPart[1:] + port, err := strconv.Atoi(portPart) + if err != nil { + return "", 0, fmt.Errorf("invalid listen: %s", listen) + } + return host, port, nil + } + + index := strings.Index(listen, ":") + if index < 0 { + port, err := strconv.Atoi(listen) + if err != nil { + return listen, 80, nil + } + return "", port, nil + } + host := listen[0:index] + portPart := listen[index+1:] + if portPart == "" { + return host, 80, nil + } + port, err := strconv.Atoi(portPart) + if err != nil { + return "", 0, fmt.Errorf("invalid listen: %s", listen) + } + return host, port, nil } diff --git a/cmd/client/commandv2/convert/nginx/env_test.go b/cmd/client/commandv2/convert/nginx/env_test.go new file mode 100644 index 0000000000..bc01e74607 --- /dev/null +++ b/cmd/client/commandv2/convert/nginx/env_test.go @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2017, MegaEase + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nginx + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestSplitAddressPort(t *testing.T) { + testCases := []struct { + listen string + address string + port int + err bool + }{ + {listen: "127.0.0.1:8000", address: "127.0.0.1", port: 8000, err: false}, + {listen: "127.0.0.1", address: "127.0.0.1", port: 80, err: false}, + {listen: "8000", address: "", port: 8000, err: false}, + {listen: "*:8000", address: "*", port: 8000, err: false}, + {listen: "localhost:8000", address: "localhost", port: 8000, err: false}, + {listen: "[::]:8000", address: "[::]", port: 8000, err: false}, + {listen: "[::1]", address: "[::1]", port: 80, err: false}, + {listen: "", address: "", port: 0, err: true}, + {listen: "[::", address: "", port: 0, err: true}, + {listen: "[::]:", address: "", port: 0, err: true}, + {listen: "[::]:8888", address: "[::]", port: 8888, err: false}, + } + for _, tc := range testCases { + msg := fmt.Sprintf("%v", tc) + address, port, err := splitAddressPort(tc.listen) + assert.Equal(t, tc.address, address, msg) + assert.Equal(t, tc.port, port, msg) + if tc.err { + assert.NotNil(t, err, msg) + } else { + assert.Nil(t, err, msg) + } + } +} diff --git a/cmd/client/commandv2/convert/nginx/parse.go b/cmd/client/commandv2/convert/nginx/parse.go new file mode 100644 index 0000000000..1e39debd28 --- /dev/null +++ b/cmd/client/commandv2/convert/nginx/parse.go @@ -0,0 +1,137 @@ +/* + * Copyright (c) 2017, MegaEase + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nginx + +import ( + "path/filepath" + + "github.com/megaease/easegress/v2/cmd/client/general" + crossplane "github.com/nginxinc/nginx-go-crossplane" +) + +const ( + DirectiveInclude = "include" + DirectiveHTTP = "http" + DirectiveServer = "server" + DirectiveLocation = "location" +) + +type Directive = crossplane.Directive +type Directives = crossplane.Directives +type Payload = crossplane.Payload + +func parsePayload(payload *Payload) { + addFilenameToPayload(payload) + directives := payload.Config[0].Parsed + directives = loadIncludes(directives, payload) + for _, d := range directives { + if d.Directive == DirectiveHTTP { + parseHTTPDirective(payload, d) + } + } +} + +func parseHTTPDirective(payload *Payload, directive *Directive) { + env := newEnv() + directives := loadIncludes(directive.Block, payload) + for _, d := range directives { + if d.Directive != DirectiveServer { + env.Update(d) + } + } + for _, d := range directives { + if d.Directive == DirectiveServer { + parseServerDirective(env.MustClone(), payload, d) + } + } +} + +func parseServerDirective(env *Env, payload *Payload, directive *Directive) { + directives := loadIncludes(directive.Block, payload) + for _, d := range directives { + if d.Directive != DirectiveLocation { + env.Update(d) + } + } + for _, d := range directives { + if d.Directive == DirectiveLocation { + parseLocationDirective(env.MustClone(), payload, d) + } + } +} + +func parseLocationDirective(env *Env, payload *Payload, directive *Directive) { + directives := loadIncludes(directive.Block, payload) + for _, d := range directives { + env.Update(d) + } + for _, d := range directives { + if d.Directive == DirectiveLocation { + parseLocationDirective(env.MustClone(), payload, d) + } + } +} + +// addFilenameToPayload adds filename to payload recursively for all nested directives. +func addFilenameToPayload(payload *crossplane.Payload) { + for _, config := range payload.Config { + filename := filepath.Base(config.File) + directives := config.Parsed + for len(directives) > 0 { + d := directives[0] + d.File = filename + directives = append(directives[1:], d.Block...) + } + } +} + +// loadIncludes loads all include files for current directives recursively but not nested. +func loadIncludes(directives crossplane.Directives, payload *crossplane.Payload) crossplane.Directives { + res := crossplane.Directives{} + for _, d := range directives { + if d.Directive != DirectiveInclude { + res = append(res, d) + continue + } + mustContainArgs(d, 1) + name := d.Args[0] + var include crossplane.Directives + for _, config := range payload.Config { + if config.File == name { + include = config.Parsed + break + } + } + if include == nil { + general.Warnf("can't find include file %s for %s", name, directiveInfo(d)) + continue + } + include = loadIncludes(include, payload) + res = append(res, include...) + } + return res +} + +func mustContainArgs(d *Directive, argNum int) { + if len(d.Args) < argNum { + general.ExitWithErrorf( + "%s must have at least %d args, please update it or remote it.", + directiveInfo(d), argNum, + ) + } +} diff --git a/cmd/client/commandv2/convert/nginx/parse_test.go b/cmd/client/commandv2/convert/nginx/parse_test.go new file mode 100644 index 0000000000..77b44ecb21 --- /dev/null +++ b/cmd/client/commandv2/convert/nginx/parse_test.go @@ -0,0 +1,140 @@ +/* + * Copyright (c) 2017, MegaEase + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nginx + +import ( + "fmt" + "testing" + + crossplane "github.com/nginxinc/nginx-go-crossplane" + "github.com/stretchr/testify/assert" +) + +func TestAddFilenameToPayload(t *testing.T) { + tempDir := newTempTestDir(t) + defer tempDir.Clean() + + var checkFilename func(ds crossplane.Directives, filename string) + checkFilename = func(ds crossplane.Directives, filename string) { + for _, d := range ds { + assert.Equal(t, filename, d.File, "line %d, directive %v", d.Line, d) + checkFilename(d.Block, filename) + } + } + + { + nginxConf := ` + events {} + http { + include %s; + server { + listen 80; + location / { + proxy_pass http://localhost:8888; + } + } + } + ` + proxyConf := `proxy_set_header Conf-One One;` + + proxyFile := tempDir.Create("proxy.conf", []byte(proxyConf)) + nginxFile := tempDir.Create("nginx.conf", []byte(fmt.Sprintf(nginxConf, proxyFile))) + + payload, err := crossplane.Parse(nginxFile, &crossplane.ParseOptions{}) + addFilenameToPayload(payload) + assert.Nil(t, err) + assert.Equal(t, 2, len(payload.Config)) + checkFilename(payload.Config[0].Parsed, "nginx.conf") + checkFilename(payload.Config[1].Parsed, "proxy.conf") + } +} + +func TestLoadIncludes(t *testing.T) { + tempDir := newTempTestDir(t) + defer tempDir.Clean() + + { + nginxConf := ` + events {} + http { + include %s; + server { + listen 80; + location / { + proxy_pass http://localhost:8888; + } + } + } + ` + proxyConf1 := `include %s; proxy_set_header Conf-One One;` + proxyConf2 := `proxy_set_header Conf-Two Two;` + + proxyFile2 := tempDir.Create("proxy2.conf", []byte(proxyConf2)) + proxyFile1 := tempDir.Create("proxy1.conf", []byte(fmt.Sprintf(proxyConf1, proxyFile2))) + nginxFile := tempDir.Create("nginx.conf", []byte(fmt.Sprintf(nginxConf, proxyFile1))) + + payload, err := crossplane.Parse(nginxFile, &crossplane.ParseOptions{}) + addFilenameToPayload(payload) + assert.Nil(t, err) + httpDirectives := payload.Config[0].Parsed[1].Block + httpDirectives = loadIncludes(httpDirectives, payload) + assert.Equal(t, 3, len(httpDirectives)) + + // first directive from conf2 + d2 := httpDirectives[0] + assert.Equal(t, "proxy_set_header", d2.Directive) + assert.Equal(t, []string{"Conf-Two", "Two"}, d2.Args) + assert.Equal(t, "proxy2.conf", d2.File) + // second directive from conf1 + d1 := httpDirectives[1] + assert.Equal(t, "proxy_set_header", d1.Directive) + assert.Equal(t, []string{"Conf-One", "One"}, d1.Args) + assert.Equal(t, "proxy1.conf", d1.File) + } + + { + // test invalid includes + nginxConf := ` + events {} + http { + include not-exist.conf; + include %s invalid-args.conf; + include; + server { + listen 80; + location / { + proxy_pass http://localhost:8888; + } + } + } + ` + proxyConf1 := `include %s; proxy_set_header Conf-One One;` + proxyConf2 := `proxy_set_header Conf-Two Two;` + + proxyFile2 := tempDir.Create("proxy2.conf", []byte(proxyConf2)) + proxyFile1 := tempDir.Create("proxy1.conf", []byte(fmt.Sprintf(proxyConf1, proxyFile2))) + nginxFile := tempDir.Create("nginx.conf", []byte(fmt.Sprintf(nginxConf, proxyFile1))) + + payload, err := crossplane.Parse(nginxFile, &crossplane.ParseOptions{}) + addFilenameToPayload(payload) + assert.Nil(t, err) + httpDirectives := payload.Config[0].Parsed[1].Block + httpDirectives = loadIncludes(httpDirectives, payload) + assert.Equal(t, 1, len(httpDirectives)) + } +} diff --git a/cmd/client/commandv2/convert/nginx/test_test.go b/cmd/client/commandv2/convert/nginx/test_test.go index 028cf1df0f..3432658747 100644 --- a/cmd/client/commandv2/convert/nginx/test_test.go +++ b/cmd/client/commandv2/convert/nginx/test_test.go @@ -20,6 +20,7 @@ package nginx import ( "encoding/json" "os" + "path/filepath" "testing" "github.com/stretchr/testify/require" @@ -27,22 +28,23 @@ import ( type tempTestDir struct { dir string + t *testing.T files []string } func newTempTestDir(t *testing.T) *tempTestDir { dir, err := os.MkdirTemp("", "test") require.Nil(t, err) - return &tempTestDir{dir: dir} + return &tempTestDir{dir: dir, t: t} } -func (dir *tempTestDir) Create(t *testing.T, content []byte) string { - file, err := os.CreateTemp(dir.dir, "test") - require.Nil(t, err) +func (dir *tempTestDir) Create(filename string, content []byte) string { + file, err := os.Create(filepath.Join(dir.dir, filename)) + require.Nil(dir.t, err) defer file.Close() _, err = file.Write(content) - require.Nil(t, err) + require.Nil(dir.t, err) dir.files = append(dir.files, file.Name()) return file.Name() } diff --git a/cmd/client/commandv2/convert/nginx/utils.go b/cmd/client/commandv2/convert/nginx/utils.go new file mode 100644 index 0000000000..1185547c2f --- /dev/null +++ b/cmd/client/commandv2/convert/nginx/utils.go @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2017, MegaEase + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nginx + +import "fmt" + +func directiveInfo(d *Directive) string { + return fmt.Sprintf("directive %s in line %d of file %s", d.Directive, d.Line, d.File) +} From 4d4c0b0e8901ce98e3153f8701aea530074fcc2a Mon Sep 17 00:00:00 2001 From: chen Date: Wed, 1 Nov 2023 17:39:07 +0800 Subject: [PATCH 04/22] finish parse part of nginx convert --- cmd/client/commandv2/convert/nginx/env.go | 201 ++++++++++++++++---- cmd/client/commandv2/convert/nginx/parse.go | 131 ++++++++++++- cmd/client/commandv2/convert/nginx/types.go | 77 ++++++++ 3 files changed, 360 insertions(+), 49 deletions(-) create mode 100644 cmd/client/commandv2/convert/nginx/types.go diff --git a/cmd/client/commandv2/convert/nginx/env.go b/cmd/client/commandv2/convert/nginx/env.go index 4fddead703..8b514c598f 100644 --- a/cmd/client/commandv2/convert/nginx/env.go +++ b/cmd/client/commandv2/convert/nginx/env.go @@ -18,11 +18,16 @@ package nginx import ( + "encoding/base64" "encoding/json" + "errors" "fmt" + "os" "regexp" "strconv" "strings" + + "github.com/megaease/easegress/v2/cmd/client/general" ) var nginxBackends = map[string]struct{}{ @@ -62,7 +67,6 @@ type ServerEnv struct { } type ProxyEnv struct { - Kind string `json:"kind"` Pass *Directive `json:"pass"` ProxySetHeader []*Directive `json:"proxy_set_header"` } @@ -126,20 +130,17 @@ func (env *Env) Update(d *Directive) { } _, ok = nginxBackends[d.Directive] if ok { - env.Proxy.Kind = d.Directive env.Proxy.Pass = d } } func (env *Env) GetServerInfo() (*ServerInfo, error) { - info := &ServerInfo{ - Port: 80, - Certs: make(map[string]string), - Keys: make(map[string]string), - } + info := &ServerInfo{} + info.Port = 80 - if env.Server.Listen != nil { - address, port, https, err := processListen(env.Server.Listen) + s := env.Server + if s.Listen != nil { + address, port, https, err := processListen(s.Listen) if err != nil { return nil, err } @@ -148,8 +149,8 @@ func (env *Env) GetServerInfo() (*ServerInfo, error) { info.HTTPS = https } - if env.Server.ServerName != nil { - serverName := env.Server.ServerName + if s.ServerName != nil { + serverName := s.ServerName hosts, err := processServerName(serverName) if err != nil { return nil, err @@ -157,18 +158,162 @@ func (env *Env) GetServerInfo() (*ServerInfo, error) { info.Hosts = hosts } + if s.SSLClientCertificate != nil { + cert := s.SSLClientCertificate + mustContainArgs(cert, 1) + caCert, err := loadCert(cert.Args[0]) + if err != nil { + return nil, fmt.Errorf("%s: %v", directiveInfo(cert), err) + } + info.CaCert = caCert + } + + if len(s.SSLCertificate) > 0 { + certs, keys, err := processSSLCertificates(s.SSLCertificate, s.SSLCertificateKey) + if err != nil { + return nil, err + } + info.Certs = certs + info.Keys = keys + } return info, nil } -func processServerName(d *Directive) ([]HostInfo, error) { - hosts := make([]HostInfo, 0) +func (env *Env) GetProxyInfo() (*ProxyInfo, error) { + p := env.Proxy + if p.Pass == nil || p.Pass.Directive != "proxy_pass" { + return nil, errors.New("no proxy_pass found") + } + servers, err := processProxyPass(p.Pass, env) + if err != nil { + return nil, err + } + + var setHeaders map[string]string + if len(p.ProxySetHeader) > 0 { + setHeaders, err = processProxySetHeader(p.ProxySetHeader) + if err != nil { + return nil, err + } + } + + return &ProxyInfo{ + Servers: servers, + SetHeaders: setHeaders, + }, nil +} + +func processProxySetHeader(ds []*Directive) (map[string]string, error) { + res := make(map[string]string) + for _, d := range ds { + mustContainArgs(d, 2) + res[d.Args[0]] = d.Args[1] + } + return res, nil +} + +func processProxyPass(d *Directive, env *Env) ([]*BackendInfo, error) { + mustContainArgs(d, 1) + url := d.Args[0] + if !strings.HasPrefix(url, "http://") && !strings.HasPrefix(url, "https://") { + return nil, fmt.Errorf("%s: proxy_pass %s is not http or https", directiveInfo(d), url) + } + prefix := "http://" + if strings.HasPrefix(url, "https://") { + prefix = "https://" + } + address := url[len(prefix):] + + var upstream *Directive + for _, u := range env.Upstream { + mustContainArgs(u, 1) + if address == u.Args[0] { + upstream = u + break + } + } + if upstream == nil { + return []*BackendInfo{{Server: url, Weight: 1}}, nil + } + res := make([]*BackendInfo, 0) + for _, block := range upstream.Block { + if block.Directive != "server" { + continue + } + mustContainArgs(block, 1) + server := block.Args[0] + weight := 1 + for _, arg := range block.Args[1:] { + if strings.HasPrefix(arg, "weight=") { + w, err := strconv.Atoi(arg[7:]) + if err != nil { + general.Warnf("%s: invalid weight %v, use default value of 1 instead", directiveInfo(block), err) + } else { + weight = w + } + } + } + res = append(res, &BackendInfo{Server: prefix + server, Weight: weight}) + } + return res, nil +} + +func processSSLCertificates(certs []*Directive, keys []*Directive) (map[string]string, map[string]string, error) { + if len(certs) != len(keys) { + var missMatch []*Directive + if len(certs) > len(keys) { + missMatch = certs[len(keys):] + } else { + missMatch = keys[len(certs):] + } + msg := "" + for _, d := range missMatch { + msg += directiveInfo(d) + "\n" + } + return nil, nil, fmt.Errorf("%s has miss certs or keys", msg) + } + + certMap := make(map[string]string) + keyMap := make(map[string]string) + for i := 0; i < len(certs); i++ { + cert := certs[i] + key := keys[i] + mustContainArgs(cert, 1) + mustContainArgs(key, 1) + certName := cert.Args[0] + keyName := key.Args[0] + certData, err := loadCert(certName) + if err != nil { + return nil, nil, fmt.Errorf("%s: %v", directiveInfo(cert), err) + } + keyData, err := loadCert(keyName) + if err != nil { + return nil, nil, fmt.Errorf("%s: %v", directiveInfo(key), err) + } + certMap[certName] = certData + keyMap[keyName] = keyData + } + return certMap, keyMap, nil +} + +func loadCert(filePath string) (string, error) { + data, err := os.ReadFile(filePath) + if err != nil { + return "", err + } + res := base64.StdEncoding.EncodeToString(data) + return res, nil +} + +func processServerName(d *Directive) ([]*HostInfo, error) { + hosts := make([]*HostInfo, 0) for _, arg := range d.Args { if strings.HasPrefix(arg, "~") { _, err := regexp.Compile(arg[1:]) if err != nil { return nil, fmt.Errorf("%s: %v", directiveInfo(d), err) } - hosts = append(hosts, HostInfo{ + hosts = append(hosts, &HostInfo{ Value: arg[1:], IsRegexp: true, }) @@ -182,7 +327,7 @@ func processServerName(d *Directive) ([]HostInfo, error) { return nil, fmt.Errorf("%s: host %s contains wildcard in the middle", directiveInfo(d), arg) } } - hosts = append(hosts, HostInfo{ + hosts = append(hosts, &HostInfo{ Value: arg, IsRegexp: false, }) @@ -205,32 +350,6 @@ func processListen(d *Directive) (address string, port int, https bool, err erro return address, port, false, nil } -type ServerInfo struct { - Port int - Address string - HTTPS bool - Hosts []HostInfo - - CaCert string - Certs map[string]string - Keys map[string]string -} - -type HostInfo struct { - Value string - IsRegexp bool -} - -type ProxyInfo struct { - Servers []*BackendInfo - SetHeaders map[string]string -} - -type BackendInfo struct { - Server string - Weight int -} - // splitAddressPort splits the listen directive into address and port. // nginx examples: // listen 127.0.0.1:8000; diff --git a/cmd/client/commandv2/convert/nginx/parse.go b/cmd/client/commandv2/convert/nginx/parse.go index 1e39debd28..6ba59f4692 100644 --- a/cmd/client/commandv2/convert/nginx/parse.go +++ b/cmd/client/commandv2/convert/nginx/parse.go @@ -18,7 +18,10 @@ package nginx import ( + "errors" + "fmt" "path/filepath" + "strings" "github.com/megaease/easegress/v2/cmd/client/general" crossplane "github.com/nginxinc/nginx-go-crossplane" @@ -35,18 +38,32 @@ type Directive = crossplane.Directive type Directives = crossplane.Directives type Payload = crossplane.Payload -func parsePayload(payload *Payload) { +func parsePayload(payload *Payload) (*Config, error) { addFilenameToPayload(payload) directives := payload.Config[0].Parsed directives = loadIncludes(directives, payload) + + config := &Config{ + Servers: make([]*Server, 0), + } for _, d := range directives { if d.Directive == DirectiveHTTP { - parseHTTPDirective(payload, d) + servers, err := parseHTTPDirective(payload, d) + if err != nil { + return nil, err + } + for _, s := range servers { + config.Servers, err = updateServers(config.Servers, s) + if err != nil { + return nil, err + } + } } } + return config, nil } -func parseHTTPDirective(payload *Payload, directive *Directive) { +func parseHTTPDirective(payload *Payload, directive *Directive) ([]*Server, error) { env := newEnv() directives := loadIncludes(directive.Block, payload) for _, d := range directives { @@ -54,37 +71,135 @@ func parseHTTPDirective(payload *Payload, directive *Directive) { env.Update(d) } } + servers := make([]*Server, 0) for _, d := range directives { if d.Directive == DirectiveServer { - parseServerDirective(env.MustClone(), payload, d) + server, err := parseServerDirective(env.MustClone(), payload, d) + if err != nil { + return nil, err + } + servers, err = updateServers(servers, server) + if err != nil { + return nil, err + } + } + } + return servers, nil +} + +func updateServers(servers []*Server, s *Server) ([]*Server, error) { + for i, server := range servers { + if server.Port != s.Port { + continue + } + // same port, try to merge. + if server.Address != s.Address { + return nil, fmt.Errorf("two server in port %d have different address %s vs %s", server.Port, server.Address, s.Address) + } + if s.HTTPS { + servers[i].HTTPS = true + } + if s.CaCert != "" { + servers[i].CaCert = s.CaCert } + for k, v := range s.Certs { + servers[i].Certs[k] = v + } + for k, v := range s.Keys { + servers[i].Keys[k] = v + } + return servers, nil } + return append(servers, s), nil } -func parseServerDirective(env *Env, payload *Payload, directive *Directive) { +func parseServerDirective(env *Env, payload *Payload, directive *Directive) (*Server, error) { directives := loadIncludes(directive.Block, payload) for _, d := range directives { if d.Directive != DirectiveLocation { env.Update(d) } } + info, err := env.GetServerInfo() + if err != nil { + return nil, err + } + res := &Server{ + ServerBase: info.ServerBase, + Rules: []*Rule{{ + Hosts: info.Hosts, + Paths: make([]*Path, 0), + }}, + } + for _, d := range directives { if d.Directive == DirectiveLocation { - parseLocationDirective(env.MustClone(), payload, d) + paths, err := parseLocationDirective(env.MustClone(), payload, d) + if err != nil { + return nil, err + } + res.Rules[0].Paths = append(res.Rules[0].Paths, paths...) } } + return res, nil } -func parseLocationDirective(env *Env, payload *Payload, directive *Directive) { +func parseLocationDirective(env *Env, payload *Payload, directive *Directive) ([]*Path, error) { directives := loadIncludes(directive.Block, payload) for _, d := range directives { env.Update(d) } + res := make([]*Path, 0) + proxyInfo, err := env.GetProxyInfo() + // for various nginx backends, we only support proxy_pass now. + // so we only warn when we can't get proxy info. + if err != nil { + general.Warnf("failed to get proxy for %s, %v", directiveInfo(directive), err) + } else { + path, pathType, err := parseLocationArgs(directive) + if err != nil { + general.Warnf("%s, %v", directiveInfo(directive), err) + } else { + res = append(res, &Path{ + Path: path, + Type: pathType, + Backend: proxyInfo, + }) + } + } + + // location can be nested. for _, d := range directives { if d.Directive == DirectiveLocation { - parseLocationDirective(env.MustClone(), payload, d) + paths, err := parseLocationDirective(env.MustClone(), payload, d) + if err != nil { + return nil, err + } + res = append(res, paths...) } } + return res, nil +} + +func parseLocationArgs(d *Directive) (string, PathType, error) { + mustContainArgs(d, 1) + arg0 := d.Args[0] + if strings.HasPrefix(arg0, "/") { + return arg0, PathTypePrefix, nil + } + mustContainArgs(d, 2) + switch arg0 { + case "=": + return d.Args[1], PathTypeExact, nil + case "~": + return d.Args[1], PathTypeRe, nil + case "~*": + return d.Args[1], PathTypeReInsensitive, nil + case "^~": + return d.Args[1], PathTypePrefix, nil + default: + return "", PathTypePrefix, errors.New("invalid location args, only support =, ~, ~*, ^~") + } } // addFilenameToPayload adds filename to payload recursively for all nested directives. diff --git a/cmd/client/commandv2/convert/nginx/types.go b/cmd/client/commandv2/convert/nginx/types.go new file mode 100644 index 0000000000..cd1e633bfa --- /dev/null +++ b/cmd/client/commandv2/convert/nginx/types.go @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2017, MegaEase + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nginx + +type Config struct { + Servers []*Server +} + +type ServerBase struct { + Port int + Address string + HTTPS bool + + CaCert string + Certs map[string]string + Keys map[string]string +} + +type Server struct { + ServerBase + Rules []*Rule +} + +type ServerInfo struct { + ServerBase + Hosts []*HostInfo +} + +type HostInfo struct { + Value string + IsRegexp bool +} + +type Rule struct { + Hosts []*HostInfo + Paths []*Path +} + +type PathType string + +const ( + PathTypePrefix PathType = "prefix" + PathTypeExact PathType = "exact" + PathTypeRe PathType = "regexp" + PathTypeReInsensitive PathType = "caseInsensitiveRegexp" +) + +type Path struct { + Path string + Type PathType + Backend *ProxyInfo +} + +type ProxyInfo struct { + Servers []*BackendInfo + SetHeaders map[string]string +} + +type BackendInfo struct { + Server string + Weight int +} From 4219c6019989e269518f70fbb899704021574cf7 Mon Sep 17 00:00:00 2001 From: chen Date: Wed, 1 Nov 2023 18:12:57 +0800 Subject: [PATCH 05/22] fix bug --- cmd/client/commandv2/convert/nginx/env.go | 2 +- .../commandv2/convert/nginx/parse_test.go | 50 +++++++++++++++++++ .../commandv2/convert/nginx/test_test.go | 8 --- cmd/client/commandv2/convert/nginx/utils.go | 10 +++- 4 files changed, 60 insertions(+), 10 deletions(-) diff --git a/cmd/client/commandv2/convert/nginx/env.go b/cmd/client/commandv2/convert/nginx/env.go index 8b514c598f..0ce114d54c 100644 --- a/cmd/client/commandv2/convert/nginx/env.go +++ b/cmd/client/commandv2/convert/nginx/env.go @@ -110,7 +110,7 @@ func (env *Env) Clone() (*Env, error) { if err != nil { return nil, err } - newEnv.updateFn = getEnvUpdateFn(env) + newEnv.updateFn = getEnvUpdateFn(&newEnv) return &newEnv, nil } diff --git a/cmd/client/commandv2/convert/nginx/parse_test.go b/cmd/client/commandv2/convert/nginx/parse_test.go index 77b44ecb21..d432e5e16a 100644 --- a/cmd/client/commandv2/convert/nginx/parse_test.go +++ b/cmd/client/commandv2/convert/nginx/parse_test.go @@ -138,3 +138,53 @@ func TestLoadIncludes(t *testing.T) { assert.Equal(t, 1, len(httpDirectives)) } } + +func TestParsePayload(t *testing.T) { + tempDir := newTempTestDir(t) + defer tempDir.Clean() + { + nginxConf := ` + events {} + http { + upstream backend { + server localhost:1234; + server localhost:2345 weight=10; + } + + server { + listen 80; + + location / { + proxy_set_header X-Path "prefix"; + proxy_pass http://localhost:8080; + } + + location /apis { + proxy_set_header X-Path "apis"; + proxy_pass http://localhost:8880; + + location /apis/v1 { + proxy_set_header X-Path "apis/v1"; + proxy_pass http://localhost:8888; + } + } + + location = /user { + proxy_pass http://localhost:8890; + } + + location /upstream { + proxy_pass http://backend; + } + } + } + ` + file := tempDir.Create("nginx.conf", []byte(nginxConf)) + payload, err := crossplane.Parse(file, &crossplane.ParseOptions{}) + assert.Nil(t, err) + + config, err := parsePayload(payload) + assert.Nil(t, err) + printJson(config) + } +} diff --git a/cmd/client/commandv2/convert/nginx/test_test.go b/cmd/client/commandv2/convert/nginx/test_test.go index 3432658747..07fe393034 100644 --- a/cmd/client/commandv2/convert/nginx/test_test.go +++ b/cmd/client/commandv2/convert/nginx/test_test.go @@ -18,7 +18,6 @@ package nginx import ( - "encoding/json" "os" "path/filepath" "testing" @@ -55,10 +54,3 @@ func (dir *tempTestDir) Clean() { } os.Remove(dir.dir) } - -func printJson(t *testing.T, v interface{}) { - t.Helper() - b, err := json.MarshalIndent(v, "", " ") - require.Nil(t, err) - t.Log(string(b)) -} diff --git a/cmd/client/commandv2/convert/nginx/utils.go b/cmd/client/commandv2/convert/nginx/utils.go index 1185547c2f..5961c0d73f 100644 --- a/cmd/client/commandv2/convert/nginx/utils.go +++ b/cmd/client/commandv2/convert/nginx/utils.go @@ -17,8 +17,16 @@ package nginx -import "fmt" +import ( + "encoding/json" + "fmt" +) func directiveInfo(d *Directive) string { return fmt.Sprintf("directive %s in line %d of file %s", d.Directive, d.Line, d.File) } + +func printJson(v interface{}) { + b, _ := json.MarshalIndent(v, "", " ") + fmt.Println(string(b)) +} From db569048d5d22f9420f5db784c51d854fe10c8e9 Mon Sep 17 00:00:00 2001 From: chen Date: Thu, 2 Nov 2023 11:38:31 +0800 Subject: [PATCH 06/22] optim --- cmd/client/commandv2/convert/nginx/env.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/cmd/client/commandv2/convert/nginx/env.go b/cmd/client/commandv2/convert/nginx/env.go index 0ce114d54c..8a57bb5b47 100644 --- a/cmd/client/commandv2/convert/nginx/env.go +++ b/cmd/client/commandv2/convert/nginx/env.go @@ -71,8 +71,8 @@ type ProxyEnv struct { ProxySetHeader []*Directive `json:"proxy_set_header"` } -func getEnvUpdateFn(env *Env) map[string]func(*Directive) { - m := map[string]func(*Directive){ +func (env *Env) init() { + env.updateFn = map[string]func(*Directive){ "listen": func(d *Directive) { env.Server.Listen = d }, "server_name": func(d *Directive) { env.Server.ServerName = d }, "ssl_client_certificate": func(d *Directive) { env.Server.SSLClientCertificate = d }, @@ -81,7 +81,6 @@ func getEnvUpdateFn(env *Env) map[string]func(*Directive) { "proxy_set_header": func(d *Directive) { env.Proxy.ProxySetHeader = append(env.Proxy.ProxySetHeader, d) }, "upstream": func(d *Directive) { env.Upstream = append(env.Upstream, d) }, } - return m } func newEnv() *Env { @@ -95,8 +94,7 @@ func newEnv() *Env { }, Upstream: make([]*Directive, 0), } - - env.updateFn = getEnvUpdateFn(env) + env.init() return env } @@ -110,7 +108,7 @@ func (env *Env) Clone() (*Env, error) { if err != nil { return nil, err } - newEnv.updateFn = getEnvUpdateFn(&newEnv) + newEnv.init() return &newEnv, nil } From f81a0c940e70eb16f9a1dfae8120909fa092f880 Mon Sep 17 00:00:00 2001 From: chen Date: Thu, 2 Nov 2023 17:07:29 +0800 Subject: [PATCH 07/22] finish convert code --- cmd/client/commandv2/common/spec.go | 88 +++++++++++++++++ cmd/client/commandv2/convert/nginx/cmd.go | 57 ++++++++++- cmd/client/commandv2/convert/nginx/convert.go | 95 +++++++++++++++++++ cmd/client/commandv2/convert/nginx/env.go | 3 +- .../commandv2/convert/nginx/parse_test.go | 10 +- cmd/client/commandv2/convert/nginx/utils.go | 7 ++ .../commandv2/create/createhttpproxy.go | 65 +++---------- .../commandv2/create/createhttpproxy_test.go | 19 ++-- pkg/object/httpserver/runtime.go | 6 +- pkg/object/httpserver/spec.go | 1 + 10 files changed, 279 insertions(+), 72 deletions(-) create mode 100644 cmd/client/commandv2/common/spec.go diff --git a/cmd/client/commandv2/common/spec.go b/cmd/client/commandv2/common/spec.go new file mode 100644 index 0000000000..3f77b9a1cd --- /dev/null +++ b/cmd/client/commandv2/common/spec.go @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2017, MegaEase + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package common + +import ( + "github.com/megaease/easegress/v2/cmd/client/general" + "github.com/megaease/easegress/v2/pkg/filters" + "github.com/megaease/easegress/v2/pkg/filters/proxies/httpproxy" + "github.com/megaease/easegress/v2/pkg/object/httpserver" + "github.com/megaease/easegress/v2/pkg/object/pipeline" + "github.com/megaease/easegress/v2/pkg/util/codectool" +) + +// HTTPServerSpec is the spec of HTTPServer. +type HTTPServerSpec struct { + Name string `json:"name"` + Kind string `json:"kind"` + + httpserver.Spec `json:",inline"` +} + +// PipelineSpec is the spec of Pipeline. +type PipelineSpec struct { + Name string `json:"name"` + Kind string `json:"kind"` + + pipeline.Spec `json:",inline"` +} + +func (p *PipelineSpec) SetFilters(filters []filters.Spec) { + data := codectool.MustMarshalYAML(filters) + maps, _ := general.UnmarshalMapInterface(data, true) + p.Filters = maps +} + +// NewHTTPServerSpec returns a new HTTPServerSpec. +func NewHTTPServerSpec(name string) *HTTPServerSpec { + return &HTTPServerSpec{ + Name: name, + Kind: httpserver.Kind, + Spec: *getDefaultHTTPServerSpec(), + } +} + +// NewPipelineSpec returns a new PipelineSpec. +func NewPipelineSpec(name string) *PipelineSpec { + return &PipelineSpec{ + Name: name, + Kind: pipeline.Kind, + Spec: *getDefaultPipelineSpec(), + } +} + +func getDefaultHTTPServerSpec() *httpserver.Spec { + return (&httpserver.HTTPServer{}).DefaultSpec().(*httpserver.Spec) +} + +func getDefaultPipelineSpec() *pipeline.Spec { + return (&pipeline.Pipeline{}).DefaultSpec().(*pipeline.Spec) +} + +// NewProxyFilterSpec returns a new ProxyFilterSpec. +func NewProxyFilterSpec(name string) *httpproxy.Spec { + spec := GetDefaultFilterSpec(httpproxy.Kind).(*httpproxy.Spec) + spec.BaseSpec.MetaSpec.Name = name + spec.BaseSpec.MetaSpec.Kind = httpproxy.Kind + return spec +} + +// GetDefaultFilterSpec returns the default filter spec of the kind. +func GetDefaultFilterSpec(kind string) filters.Spec { + return filters.GetKind(kind).DefaultSpec() +} diff --git a/cmd/client/commandv2/convert/nginx/cmd.go b/cmd/client/commandv2/convert/nginx/cmd.go index 4f57bd4675..659342dc21 100644 --- a/cmd/client/commandv2/convert/nginx/cmd.go +++ b/cmd/client/commandv2/convert/nginx/cmd.go @@ -20,15 +20,24 @@ package nginx import ( "encoding/json" "fmt" + "math/rand" "github.com/megaease/easegress/v2/cmd/client/general" crossplane "github.com/nginxinc/nginx-go-crossplane" "github.com/spf13/cobra" ) +type Options struct { + NginxConf string + Output string + Prefix string + usedNames map[string]struct{} +} + // Cmd returns convert nginx.conf command. func Cmd() *cobra.Command { - var nginxConf string + flags := &Options{} + flags.init() cmd := &cobra.Command{ Use: "nginx", Short: "Convert nginx.conf to easegress yaml file", @@ -36,11 +45,19 @@ func Cmd() *cobra.Command { return nil }, Run: func(cmd *cobra.Command, args []string) { - payload, err := crossplane.Parse(nginxConf, &crossplane.ParseOptions{}) + payload, err := crossplane.Parse(flags.NginxConf, &crossplane.ParseOptions{}) if err != nil { general.ExitWithErrorf("parse nginx.conf failed: %v", err) } - parsePayload(payload) + config, err := parsePayload(payload) + if err != nil { + general.ExitWithError(err) + } + hs, pls, err := convertConfig(flags, config) + if err != nil { + general.ExitWithError(err) + } + fmt.Println(hs, pls) data, err := json.Marshal(payload) if err != nil { general.ExitWithError(err) @@ -48,6 +65,38 @@ func Cmd() *cobra.Command { fmt.Println(string(data)) }, } - cmd.Flags().StringVarP(&nginxConf, "file", "f", "", "nginx.conf file path") + cmd.Flags().StringVarP(&flags.NginxConf, "file", "f", "", "nginx.conf file path") + cmd.Flags().StringVarP(&flags.Output, "output", "o", "", "output yaml file path") + cmd.Flags().StringVar(&flags.Prefix, "prefix", "", "prefix of output yaml resources") return cmd } + +func (opt *Options) init() { + opt.usedNames = make(map[string]struct{}) + opt.usedNames[""] = struct{}{} +} + +// GetPipelineName create a global uniq name for pipeline based on path. +func (opt *Options) GetPipelineName(path string) string { + letters := []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") + nameRunes := make([]rune, 0) + for _, r := range path { + if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') { + nameRunes = append(nameRunes, r) + } + } + name := string(nameRunes) + if _, ok := opt.usedNames[name]; !ok { + opt.usedNames[name] = struct{}{} + return fmt.Sprintf("%s-%s", opt.Prefix, name) + } + for i := 0; i < 8; i++ { + nameRunes = append(nameRunes, letters[rand.Intn(len(letters))]) + } + name = string(nameRunes) + if _, ok := opt.usedNames[name]; !ok { + opt.usedNames[name] = struct{}{} + return fmt.Sprintf("%s-%s", opt.Prefix, name) + } + return opt.GetPipelineName(path) +} diff --git a/cmd/client/commandv2/convert/nginx/convert.go b/cmd/client/commandv2/convert/nginx/convert.go index 0d041760e3..7fda105e98 100644 --- a/cmd/client/commandv2/convert/nginx/convert.go +++ b/cmd/client/commandv2/convert/nginx/convert.go @@ -16,3 +16,98 @@ */ package nginx + +import ( + "fmt" + + "github.com/megaease/easegress/v2/cmd/client/commandv2/common" + "github.com/megaease/easegress/v2/cmd/client/general" + "github.com/megaease/easegress/v2/pkg/filters" + "github.com/megaease/easegress/v2/pkg/object/httpserver/routers" +) + +func convertConfig(options *Options, config *Config) ([]*common.HTTPServerSpec, []*common.PipelineSpec, error) { + httpServers := make([]*common.HTTPServerSpec, 0) + pipelines := make([]*common.PipelineSpec, 0) + for _, server := range config.Servers { + s, p, err := convertServer(options, server) + if err != nil { + return nil, nil, err + } + httpServers = append(httpServers, s) + pipelines = append(pipelines, p...) + } + return httpServers, pipelines, nil +} + +func convertServer(options *Options, server *Server) (*common.HTTPServerSpec, []*common.PipelineSpec, error) { + pipelines := make([]*common.PipelineSpec, 0) + httpServer := common.NewHTTPServerSpec(fmt.Sprintf("%s-%d", options.Prefix, server.Port)) + httpServer = convertServerBase(httpServer, server.ServerBase) + + httpServer.Rules = make([]*routers.Rule, 0) + for _, rule := range server.Rules { + routerRule, pls := convertRule(options, rule) + httpServer.Rules = append(httpServer.Rules, routerRule) + pipelines = append(pipelines, pls...) + } + return httpServer, pipelines, nil +} + +func convertServerBase(spec *common.HTTPServerSpec, base ServerBase) *common.HTTPServerSpec { + spec.Port = uint16(base.Port) + spec.Address = base.Address + spec.HTTPS = base.HTTPS + spec.CaCertBase64 = base.CaCert + for k, v := range base.Certs { + spec.Certs[k] = v + } + for k, v := range base.Keys { + spec.Keys[k] = v + } + return spec +} + +func convertRule(options *Options, rule *Rule) (*routers.Rule, []*common.PipelineSpec) { + router := &routers.Rule{ + Hosts: make([]routers.Host, 0), + Paths: make([]*routers.Path, 0), + } + for _, h := range rule.Hosts { + router.Hosts = append(router.Hosts, routers.Host{ + Value: h.Value, + IsRegexp: h.IsRegexp, + }) + } + pipelines := make([]*common.PipelineSpec, 0) + for _, p := range rule.Paths { + name := options.GetPipelineName(p.Path) + path := routers.Path{ + Backend: name, + } + switch p.Type { + case PathTypeExact: + path.Path = p.Path + case PathTypePrefix: + path.PathPrefix = p.Path + case PathTypeRe: + path.PathRegexp = p.Path + case PathTypeReInsensitive: + path.PathRegexp = fmt.Sprintf("(?i)%s", p.Path) + default: + general.Warnf("unknown path type: %s", p.Type) + } + router.Paths = append(router.Paths, &path) + pipelineSpec := convertProxy(name, p.Backend) + pipelines = append(pipelines, pipelineSpec) + } + return router, pipelines +} + +func convertProxy(name string, info *ProxyInfo) *common.PipelineSpec { + pipeline := common.NewPipelineSpec(name) + // TODO: set header, update proxy filter + proxy := common.NewProxyFilterSpec("proxy") + pipeline.SetFilters([]filters.Spec{proxy}) + return pipeline +} diff --git a/cmd/client/commandv2/convert/nginx/env.go b/cmd/client/commandv2/convert/nginx/env.go index 8a57bb5b47..ce6b0dac42 100644 --- a/cmd/client/commandv2/convert/nginx/env.go +++ b/cmd/client/commandv2/convert/nginx/env.go @@ -288,8 +288,9 @@ func processSSLCertificates(certs []*Directive, keys []*Directive) (map[string]s if err != nil { return nil, nil, fmt.Errorf("%s: %v", directiveInfo(key), err) } + // cert and keys should have the same key to match. certMap[certName] = certData - keyMap[keyName] = keyData + keyMap[certName] = keyData } return certMap, keyMap, nil } diff --git a/cmd/client/commandv2/convert/nginx/parse_test.go b/cmd/client/commandv2/convert/nginx/parse_test.go index d432e5e16a..1b7381a0d9 100644 --- a/cmd/client/commandv2/convert/nginx/parse_test.go +++ b/cmd/client/commandv2/convert/nginx/parse_test.go @@ -182,9 +182,15 @@ func TestParsePayload(t *testing.T) { file := tempDir.Create("nginx.conf", []byte(nginxConf)) payload, err := crossplane.Parse(file, &crossplane.ParseOptions{}) assert.Nil(t, err) - config, err := parsePayload(payload) assert.Nil(t, err) - printJson(config) + // printJson(config) + + opt := &Options{Prefix: "test"} + opt.init() + hs, pls, err := convertConfig(opt, config) + assert.Nil(t, err) + printYaml(hs) + printYaml(pls) } } diff --git a/cmd/client/commandv2/convert/nginx/utils.go b/cmd/client/commandv2/convert/nginx/utils.go index 5961c0d73f..e29a4bbcf6 100644 --- a/cmd/client/commandv2/convert/nginx/utils.go +++ b/cmd/client/commandv2/convert/nginx/utils.go @@ -20,6 +20,8 @@ package nginx import ( "encoding/json" "fmt" + + "gopkg.in/yaml.v3" ) func directiveInfo(d *Directive) string { @@ -30,3 +32,8 @@ func printJson(v interface{}) { b, _ := json.MarshalIndent(v, "", " ") fmt.Println(string(b)) } + +func printYaml(v interface{}) { + b, _ := yaml.Marshal(v) + fmt.Println(string(b)) +} diff --git a/cmd/client/commandv2/create/createhttpproxy.go b/cmd/client/commandv2/create/createhttpproxy.go index 8f859b0af4..350d20cc37 100644 --- a/cmd/client/commandv2/create/createhttpproxy.go +++ b/cmd/client/commandv2/create/createhttpproxy.go @@ -24,14 +24,13 @@ import ( "path/filepath" "strings" + "github.com/megaease/easegress/v2/cmd/client/commandv2/common" "github.com/megaease/easegress/v2/cmd/client/general" "github.com/megaease/easegress/v2/cmd/client/resources" "github.com/megaease/easegress/v2/pkg/filters" "github.com/megaease/easegress/v2/pkg/filters/proxies" "github.com/megaease/easegress/v2/pkg/filters/proxies/httpproxy" - "github.com/megaease/easegress/v2/pkg/object/httpserver" "github.com/megaease/easegress/v2/pkg/object/httpserver/routers" - "github.com/megaease/easegress/v2/pkg/object/pipeline" "github.com/megaease/easegress/v2/pkg/util/codectool" "github.com/spf13/cobra" ) @@ -147,22 +146,6 @@ func httpProxyRun(cmd *cobra.Command, args []string) error { return nil } -// HTTPServerSpec is the spec of HTTPServer. -type HTTPServerSpec struct { - Name string `json:"name"` - Kind string `json:"kind"` - - httpserver.Spec `json:",inline"` -} - -// PipelineSpec is the spec of Pipeline. -type PipelineSpec struct { - Name string `json:"name"` - Kind string `json:"kind"` - - pipeline.Spec `json:",inline"` -} - // Complete completes all the required options. func (o *HTTPProxyOptions) Complete(args []string) { o.Name = args[0] @@ -223,12 +206,8 @@ func (o *HTTPProxyOptions) getPipelineName(id int) string { } // Translate translates HTTPProxyOptions to HTTPServerSpec and PipelineSpec. -func (o *HTTPProxyOptions) Translate() (*HTTPServerSpec, []*PipelineSpec) { - hs := &HTTPServerSpec{ - Name: o.getServerName(), - Kind: httpserver.Kind, - Spec: *getDefaultHTTPServerSpec(), - } +func (o *HTTPProxyOptions) Translate() (*common.HTTPServerSpec, []*common.PipelineSpec) { + hs := common.NewHTTPServerSpec(o.getServerName()) hs.Port = uint16(o.Port) if o.TLS { hs.HTTPS = true @@ -247,9 +226,9 @@ func (o *HTTPProxyOptions) Translate() (*HTTPServerSpec, []*PipelineSpec) { return hs, pipelines } -func (o *HTTPProxyOptions) translateRules() (routers.Rules, []*PipelineSpec) { +func (o *HTTPProxyOptions) translateRules() (routers.Rules, []*common.PipelineSpec) { var rules routers.Rules - var pipelines []*PipelineSpec + var pipelines []*common.PipelineSpec pipelineID := 0 for _, rule := range o.rules { @@ -261,11 +240,10 @@ func (o *HTTPProxyOptions) translateRules() (routers.Rules, []*PipelineSpec) { PathPrefix: rule.PathPrefix, Backend: pipelineName, } - pipelines = append(pipelines, &PipelineSpec{ - Name: pipelineName, - Kind: pipeline.Kind, - Spec: *translateToPipeline(rule.Endpoints), - }) + + pipelineSpec := common.NewPipelineSpec(pipelineName) + translateToPipeline(rule.Endpoints, pipelineSpec) + pipelines = append(pipelines, pipelineSpec) l := len(rules) if l != 0 && rules[l-1].Host == rule.Host { @@ -306,20 +284,13 @@ func loadCertFile(filePath string) (string, error) { return base64.StdEncoding.EncodeToString(data), nil } -func translateToPipeline(endpoints []string) *pipeline.Spec { +func translateToPipeline(endpoints []string, spec *common.PipelineSpec) { proxy := translateToProxyFilter(endpoints) - data := codectool.MustMarshalYAML(proxy) - maps, _ := general.UnmarshalMapInterface(data, false) - - spec := getDefaultPipelineSpec() - spec.Filters = maps - return spec + spec.SetFilters([]filters.Spec{proxy}) } func translateToProxyFilter(endpoints []string) *httpproxy.Spec { - spec := getDefaultProxyFilterSpec() - spec.BaseSpec.MetaSpec.Name = "proxy" - spec.BaseSpec.MetaSpec.Kind = httpproxy.Kind + spec := common.NewProxyFilterSpec("proxy") servers := make([]*proxies.Server, len(endpoints)) for i, endpoint := range endpoints { @@ -381,15 +352,3 @@ type HTTPProxyRule struct { PathPrefix string Endpoints []string } - -func getDefaultHTTPServerSpec() *httpserver.Spec { - return (&httpserver.HTTPServer{}).DefaultSpec().(*httpserver.Spec) -} - -func getDefaultPipelineSpec() *pipeline.Spec { - return (&pipeline.Pipeline{}).DefaultSpec().(*pipeline.Spec) -} - -func getDefaultProxyFilterSpec() *httpproxy.Spec { - return filters.GetKind(httpproxy.Kind).DefaultSpec().(*httpproxy.Spec) -} diff --git a/cmd/client/commandv2/create/createhttpproxy_test.go b/cmd/client/commandv2/create/createhttpproxy_test.go index c05f194282..f0ffd8ce13 100644 --- a/cmd/client/commandv2/create/createhttpproxy_test.go +++ b/cmd/client/commandv2/create/createhttpproxy_test.go @@ -24,7 +24,7 @@ import ( "path/filepath" "testing" - "github.com/megaease/easegress/v2/pkg/filters" + "github.com/megaease/easegress/v2/cmd/client/commandv2/common" "github.com/megaease/easegress/v2/pkg/filters/proxies/httpproxy" "github.com/megaease/easegress/v2/pkg/object/httpserver/routers" "github.com/megaease/easegress/v2/pkg/util/codectool" @@ -44,7 +44,7 @@ pools: loadBalance: policy: roundRobin ` - expected := getDefaultProxyFilterSpec() + expected := common.NewProxyFilterSpec("proxy") err := codectool.UnmarshalYAML([]byte(yamlStr), expected) assert.Nil(err) @@ -71,12 +71,13 @@ filters: policy: roundRobin ` // compare expected and got pipeline - expected := getDefaultPipelineSpec() + expected := common.NewPipelineSpec("pipeline") err := codectool.UnmarshalYAML([]byte(yamlStr), expected) assert.Nil(err) endpoints := []string{"http://127.0.0.1:9095", "http://127.0.0.1:9096"} - got := translateToPipeline(endpoints) + got := common.NewPipelineSpec("pipeline") + translateToPipeline(endpoints, got) // filters part is not compare here, because the filter part is map[string]interface{}, // the expected map[string]interface{} is unmarshal from yaml, @@ -92,7 +93,7 @@ filters: // if marshal it once, some part of expectedFilter will be nil. // but gotFilter will be empty. for example []string{} vs nil. // []string{} and nil are actually same in this case. - expectedFilter := getDefaultProxyFilterSpec() + expectedFilter := common.NewProxyFilterSpec("proxy") filterYaml := codectool.MustMarshalYAML(expected.Filters[0]) err = codectool.UnmarshalYAML(filterYaml, expectedFilter) assert.Nil(err) @@ -100,7 +101,7 @@ filters: err = codectool.UnmarshalYAML(filterYaml, expectedFilter) assert.Nil(err) - gotFilter := filters.GetKind(httpproxy.Kind).DefaultSpec().(*httpproxy.Spec) + gotFilter := common.NewProxyFilterSpec("proxy") filterYaml = codectool.MustMarshalYAML(got.Filters[0]) err = codectool.UnmarshalYAML(filterYaml, gotFilter) assert.Nil(err) @@ -311,11 +312,11 @@ filters: policy: roundRobin ` expectedFilter := func() *httpproxy.Spec { - expected := getDefaultPipelineSpec() + expected := common.NewPipelineSpec("pipeline") err = codectool.UnmarshalYAML([]byte(yamlStr), expected) assert.Nil(err) - expectedFilter := getDefaultProxyFilterSpec() + expectedFilter := common.NewProxyFilterSpec("proxy") filterYaml := codectool.MustMarshalYAML(expected.Filters[0]) err = codectool.UnmarshalYAML(filterYaml, expectedFilter) assert.Nil(err) @@ -326,7 +327,7 @@ filters: }() for i, p := range pls { - gotFilter := getDefaultProxyFilterSpec() + gotFilter := common.NewProxyFilterSpec("proxy") filterYaml := codectool.MustMarshalYAML(p.Filters[0]) err = codectool.UnmarshalYAML(filterYaml, gotFilter) assert.Nil(err) diff --git a/pkg/object/httpserver/runtime.go b/pkg/object/httpserver/runtime.go index c25f1f4998..33abe988cd 100644 --- a/pkg/object/httpserver/runtime.go +++ b/pkg/object/httpserver/runtime.go @@ -268,7 +268,7 @@ func (r *runtime) startHTTP3Server() { } r.server3 = &http3.Server{ - Addr: fmt.Sprintf(":%d", r.spec.Port), + Addr: fmt.Sprintf("%s:%d", r.spec.Address, r.spec.Port), Handler: r.mux, TLSConfig: tlsConfig, QuicConfig: &quic.Config{ @@ -303,14 +303,14 @@ func (r *runtime) startHTTP1And2Server() { return !bytes.Contains(p, []byte("TLS handshake error")) }) r.server = &http.Server{ - Addr: fmt.Sprintf(":%d", r.spec.Port), + Addr: fmt.Sprintf("%s:%d", r.spec.Address, r.spec.Port), Handler: r.mux, IdleTimeout: keepAliveTimeout, ErrorLog: log.New(fw, "", log.LstdFlags), } r.server.SetKeepAlivesEnabled(r.spec.KeepAlive) - listener, err := gnet.Listen("tcp", fmt.Sprintf(":%d", r.spec.Port)) + listener, err := gnet.Listen("tcp", fmt.Sprintf("%s:%d", r.spec.Address, r.spec.Port)) if err != nil { logger.Errorf("httpserver %s failed to listen: %v", r.superSpec.Name(), err) r.setState(stateFailed) diff --git a/pkg/object/httpserver/spec.go b/pkg/object/httpserver/spec.go index 3a92d90e60..c79c6ce79b 100644 --- a/pkg/object/httpserver/spec.go +++ b/pkg/object/httpserver/spec.go @@ -37,6 +37,7 @@ type ( HTTPS bool `json:"https" jsonschema:"required"` AutoCert bool `json:"autoCert" jsonschema:"omitempty"` XForwardedFor bool `json:"xForwardedFor" jsonschema:"omitempty"` + Address string `json:"address" jsonschema:"omitempty"` Port uint16 `json:"port" jsonschema:"required,minimum=1"` ClientMaxBodySize int64 `json:"clientMaxBodySize" jsonschema:"omitempty"` KeepAliveTimeout string `json:"keepAliveTimeout" jsonschema:"omitempty,format=duration"` From 09a055b5f11d8cbcffff55111a66da5db906057c Mon Sep 17 00:00:00 2001 From: chen Date: Thu, 2 Nov 2023 18:37:26 +0800 Subject: [PATCH 08/22] mv omitempty from jsonschema to json, to the generated yaml is more clean --- pkg/filters/proxies/httpproxy/pool.go | 14 +++++++------- pkg/filters/proxies/httpproxy/proxy.go | 8 ++++---- pkg/filters/proxies/loadbalance.go | 10 +++++----- pkg/filters/proxies/server.go | 6 +++--- pkg/filters/proxies/serverpool.go | 10 +++++----- pkg/supervisor/spec.go | 2 +- 6 files changed, 25 insertions(+), 25 deletions(-) diff --git a/pkg/filters/proxies/httpproxy/pool.go b/pkg/filters/proxies/httpproxy/pool.go index 6ed24f3c5e..6abd9d4251 100644 --- a/pkg/filters/proxies/httpproxy/pool.go +++ b/pkg/filters/proxies/httpproxy/pool.go @@ -181,16 +181,16 @@ type ServerPool struct { type ServerPoolSpec struct { BaseServerPoolSpec `json:",inline"` - Filter *RequestMatcherSpec `json:"filter" jsonschema:"omitempty"` - SpanName string `json:"spanName" jsonschema:"omitempty"` - ServerMaxBodySize int64 `json:"serverMaxBodySize" jsonschema:"omitempty"` - Timeout string `json:"timeout" jsonschema:"omitempty,format=duration"` - RetryPolicy string `json:"retryPolicy" jsonschema:"omitempty"` - CircuitBreakerPolicy string `json:"circuitBreakerPolicy" jsonschema:"omitempty"` + Filter *RequestMatcherSpec `json:"filter,omitempty" jsonschema:"omitempty"` + SpanName string `json:"spanName,omitempty" jsonschema:"omitempty"` + ServerMaxBodySize int64 `json:"serverMaxBodySize,omitempty" jsonschema:"omitempty"` + Timeout string `json:"timeout,omitempty" jsonschema:"omitempty,format=duration"` + RetryPolicy string `json:"retryPolicy,omitempty" jsonschema:"omitempty"` + CircuitBreakerPolicy string `json:"circuitBreakerPolicy,omitempty" jsonschema:"omitempty"` MemoryCache *MemoryCacheSpec `json:"memoryCache,omitempty" jsonschema:"omitempty"` // FailureCodes would be 5xx if it isn't assigned any value. - FailureCodes []int `json:"failureCodes" jsonschema:"omitempty,uniqueItems=true"` + FailureCodes []int `json:"failureCodes,omitempty" jsonschema:"omitempty,uniqueItems=true"` } // ServerPoolStatus is the status of Pool. diff --git a/pkg/filters/proxies/httpproxy/proxy.go b/pkg/filters/proxies/httpproxy/proxy.go index e67cb19eba..834ee2a0de 100644 --- a/pkg/filters/proxies/httpproxy/proxy.go +++ b/pkg/filters/proxies/httpproxy/proxy.go @@ -110,10 +110,10 @@ type ( MirrorPool *ServerPoolSpec `json:"mirrorPool,omitempty" jsonschema:"omitempty"` Compression *CompressionSpec `json:"compression,omitempty" jsonschema:"omitempty"` MTLS *MTLS `json:"mtls,omitempty" jsonschema:"omitempty"` - MaxIdleConns int `json:"maxIdleConns" jsonschema:"omitempty"` - MaxIdleConnsPerHost int `json:"maxIdleConnsPerHost" jsonschema:"omitempty"` - MaxRedirection int `json:"maxRedirection" jsonschema:"omitempty"` - ServerMaxBodySize int64 `json:"serverMaxBodySize" jsonschema:"omitempty"` + MaxIdleConns int `json:"maxIdleConns,omitempty" jsonschema:"omitempty"` + MaxIdleConnsPerHost int `json:"maxIdleConnsPerHost,omitempty" jsonschema:"omitempty"` + MaxRedirection int `json:"maxRedirection,omitempty" jsonschema:"omitempty"` + ServerMaxBodySize int64 `json:"serverMaxBodySize,omitempty" jsonschema:"omitempty"` } // Status is the status of Proxy. diff --git a/pkg/filters/proxies/loadbalance.go b/pkg/filters/proxies/loadbalance.go index 8a31fca916..e591ffb064 100644 --- a/pkg/filters/proxies/loadbalance.go +++ b/pkg/filters/proxies/loadbalance.go @@ -54,11 +54,11 @@ type LoadBalancer interface { // this is not good as new policies could be added in the future, we should // convert it to a map later. type LoadBalanceSpec struct { - Policy string `json:"policy" jsonschema:"omitempty"` - HeaderHashKey string `json:"headerHashKey" jsonschema:"omitempty"` - ForwardKey string `json:"forwardKey" jsonschema:"omitempty"` - StickySession *StickySessionSpec `json:"stickySession" jsonschema:"omitempty"` - HealthCheck *HealthCheckSpec `json:"healthCheck" jsonschema:"omitempty"` + Policy string `json:"policy,omitempty" jsonschema:"omitempty"` + HeaderHashKey string `json:"headerHashKey,omitempty" jsonschema:"omitempty"` + ForwardKey string `json:"forwardKey,omitempty" jsonschema:"omitempty"` + StickySession *StickySessionSpec `json:"stickySession,omitempty" jsonschema:"omitempty"` + HealthCheck *HealthCheckSpec `json:"healthCheck,omitempty" jsonschema:"omitempty"` } // LoadBalancePolicy is the interface of a load balance policy. diff --git a/pkg/filters/proxies/server.go b/pkg/filters/proxies/server.go index 6ea5ff2a28..76bfd44493 100644 --- a/pkg/filters/proxies/server.go +++ b/pkg/filters/proxies/server.go @@ -27,9 +27,9 @@ import ( // Server is a backend proxy server. type Server struct { URL string `json:"url" jsonschema:"required,format=url"` - Tags []string `json:"tags" jsonschema:"omitempty,uniqueItems=true"` - Weight int `json:"weight" jsonschema:"omitempty,minimum=0,maximum=100"` - KeepHost bool `json:"keepHost" jsonschema:"omitempty,default=false"` + Tags []string `json:"tags,omitempty" jsonschema:"omitempty,uniqueItems=true"` + Weight int `json:"weight,omitempty" jsonschema:"omitempty,minimum=0,maximum=100"` + KeepHost bool `json:"keepHost,omitempty" jsonschema:"omitempty,default=false"` AddrIsHostName bool `json:"-"` Unhealth bool `json:"-"` // HealthCounter is used to count the number of successive health checks diff --git a/pkg/filters/proxies/serverpool.go b/pkg/filters/proxies/serverpool.go index 720943df45..c0d152f137 100644 --- a/pkg/filters/proxies/serverpool.go +++ b/pkg/filters/proxies/serverpool.go @@ -45,11 +45,11 @@ type ServerPoolBase struct { // ServerPoolBaseSpec is the spec for a base server pool. type ServerPoolBaseSpec struct { - ServerTags []string `json:"serverTags" jsonschema:"omitempty,uniqueItems=true"` - Servers []*Server `json:"servers" jsonschema:"omitempty"` - ServiceRegistry string `json:"serviceRegistry" jsonschema:"omitempty"` - ServiceName string `json:"serviceName" jsonschema:"omitempty"` - LoadBalance *LoadBalanceSpec `json:"loadBalance" jsonschema:"omitempty"` + ServerTags []string `json:"serverTags,omitempty" jsonschema:"omitempty,uniqueItems=true"` + Servers []*Server `json:"servers,omitempty" jsonschema:"omitempty"` + ServiceRegistry string `json:"serviceRegistry,omitempty" jsonschema:"omitempty"` + ServiceName string `json:"serviceName,omitempty" jsonschema:"omitempty"` + LoadBalance *LoadBalanceSpec `json:"loadBalance,omitempty" jsonschema:"omitempty"` } // Validate validates ServerPoolSpec. diff --git a/pkg/supervisor/spec.go b/pkg/supervisor/spec.go index 09695fbd76..0bed2ae20d 100644 --- a/pkg/supervisor/spec.go +++ b/pkg/supervisor/spec.go @@ -47,7 +47,7 @@ type ( Version string `json:"version" jsonschema:"required"` // RFC3339 format - CreatedAt string `json:"createdAt" jsonschema:"omitempty"` + CreatedAt string `json:"createdAt,omitempty" jsonschema:"omitempty"` } ) From 977554fd89ba7f015ea70d9fba8f381cc4e3b075 Mon Sep 17 00:00:00 2001 From: chen Date: Fri, 3 Nov 2023 17:46:47 +0800 Subject: [PATCH 09/22] finish nginx convert code --- cmd/client/commandv2/common/spec.go | 17 +++ cmd/client/commandv2/convert/nginx/cmd.go | 14 +- cmd/client/commandv2/convert/nginx/convert.go | 143 +++++++++++++++++- .../commandv2/convert/nginx/convert_test.go | 58 +++++++ .../commandv2/convert/nginx/parse_test.go | 9 +- .../commandv2/convert/nginx/test_test.go | 18 +++ cmd/client/commandv2/convert/nginx/utils.go | 3 +- pkg/filters/builder/builder.go | 15 +- pkg/filters/builder/extrafuncs.go | 13 ++ pkg/filters/builder/requestadaptor.go | 10 +- pkg/filters/proxies/httpproxy/wspool.go | 10 +- .../httpprot/httpheader/httpheader.go | 6 +- 12 files changed, 286 insertions(+), 30 deletions(-) diff --git a/cmd/client/commandv2/common/spec.go b/cmd/client/commandv2/common/spec.go index 3f77b9a1cd..4bf6baf027 100644 --- a/cmd/client/commandv2/common/spec.go +++ b/cmd/client/commandv2/common/spec.go @@ -20,6 +20,7 @@ package common import ( "github.com/megaease/easegress/v2/cmd/client/general" "github.com/megaease/easegress/v2/pkg/filters" + "github.com/megaease/easegress/v2/pkg/filters/builder" "github.com/megaease/easegress/v2/pkg/filters/proxies/httpproxy" "github.com/megaease/easegress/v2/pkg/object/httpserver" "github.com/megaease/easegress/v2/pkg/object/pipeline" @@ -82,6 +83,22 @@ func NewProxyFilterSpec(name string) *httpproxy.Spec { return spec } +// NewWebsocketFilterSpec returns a new WebsocketFilterSpec. +func NewWebsocketFilterSpec(name string) *httpproxy.WebSocketProxySpec { + spec := GetDefaultFilterSpec(httpproxy.WebSocketProxyKind).(*httpproxy.WebSocketProxySpec) + spec.BaseSpec.MetaSpec.Name = name + spec.BaseSpec.MetaSpec.Kind = httpproxy.WebSocketProxyKind + return spec +} + +// NewRequestAdaptorFilterSpec returns a new RequestAdaptorFilterSpec. +func NewRequestAdaptorFilterSpec(name string) *builder.RequestAdaptorSpec { + spec := GetDefaultFilterSpec(builder.RequestAdaptorKind).(*builder.RequestAdaptorSpec) + spec.BaseSpec.MetaSpec.Name = name + spec.BaseSpec.MetaSpec.Kind = builder.RequestAdaptorKind + return spec +} + // GetDefaultFilterSpec returns the default filter spec of the kind. func GetDefaultFilterSpec(kind string) filters.Spec { return filters.GetKind(kind).DefaultSpec() diff --git a/cmd/client/commandv2/convert/nginx/cmd.go b/cmd/client/commandv2/convert/nginx/cmd.go index 659342dc21..cfebb939ff 100644 --- a/cmd/client/commandv2/convert/nginx/cmd.go +++ b/cmd/client/commandv2/convert/nginx/cmd.go @@ -18,7 +18,6 @@ package nginx import ( - "encoding/json" "fmt" "math/rand" @@ -57,12 +56,13 @@ func Cmd() *cobra.Command { if err != nil { general.ExitWithError(err) } - fmt.Println(hs, pls) - data, err := json.Marshal(payload) - if err != nil { - general.ExitWithError(err) - } - fmt.Println(string(data)) + printYaml(hs) + printYaml(pls) + // data, err := json.Marshal(payload) + // if err != nil { + // general.ExitWithError(err) + // } + // fmt.Println(string(data)) }, } cmd.Flags().StringVarP(&flags.NginxConf, "file", "f", "", "nginx.conf file path") diff --git a/cmd/client/commandv2/convert/nginx/convert.go b/cmd/client/commandv2/convert/nginx/convert.go index 7fda105e98..971b476b6a 100644 --- a/cmd/client/commandv2/convert/nginx/convert.go +++ b/cmd/client/commandv2/convert/nginx/convert.go @@ -19,11 +19,19 @@ package nginx import ( "fmt" + "regexp" + "sort" + "strings" "github.com/megaease/easegress/v2/cmd/client/commandv2/common" "github.com/megaease/easegress/v2/cmd/client/general" "github.com/megaease/easegress/v2/pkg/filters" + "github.com/megaease/easegress/v2/pkg/filters/builder" + "github.com/megaease/easegress/v2/pkg/filters/proxies" + "github.com/megaease/easegress/v2/pkg/filters/proxies/httpproxy" "github.com/megaease/easegress/v2/pkg/object/httpserver/routers" + "github.com/megaease/easegress/v2/pkg/protocols/httpprot/httpheader" + "github.com/megaease/easegress/v2/pkg/util/codectool" ) func convertConfig(options *Options, config *Config) ([]*common.HTTPServerSpec, []*common.PipelineSpec, error) { @@ -68,6 +76,9 @@ func convertServerBase(spec *common.HTTPServerSpec, base ServerBase) *common.HTT return spec } +// convertRule converts nginx conf to easegress rule. +// exact path > prefix path > regexp path. +// prefix path should be sorted by path length. func convertRule(options *Options, rule *Rule) (*routers.Rule, []*common.PipelineSpec) { router := &routers.Rule{ Hosts: make([]routers.Host, 0), @@ -79,35 +90,157 @@ func convertRule(options *Options, rule *Rule) (*routers.Rule, []*common.Pipelin IsRegexp: h.IsRegexp, }) } + pipelines := make([]*common.PipelineSpec, 0) + exactPaths := make([]*routers.Path, 0) + prefixPaths := make([]*routers.Path, 0) + rePaths := make([]*routers.Path, 0) for _, p := range rule.Paths { name := options.GetPipelineName(p.Path) - path := routers.Path{ + path := &routers.Path{ Backend: name, } + // path for websocket should not limit body size. + if isWebsocket(p.Backend) { + path.ClientMaxBodySize = -1 + } switch p.Type { case PathTypeExact: path.Path = p.Path + exactPaths = append(exactPaths, path) case PathTypePrefix: path.PathPrefix = p.Path + prefixPaths = append(prefixPaths, path) case PathTypeRe: path.PathRegexp = p.Path + rePaths = append(rePaths, path) case PathTypeReInsensitive: path.PathRegexp = fmt.Sprintf("(?i)%s", p.Path) + rePaths = append(rePaths, path) default: general.Warnf("unknown path type: %s", p.Type) } - router.Paths = append(router.Paths, &path) pipelineSpec := convertProxy(name, p.Backend) pipelines = append(pipelines, pipelineSpec) } + sort.Slice(prefixPaths, func(i, j int) bool { + return prefixPaths[i].Path > prefixPaths[j].Path + }) + router.Paths = append(router.Paths, exactPaths...) + router.Paths = append(router.Paths, prefixPaths...) + router.Paths = append(router.Paths, rePaths...) return router, pipelines } func convertProxy(name string, info *ProxyInfo) *common.PipelineSpec { pipeline := common.NewPipelineSpec(name) - // TODO: set header, update proxy filter - proxy := common.NewProxyFilterSpec("proxy") - pipeline.SetFilters([]filters.Spec{proxy}) + + flow := make([]filters.Spec, 0) + if len(info.SetHeaders) != 0 { + adaptor := getRequestAdaptor(info) + flow = append(flow, adaptor) + } + + var proxy filters.Spec + if isWebsocket(info) { + proxy = getWebsocketFilter(info) + } else { + proxy = getProxyFilter(info) + } + flow = append(flow, proxy) + pipeline.SetFilters(flow) return pipeline } + +var nginxEmbeddedVarRe *regexp.Regexp +var nginxToTemplateMap = map[string]string{ + "$host": ".req.Host", + "$hostname": ".req.Host", + "$content_length": `header .req.Header "Content-Length"`, + "$content_type": `header .req.Header "Content-Type"`, + "$remote_addr": ".req.RemoteAddr", + "$remote_user": "username .req", + "$request_body": ".req.Body", + "$request_method": ".req.Method", + "$request_uri": ".req.RequestURI", + "$scheme": ".req.URL.Scheme", +} + +func translateNginxEmbeddedVar(s string) string { + if nginxEmbeddedVarRe == nil { + nginxEmbeddedVarRe = regexp.MustCompile(`\$[a-zA-Z0-9_]+`) + } + return nginxEmbeddedVarRe.ReplaceAllStringFunc(s, func(s string) string { + newValue := nginxToTemplateMap[s] + if newValue == "" { + newValue = s + msg := "nginx embedded value %s is not supported now, " + msg += "please check easegress RequestAdaptor filter for more information about template." + general.Warnf(msg, s) + } + return fmt.Sprintf("{{ %s }}", newValue) + }) +} + +func getRequestAdaptor(info *ProxyInfo) *builder.RequestAdaptorSpec { + spec := common.NewRequestAdaptorFilterSpec("request-adaptor") + template := &builder.RequestAdaptorTemplate{ + Header: &httpheader.AdaptSpec{ + Set: make(map[string]string), + }, + } + for k, v := range info.SetHeaders { + if k == "Upgrade" || k == "Connection" { + continue + } + template.Header.Set[k] = translateNginxEmbeddedVar(v) + } + data := codectool.MustMarshalYAML(template) + spec.Template = string(data) + return spec +} + +func isWebsocket(info *ProxyInfo) bool { + return info.SetHeaders["Upgrade"] != "" && info.SetHeaders["Connection"] != "" +} + +func getWebsocketFilter(info *ProxyInfo) *httpproxy.WebSocketProxySpec { + for i, s := range info.Servers { + s.Server = strings.Replace(s.Server, "http://", "ws://", 1) + s.Server = strings.Replace(s.Server, "https://", "wss://", 1) + info.Servers[i] = s + } + spec := common.NewWebsocketFilterSpec("websocket") + spec.Pools = []*httpproxy.WebSocketServerPoolSpec{{ + BaseServerPoolSpec: getBaseServerPool(info), + }} + return spec +} + +func getProxyFilter(info *ProxyInfo) *httpproxy.Spec { + spec := common.NewProxyFilterSpec("proxy") + spec.Pools = []*httpproxy.ServerPoolSpec{{ + BaseServerPoolSpec: getBaseServerPool(info), + }} + return spec +} + +func getBaseServerPool(info *ProxyInfo) httpproxy.BaseServerPoolSpec { + servers := make([]*proxies.Server, len(info.Servers)) + policy := proxies.LoadBalancePolicyRoundRobin + for i, s := range info.Servers { + servers[i] = &proxies.Server{ + URL: s.Server, + Weight: s.Weight, + } + if s.Weight != 1 { + policy = proxies.LoadBalancePolicyWeightedRandom + } + } + return httpproxy.BaseServerPoolSpec{ + Servers: servers, + LoadBalance: &proxies.LoadBalanceSpec{ + Policy: policy, + }, + } +} diff --git a/cmd/client/commandv2/convert/nginx/convert_test.go b/cmd/client/commandv2/convert/nginx/convert_test.go index 0d041760e3..67af3bd23d 100644 --- a/cmd/client/commandv2/convert/nginx/convert_test.go +++ b/cmd/client/commandv2/convert/nginx/convert_test.go @@ -16,3 +16,61 @@ */ package nginx + +import ( + "fmt" + "net/http" + "strings" + "testing" + + "github.com/megaease/easegress/v2/pkg/filters" + "github.com/megaease/easegress/v2/pkg/filters/builder" + "github.com/megaease/easegress/v2/pkg/protocols/httpprot" + "github.com/stretchr/testify/assert" +) + +func TestGetRequestAdaptor(t *testing.T) { + assert := assert.New(t) + req, err := http.NewRequest("GET", "http://example.com", strings.NewReader("test")) + req.Header.Set("Content-Length", "4") + req.Header.Set("Content-Type", "text/plain") + req.SetBasicAuth("user", "pass") + req.RemoteAddr = "localhost:8080" + req.RequestURI = "/apis/v1" + assert.Nil(err) + ctx := newContext(t, req) + info := &ProxyInfo{ + SetHeaders: map[string]string{ + "X-Host": "$host", + "X-Hostname": "$hostname", + "X-Content": "$content_length", + "X-Content-Type": "$content_type", + "X-Remote-Addr": "$remote_addr", + "X-Remote-User": "$remote_user", + "X-Request-Body": "$request_body", + "X-Method": "$request_method", + "X-Request-URI": "$request_uri", + "X-Scheme": "$scheme", + }, + } + spec := getRequestAdaptor(info) + ra := filters.GetKind(builder.RequestAdaptorKind).CreateInstance(spec) + ra.Init() + ra.Handle(ctx) + h := ctx.GetInputRequest().(*httpprot.Request).Header() + expected := map[string]string{ + "X-Host": "example.com", + "X-Hostname": "example.com", + "X-Content": "4", + "X-Content-Type": "text/plain", + "X-Remote-Addr": "localhost:8080", + "X-Remote-User": "user", + "X-Request-Body": "test", + "X-Method": "GET", + "X-Request-URI": "/apis/v1", + "X-Scheme": "http", + } + for k, v := range expected { + assert.Equal(v, h.Get(k), fmt.Sprintf("header %s", k)) + } +} diff --git a/cmd/client/commandv2/convert/nginx/parse_test.go b/cmd/client/commandv2/convert/nginx/parse_test.go index 1b7381a0d9..1bbdabe28a 100644 --- a/cmd/client/commandv2/convert/nginx/parse_test.go +++ b/cmd/client/commandv2/convert/nginx/parse_test.go @@ -176,6 +176,13 @@ func TestParsePayload(t *testing.T) { location /upstream { proxy_pass http://backend; } + + location /websocket { + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + proxy_pass http://localhost:9090; + } } } ` @@ -190,7 +197,7 @@ func TestParsePayload(t *testing.T) { opt.init() hs, pls, err := convertConfig(opt, config) assert.Nil(t, err) - printYaml(hs) + printJson(hs) printYaml(pls) } } diff --git a/cmd/client/commandv2/convert/nginx/test_test.go b/cmd/client/commandv2/convert/nginx/test_test.go index 07fe393034..1e8aba4552 100644 --- a/cmd/client/commandv2/convert/nginx/test_test.go +++ b/cmd/client/commandv2/convert/nginx/test_test.go @@ -18,13 +18,22 @@ package nginx import ( + "net/http" "os" "path/filepath" "testing" + "github.com/megaease/easegress/v2/pkg/context" + "github.com/megaease/easegress/v2/pkg/logger" + "github.com/megaease/easegress/v2/pkg/protocols/httpprot" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) +func init() { + logger.InitMock() +} + type tempTestDir struct { dir string t *testing.T @@ -54,3 +63,12 @@ func (dir *tempTestDir) Clean() { } os.Remove(dir.dir) } + +func newContext(t *testing.T, req *http.Request) *context.Context { + ctx := context.New(nil) + r, err := httpprot.NewRequest(req) + assert.Nil(t, err) + r.FetchPayload(0) + ctx.SetRequest(context.DefaultNamespace, r) + return ctx +} diff --git a/cmd/client/commandv2/convert/nginx/utils.go b/cmd/client/commandv2/convert/nginx/utils.go index e29a4bbcf6..b5aeede62d 100644 --- a/cmd/client/commandv2/convert/nginx/utils.go +++ b/cmd/client/commandv2/convert/nginx/utils.go @@ -20,12 +20,13 @@ package nginx import ( "encoding/json" "fmt" + "strings" "gopkg.in/yaml.v3" ) func directiveInfo(d *Directive) string { - return fmt.Sprintf("directive %s in line %d of file %s", d.Directive, d.Line, d.File) + return fmt.Sprintf("directive <%s %s> of %s:%d", d.Directive, strings.Join(d.Args, " "), d.File, d.Line) } func printJson(v interface{}) { diff --git a/pkg/filters/builder/builder.go b/pkg/filters/builder/builder.go index ebabbf763f..d675283f79 100644 --- a/pkg/filters/builder/builder.go +++ b/pkg/filters/builder/builder.go @@ -39,9 +39,9 @@ type ( // Spec is the spec of Builder. Spec struct { - LeftDelim string `json:"leftDelim" jsonschema:"omitempty"` - RightDelim string `json:"rightDelim" jsonschema:"omitempty"` - Template string `json:"template" jsonschema:"omitempty"` + LeftDelim string `json:"leftDelim,omitempty" jsonschema:"omitempty"` + RightDelim string `json:"rightDelim,omitempty" jsonschema:"omitempty"` + Template string `json:"template,omitempty" jsonschema:"omitempty"` } ) @@ -79,15 +79,24 @@ func prepareBuilderData(ctx *context.Context) (map[string]interface{}, error) { requests := make(map[string]interface{}) responses := make(map[string]interface{}) + var defaultReq, defaultResp interface{} for k, v := range ctx.Requests() { requests[k] = v.ToBuilderRequest(k) + if k == context.DefaultNamespace { + defaultReq = requests[k] + } } for k, v := range ctx.Responses() { responses[k] = v.ToBuilderResponse(k) + if k == context.DefaultNamespace { + defaultResp = responses[k] + } } return map[string]interface{}{ + "req": defaultReq, + "resp": defaultResp, "requests": requests, "responses": responses, "data": ctx.Data(), diff --git a/pkg/filters/builder/extrafuncs.go b/pkg/filters/builder/extrafuncs.go index 65c642eb88..ea90b7a168 100644 --- a/pkg/filters/builder/extrafuncs.go +++ b/pkg/filters/builder/extrafuncs.go @@ -20,6 +20,7 @@ package builder import ( "encoding/json" "fmt" + "net/http" "strconv" "strings" "text/template" @@ -141,4 +142,16 @@ var extraFuncs = template.FuncMap{ "panic": func(v interface{}) interface{} { panic(v) }, + + "header": func(header http.Header, key string) string { + return header.Get(key) + }, + + "username": func(req interface{}) string { + type BasicAuth interface { + BasicAuth() (username, password string, ok bool) + } + username, _, _ := req.(BasicAuth).BasicAuth() + return username + }, } diff --git a/pkg/filters/builder/requestadaptor.go b/pkg/filters/builder/requestadaptor.go index ef9cb8f7b0..a74d1f0d8c 100644 --- a/pkg/filters/builder/requestadaptor.go +++ b/pkg/filters/builder/requestadaptor.go @@ -145,18 +145,18 @@ type ( Spec `json:",inline"` RequestAdaptorTemplate `json:",inline"` - Compress string `json:"compress" jsonschema:"omitempty"` - Decompress string `json:"decompress" jsonschema:"omitempty"` + Compress string `json:"compress,omitempty" jsonschema:"omitempty"` + Decompress string `json:"decompress,omitempty" jsonschema:"omitempty"` Sign *SignerSpec `json:"sign,omitempty" jsonschema:"omitempty"` } // RequestAdaptorTemplate is the template of the request adaptor. RequestAdaptorTemplate struct { - Host string `json:"host" jsonschema:"omitempty"` - Method string `json:"method" jsonschema:"omitempty,format=httpmethod"` + Host string `json:"host,omitempty" jsonschema:"omitempty"` + Method string `json:"method,omitempty" jsonschema:"omitempty,format=httpmethod"` Path *pathadaptor.Spec `json:"path,omitempty" jsonschema:"omitempty"` Header *httpheader.AdaptSpec `json:"header,omitempty" jsonschema:"omitempty"` - Body string `json:"body" jsonschema:"omitempty"` + Body string `json:"body,omitempty" jsonschema:"omitempty"` } // SignerSpec is the spec of the request signer. diff --git a/pkg/filters/proxies/httpproxy/wspool.go b/pkg/filters/proxies/httpproxy/wspool.go index 35a4a9ce49..d645f0991c 100644 --- a/pkg/filters/proxies/httpproxy/wspool.go +++ b/pkg/filters/proxies/httpproxy/wspool.go @@ -47,11 +47,11 @@ type WebSocketServerPool struct { // WebSocketServerPoolSpec is the spec for a server pool. type WebSocketServerPoolSpec struct { BaseServerPoolSpec `json:",inline"` - ClientMaxMsgSize int64 `json:"clientMaxMsgSize" jsonschema:"omitempty"` - ServerMaxMsgSize int64 `json:"serverMaxMsgSize" jsonschema:"omitempty"` - Filter *RequestMatcherSpec `json:"filter" jsonschema:"omitempty"` - InsecureSkipVerify bool `json:"insecureSkipVerify" jsonschema:"omitempty"` - OriginPatterns []string `json:"originPatterns" jsonschema:"omitempty"` + ClientMaxMsgSize int64 `json:"clientMaxMsgSize,omitempty" jsonschema:"omitempty"` + ServerMaxMsgSize int64 `json:"serverMaxMsgSize,omitempty" jsonschema:"omitempty"` + Filter *RequestMatcherSpec `json:"filter,omitempty" jsonschema:"omitempty"` + InsecureSkipVerify bool `json:"insecureSkipVerify,omitempty" jsonschema:"omitempty"` + OriginPatterns []string `json:"originPatterns,omitempty" jsonschema:"omitempty"` } // NewWebSocketServerPool creates a new server pool according to spec. diff --git a/pkg/protocols/httpprot/httpheader/httpheader.go b/pkg/protocols/httpprot/httpheader/httpheader.go index 140d7d0613..755ad10745 100644 --- a/pkg/protocols/httpprot/httpheader/httpheader.go +++ b/pkg/protocols/httpprot/httpheader/httpheader.go @@ -31,11 +31,11 @@ type ( // AdaptSpec describes rules for adapting. AdaptSpec struct { - Del []string `json:"del" jsonschema:"omitempty,uniqueItems=true"` + Del []string `json:"del,omitempty" jsonschema:"omitempty,uniqueItems=true"` // NOTE: Set and Add allow empty value. - Set map[string]string `json:"set" jsonschema:"omitempty"` - Add map[string]string `json:"add" jsonschema:"omitempty"` + Set map[string]string `json:"set,omitempty" jsonschema:"omitempty"` + Add map[string]string `json:"add,omitempty" jsonschema:"omitempty"` } ) From 17d5e70f76b718c927dc76ffbe28b069ef0f82fc Mon Sep 17 00:00:00 2001 From: chen Date: Fri, 3 Nov 2023 17:54:20 +0800 Subject: [PATCH 10/22] fix go.mod --- go.mod | 2 +- go.sum | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 7f433f34af..54ec61c0ed 100644 --- a/go.mod +++ b/go.mod @@ -40,6 +40,7 @@ require ( github.com/megaease/yaml v0.0.0-20220804061446-4f18d6510aed github.com/mitchellh/mapstructure v1.5.0 github.com/nacos-group/nacos-sdk-go v1.1.0 + github.com/nginxinc/nginx-go-crossplane v0.4.33 github.com/open-policy-agent/opa v0.49.2 github.com/openzipkin/zipkin-go v0.4.1 github.com/patrickmn/go-cache v2.1.0+incompatible @@ -106,7 +107,6 @@ require ( github.com/jstemmer/go-junit-report v1.0.0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/maxbrunsfeld/counterfeiter/v6 v6.6.1 // indirect - github.com/nginxinc/nginx-go-crossplane v0.4.33 // indirect github.com/onsi/ginkgo/v2 v2.9.5 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b // indirect diff --git a/go.sum b/go.sum index b55b7e04cb..15178d58f8 100644 --- a/go.sum +++ b/go.sum @@ -726,6 +726,7 @@ github.com/rs/cors v1.9.0 h1:l9HGsTsHJcvW14Nk7J9KFz8bzeAWXn3CG6bgt7LsrAE= github.com/rs/cors v1.9.0/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/sclevine/spec v1.4.0 h1:z/Q9idDcay5m5irkZ28M7PtQM4aOISzOpj4bUPkDee8= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= From 970e3aff62fe381e6f47e329473305429a0c2e10 Mon Sep 17 00:00:00 2001 From: chen Date: Tue, 7 Nov 2023 16:11:30 +0800 Subject: [PATCH 11/22] add more test for parse --- cmd/client/commandv2/convert/nginx/cmd.go | 44 +++++- .../commandv2/convert/nginx/cmd_test.go | 29 ++++ cmd/client/commandv2/convert/nginx/convert.go | 3 + cmd/client/commandv2/convert/nginx/env.go | 41 ++++- .../commandv2/convert/nginx/parse_test.go | 142 ++++++++++++++++-- cmd/client/commandv2/convert/nginx/types.go | 45 +++--- cmd/client/commandv2/convert/nginx/utils.go | 10 +- .../commandv2/convert/nginx/utils_test.go | 34 +++++ 8 files changed, 294 insertions(+), 54 deletions(-) create mode 100644 cmd/client/commandv2/convert/nginx/cmd_test.go create mode 100644 cmd/client/commandv2/convert/nginx/utils_test.go diff --git a/cmd/client/commandv2/convert/nginx/cmd.go b/cmd/client/commandv2/convert/nginx/cmd.go index cfebb939ff..763f690fae 100644 --- a/cmd/client/commandv2/convert/nginx/cmd.go +++ b/cmd/client/commandv2/convert/nginx/cmd.go @@ -20,8 +20,12 @@ package nginx import ( "fmt" "math/rand" + "os" + "path/filepath" + "github.com/megaease/easegress/v2/cmd/client/commandv2/common" "github.com/megaease/easegress/v2/cmd/client/general" + "github.com/megaease/easegress/v2/pkg/util/codectool" crossplane "github.com/nginxinc/nginx-go-crossplane" "github.com/spf13/cobra" ) @@ -56,13 +60,9 @@ func Cmd() *cobra.Command { if err != nil { general.ExitWithError(err) } - printYaml(hs) - printYaml(pls) - // data, err := json.Marshal(payload) - // if err != nil { - // general.ExitWithError(err) - // } - // fmt.Println(string(data)) + if err := writeYaml(flags.Output, hs, pls); err != nil { + general.ExitWithError(err) + } }, } cmd.Flags().StringVarP(&flags.NginxConf, "file", "f", "", "nginx.conf file path") @@ -100,3 +100,33 @@ func (opt *Options) GetPipelineName(path string) string { } return opt.GetPipelineName(path) } + +func writeYaml(filename string, servers []*common.HTTPServerSpec, pipelines []*common.PipelineSpec) error { + absPath, err := filepath.Abs(filename) + if err != nil { + return err + } + file, err := os.Create(absPath) + if err != nil { + return err + } + defer file.Close() + + for _, s := range servers { + data, err := codectool.MarshalYAML(s) + if err != nil { + return err + } + file.WriteString(string(data)) + file.WriteString("\n---\n") + } + for _, p := range pipelines { + data, err := codectool.MarshalYAML(p) + if err != nil { + return err + } + file.WriteString(string(data)) + file.WriteString("\n---\n") + } + return nil +} diff --git a/cmd/client/commandv2/convert/nginx/cmd_test.go b/cmd/client/commandv2/convert/nginx/cmd_test.go new file mode 100644 index 0000000000..ec50b5ff68 --- /dev/null +++ b/cmd/client/commandv2/convert/nginx/cmd_test.go @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2017, MegaEase + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nginx + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestCmd(t *testing.T) { + cmd := Cmd() + assert.NotNil(t, cmd) +} diff --git a/cmd/client/commandv2/convert/nginx/convert.go b/cmd/client/commandv2/convert/nginx/convert.go index 971b476b6a..8faf5b4a8b 100644 --- a/cmd/client/commandv2/convert/nginx/convert.go +++ b/cmd/client/commandv2/convert/nginx/convert.go @@ -219,6 +219,9 @@ func getWebsocketFilter(info *ProxyInfo) *httpproxy.WebSocketProxySpec { func getProxyFilter(info *ProxyInfo) *httpproxy.Spec { spec := common.NewProxyFilterSpec("proxy") + spec.Compression = &httpproxy.CompressionSpec{ + MinLength: uint32(info.GzipMinLength), + } spec.Pools = []*httpproxy.ServerPoolSpec{{ BaseServerPoolSpec: getBaseServerPool(info), }} diff --git a/cmd/client/commandv2/convert/nginx/env.go b/cmd/client/commandv2/convert/nginx/env.go index ce6b0dac42..fdce8b1862 100644 --- a/cmd/client/commandv2/convert/nginx/env.go +++ b/cmd/client/commandv2/convert/nginx/env.go @@ -69,6 +69,12 @@ type ServerEnv struct { type ProxyEnv struct { Pass *Directive `json:"pass"` ProxySetHeader []*Directive `json:"proxy_set_header"` + Gzip *GzipEnv `json:"gzip"` +} + +type GzipEnv struct { + Gzip *Directive `json:"gzip"` + GzipMinLength *Directive `json:"gzip_min_length"` } func (env *Env) init() { @@ -80,6 +86,8 @@ func (env *Env) init() { "ssl_certificate_key": func(d *Directive) { env.Server.SSLCertificateKey = append(env.Server.SSLCertificateKey, d) }, "proxy_set_header": func(d *Directive) { env.Proxy.ProxySetHeader = append(env.Proxy.ProxySetHeader, d) }, "upstream": func(d *Directive) { env.Upstream = append(env.Upstream, d) }, + "gzip": func(d *Directive) { env.Proxy.Gzip.Gzip = d }, + "gzip_min_length": func(d *Directive) { env.Proxy.Gzip.GzipMinLength = d }, } } @@ -91,6 +99,7 @@ func newEnv() *Env { }, Proxy: &ProxyEnv{ ProxySetHeader: make([]*Directive, 0), + Gzip: &GzipEnv{}, }, Upstream: make([]*Directive, 0), } @@ -195,12 +204,40 @@ func (env *Env) GetProxyInfo() (*ProxyInfo, error) { } } + gzipMinLength := processGzip(p.Gzip) + return &ProxyInfo{ - Servers: servers, - SetHeaders: setHeaders, + Servers: servers, + SetHeaders: setHeaders, + GzipMinLength: gzipMinLength, }, nil } +func processGzip(gzip *GzipEnv) int { + if gzip.Gzip == nil { + return 0 + } + mustContainArgs(gzip.Gzip, 1) + if gzip.Gzip.Args[0] != "on" { + return 0 + } + if gzip.GzipMinLength == nil { + // nginx default value + return 20 + } + mustContainArgs(gzip.GzipMinLength, 1) + minLength, err := strconv.Atoi(gzip.GzipMinLength.Args[0]) + if err != nil { + general.Warnf("%s: invalid number %v, use default value of 20 instead", directiveInfo(gzip.GzipMinLength), err) + return 20 + } + if minLength < 0 { + general.Warnf("%s: negative number, use default value of 20 instead", directiveInfo(gzip.GzipMinLength)) + return 20 + } + return minLength +} + func processProxySetHeader(ds []*Directive) (map[string]string, error) { res := make(map[string]string) for _, d := range ds { diff --git a/cmd/client/commandv2/convert/nginx/parse_test.go b/cmd/client/commandv2/convert/nginx/parse_test.go index 1bbdabe28a..3e88b2a71e 100644 --- a/cmd/client/commandv2/convert/nginx/parse_test.go +++ b/cmd/client/commandv2/convert/nginx/parse_test.go @@ -18,9 +18,12 @@ package nginx import ( + "bytes" "fmt" + "html/template" "testing" + "github.com/megaease/easegress/v2/pkg/util/codectool" crossplane "github.com/nginxinc/nginx-go-crossplane" "github.com/stretchr/testify/assert" ) @@ -154,10 +157,7 @@ func TestParsePayload(t *testing.T) { server { listen 80; - location / { - proxy_set_header X-Path "prefix"; - proxy_pass http://localhost:8080; - } + server_name www.example.com *.example.com; location /apis { proxy_set_header X-Path "apis"; @@ -169,11 +169,9 @@ func TestParsePayload(t *testing.T) { } } - location = /user { - proxy_pass http://localhost:8890; - } - location /upstream { + gzip on; + gzip_min_length 1000; proxy_pass http://backend; } @@ -184,20 +182,134 @@ func TestParsePayload(t *testing.T) { proxy_pass http://localhost:9090; } } + + server { + listen 127.0.0.1:443 ssl; + ssl_client_certificate {{ .CaFile }}; + ssl_certificate {{ .CertFile }}; + ssl_certificate_key {{ .KeyFile }}; + + location = /user { + proxy_pass http://localhost:9999; + } + + location ^~ /user/admin { + proxy_pass http://localhost:9991; + } + + location ~* /user/.* { + proxy_pass http://localhost:9992; + } + + location ~ /user/.* { + proxy_pass http://localhost:9993; + } + } } ` - file := tempDir.Create("nginx.conf", []byte(nginxConf)) + tmplValue := map[string]string{ + "CaFile": tempDir.Create("ca.crt", []byte("ca")), + "CertFile": tempDir.Create("cert.crt", []byte("cert")), + "KeyFile": tempDir.Create("key.crt", []byte("key")), + } + + tmpl, err := template.New("nginx").Parse(nginxConf) + assert.Nil(t, err) + var buffer bytes.Buffer + tmpl.Execute(&buffer, tmplValue) + + file := tempDir.Create("nginx.conf", buffer.Bytes()) payload, err := crossplane.Parse(file, &crossplane.ParseOptions{}) assert.Nil(t, err) config, err := parsePayload(payload) assert.Nil(t, err) - // printJson(config) - opt := &Options{Prefix: "test"} - opt.init() - hs, pls, err := convertConfig(opt, config) + proxyInfo := ` +servers: + - port: 80 + rules: + - hosts: + - value: www.example.com + isRegexp: false + - value: '*.example.com' + isRegexp: false + paths: + - path: /apis + type: prefix + backend: + servers: + - server: http://localhost:8880 + weight: 1 + setHeaders: + X-Path: apis + - path: /apis/v1 + type: prefix + backend: + servers: + - server: http://localhost:8888 + weight: 1 + setHeaders: + X-Path: apis/v1 + - path: /upstream + type: prefix + backend: + servers: + - server: http://localhost:1234 + weight: 1 + - server: http://localhost:2345 + weight: 10 + gzipMinLength: 1000 + - path: /websocket + type: prefix + backend: + servers: + - server: http://localhost:9090 + weight: 1 + setHeaders: + Connection: $connection_upgrade + Upgrade: $http_upgrade + - port: 443 + address: 127.0.0.1 + https: true + caCert: Y2E= + certs: + {{ .CertFile }}: Y2VydA== + keys: + {{ .CertFile }}: a2V5 + rules: + - paths: + - path: /user + type: exact + backend: + servers: + - server: http://localhost:9999 + weight: 1 + - path: /user/admin + type: prefix + backend: + servers: + - server: http://localhost:9991 + weight: 1 + - path: /user/.* + type: caseInsensitiveRegexp + backend: + servers: + - server: http://localhost:9992 + weight: 1 + - path: /user/.* + type: regexp + backend: + servers: + - server: http://localhost:9993 + weight: 1 +` + tmp, err := template.New("proxyInfo").Parse(proxyInfo) + assert.Nil(t, err) + var proxyBuffer bytes.Buffer + tmp.Execute(&proxyBuffer, tmplValue) + expected := &Config{} + err = codectool.UnmarshalYAML(proxyBuffer.Bytes(), expected) assert.Nil(t, err) - printJson(hs) - printYaml(pls) + assert.Equal(t, expected, config) } } diff --git a/cmd/client/commandv2/convert/nginx/types.go b/cmd/client/commandv2/convert/nginx/types.go index cd1e633bfa..04e2c3c62c 100644 --- a/cmd/client/commandv2/convert/nginx/types.go +++ b/cmd/client/commandv2/convert/nginx/types.go @@ -18,37 +18,37 @@ package nginx type Config struct { - Servers []*Server + Servers []*Server `json:"servers"` } type ServerBase struct { - Port int - Address string - HTTPS bool + Port int `json:"port"` + Address string `json:"address"` + HTTPS bool `json:"https"` - CaCert string - Certs map[string]string - Keys map[string]string + CaCert string `json:"caCert"` + Certs map[string]string `json:"certs"` + Keys map[string]string `json:"keys"` } type Server struct { - ServerBase - Rules []*Rule + ServerBase `json:",inline"` + Rules []*Rule `json:"rules"` } type ServerInfo struct { - ServerBase - Hosts []*HostInfo + ServerBase `json:",inline"` + Hosts []*HostInfo `json:"hosts"` } type HostInfo struct { - Value string - IsRegexp bool + Value string `json:"value"` + IsRegexp bool `json:"isRegexp"` } type Rule struct { - Hosts []*HostInfo - Paths []*Path + Hosts []*HostInfo `json:"hosts"` + Paths []*Path `json:"paths"` } type PathType string @@ -61,17 +61,18 @@ const ( ) type Path struct { - Path string - Type PathType - Backend *ProxyInfo + Path string `json:"path"` + Type PathType `json:"type"` + Backend *ProxyInfo `json:"backend"` } type ProxyInfo struct { - Servers []*BackendInfo - SetHeaders map[string]string + Servers []*BackendInfo `json:"servers"` + SetHeaders map[string]string `json:"setHeaders"` + GzipMinLength int `json:"gzipMinLength"` } type BackendInfo struct { - Server string - Weight int + Server string `json:"server"` + Weight int `json:"weight"` } diff --git a/cmd/client/commandv2/convert/nginx/utils.go b/cmd/client/commandv2/convert/nginx/utils.go index b5aeede62d..8f05c9b4ee 100644 --- a/cmd/client/commandv2/convert/nginx/utils.go +++ b/cmd/client/commandv2/convert/nginx/utils.go @@ -18,23 +18,17 @@ package nginx import ( - "encoding/json" "fmt" "strings" - "gopkg.in/yaml.v3" + "github.com/megaease/easegress/v2/pkg/util/codectool" ) func directiveInfo(d *Directive) string { return fmt.Sprintf("directive <%s %s> of %s:%d", d.Directive, strings.Join(d.Args, " "), d.File, d.Line) } -func printJson(v interface{}) { - b, _ := json.MarshalIndent(v, "", " ") - fmt.Println(string(b)) -} - func printYaml(v interface{}) { - b, _ := yaml.Marshal(v) + b, _ := codectool.MarshalYAML(v) fmt.Println(string(b)) } diff --git a/cmd/client/commandv2/convert/nginx/utils_test.go b/cmd/client/commandv2/convert/nginx/utils_test.go new file mode 100644 index 0000000000..31f06c02b5 --- /dev/null +++ b/cmd/client/commandv2/convert/nginx/utils_test.go @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2017, MegaEase + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nginx + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestDirectiveInfo(t *testing.T) { + d := &Directive{ + Directive: "test", + Args: []string{"a", "b"}, + File: "test.conf", + Line: 100, + } + assert.Equal(t, "directive of test.conf:100", directiveInfo(d)) +} From 0d472e4d73a1a0b880ef4232337deed0133be6d0 Mon Sep 17 00:00:00 2001 From: chen Date: Tue, 7 Nov 2023 17:51:11 +0800 Subject: [PATCH 12/22] add more test for convert --- cmd/client/commandv2/common/spec.go | 5 +- cmd/client/commandv2/convert/nginx/convert.go | 6 +- .../commandv2/convert/nginx/convert_test.go | 201 ++++++++++++++++++ cmd/client/commandv2/convert/nginx/utils.go | 7 - 4 files changed, 209 insertions(+), 10 deletions(-) diff --git a/cmd/client/commandv2/common/spec.go b/cmd/client/commandv2/common/spec.go index 4bf6baf027..e893fe5d26 100644 --- a/cmd/client/commandv2/common/spec.go +++ b/cmd/client/commandv2/common/spec.go @@ -51,11 +51,14 @@ func (p *PipelineSpec) SetFilters(filters []filters.Spec) { // NewHTTPServerSpec returns a new HTTPServerSpec. func NewHTTPServerSpec(name string) *HTTPServerSpec { - return &HTTPServerSpec{ + spec := &HTTPServerSpec{ Name: name, Kind: httpserver.Kind, Spec: *getDefaultHTTPServerSpec(), } + spec.Spec.Certs = map[string]string{} + spec.Spec.Keys = map[string]string{} + return spec } // NewPipelineSpec returns a new PipelineSpec. diff --git a/cmd/client/commandv2/convert/nginx/convert.go b/cmd/client/commandv2/convert/nginx/convert.go index 8faf5b4a8b..09ac5a0506 100644 --- a/cmd/client/commandv2/convert/nginx/convert.go +++ b/cmd/client/commandv2/convert/nginx/convert.go @@ -219,8 +219,10 @@ func getWebsocketFilter(info *ProxyInfo) *httpproxy.WebSocketProxySpec { func getProxyFilter(info *ProxyInfo) *httpproxy.Spec { spec := common.NewProxyFilterSpec("proxy") - spec.Compression = &httpproxy.CompressionSpec{ - MinLength: uint32(info.GzipMinLength), + if info.GzipMinLength != 0 { + spec.Compression = &httpproxy.CompressionSpec{ + MinLength: uint32(info.GzipMinLength), + } } spec.Pools = []*httpproxy.ServerPoolSpec{{ BaseServerPoolSpec: getBaseServerPool(info), diff --git a/cmd/client/commandv2/convert/nginx/convert_test.go b/cmd/client/commandv2/convert/nginx/convert_test.go index 67af3bd23d..10643f2b53 100644 --- a/cmd/client/commandv2/convert/nginx/convert_test.go +++ b/cmd/client/commandv2/convert/nginx/convert_test.go @@ -23,9 +23,11 @@ import ( "strings" "testing" + "github.com/megaease/easegress/v2/cmd/client/commandv2/common" "github.com/megaease/easegress/v2/pkg/filters" "github.com/megaease/easegress/v2/pkg/filters/builder" "github.com/megaease/easegress/v2/pkg/protocols/httpprot" + "github.com/megaease/easegress/v2/pkg/util/codectool" "github.com/stretchr/testify/assert" ) @@ -74,3 +76,202 @@ func TestGetRequestAdaptor(t *testing.T) { assert.Equal(v, h.Get(k), fmt.Sprintf("header %s", k)) } } + +func TestConvertConfig(t *testing.T) { + options := &Options{ + Prefix: "test-convert", + } + options.init() + conf := ` +servers: +- port: 8080 + address: localhost + https: true + caCert: caCertBase64Str + certs: + cert1: cert1Base64Str + keys: + cert1: key1Base64Str + rules: + - hosts: + - value: www.example.com + isRegexp: false + - isRegexp: true + value: '.*\.example\.com' + paths: + - path: /apis + type: prefix + backend: + servers: + - server: http://localhost:8880 + weight: 1 + - server: http://localhost:8881 + weight: 2 + setHeaders: + X-Path: apis + gzipMinLength: 1000 + - path: /exact + type: exact + backend: + servers: + - server: http://localhost:9999 + weight: 1 + - path: /regexp + type: regexp + backend: + servers: + - server: http://localhost:7777 + weight: 1 + - path: /case-insensitive-regexp + type: caseInsensitiveRegexp + backend: + servers: + - server: http://localhost:6666 + weight: 1 +` + config := &Config{} + err := codectool.Unmarshal([]byte(conf), config) + assert.Nil(t, err) + httpServers, pipelines, err := convertConfig(options, config) + assert.Nil(t, err) + assert.Equal(t, 1, len(httpServers)) + assert.Equal(t, 4, len(pipelines)) + serverYaml := ` +name: test-convert-8080 +kind: HTTPServer +https: true +caCertBase64: caCertBase64Str +certs: + cert1: cert1Base64Str +keys: + cert1: key1Base64Str +port: 8080 +address: localhost +rules: +- hosts: + - value: www.example.com + isRegexp: false + - isRegexp: true + value: '.*\.example\.com' + paths: + - path: /exact + backend: test-convert-exact + - pathPrefix: /apis + backend: test-convert-apis + - pathRegexp: /regexp + backend: test-convert-regexp + - pathRegexp: (?i)/case-insensitive-regexp + backend: test-convert-caseinsensitiveregexp +` + expected := common.NewHTTPServerSpec("test-convert-8080") + err = codectool.UnmarshalYAML([]byte(serverYaml), expected) + assert.Nil(t, err) + assert.Equal(t, expected, httpServers[0]) + + pipelineApis := ` +name: test-convert-apis +kind: Pipeline +filters: + - kind: RequestAdaptor + name: request-adaptor + template: | + header: + set: + X-Path: apis + - compression: + minLength: 1000 + kind: Proxy + name: proxy + pools: + - loadBalance: + policy: weightedRandom + servers: + - url: http://localhost:8880 + weight: 1 + - url: http://localhost:8881 + weight: 2 +` + pipelineExact := ` +name: test-convert-exact +kind: Pipeline +filters: + - kind: Proxy + name: proxy + pools: + - loadBalance: + policy: roundRobin + servers: + - url: http://localhost:9999 + weight: 1 +` + pipelineRegexp := ` +name: test-convert-regexp +kind: Pipeline +filters: + - kind: Proxy + name: proxy + pools: + - loadBalance: + policy: roundRobin + servers: + - url: http://localhost:7777 + weight: 1 +` + pipelineCIReg := ` +name: test-convert-caseinsensitiveregexp +kind: Pipeline +filters: + - kind: Proxy + name: proxy + pools: + - loadBalance: + policy: roundRobin + servers: + - url: http://localhost:6666 + weight: 1 +` + for i, yamlStr := range []string{pipelineApis, pipelineExact, pipelineRegexp, pipelineCIReg} { + spec := common.NewPipelineSpec("") + err = codectool.UnmarshalYAML([]byte(yamlStr), spec) + assert.Nil(t, err, i) + for j, f := range spec.Filters { + compareFilter(t, f, pipelines[i].Filters[j], fmt.Sprintf("%d filter in %d pipeline", j, i)) + } + } +} + +func compareFilter(t *testing.T, f1 map[string]interface{}, f2 map[string]interface{}, msg string) { + d1, err := codectool.MarshalYAML(f1) + assert.Nil(t, err, msg) + d2, err := codectool.MarshalYAML(f2) + assert.Nil(t, err, msg) + + var specFn func() interface{} + switch f1["kind"] { + case "Proxy": + specFn = func() interface{} { + return common.NewProxyFilterSpec("") + } + s1 := common.NewProxyFilterSpec("") + err = codectool.UnmarshalYAML(d1, s1) + assert.Nil(t, err, msg) + s2 := common.NewProxyFilterSpec("") + err = codectool.Unmarshal(d2, s2) + assert.Nil(t, err, msg) + assert.Equal(t, s1, s2) + case "RequestAdaptor": + specFn = func() interface{} { + return common.NewRequestAdaptorFilterSpec("") + } + default: + t.Errorf("filter kind %s is not compared", f1["kind"]) + return + } + s1 := specFn() + err = codectool.UnmarshalYAML(d1, s1) + assert.Nil(t, err, msg) + s2 := specFn() + err = codectool.Unmarshal(d2, s2) + assert.Nil(t, err, msg) + assert.Equal(t, s1, s2) +} diff --git a/cmd/client/commandv2/convert/nginx/utils.go b/cmd/client/commandv2/convert/nginx/utils.go index 8f05c9b4ee..5aad3ae5e2 100644 --- a/cmd/client/commandv2/convert/nginx/utils.go +++ b/cmd/client/commandv2/convert/nginx/utils.go @@ -20,15 +20,8 @@ package nginx import ( "fmt" "strings" - - "github.com/megaease/easegress/v2/pkg/util/codectool" ) func directiveInfo(d *Directive) string { return fmt.Sprintf("directive <%s %s> of %s:%d", d.Directive, strings.Join(d.Args, " "), d.File, d.Line) } - -func printYaml(v interface{}) { - b, _ := codectool.MarshalYAML(v) - fmt.Println(string(b)) -} From 5a8c60604aed45c4f4d27cb7cd99c78576ba55ed Mon Sep 17 00:00:00 2001 From: chen Date: Tue, 7 Nov 2023 18:30:13 +0800 Subject: [PATCH 13/22] add more test in cmd --- cmd/client/commandv2/convert/nginx/cmd.go | 10 +++++ .../commandv2/convert/nginx/cmd_test.go | 40 +++++++++++++++++++ cmd/client/commandv2/create/create_test.go | 1 + 3 files changed, 51 insertions(+) diff --git a/cmd/client/commandv2/convert/nginx/cmd.go b/cmd/client/commandv2/convert/nginx/cmd.go index 763f690fae..e223458632 100644 --- a/cmd/client/commandv2/convert/nginx/cmd.go +++ b/cmd/client/commandv2/convert/nginx/cmd.go @@ -45,6 +45,15 @@ func Cmd() *cobra.Command { Use: "nginx", Short: "Convert nginx.conf to easegress yaml file", Args: func(cmd *cobra.Command, args []string) error { + if flags.NginxConf == "" { + return fmt.Errorf("nginx.conf file path is required") + } + if flags.Output == "" { + return fmt.Errorf("output yaml file path is required") + } + if flags.Prefix == "" { + return fmt.Errorf("prefix is required") + } return nil }, Run: func(cmd *cobra.Command, args []string) { @@ -128,5 +137,6 @@ func writeYaml(filename string, servers []*common.HTTPServerSpec, pipelines []*c file.WriteString(string(data)) file.WriteString("\n---\n") } + file.Sync() return nil } diff --git a/cmd/client/commandv2/convert/nginx/cmd_test.go b/cmd/client/commandv2/convert/nginx/cmd_test.go index ec50b5ff68..cb60b938c5 100644 --- a/cmd/client/commandv2/convert/nginx/cmd_test.go +++ b/cmd/client/commandv2/convert/nginx/cmd_test.go @@ -18,6 +18,8 @@ package nginx import ( + "io" + "os" "testing" "github.com/stretchr/testify/assert" @@ -26,4 +28,42 @@ import ( func TestCmd(t *testing.T) { cmd := Cmd() assert.NotNil(t, cmd) + cmd.ParseFlags([]string{""}) + assert.NotNil(t, cmd.Args(cmd, []string{})) + + cmd.ParseFlags([]string{"-f", "test.conf"}) + assert.NotNil(t, cmd.Args(cmd, []string{})) + + cmd.ParseFlags([]string{"-o", "test.yaml"}) + assert.NotNil(t, cmd.Args(cmd, []string{})) + + cmd.ParseFlags([]string{"--prefix", "test"}) + assert.Nil(t, cmd.Args(cmd, []string{})) + + tempDir := newTempTestDir(t) + defer tempDir.Clean() + + nginxConf := ` + events {} + http { + server { + listen 127.0.0.1:8080; + + location = /user { + proxy_pass http://localhost:9999; + } + } + } + ` + nginxFile := tempDir.Create("nginx.conf", []byte(nginxConf)) + outputFile := tempDir.Create("test.yaml", []byte("")) + cmd.ParseFlags([]string{"-f", nginxFile, "-o", outputFile, "--prefix", "test"}) + cmd.Run(cmd, []string{}) + file, err := os.Open(outputFile) + assert.Nil(t, err) + defer file.Close() + data, err := io.ReadAll(file) + assert.Nil(t, err) + assert.Contains(t, string(data), "test-8080") + assert.Contains(t, string(data), "test-user") } diff --git a/cmd/client/commandv2/create/create_test.go b/cmd/client/commandv2/create/create_test.go index 5720904f77..0136637de4 100644 --- a/cmd/client/commandv2/create/create_test.go +++ b/cmd/client/commandv2/create/create_test.go @@ -27,4 +27,5 @@ import ( func TestCmd(t *testing.T) { cmd := Cmd() assert.NotNil(t, cmd) + assert.Error(t, cmd.Args(cmd, nil)) } From 6d47267e726a84981bb9f2b0356c0d281fd7379c Mon Sep 17 00:00:00 2001 From: chen Date: Tue, 7 Nov 2023 18:58:03 +0800 Subject: [PATCH 14/22] add tests --- .../commandv2/convert/nginx/cmd_test.go | 17 ++++++++ cmd/client/commandv2/convert/nginx/convert.go | 9 ++++- .../commandv2/convert/nginx/convert_test.go | 40 ++++++++++++++----- 3 files changed, 55 insertions(+), 11 deletions(-) diff --git a/cmd/client/commandv2/convert/nginx/cmd_test.go b/cmd/client/commandv2/convert/nginx/cmd_test.go index cb60b938c5..077ec05271 100644 --- a/cmd/client/commandv2/convert/nginx/cmd_test.go +++ b/cmd/client/commandv2/convert/nginx/cmd_test.go @@ -67,3 +67,20 @@ func TestCmd(t *testing.T) { assert.Contains(t, string(data), "test-8080") assert.Contains(t, string(data), "test-user") } + +func TestOption(t *testing.T) { + option := &Options{ + NginxConf: "test.conf", + Output: "test.yaml", + Prefix: "test", + } + option.init() + path := option.GetPipelineName("/user") + assert.Equal(t, "test-user", path) + path = option.GetPipelineName("/apis/v1") + assert.Equal(t, "test-apisv1", path) + + path = option.GetPipelineName("/apis/v1/") + assert.Contains(t, path, "test-apisv1") + assert.NotEqual(t, "test-apisv1", path) +} diff --git a/cmd/client/commandv2/convert/nginx/convert.go b/cmd/client/commandv2/convert/nginx/convert.go index 09ac5a0506..f50575e431 100644 --- a/cmd/client/commandv2/convert/nginx/convert.go +++ b/cmd/client/commandv2/convert/nginx/convert.go @@ -124,7 +124,7 @@ func convertRule(options *Options, rule *Rule) (*routers.Rule, []*common.Pipelin pipelines = append(pipelines, pipelineSpec) } sort.Slice(prefixPaths, func(i, j int) bool { - return prefixPaths[i].Path > prefixPaths[j].Path + return len(prefixPaths[i].PathPrefix) > len(prefixPaths[j].PathPrefix) }) router.Paths = append(router.Paths, exactPaths...) router.Paths = append(router.Paths, prefixPaths...) @@ -138,7 +138,9 @@ func convertProxy(name string, info *ProxyInfo) *common.PipelineSpec { flow := make([]filters.Spec, 0) if len(info.SetHeaders) != 0 { adaptor := getRequestAdaptor(info) - flow = append(flow, adaptor) + if adaptor != nil { + flow = append(flow, adaptor) + } } var proxy filters.Spec @@ -195,6 +197,9 @@ func getRequestAdaptor(info *ProxyInfo) *builder.RequestAdaptorSpec { } template.Header.Set[k] = translateNginxEmbeddedVar(v) } + if len(template.Header.Set) == 0 { + return nil + } data := codectool.MustMarshalYAML(template) spec.Template = string(data) return spec diff --git a/cmd/client/commandv2/convert/nginx/convert_test.go b/cmd/client/commandv2/convert/nginx/convert_test.go index 10643f2b53..299c86dc69 100644 --- a/cmd/client/commandv2/convert/nginx/convert_test.go +++ b/cmd/client/commandv2/convert/nginx/convert_test.go @@ -128,6 +128,15 @@ servers: servers: - server: http://localhost:6666 weight: 1 + - path: /websocket + type: prefix + backend: + servers: + - server: http://localhost:9090 + weight: 1 + setHeaders: + Connection: $connection_upgrade + Upgrade: $http_upgrade ` config := &Config{} err := codectool.Unmarshal([]byte(conf), config) @@ -135,7 +144,7 @@ servers: httpServers, pipelines, err := convertConfig(options, config) assert.Nil(t, err) assert.Equal(t, 1, len(httpServers)) - assert.Equal(t, 4, len(pipelines)) + assert.Equal(t, 5, len(pipelines)) serverYaml := ` name: test-convert-8080 kind: HTTPServer @@ -156,6 +165,9 @@ rules: paths: - path: /exact backend: test-convert-exact + - pathPrefix: /websocket + backend: test-convert-websocket + clientMaxBodySize: -1 - pathPrefix: /apis backend: test-convert-apis - pathRegexp: /regexp @@ -230,7 +242,20 @@ filters: - url: http://localhost:6666 weight: 1 ` - for i, yamlStr := range []string{pipelineApis, pipelineExact, pipelineRegexp, pipelineCIReg} { + pipelineWebsocket := ` +name: test-convert-websocket +kind: Pipeline +filters: + - kind: WebSocketProxy + name: websocket + pools: + - loadBalance: + policy: roundRobin + servers: + - url: ws://localhost:9090 + weight: 1 +` + for i, yamlStr := range []string{pipelineApis, pipelineExact, pipelineRegexp, pipelineCIReg, pipelineWebsocket} { spec := common.NewPipelineSpec("") err = codectool.UnmarshalYAML([]byte(yamlStr), spec) assert.Nil(t, err, i) @@ -252,17 +277,14 @@ func compareFilter(t *testing.T, f1 map[string]interface{}, f2 map[string]interf specFn = func() interface{} { return common.NewProxyFilterSpec("") } - s1 := common.NewProxyFilterSpec("") - err = codectool.UnmarshalYAML(d1, s1) - assert.Nil(t, err, msg) - s2 := common.NewProxyFilterSpec("") - err = codectool.Unmarshal(d2, s2) - assert.Nil(t, err, msg) - assert.Equal(t, s1, s2) case "RequestAdaptor": specFn = func() interface{} { return common.NewRequestAdaptorFilterSpec("") } + case "WebSocketProxy": + specFn = func() interface{} { + return common.NewWebsocketFilterSpec("") + } default: t.Errorf("filter kind %s is not compared", f1["kind"]) return From d966e40d5c8651646e442b790e211dec78143353 Mon Sep 17 00:00:00 2001 From: chen Date: Tue, 7 Nov 2023 19:06:03 +0800 Subject: [PATCH 15/22] fix conflict --- go.mod | 6 +++--- go.sum | 14 +++++--------- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 86a1c4ddb5..400bf03690 100644 --- a/go.mod +++ b/go.mod @@ -39,9 +39,9 @@ require ( github.com/megaease/jsonschema v0.5.1 github.com/megaease/yaml v0.0.0-20220804061446-4f18d6510aed github.com/mitchellh/mapstructure v1.5.0 - github.com/nginxinc/nginx-go-crossplane v0.4.33 github.com/nacos-group/nacos-sdk-go v1.1.4 github.com/nacos-group/nacos-sdk-go/v2 v2.2.3 + github.com/nginxinc/nginx-go-crossplane v0.4.33 github.com/open-policy-agent/opa v0.58.0 github.com/openzipkin/zipkin-go v0.4.2 github.com/patrickmn/go-cache v2.1.0+incompatible @@ -108,11 +108,11 @@ require ( github.com/gorilla/mux v1.8.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.0 // indirect github.com/hashicorp/go-version v1.6.0 // indirect - github.com/jstemmer/go-junit-report v1.0.0 // indirect - github.com/maxbrunsfeld/counterfeiter/v6 v6.6.1 // indirect github.com/huandu/xstrings v1.4.0 // indirect + github.com/jstemmer/go-junit-report v1.0.0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect + github.com/maxbrunsfeld/counterfeiter/v6 v6.6.1 // indirect github.com/onsi/ginkgo/v2 v2.13.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b // indirect diff --git a/go.sum b/go.sum index e74e030934..a87fe79a27 100644 --- a/go.sum +++ b/go.sum @@ -573,12 +573,10 @@ github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= -github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/maxbrunsfeld/counterfeiter/v6 v6.6.1 h1:9XE5ykDiC8eNSqIPkxx0EsV3kMX1oe4kQWRZjIgytUA= -github.com/maxbrunsfeld/counterfeiter/v6 v6.6.1/go.mod h1:qbKwBR+qQODzH2WD/s53mdgp/xVcXMlJb59GRFOp6Z4= github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k= +github.com/maxbrunsfeld/counterfeiter/v6 v6.6.1 h1:9XE5ykDiC8eNSqIPkxx0EsV3kMX1oe4kQWRZjIgytUA= +github.com/maxbrunsfeld/counterfeiter/v6 v6.6.1/go.mod h1:qbKwBR+qQODzH2WD/s53mdgp/xVcXMlJb59GRFOp6Z4= github.com/megaease/easemesh-api v1.4.4 h1:E18mtLfj8ffuPTeN7MqZeakJgT/tJ92JNIZsY2k2GE0= github.com/megaease/easemesh-api v1.4.4/go.mod h1:GuAE5DwqK6lI/ovoRKjyPxBCSoMhj0NLp9PRejj0Hnw= github.com/megaease/grace v1.0.0 h1:b44R3j6e/iaN62F4ZUnru9nzL1VaIcxxUZjSPVtTVzI= @@ -611,14 +609,12 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/nacos-group/nacos-sdk-go v1.1.0 h1:6ESrAegx2pqp3Vi8mqDi7s2Vq+I+u0oYLn646K4wx6o= -github.com/nacos-group/nacos-sdk-go v1.1.0/go.mod h1:Y/9Dj0Bl04hWUO1DaL4+r+fLzv5nl9kn58Vt1OGvWdw= -github.com/nginxinc/nginx-go-crossplane v0.4.33 h1:Sr6ptE+xzQ7Vc5xs1SXNDDBWvVwKA7BTtQJZIGTS7RY= -github.com/nginxinc/nginx-go-crossplane v0.4.33/go.mod h1:UzbZnyFv0vPlt1Urbnp/mrFCzBL4tYCReFuNBpFQEfI= github.com/nacos-group/nacos-sdk-go v1.1.4 h1:qyrZ7HTWM4aeymFfqnbgNRERh7TWuER10pCB7ddRcTY= github.com/nacos-group/nacos-sdk-go v1.1.4/go.mod h1:cBv9wy5iObs7khOqov1ERFQrCuTR4ILpgaiaVMxEmGI= github.com/nacos-group/nacos-sdk-go/v2 v2.2.3 h1:sUQx4f1bXDeeOOEQZjGAitzxYApbYY9fVDbxVCaBW+I= github.com/nacos-group/nacos-sdk-go/v2 v2.2.3/go.mod h1:UL4U89WYdnyajgKJUMpuT1Rr6iNmbjrxOO40JRgtA00= +github.com/nginxinc/nginx-go-crossplane v0.4.33 h1:Sr6ptE+xzQ7Vc5xs1SXNDDBWvVwKA7BTtQJZIGTS7RY= +github.com/nginxinc/nginx-go-crossplane v0.4.33/go.mod h1:UzbZnyFv0vPlt1Urbnp/mrFCzBL4tYCReFuNBpFQEfI= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nrdcg/dnspod-go v0.4.0 h1:c/jn1mLZNKF3/osJ6mz3QPxTudvPArXTjpkmYj0uK6U= github.com/nrdcg/dnspod-go v0.4.0/go.mod h1:vZSoFSFeQVm2gWLMkyX61LZ8HI3BaqtHZWgPTGKr6KQ= @@ -717,11 +713,11 @@ github.com/rs/cors v1.10.1 h1:L0uuZVXIKlI1SShY2nhFfo44TYvDPQ1w4oFkUJNfhyo= github.com/rs/cors v1.10.1/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/sclevine/spec v1.4.0 h1:z/Q9idDcay5m5irkZ28M7PtQM4aOISzOpj4bUPkDee8= github.com/sagikazarmark/locafero v0.3.0 h1:zT7VEGWC2DTflmccN/5T1etyKvxSxpHsjb9cJvm4SvQ= github.com/sagikazarmark/locafero v0.3.0/go.mod h1:w+v7UsPNFwzF1cHuOajOOzoq4U7v/ig1mpRjqV+Bu1U= github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= +github.com/sclevine/spec v1.4.0 h1:z/Q9idDcay5m5irkZ28M7PtQM4aOISzOpj4bUPkDee8= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= From 3fdb6fd7a0fdaf391f0d4ccdb4c3637c810b40bf Mon Sep 17 00:00:00 2001 From: chen Date: Wed, 8 Nov 2023 14:19:08 +0800 Subject: [PATCH 16/22] add more test --- cmd/client/commandv2/convert/nginx/cmd.go | 3 + .../commandv2/convert/nginx/convert_test.go | 4 +- .../commandv2/convert/nginx/env_test.go | 77 +++++++++++++++++ cmd/client/commandv2/convert/nginx/parse.go | 1 + .../commandv2/convert/nginx/parse_test.go | 84 +++++++++++++++++++ 5 files changed, 167 insertions(+), 2 deletions(-) diff --git a/cmd/client/commandv2/convert/nginx/cmd.go b/cmd/client/commandv2/convert/nginx/cmd.go index e223458632..7fdb976813 100644 --- a/cmd/client/commandv2/convert/nginx/cmd.go +++ b/cmd/client/commandv2/convert/nginx/cmd.go @@ -61,6 +61,9 @@ func Cmd() *cobra.Command { if err != nil { general.ExitWithErrorf("parse nginx.conf failed: %v", err) } + for _, e := range payload.Errors { + general.Warnf("parse nginx.conf error: %v in %s of %s", e.Error, e.Line, e.File) + } config, err := parsePayload(payload) if err != nil { general.ExitWithError(err) diff --git a/cmd/client/commandv2/convert/nginx/convert_test.go b/cmd/client/commandv2/convert/nginx/convert_test.go index 299c86dc69..87bedeb623 100644 --- a/cmd/client/commandv2/convert/nginx/convert_test.go +++ b/cmd/client/commandv2/convert/nginx/convert_test.go @@ -132,7 +132,7 @@ servers: type: prefix backend: servers: - - server: http://localhost:9090 + - server: https://localhost:9090 weight: 1 setHeaders: Connection: $connection_upgrade @@ -252,7 +252,7 @@ filters: - loadBalance: policy: roundRobin servers: - - url: ws://localhost:9090 + - url: wss://localhost:9090 weight: 1 ` for i, yamlStr := range []string{pipelineApis, pipelineExact, pipelineRegexp, pipelineCIReg, pipelineWebsocket} { diff --git a/cmd/client/commandv2/convert/nginx/env_test.go b/cmd/client/commandv2/convert/nginx/env_test.go index bc01e74607..85223cdce4 100644 --- a/cmd/client/commandv2/convert/nginx/env_test.go +++ b/cmd/client/commandv2/convert/nginx/env_test.go @@ -55,3 +55,80 @@ func TestSplitAddressPort(t *testing.T) { } } } + +func newDirective(d string, args ...string) *Directive { + return &Directive{ + Directive: d, + Args: args, + } +} + +func TestEnvProcessErrors(t *testing.T) { + // gzip + { + testCases := []struct { + gzip *Directive + len *Directive + res int + }{ + {gzip: newDirective("gzip", "invalid"), len: nil, res: 0}, + {gzip: newDirective("gzip", "on"), len: nil, res: 20}, + {gzip: newDirective("gzip", "on"), len: newDirective("gzip_min_length", "200"), res: 200}, + {gzip: newDirective("gzip", "on"), len: newDirective("gzip_min_length", "invalid"), res: 20}, + {gzip: newDirective("gzip", "on"), len: newDirective("gzip_min_length", "-1"), res: 20}, + } + for i, tc := range testCases { + gzip := &GzipEnv{ + Gzip: tc.gzip, + GzipMinLength: tc.len, + } + got := processGzip(gzip) + assert.Equal(t, tc.res, got, "case", i) + } + } + + // ssl + { + certs := []*Directive{newDirective("ssl_certificate", "cert1"), newDirective("ssl_certificate", "cert2")} + keys := []*Directive{newDirective("ssl_certificate_key", "key1")} + _, _, err := processSSLCertificates(certs, keys) + assert.NotNil(t, err) + + certs = []*Directive{newDirective("ssl_certificate", "cert1")} + keys = []*Directive{newDirective("ssl_certificate_key", "key1"), newDirective("ssl_certificate_key", "key2")} + _, _, err = processSSLCertificates(certs, keys) + assert.NotNil(t, err) + + certs = []*Directive{newDirective("ssl_certificate", "cert1")} + keys = []*Directive{newDirective("ssl_certificate_key", "key1")} + _, _, err = processSSLCertificates(certs, keys) + assert.NotNil(t, err) + } + + // server name + { + testCases := []struct { + server *Directive + hostValues []string + isRegexp []bool + err bool + }{ + {server: newDirective("server_name", "~www.example.com$"), hostValues: []string{"www.example.com$"}, isRegexp: []bool{true}, err: false}, + {server: newDirective("server_name", "~["), hostValues: []string{}, isRegexp: []bool{}, err: true}, + {server: newDirective("server_name", "*.example.*"), hostValues: []string{}, isRegexp: []bool{}, err: true}, + } + for i, tc := range testCases { + serverNames, err := processServerName(tc.server) + assert.Equal(t, len(tc.hostValues), len(serverNames), "case", i) + for i := 0; i < len(tc.hostValues); i++ { + assert.Equal(t, tc.hostValues[i], serverNames[i].Value, "case", i) + assert.Equal(t, tc.isRegexp[i], serverNames[i].IsRegexp, "case", i) + } + if tc.err { + assert.NotNil(t, err, "case", i) + } else { + assert.Nil(t, err, "case", i) + } + } + } +} diff --git a/cmd/client/commandv2/convert/nginx/parse.go b/cmd/client/commandv2/convert/nginx/parse.go index 6ba59f4692..0cf43e7176 100644 --- a/cmd/client/commandv2/convert/nginx/parse.go +++ b/cmd/client/commandv2/convert/nginx/parse.go @@ -108,6 +108,7 @@ func updateServers(servers []*Server, s *Server) ([]*Server, error) { for k, v := range s.Keys { servers[i].Keys[k] = v } + servers[i].Rules = append(servers[i].Rules, s.Rules...) return servers, nil } return append(servers, s), nil diff --git a/cmd/client/commandv2/convert/nginx/parse_test.go b/cmd/client/commandv2/convert/nginx/parse_test.go index 3e88b2a71e..12d3d61f5b 100644 --- a/cmd/client/commandv2/convert/nginx/parse_test.go +++ b/cmd/client/commandv2/convert/nginx/parse_test.go @@ -313,3 +313,87 @@ servers: assert.Equal(t, expected, config) } } + +func TestUpdateServer(t *testing.T) { + tempDir := newTempTestDir(t) + defer tempDir.Clean() + { + nginxConf := ` + events {} + http { + server { + listen 80; + + location /apis { + proxy_pass http://localhost:8880; + } + } + + server { + listen 80; + + location /user { + proxy_pass http://localhost:9999; + } + } + } + ` + file := tempDir.Create("nginx.conf", []byte(nginxConf)) + payload, err := crossplane.Parse(file, &crossplane.ParseOptions{}) + assert.Nil(t, err) + config, err := parsePayload(payload) + assert.Nil(t, err) + + proxyInfo := ` +servers: + - port: 80 + rules: + - paths: + - path: /apis + type: prefix + backend: + servers: + - server: http://localhost:8880 + weight: 1 + - paths: + - path: /user + type: prefix + backend: + servers: + - server: http://localhost:9999 + weight: 1 +` + expected := &Config{} + err = codectool.UnmarshalYAML([]byte(proxyInfo), expected) + assert.Nil(t, err) + assert.Equal(t, expected, config) + } + { + servers := []*Server{ + { + ServerBase: ServerBase{ + Port: 80, + Certs: map[string]string{}, + Keys: map[string]string{}, + }, + }, + } + server := &Server{ + ServerBase: ServerBase{ + Port: 80, + HTTPS: true, + CaCert: "ca", + Certs: map[string]string{"cert": "cert"}, + Keys: map[string]string{"cert": "key"}, + }, + } + servers, err := updateServers(servers, server) + assert.Nil(t, err) + assert.Equal(t, 1, len(servers)) + got := servers[0] + assert.True(t, got.HTTPS) + assert.Equal(t, "ca", got.CaCert) + assert.Equal(t, map[string]string{"cert": "cert"}, got.Certs) + assert.Equal(t, map[string]string{"cert": "key"}, got.Keys) + } +} From 1f886758458d9244134abf7f067ede74c9728253 Mon Sep 17 00:00:00 2001 From: chen Date: Wed, 8 Nov 2023 16:53:42 +0800 Subject: [PATCH 17/22] add doc --- cmd/client/commandv2/convert/nginx/cmd.go | 11 +- docs/02.Tutorials/2.2.HTTP-Proxy-Usage.md | 7 +- docs/03.Advanced-Cookbook/3.13.Nginx.md | 342 ++++++++++++++++++++++ docs/03.Advanced-Cookbook/README.md | 1 + docs/07.Reference/7.01.Controllers.md | 4 +- docs/07.Reference/7.02.Filters.md | 34 ++- 6 files changed, 393 insertions(+), 6 deletions(-) create mode 100644 docs/03.Advanced-Cookbook/3.13.Nginx.md diff --git a/cmd/client/commandv2/convert/nginx/cmd.go b/cmd/client/commandv2/convert/nginx/cmd.go index 7fdb976813..336dc71fb6 100644 --- a/cmd/client/commandv2/convert/nginx/cmd.go +++ b/cmd/client/commandv2/convert/nginx/cmd.go @@ -41,9 +41,16 @@ type Options struct { func Cmd() *cobra.Command { flags := &Options{} flags.init() + examples := []general.Example{ + { + Desc: "Convert nginx config to easegress yamls", + Command: "egctl convert nginx -f -o --prefix ", + }, + } cmd := &cobra.Command{ - Use: "nginx", - Short: "Convert nginx.conf to easegress yaml file", + Use: "nginx", + Short: "Convert nginx.conf to easegress yaml file", + Example: general.CreateMultiExample(examples), Args: func(cmd *cobra.Command, args []string) error { if flags.NginxConf == "" { return fmt.Errorf("nginx.conf file path is required") diff --git a/docs/02.Tutorials/2.2.HTTP-Proxy-Usage.md b/docs/02.Tutorials/2.2.HTTP-Proxy-Usage.md index ee216c260f..71a8cf3c44 100644 --- a/docs/02.Tutorials/2.2.HTTP-Proxy-Usage.md +++ b/docs/02.Tutorials/2.2.HTTP-Proxy-Usage.md @@ -98,9 +98,14 @@ port: 10080 rules: # Rules for host matching. # If not match, HTTPServer will check next rule. +# wildcard is supported in front or end of hostnames. - host: - hosts: [, ] hostRegexp: + hosts: + - value: *.example.com + isRegexp: false + - value: www.example.* + isRegexp: false # IP-based filtering. ipFilter: {} diff --git a/docs/03.Advanced-Cookbook/3.13.Nginx.md b/docs/03.Advanced-Cookbook/3.13.Nginx.md new file mode 100644 index 0000000000..7baacc88b9 --- /dev/null +++ b/docs/03.Advanced-Cookbook/3.13.Nginx.md @@ -0,0 +1,342 @@ +# Nginx Configuration Conversion to Easegress YAML + +- [Nginx Configuration Conversion to Easegress YAML](#nginx-configuration-conversion-to-easegress-yaml) + - [Nginx Configurations Supported for Conversion](#nginx-configurations-supported-for-conversion) + - [listen](#listen) + - [server\_name](#server_name) + - [location](#location) + - [proxy\_pass and upstream](#proxy_pass-and-upstream) + - [websocket](#websocket) + - [HTTPS](#https) + - [proxy\_set\_header](#proxy_set_header) + - [gzip](#gzip) + - [Example](#example) + +This document serves as a guide for converting Nginx configurations to Easegress YAML format using the `egctl convert nginx` command. + +```bash +egctl convert nginx -f -o --prefix +``` + +## Nginx Configurations Supported for Conversion + +### listen +The `listen` directive supports `port`, `address:port`, and `ssl` configurations. It corresponds to the `port` and `address` properties of an Easegress `HTTPServer`. + +``` +listen: 80; +listen: 127.0.0.1:80; +listen: 443 ssl; +``` + +### server_name +Supports exact hostnames, prefixed hostnames, suffixed hostnames, and regular expressions. Translates to the `hosts` attribute in Easegress `HTTPServer`. + +``` +server_name www.example.com *.example.com; +server_name www.example.* ~^(?.+)\.example\.net$ +``` + +### location +Handles `location` with prefix paths, exact paths, and regular expression paths (case-insensitive included). The ordering in Easegress follows Nginx's precedence: exact paths first, prefix paths next (sorted by length), and regex paths last. Nested locations are also supported. + +This translates to `rules` and `paths` in an Easegress `HTTPServer`. + +``` +location /apis { + location /apis/v1 { + ... + } + ... +} + +location = /user { + ... +} + +location ~* \.(gif|jpg|jpeg)$ { + ... +} + +location ^~ /admin/ { + ... +} +``` + +The resulting path order in Easegress would be: + +``` +- path: /user +- pathPrefix: /apis/v1 +- pathPrefix: /admin/ +- pathPrefix: /apis +- pathRegexp: \.(gif|jpg|jpeg)$ +``` + +### proxy_pass and upstream +Currently, only reverse proxy functionalities are supported. For `upstream`, the default load-balancing strategy is `roundRobin`. If a server weight is specified, `weightRandom` is used for load balancing. + +Equal to Easegress `Pipeline` with `Proxy` filter. + +``` +proxy_pass http://user.example.com + +proxy_pass http:backend + +upstream backend { + server 127.0.0.1:8080; + server 127.0.0.2:9090 weight=10; +} +``` + +### websocket +Settings recognized as `WebSocket` configurations correspond to an Easegress `Pipeline` with a `WebsocketProxy` filter. + +``` +location /websocket { + proxy_pass http://wsbackend; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "Upgrade"; +} +``` + +### HTTPS +HTTPS configurations are supported as per Easegress `HTTPServer` settings. + +``` +listen 443 ssl; + +ssl_certificate you-cert.crt; +ssl_certificate_key your-cert-key.key; + +ssl_certificate you-cert2.crt; +ssl_certificate_key your-cert-key2.key; + +ssl_client_certificate your-client-cert.crt; +``` + +### proxy_set_header +Translates to an Easegress `Pipeline` with a `RequestAdaptor` filter. + +The following Nginx embedded variables are supported: + +- `$host` +- `$hostname` +- `$content_length` +- `$content_type` +- `$remote_addr` +- `$remote_user` +- `$request_body` +- `$request_method` +- `$request_uri` +- `$scheme` + +For example: +``` +proxy_set_header Host $host +``` + +### gzip +Corresponds to an Easegress `Pipeline` with a `Proxy` filter, including `compression` settings. + +``` +gzip on; +gzip_min_length 1000; +``` + +## Example + +Given following nginx configuration: +``` +events {} +http { + upstream backend { + server localhost:1234; + server localhost:2345 weight=10; + } + + server { + listen 80; + + server_name www.example.com *.example.com; + + location /apis { + proxy_set_header X-Path "apis"; + proxy_pass http://localhost:8880; + + location /apis/v1 { + proxy_set_header X-Path "apis/v1"; + proxy_pass http://localhost:8888; + } + } + + location /upstream { + gzip on; + gzip_min_length 1000; + proxy_pass http://backend; + } + + location /websocket { + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + proxy_pass http://localhost:9090; + } + } + + server { + listen 127.0.0.1:443 ssl; + ssl_client_certificate ; + ssl_certificate ; + ssl_certificate_key ; + + location = /user { + proxy_pass http://localhost:9999; + } + } +} +``` + +then run + +```bash +egctl convert ngxin -f nginx.conf -o nginx.yaml --prefix convert-nginx +``` + +will generate following Easegress YAML: + +```yaml +name: convert-nginx-80 +kind: HTTPServer +address: "" +port: 80 +keepAliveTimeout: 60s +maxConnections: 10240 +rules: + - hosts: + - isRegexp: false + value: www.example.com + - isRegexp: false + value: '*.example.com' + paths: + - pathPrefix: /websocket + backend: convert-nginx-websocket + clientMaxBodySize: -1 + - pathPrefix: /upstream + backend: convert-nginx-upstream + - pathPrefix: /apis/v1 + backend: convert-nginx-apisv1 + - pathPrefix: /apis + backend: convert-nginx-apis + +--- +name: convert-nginx-443 +kind: HTTPServer +https: true +address: 127.0.0.1 +port: 443 +caCertBase64: +certs: + : +keys: + : +rules: + - paths: + - path: /user + backend: convert-nginx-user +--- +name: convert-nginx-apis +kind: Pipeline +flow: [] +filters: + - kind: RequestAdaptor + name: request-adaptor + template: | + header: + set: + X-Path: apis + - kind: Proxy + maxIdleConns: 10240 + maxIdleConnsPerHost: 1024 + name: proxy + pools: + - loadBalance: + policy: roundRobin + servers: + - url: http://localhost:8880 + weight: 1 + +--- +name: convert-nginx-apisv1 +kind: Pipeline +flow: [] +filters: + - kind: RequestAdaptor + name: request-adaptor + template: | + header: + set: + X-Path: apis/v1 + version: "" + - kind: Proxy + maxIdleConns: 10240 + maxIdleConnsPerHost: 1024 + name: proxy + pools: + - loadBalance: + policy: roundRobin + servers: + - url: http://localhost:8888 + weight: 1 + +--- +name: convert-nginx-upstream +kind: Pipeline +flow: [] +filters: + - compression: + minLength: 1000 + kind: Proxy + maxIdleConns: 10240 + maxIdleConnsPerHost: 1024 + name: proxy + pools: + - loadBalance: + policy: weightedRandom + servers: + - url: http://localhost:1234 + weight: 1 + - url: http://localhost:2345 + weight: 10 + +--- +name: convert-nginx-websocket +kind: Pipeline +flow: [] +filters: + - kind: WebSocketProxy + name: websocket + pools: + - loadBalance: + policy: roundRobin + servers: + - url: ws://localhost:9090 + weight: 1 + +--- +name: convert-nginx-user +kind: Pipeline +flow: [] +filters: + - kind: Proxy + maxIdleConns: 10240 + maxIdleConnsPerHost: 1024 + name: proxy + pools: + - loadBalance: + policy: roundRobin + servers: + - url: http://localhost:9999 + weight: 1 +``` diff --git a/docs/03.Advanced-Cookbook/README.md b/docs/03.Advanced-Cookbook/README.md index 7e75ed84de..74d2f9de9e 100644 --- a/docs/03.Advanced-Cookbook/README.md +++ b/docs/03.Advanced-Cookbook/README.md @@ -12,3 +12,4 @@ ### [Workflow](3.10.Workflow.md) ### [Performance](3.11.Performance.md) ### [Migrate v1.x Filter To v2.x](3.12.Migrate.md) +### [Nginx Configuration Conversion to Easegress YAML](3.13.Nginx.md) diff --git a/docs/07.Reference/7.01.Controllers.md b/docs/07.Reference/7.01.Controllers.md index b19622ba4a..1a14f962bc 100644 --- a/docs/07.Reference/7.01.Controllers.md +++ b/docs/07.Reference/7.01.Controllers.md @@ -678,7 +678,7 @@ domains: | Name | Type | Description | Required | | ---------- | ----------------------------------- | ------------------------------------------------------------- | -------- | | ipFilter | [ipfilter.Spec](#ipfilterSpec) | IP Filter for all traffic under the rule | No | -| host | string | Exact host to match | No | +| host | string | Exact host or wildcard. For example "*.example.com" or "www.example.com". | No | | hostRegexp | string | Host in regular expression to match | No | | hosts | [][httpserver.Host](#httpserverhost) | Hosts to match | No | | paths | [][httpserver.Path](#httpserverPath) | Path matching rules, empty means to match nothing. Note that multiple paths are matched in the order of their appearance in the spec, this is different from Nginx. | No | @@ -691,7 +691,7 @@ domains: | Name | Type | Description | Required | | ------------- | ------------------------ | ---------------------------------------------------------------------- | -------- | | isRegexp | bool | Whether `value` is regular expression or exact value, default is false | No | -| value | string | Host value to match | Yes | +| value | string | Host value to match. Wildcard is supported. | Yes | ### httpserver.Path diff --git a/docs/07.Reference/7.02.Filters.md b/docs/07.Reference/7.02.Filters.md index 254093cdf6..b0ebed1733 100644 --- a/docs/07.Reference/7.02.Filters.md +++ b/docs/07.Reference/7.02.Filters.md @@ -449,7 +449,7 @@ path: addPrefix: /v3 ``` -The example configuration below removes header `X-Version` from all `GET` requests. +The example configuration below removes header `X-Version` from all `GET` requests and set header with key `Host` and value from current request host. See more details about template in [here](#template-of-builder-filters). ```yaml kind: RequestAdaptor @@ -457,7 +457,13 @@ name: request-adaptor-example method: GET header: del: ["X-Version"] +template: | + method: '{{ .req.Method }}' + header: + set: + Host: '{{ .req.Host }}' ``` +The structure of template follows the structure of `Configuration` below. The example configuration below modifies the request path using regular expressions. @@ -1897,6 +1903,13 @@ and please refer [Pipeline](7.01.Controllers.md#pipeline) for what is `namespace For example, if the request of the `DEFAULT` namespace is an HTTP one, we can access its method via `.requests.DEFAULT.Method`. +For convenience, shorthand notations are used to simplify the access of default request and response properties. In this notation: + +- `.req.Host` is equal to `.requests.DEFAULT.Host` +- `.resp.Body` is equal to `.responses.DEFAULT.Body` + +Here, `.req` is a shorthand for `.requests.DEFAULT`, and similarly, `.resp` is shorthand for `.requests.DEFAULT`. + Easegress also injects other data into the template engine, which can be accessed with `.data.`, for example, we can use `.data.PIPELINE` to read the data defined in the pipeline spec. @@ -1904,6 +1917,25 @@ read the data defined in the pipeline spec. The `template` should generate a string in YAML format, the schema of the result YAML varies from filters and protocols. +Use `RequestAdaptor` as an example: + +```yaml +kind: RequestAdaptor +name: request-adaptor +template: | + header: + set: + Content: '{{ header .req.Header "Content-Length" }}' + Content-Type: '{{ header .req.Header "Content-Type" }}' + Host: '{{ .req.Host }}' + Method: '{{ .req.Method }}' + Remote-Addr: '{{ .req.RemoteAddr }}' + Remote-User: '{{ username .req }}' + Request-Body: '{{ .req.Body }}' + Request-URI: '{{ .req.RequestURI }}' + Scheme: '{{ .req.URL.Scheme }}' +``` + #### HTTP Specific * **Available fields of existing requests** From 423867c2909e78af5d944d3029904247d3098656 Mon Sep 17 00:00:00 2001 From: chen Date: Wed, 8 Nov 2023 17:07:08 +0800 Subject: [PATCH 18/22] add comments to variables and functions --- cmd/client/commandv2/common/spec.go | 1 + cmd/client/commandv2/convert/nginx/cmd.go | 1 + cmd/client/commandv2/convert/nginx/env.go | 7 +++++++ cmd/client/commandv2/convert/nginx/parse.go | 15 ++++++++++++--- cmd/client/commandv2/convert/nginx/types.go | 20 +++++++++++++++++--- 5 files changed, 38 insertions(+), 6 deletions(-) diff --git a/cmd/client/commandv2/common/spec.go b/cmd/client/commandv2/common/spec.go index e893fe5d26..858766327b 100644 --- a/cmd/client/commandv2/common/spec.go +++ b/cmd/client/commandv2/common/spec.go @@ -43,6 +43,7 @@ type PipelineSpec struct { pipeline.Spec `json:",inline"` } +// SetFilters sets the filters of PipelineSpec. func (p *PipelineSpec) SetFilters(filters []filters.Spec) { data := codectool.MustMarshalYAML(filters) maps, _ := general.UnmarshalMapInterface(data, true) diff --git a/cmd/client/commandv2/convert/nginx/cmd.go b/cmd/client/commandv2/convert/nginx/cmd.go index 336dc71fb6..a864bdab7d 100644 --- a/cmd/client/commandv2/convert/nginx/cmd.go +++ b/cmd/client/commandv2/convert/nginx/cmd.go @@ -30,6 +30,7 @@ import ( "github.com/spf13/cobra" ) +// Options contains the options for convert nginx.conf. type Options struct { NginxConf string Output string diff --git a/cmd/client/commandv2/convert/nginx/env.go b/cmd/client/commandv2/convert/nginx/env.go index fdce8b1862..344763a2e4 100644 --- a/cmd/client/commandv2/convert/nginx/env.go +++ b/cmd/client/commandv2/convert/nginx/env.go @@ -66,12 +66,14 @@ type ServerEnv struct { SSLCertificateKey []*Directive `json:"ssl_certificate_key"` } +// ProxyEnv is the environment for creating proxy. type ProxyEnv struct { Pass *Directive `json:"pass"` ProxySetHeader []*Directive `json:"proxy_set_header"` Gzip *GzipEnv `json:"gzip"` } +// GzipEnv is the environment for creating gzip. type GzipEnv struct { Gzip *Directive `json:"gzip"` GzipMinLength *Directive `json:"gzip_min_length"` @@ -107,6 +109,7 @@ func newEnv() *Env { return env } +// Clone clones the environment. func (env *Env) Clone() (*Env, error) { data, err := json.Marshal(env) if err != nil { @@ -121,6 +124,7 @@ func (env *Env) Clone() (*Env, error) { return &newEnv, nil } +// MustClone clones the environment. func (env *Env) MustClone() *Env { newEnv, err := env.Clone() if err != nil { @@ -129,6 +133,7 @@ func (env *Env) MustClone() *Env { return newEnv } +// Update updates the environment. func (env *Env) Update(d *Directive) { fn, ok := env.updateFn[d.Directive] if ok { @@ -141,6 +146,7 @@ func (env *Env) Update(d *Directive) { } } +// GetServerInfo gets the server info from environment. func (env *Env) GetServerInfo() (*ServerInfo, error) { info := &ServerInfo{} info.Port = 80 @@ -186,6 +192,7 @@ func (env *Env) GetServerInfo() (*ServerInfo, error) { return info, nil } +// GetProxyInfo gets the proxy info from environment. func (env *Env) GetProxyInfo() (*ProxyInfo, error) { p := env.Proxy if p.Pass == nil || p.Pass.Directive != "proxy_pass" { diff --git a/cmd/client/commandv2/convert/nginx/parse.go b/cmd/client/commandv2/convert/nginx/parse.go index 0cf43e7176..0a8a46b678 100644 --- a/cmd/client/commandv2/convert/nginx/parse.go +++ b/cmd/client/commandv2/convert/nginx/parse.go @@ -28,14 +28,23 @@ import ( ) const ( - DirectiveInclude = "include" - DirectiveHTTP = "http" - DirectiveServer = "server" + // DirectiveInclude is the include directive. + DirectiveInclude = "include" + // DirectiveHTTP is the http directive. + DirectiveHTTP = "http" + // DirectiveServer is the server directive. + DirectiveServer = "server" + // DirectiveLocation is the location directive. DirectiveLocation = "location" ) +// Directive is the nginx directive. type Directive = crossplane.Directive + +// Directives is the nginx directives. type Directives = crossplane.Directives + +// Payload is the nginx payload. type Payload = crossplane.Payload func parsePayload(payload *Payload) (*Config, error) { diff --git a/cmd/client/commandv2/convert/nginx/types.go b/cmd/client/commandv2/convert/nginx/types.go index 04e2c3c62c..1fc4acc85b 100644 --- a/cmd/client/commandv2/convert/nginx/types.go +++ b/cmd/client/commandv2/convert/nginx/types.go @@ -17,10 +17,12 @@ package nginx +// Config is config for convert. type Config struct { Servers []*Server `json:"servers"` } +// ServerBase is the base config for server. type ServerBase struct { Port int `json:"port"` Address string `json:"address"` @@ -31,47 +33,59 @@ type ServerBase struct { Keys map[string]string `json:"keys"` } +// Server is the config for server. type Server struct { ServerBase `json:",inline"` Rules []*Rule `json:"rules"` } +// ServerInfo is the info config for server. type ServerInfo struct { ServerBase `json:",inline"` Hosts []*HostInfo `json:"hosts"` } +// HostInfo is the info config for host. type HostInfo struct { Value string `json:"value"` IsRegexp bool `json:"isRegexp"` } +// Rule is the config for rule. type Rule struct { Hosts []*HostInfo `json:"hosts"` Paths []*Path `json:"paths"` } +// PathType is the type of path. type PathType string const ( - PathTypePrefix PathType = "prefix" - PathTypeExact PathType = "exact" - PathTypeRe PathType = "regexp" + // PathTypePrefix is the prefix type of path. + PathTypePrefix PathType = "prefix" + // PathTypeExact is the exact type of path. + PathTypeExact PathType = "exact" + // PathTypeRe is the regexp type of path. + PathTypeRe PathType = "regexp" + // PathTypeReInsensitive is the case insensitive regexp type of path. PathTypeReInsensitive PathType = "caseInsensitiveRegexp" ) +// Path is the config for path. type Path struct { Path string `json:"path"` Type PathType `json:"type"` Backend *ProxyInfo `json:"backend"` } +// ProxyInfo is the config for proxy. type ProxyInfo struct { Servers []*BackendInfo `json:"servers"` SetHeaders map[string]string `json:"setHeaders"` GzipMinLength int `json:"gzipMinLength"` } +// BackendInfo is the config for backend. type BackendInfo struct { Server string `json:"server"` Weight int `json:"weight"` From 64689a2c245def18e789545c55952ca2d82a1cda Mon Sep 17 00:00:00 2001 From: su chen Date: Fri, 10 Nov 2023 13:50:45 +0800 Subject: [PATCH 19/22] Update cmd/client/commandv2/convert.go Co-authored-by: Yun Long --- cmd/client/commandv2/convert.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/client/commandv2/convert.go b/cmd/client/commandv2/convert.go index c3d55bfa19..1dc6a2bc21 100644 --- a/cmd/client/commandv2/convert.go +++ b/cmd/client/commandv2/convert.go @@ -27,7 +27,7 @@ import ( func ConvertCmd() *cobra.Command { cmd := &cobra.Command{ Use: "convert", - Short: "Convert other format to easegress yaml file", + Short: "Convert other kinds of config to Easegress yaml file", } cmd.AddCommand(nginx.Cmd()) return cmd From 1f9f2cb84bc020fda22cf13eea17097e0911941e Mon Sep 17 00:00:00 2001 From: su chen Date: Fri, 10 Nov 2023 13:51:09 +0800 Subject: [PATCH 20/22] Update cmd/client/commandv2/convert/nginx/cmd.go Co-authored-by: Yun Long --- cmd/client/commandv2/convert/nginx/cmd.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/client/commandv2/convert/nginx/cmd.go b/cmd/client/commandv2/convert/nginx/cmd.go index a864bdab7d..3c80bbbd08 100644 --- a/cmd/client/commandv2/convert/nginx/cmd.go +++ b/cmd/client/commandv2/convert/nginx/cmd.go @@ -96,7 +96,7 @@ func (opt *Options) init() { opt.usedNames[""] = struct{}{} } -// GetPipelineName create a global uniq name for pipeline based on path. +// GetPipelineName creates a globally unique name for the pipeline based on the path. func (opt *Options) GetPipelineName(path string) string { letters := []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") nameRunes := make([]rune, 0) From f7c6ba2591a4e1b8d31dc9e1082698e5a50a1884 Mon Sep 17 00:00:00 2001 From: chen Date: Fri, 10 Nov 2023 14:18:58 +0800 Subject: [PATCH 21/22] udpate based on review advice --- cmd/client/commandv2/convert/nginx/cmd.go | 22 +++++++------- .../commandv2/convert/nginx/cmd_test.go | 6 ++-- cmd/client/commandv2/convert/nginx/convert.go | 30 +++++++++---------- .../commandv2/convert/nginx/convert_test.go | 14 ++++----- .../commandv2/create/createhttpproxy.go | 16 +++++----- .../commandv2/create/createhttpproxy_test.go | 18 +++++------ .../commandv2/{common => specs}/spec.go | 2 +- 7 files changed, 54 insertions(+), 54 deletions(-) rename cmd/client/commandv2/{common => specs}/spec.go (99%) diff --git a/cmd/client/commandv2/convert/nginx/cmd.go b/cmd/client/commandv2/convert/nginx/cmd.go index 3c80bbbd08..ece278396b 100644 --- a/cmd/client/commandv2/convert/nginx/cmd.go +++ b/cmd/client/commandv2/convert/nginx/cmd.go @@ -23,7 +23,7 @@ import ( "os" "path/filepath" - "github.com/megaease/easegress/v2/cmd/client/commandv2/common" + "github.com/megaease/easegress/v2/cmd/client/commandv2/specs" "github.com/megaease/easegress/v2/cmd/client/general" "github.com/megaease/easegress/v2/pkg/util/codectool" crossplane "github.com/nginxinc/nginx-go-crossplane" @@ -32,10 +32,10 @@ import ( // Options contains the options for convert nginx.conf. type Options struct { - NginxConf string - Output string - Prefix string - usedNames map[string]struct{} + NginxConf string + Output string + ResourcePrefix string + usedNames map[string]struct{} } // Cmd returns convert nginx.conf command. @@ -45,7 +45,7 @@ func Cmd() *cobra.Command { examples := []general.Example{ { Desc: "Convert nginx config to easegress yamls", - Command: "egctl convert nginx -f -o --prefix ", + Command: "egctl convert nginx -f -o --resource-prefix ", }, } cmd := &cobra.Command{ @@ -59,7 +59,7 @@ func Cmd() *cobra.Command { if flags.Output == "" { return fmt.Errorf("output yaml file path is required") } - if flags.Prefix == "" { + if flags.ResourcePrefix == "" { return fmt.Errorf("prefix is required") } return nil @@ -87,7 +87,7 @@ func Cmd() *cobra.Command { } cmd.Flags().StringVarP(&flags.NginxConf, "file", "f", "", "nginx.conf file path") cmd.Flags().StringVarP(&flags.Output, "output", "o", "", "output yaml file path") - cmd.Flags().StringVar(&flags.Prefix, "prefix", "", "prefix of output yaml resources") + cmd.Flags().StringVar(&flags.ResourcePrefix, "resource-prefix", "nginx", "prefix of output yaml resources") return cmd } @@ -108,7 +108,7 @@ func (opt *Options) GetPipelineName(path string) string { name := string(nameRunes) if _, ok := opt.usedNames[name]; !ok { opt.usedNames[name] = struct{}{} - return fmt.Sprintf("%s-%s", opt.Prefix, name) + return fmt.Sprintf("%s-%s", opt.ResourcePrefix, name) } for i := 0; i < 8; i++ { nameRunes = append(nameRunes, letters[rand.Intn(len(letters))]) @@ -116,12 +116,12 @@ func (opt *Options) GetPipelineName(path string) string { name = string(nameRunes) if _, ok := opt.usedNames[name]; !ok { opt.usedNames[name] = struct{}{} - return fmt.Sprintf("%s-%s", opt.Prefix, name) + return fmt.Sprintf("%s-%s", opt.ResourcePrefix, name) } return opt.GetPipelineName(path) } -func writeYaml(filename string, servers []*common.HTTPServerSpec, pipelines []*common.PipelineSpec) error { +func writeYaml(filename string, servers []*specs.HTTPServerSpec, pipelines []*specs.PipelineSpec) error { absPath, err := filepath.Abs(filename) if err != nil { return err diff --git a/cmd/client/commandv2/convert/nginx/cmd_test.go b/cmd/client/commandv2/convert/nginx/cmd_test.go index 077ec05271..4dbfef6fcb 100644 --- a/cmd/client/commandv2/convert/nginx/cmd_test.go +++ b/cmd/client/commandv2/convert/nginx/cmd_test.go @@ -70,9 +70,9 @@ func TestCmd(t *testing.T) { func TestOption(t *testing.T) { option := &Options{ - NginxConf: "test.conf", - Output: "test.yaml", - Prefix: "test", + NginxConf: "test.conf", + Output: "test.yaml", + ResourcePrefix: "test", } option.init() path := option.GetPipelineName("/user") diff --git a/cmd/client/commandv2/convert/nginx/convert.go b/cmd/client/commandv2/convert/nginx/convert.go index f50575e431..7eec300c16 100644 --- a/cmd/client/commandv2/convert/nginx/convert.go +++ b/cmd/client/commandv2/convert/nginx/convert.go @@ -23,7 +23,7 @@ import ( "sort" "strings" - "github.com/megaease/easegress/v2/cmd/client/commandv2/common" + "github.com/megaease/easegress/v2/cmd/client/commandv2/specs" "github.com/megaease/easegress/v2/cmd/client/general" "github.com/megaease/easegress/v2/pkg/filters" "github.com/megaease/easegress/v2/pkg/filters/builder" @@ -34,9 +34,9 @@ import ( "github.com/megaease/easegress/v2/pkg/util/codectool" ) -func convertConfig(options *Options, config *Config) ([]*common.HTTPServerSpec, []*common.PipelineSpec, error) { - httpServers := make([]*common.HTTPServerSpec, 0) - pipelines := make([]*common.PipelineSpec, 0) +func convertConfig(options *Options, config *Config) ([]*specs.HTTPServerSpec, []*specs.PipelineSpec, error) { + httpServers := make([]*specs.HTTPServerSpec, 0) + pipelines := make([]*specs.PipelineSpec, 0) for _, server := range config.Servers { s, p, err := convertServer(options, server) if err != nil { @@ -48,9 +48,9 @@ func convertConfig(options *Options, config *Config) ([]*common.HTTPServerSpec, return httpServers, pipelines, nil } -func convertServer(options *Options, server *Server) (*common.HTTPServerSpec, []*common.PipelineSpec, error) { - pipelines := make([]*common.PipelineSpec, 0) - httpServer := common.NewHTTPServerSpec(fmt.Sprintf("%s-%d", options.Prefix, server.Port)) +func convertServer(options *Options, server *Server) (*specs.HTTPServerSpec, []*specs.PipelineSpec, error) { + pipelines := make([]*specs.PipelineSpec, 0) + httpServer := specs.NewHTTPServerSpec(fmt.Sprintf("%s-%d", options.ResourcePrefix, server.Port)) httpServer = convertServerBase(httpServer, server.ServerBase) httpServer.Rules = make([]*routers.Rule, 0) @@ -62,7 +62,7 @@ func convertServer(options *Options, server *Server) (*common.HTTPServerSpec, [] return httpServer, pipelines, nil } -func convertServerBase(spec *common.HTTPServerSpec, base ServerBase) *common.HTTPServerSpec { +func convertServerBase(spec *specs.HTTPServerSpec, base ServerBase) *specs.HTTPServerSpec { spec.Port = uint16(base.Port) spec.Address = base.Address spec.HTTPS = base.HTTPS @@ -79,7 +79,7 @@ func convertServerBase(spec *common.HTTPServerSpec, base ServerBase) *common.HTT // convertRule converts nginx conf to easegress rule. // exact path > prefix path > regexp path. // prefix path should be sorted by path length. -func convertRule(options *Options, rule *Rule) (*routers.Rule, []*common.PipelineSpec) { +func convertRule(options *Options, rule *Rule) (*routers.Rule, []*specs.PipelineSpec) { router := &routers.Rule{ Hosts: make([]routers.Host, 0), Paths: make([]*routers.Path, 0), @@ -91,7 +91,7 @@ func convertRule(options *Options, rule *Rule) (*routers.Rule, []*common.Pipelin }) } - pipelines := make([]*common.PipelineSpec, 0) + pipelines := make([]*specs.PipelineSpec, 0) exactPaths := make([]*routers.Path, 0) prefixPaths := make([]*routers.Path, 0) rePaths := make([]*routers.Path, 0) @@ -132,8 +132,8 @@ func convertRule(options *Options, rule *Rule) (*routers.Rule, []*common.Pipelin return router, pipelines } -func convertProxy(name string, info *ProxyInfo) *common.PipelineSpec { - pipeline := common.NewPipelineSpec(name) +func convertProxy(name string, info *ProxyInfo) *specs.PipelineSpec { + pipeline := specs.NewPipelineSpec(name) flow := make([]filters.Spec, 0) if len(info.SetHeaders) != 0 { @@ -185,7 +185,7 @@ func translateNginxEmbeddedVar(s string) string { } func getRequestAdaptor(info *ProxyInfo) *builder.RequestAdaptorSpec { - spec := common.NewRequestAdaptorFilterSpec("request-adaptor") + spec := specs.NewRequestAdaptorFilterSpec("request-adaptor") template := &builder.RequestAdaptorTemplate{ Header: &httpheader.AdaptSpec{ Set: make(map[string]string), @@ -215,7 +215,7 @@ func getWebsocketFilter(info *ProxyInfo) *httpproxy.WebSocketProxySpec { s.Server = strings.Replace(s.Server, "https://", "wss://", 1) info.Servers[i] = s } - spec := common.NewWebsocketFilterSpec("websocket") + spec := specs.NewWebsocketFilterSpec("websocket") spec.Pools = []*httpproxy.WebSocketServerPoolSpec{{ BaseServerPoolSpec: getBaseServerPool(info), }} @@ -223,7 +223,7 @@ func getWebsocketFilter(info *ProxyInfo) *httpproxy.WebSocketProxySpec { } func getProxyFilter(info *ProxyInfo) *httpproxy.Spec { - spec := common.NewProxyFilterSpec("proxy") + spec := specs.NewProxyFilterSpec("proxy") if info.GzipMinLength != 0 { spec.Compression = &httpproxy.CompressionSpec{ MinLength: uint32(info.GzipMinLength), diff --git a/cmd/client/commandv2/convert/nginx/convert_test.go b/cmd/client/commandv2/convert/nginx/convert_test.go index 87bedeb623..4063e92b1b 100644 --- a/cmd/client/commandv2/convert/nginx/convert_test.go +++ b/cmd/client/commandv2/convert/nginx/convert_test.go @@ -23,7 +23,7 @@ import ( "strings" "testing" - "github.com/megaease/easegress/v2/cmd/client/commandv2/common" + "github.com/megaease/easegress/v2/cmd/client/commandv2/specs" "github.com/megaease/easegress/v2/pkg/filters" "github.com/megaease/easegress/v2/pkg/filters/builder" "github.com/megaease/easegress/v2/pkg/protocols/httpprot" @@ -79,7 +79,7 @@ func TestGetRequestAdaptor(t *testing.T) { func TestConvertConfig(t *testing.T) { options := &Options{ - Prefix: "test-convert", + ResourcePrefix: "test-convert", } options.init() conf := ` @@ -175,7 +175,7 @@ rules: - pathRegexp: (?i)/case-insensitive-regexp backend: test-convert-caseinsensitiveregexp ` - expected := common.NewHTTPServerSpec("test-convert-8080") + expected := specs.NewHTTPServerSpec("test-convert-8080") err = codectool.UnmarshalYAML([]byte(serverYaml), expected) assert.Nil(t, err) assert.Equal(t, expected, httpServers[0]) @@ -256,7 +256,7 @@ filters: weight: 1 ` for i, yamlStr := range []string{pipelineApis, pipelineExact, pipelineRegexp, pipelineCIReg, pipelineWebsocket} { - spec := common.NewPipelineSpec("") + spec := specs.NewPipelineSpec("") err = codectool.UnmarshalYAML([]byte(yamlStr), spec) assert.Nil(t, err, i) for j, f := range spec.Filters { @@ -275,15 +275,15 @@ func compareFilter(t *testing.T, f1 map[string]interface{}, f2 map[string]interf switch f1["kind"] { case "Proxy": specFn = func() interface{} { - return common.NewProxyFilterSpec("") + return specs.NewProxyFilterSpec("") } case "RequestAdaptor": specFn = func() interface{} { - return common.NewRequestAdaptorFilterSpec("") + return specs.NewRequestAdaptorFilterSpec("") } case "WebSocketProxy": specFn = func() interface{} { - return common.NewWebsocketFilterSpec("") + return specs.NewWebsocketFilterSpec("") } default: t.Errorf("filter kind %s is not compared", f1["kind"]) diff --git a/cmd/client/commandv2/create/createhttpproxy.go b/cmd/client/commandv2/create/createhttpproxy.go index 350d20cc37..5f273ce6fe 100644 --- a/cmd/client/commandv2/create/createhttpproxy.go +++ b/cmd/client/commandv2/create/createhttpproxy.go @@ -24,7 +24,7 @@ import ( "path/filepath" "strings" - "github.com/megaease/easegress/v2/cmd/client/commandv2/common" + "github.com/megaease/easegress/v2/cmd/client/commandv2/specs" "github.com/megaease/easegress/v2/cmd/client/general" "github.com/megaease/easegress/v2/cmd/client/resources" "github.com/megaease/easegress/v2/pkg/filters" @@ -206,8 +206,8 @@ func (o *HTTPProxyOptions) getPipelineName(id int) string { } // Translate translates HTTPProxyOptions to HTTPServerSpec and PipelineSpec. -func (o *HTTPProxyOptions) Translate() (*common.HTTPServerSpec, []*common.PipelineSpec) { - hs := common.NewHTTPServerSpec(o.getServerName()) +func (o *HTTPProxyOptions) Translate() (*specs.HTTPServerSpec, []*specs.PipelineSpec) { + hs := specs.NewHTTPServerSpec(o.getServerName()) hs.Port = uint16(o.Port) if o.TLS { hs.HTTPS = true @@ -226,9 +226,9 @@ func (o *HTTPProxyOptions) Translate() (*common.HTTPServerSpec, []*common.Pipeli return hs, pipelines } -func (o *HTTPProxyOptions) translateRules() (routers.Rules, []*common.PipelineSpec) { +func (o *HTTPProxyOptions) translateRules() (routers.Rules, []*specs.PipelineSpec) { var rules routers.Rules - var pipelines []*common.PipelineSpec + var pipelines []*specs.PipelineSpec pipelineID := 0 for _, rule := range o.rules { @@ -241,7 +241,7 @@ func (o *HTTPProxyOptions) translateRules() (routers.Rules, []*common.PipelineSp Backend: pipelineName, } - pipelineSpec := common.NewPipelineSpec(pipelineName) + pipelineSpec := specs.NewPipelineSpec(pipelineName) translateToPipeline(rule.Endpoints, pipelineSpec) pipelines = append(pipelines, pipelineSpec) @@ -284,13 +284,13 @@ func loadCertFile(filePath string) (string, error) { return base64.StdEncoding.EncodeToString(data), nil } -func translateToPipeline(endpoints []string, spec *common.PipelineSpec) { +func translateToPipeline(endpoints []string, spec *specs.PipelineSpec) { proxy := translateToProxyFilter(endpoints) spec.SetFilters([]filters.Spec{proxy}) } func translateToProxyFilter(endpoints []string) *httpproxy.Spec { - spec := common.NewProxyFilterSpec("proxy") + spec := specs.NewProxyFilterSpec("proxy") servers := make([]*proxies.Server, len(endpoints)) for i, endpoint := range endpoints { diff --git a/cmd/client/commandv2/create/createhttpproxy_test.go b/cmd/client/commandv2/create/createhttpproxy_test.go index f0ffd8ce13..a8aec49c6a 100644 --- a/cmd/client/commandv2/create/createhttpproxy_test.go +++ b/cmd/client/commandv2/create/createhttpproxy_test.go @@ -24,7 +24,7 @@ import ( "path/filepath" "testing" - "github.com/megaease/easegress/v2/cmd/client/commandv2/common" + "github.com/megaease/easegress/v2/cmd/client/commandv2/specs" "github.com/megaease/easegress/v2/pkg/filters/proxies/httpproxy" "github.com/megaease/easegress/v2/pkg/object/httpserver/routers" "github.com/megaease/easegress/v2/pkg/util/codectool" @@ -44,7 +44,7 @@ pools: loadBalance: policy: roundRobin ` - expected := common.NewProxyFilterSpec("proxy") + expected := specs.NewProxyFilterSpec("proxy") err := codectool.UnmarshalYAML([]byte(yamlStr), expected) assert.Nil(err) @@ -71,12 +71,12 @@ filters: policy: roundRobin ` // compare expected and got pipeline - expected := common.NewPipelineSpec("pipeline") + expected := specs.NewPipelineSpec("pipeline") err := codectool.UnmarshalYAML([]byte(yamlStr), expected) assert.Nil(err) endpoints := []string{"http://127.0.0.1:9095", "http://127.0.0.1:9096"} - got := common.NewPipelineSpec("pipeline") + got := specs.NewPipelineSpec("pipeline") translateToPipeline(endpoints, got) // filters part is not compare here, because the filter part is map[string]interface{}, @@ -93,7 +93,7 @@ filters: // if marshal it once, some part of expectedFilter will be nil. // but gotFilter will be empty. for example []string{} vs nil. // []string{} and nil are actually same in this case. - expectedFilter := common.NewProxyFilterSpec("proxy") + expectedFilter := specs.NewProxyFilterSpec("proxy") filterYaml := codectool.MustMarshalYAML(expected.Filters[0]) err = codectool.UnmarshalYAML(filterYaml, expectedFilter) assert.Nil(err) @@ -101,7 +101,7 @@ filters: err = codectool.UnmarshalYAML(filterYaml, expectedFilter) assert.Nil(err) - gotFilter := common.NewProxyFilterSpec("proxy") + gotFilter := specs.NewProxyFilterSpec("proxy") filterYaml = codectool.MustMarshalYAML(got.Filters[0]) err = codectool.UnmarshalYAML(filterYaml, gotFilter) assert.Nil(err) @@ -312,11 +312,11 @@ filters: policy: roundRobin ` expectedFilter := func() *httpproxy.Spec { - expected := common.NewPipelineSpec("pipeline") + expected := specs.NewPipelineSpec("pipeline") err = codectool.UnmarshalYAML([]byte(yamlStr), expected) assert.Nil(err) - expectedFilter := common.NewProxyFilterSpec("proxy") + expectedFilter := specs.NewProxyFilterSpec("proxy") filterYaml := codectool.MustMarshalYAML(expected.Filters[0]) err = codectool.UnmarshalYAML(filterYaml, expectedFilter) assert.Nil(err) @@ -327,7 +327,7 @@ filters: }() for i, p := range pls { - gotFilter := common.NewProxyFilterSpec("proxy") + gotFilter := specs.NewProxyFilterSpec("proxy") filterYaml := codectool.MustMarshalYAML(p.Filters[0]) err = codectool.UnmarshalYAML(filterYaml, gotFilter) assert.Nil(err) diff --git a/cmd/client/commandv2/common/spec.go b/cmd/client/commandv2/specs/spec.go similarity index 99% rename from cmd/client/commandv2/common/spec.go rename to cmd/client/commandv2/specs/spec.go index 858766327b..f4d91ab32c 100644 --- a/cmd/client/commandv2/common/spec.go +++ b/cmd/client/commandv2/specs/spec.go @@ -15,7 +15,7 @@ * limitations under the License. */ -package common +package specs import ( "github.com/megaease/easegress/v2/cmd/client/general" From 847154464f9a62a91eb618d61cd3ccd25599efbc Mon Sep 17 00:00:00 2001 From: chen Date: Fri, 10 Nov 2023 14:27:15 +0800 Subject: [PATCH 22/22] fix test --- cmd/client/commandv2/convert/nginx/cmd_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/client/commandv2/convert/nginx/cmd_test.go b/cmd/client/commandv2/convert/nginx/cmd_test.go index 4dbfef6fcb..dc6c93f1ab 100644 --- a/cmd/client/commandv2/convert/nginx/cmd_test.go +++ b/cmd/client/commandv2/convert/nginx/cmd_test.go @@ -35,9 +35,9 @@ func TestCmd(t *testing.T) { assert.NotNil(t, cmd.Args(cmd, []string{})) cmd.ParseFlags([]string{"-o", "test.yaml"}) - assert.NotNil(t, cmd.Args(cmd, []string{})) + assert.Nil(t, cmd.Args(cmd, []string{})) - cmd.ParseFlags([]string{"--prefix", "test"}) + cmd.ParseFlags([]string{"--resource-prefix", "test"}) assert.Nil(t, cmd.Args(cmd, []string{})) tempDir := newTempTestDir(t)