diff --git a/admin/test/utils.go b/admin/test/utils.go index 086ba2e6e47..26391c41ee0 100644 --- a/admin/test/utils.go +++ b/admin/test/utils.go @@ -26,12 +26,10 @@ import ( "testing" "time" - "github.com/google/uuid" "google.golang.org/grpc" "google.golang.org/grpc/admin" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials/insecure" - "google.golang.org/grpc/internal/testutils/xds/bootstrap" "google.golang.org/grpc/status" v3statusgrpc "github.com/envoyproxy/go-control-plane/envoy/service/status/v3" @@ -54,16 +52,6 @@ type ExpectedStatusCodes struct { // RunRegisterTests makes a client, runs the RPCs, and compares the status // codes. func RunRegisterTests(t *testing.T, ec ExpectedStatusCodes) { - nodeID := uuid.New().String() - bootstrapCleanup, err := bootstrap.CreateFile(bootstrap.Options{ - NodeID: nodeID, - ServerURI: "no.need.for.a.server", - }) - if err != nil { - t.Fatal(err) - } - defer bootstrapCleanup() - lis, err := net.Listen("tcp", "localhost:0") if err != nil { t.Fatalf("cannot create listener: %v", err) diff --git a/internal/testutils/xds/e2e/setup_certs.go b/internal/testutils/tls_creds.go similarity index 62% rename from internal/testutils/xds/e2e/setup_certs.go rename to internal/testutils/tls_creds.go index fb289487a4a..d93151380c1 100644 --- a/internal/testutils/xds/e2e/setup_certs.go +++ b/internal/testutils/tls_creds.go @@ -1,6 +1,6 @@ /* * - * Copyright 2022 gRPC authors. + * Copyright 2024 gRPC authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,65 +13,20 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ -package e2e +package testutils import ( "crypto/tls" "crypto/x509" - "fmt" "os" - "path" "testing" "google.golang.org/grpc/credentials" "google.golang.org/grpc/testdata" ) -const ( - // Names of files inside tempdir, for certprovider plugin to watch. - certFile = "cert.pem" - keyFile = "key.pem" - rootFile = "ca.pem" -) - -func createTmpFile(src, dst string) error { - data, err := os.ReadFile(src) - if err != nil { - return fmt.Errorf("os.ReadFile(%q) failed: %v", src, err) - } - if err := os.WriteFile(dst, data, os.ModePerm); err != nil { - return fmt.Errorf("os.WriteFile(%q) failed: %v", dst, err) - } - return nil -} - -// createTempDirWithFiles creates a temporary directory under the system default -// tempDir with the given dirSuffix. It also reads from certSrc, keySrc and -// rootSrc files are creates appropriate files under the newly create tempDir. -// Returns the name of the created tempDir. -func createTmpDirWithFiles(dirSuffix, certSrc, keySrc, rootSrc string) (string, error) { - // Create a temp directory. Passing an empty string for the first argument - // uses the system temp directory. - dir, err := os.MkdirTemp("", dirSuffix) - if err != nil { - return "", fmt.Errorf("os.MkdirTemp() failed: %v", err) - } - - if err := createTmpFile(testdata.Path(certSrc), path.Join(dir, certFile)); err != nil { - return "", err - } - if err := createTmpFile(testdata.Path(keySrc), path.Join(dir, keyFile)); err != nil { - return "", err - } - if err := createTmpFile(testdata.Path(rootSrc), path.Join(dir, rootFile)); err != nil { - return "", err - } - return dir, nil -} - // CreateClientTLSCredentials creates client-side TLS transport credentials // using certificate and key files from testdata/x509 directory. func CreateClientTLSCredentials(t *testing.T) credentials.TransportCredentials { diff --git a/internal/testutils/xds/bootstrap/bootstrap.go b/internal/testutils/xds/bootstrap/bootstrap.go deleted file mode 100644 index 2ecd46d9aa6..00000000000 --- a/internal/testutils/xds/bootstrap/bootstrap.go +++ /dev/null @@ -1,169 +0,0 @@ -/* - * - * Copyright 2021 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -// Package bootstrap provides functionality to generate bootstrap configuration. -package bootstrap - -import ( - "encoding/json" - "fmt" - "os" - - "google.golang.org/grpc/grpclog" - "google.golang.org/grpc/internal/envconfig" -) - -var logger = grpclog.Component("internal/xds") - -// Options wraps the parameters used to generate bootstrap configuration. -type Options struct { - // NodeID is the node identifier of the gRPC client/server node in the - // proxyless service mesh. - NodeID string - // ServerURI is the address of the management server. - ServerURI string - // IgnoreResourceDeletion, if true, results in a bootstrap config with the - // `server_features` list containing `ignore_resource_deletion`. This results - // in gRPC ignoring resource deletions from the management server, as per A53. - IgnoreResourceDeletion bool - // ClientDefaultListenerResourceNameTemplate is the default listener - // resource name template to be used on the gRPC client. - ClientDefaultListenerResourceNameTemplate string - // ServerListenerResourceNameTemplate is the listener resource name template - // to be used on the gRPC server. - ServerListenerResourceNameTemplate string - // CertificateProviders is the certificate providers configuration. - CertificateProviders map[string]json.RawMessage - // Authorities is a list of non-default authorities. - // - // In the config, an authority contains {ServerURI, xds-version, creds, - // features, etc}. Note that this fields only has ServerURI (it's a - // map[authority-name]ServerURI). The other fields (version, creds, - // features) are assumed to be the same as the default authority (they can - // be added later if needed). - // - // If the env var corresponding to federation (envconfig.XDSFederation) is - // set, an entry with empty string as the key and empty server config as - // value will be added. This will be used by new style resource names with - // an empty authority. - Authorities map[string]string -} - -// CreateFile creates a temporary file with bootstrap contents, based on the -// passed in options, and updates the bootstrap environment variable to point to -// this file. -// -// Returns a cleanup function which will be non-nil if the setup process was -// completed successfully. It is the responsibility of the caller to invoke the -// cleanup function at the end of the test. -func CreateFile(opts Options) (func(), error) { - bootstrapContents, err := Contents(opts) - if err != nil { - return nil, err - } - f, err := os.CreateTemp("", "test_xds_bootstrap_*") - if err != nil { - return nil, fmt.Errorf("failed to created bootstrap file: %v", err) - } - - if err := os.WriteFile(f.Name(), bootstrapContents, 0644); err != nil { - return nil, fmt.Errorf("failed to created bootstrap file: %v", err) - } - logger.Infof("Created bootstrap file at %q with contents: %s\n", f.Name(), bootstrapContents) - - origBootstrapFileName := envconfig.XDSBootstrapFileName - envconfig.XDSBootstrapFileName = f.Name() - return func() { - os.Remove(f.Name()) - envconfig.XDSBootstrapFileName = origBootstrapFileName - }, nil -} - -// Contents returns the contents to go into a bootstrap file, environment, or -// configuration passed to xds.NewXDSResolverWithConfigForTesting. -func Contents(opts Options) ([]byte, error) { - cfg := &bootstrapConfig{ - XdsServers: []server{ - { - ServerURI: opts.ServerURI, - ChannelCreds: []creds{{Type: "insecure"}}, - }, - }, - Node: node{ - ID: opts.NodeID, - }, - CertificateProviders: opts.CertificateProviders, - ClientDefaultListenerResourceNameTemplate: opts.ClientDefaultListenerResourceNameTemplate, - ServerListenerResourceNameTemplate: opts.ServerListenerResourceNameTemplate, - } - if opts.IgnoreResourceDeletion { - cfg.XdsServers[0].ServerFeatures = append(cfg.XdsServers[0].ServerFeatures, "ignore_resource_deletion") - } - - // This will end up using the top-level server list for new style - // resources with empty authority. - auths := map[string]authority{"": {}} - for n, auURI := range opts.Authorities { - // If the authority server URI is empty, set it to an empty authority - // config, resulting in it using the top-level xds server config. - a := authority{} - if auURI != "" { - a = authority{XdsServers: []server{{ - ServerURI: auURI, - ChannelCreds: []creds{{Type: "insecure"}}, - ServerFeatures: cfg.XdsServers[0].ServerFeatures, - }}} - } - auths[n] = a - } - cfg.Authorities = auths - - bootstrapContents, err := json.MarshalIndent(cfg, "", " ") - if err != nil { - return nil, fmt.Errorf("failed to created bootstrap file: %v", err) - } - return bootstrapContents, nil -} - -type bootstrapConfig struct { - XdsServers []server `json:"xds_servers,omitempty"` - Node node `json:"node,omitempty"` - CertificateProviders map[string]json.RawMessage `json:"certificate_providers,omitempty"` - ClientDefaultListenerResourceNameTemplate string `json:"client_default_listener_resource_name_template,omitempty"` - ServerListenerResourceNameTemplate string `json:"server_listener_resource_name_template,omitempty"` - Authorities map[string]authority `json:"authorities,omitempty"` -} - -type authority struct { - XdsServers []server `json:"xds_servers,omitempty"` -} - -type server struct { - ServerURI string `json:"server_uri,omitempty"` - ChannelCreds []creds `json:"channel_creds,omitempty"` - ServerFeatures []string `json:"server_features,omitempty"` -} - -type creds struct { - Type string `json:"type,omitempty"` - Config any `json:"config,omitempty"` -} - -type node struct { - ID string `json:"id,omitempty"` -} diff --git a/internal/testutils/xds/e2e/bootstrap.go b/internal/testutils/xds/e2e/bootstrap.go index 99702032f81..907a7b39617 100644 --- a/internal/testutils/xds/e2e/bootstrap.go +++ b/internal/testutils/xds/e2e/bootstrap.go @@ -21,6 +21,12 @@ package e2e import ( "encoding/json" "fmt" + "os" + "path" + "testing" + + "google.golang.org/grpc/internal/xds/bootstrap" + "google.golang.org/grpc/testdata" ) // DefaultFileWatcherConfig is a helper function to create a default certificate @@ -37,3 +43,88 @@ func DefaultFileWatcherConfig(certPath, keyPath, caPath string) json.RawMessage } }`, certPath, keyPath, caPath)) } + +// DefaultBootstrapContents creates a default bootstrap configuration with the +// given node ID and server URI. It also creates certificate provider +// configuration and sets the listener resource name template to be used on the +// server side. +func DefaultBootstrapContents(t *testing.T, nodeID, serverURI string) []byte { + t.Helper() + + // Create a directory to hold certs and key files used on the server side. + serverDir, err := createTmpDirWithCerts("testServerSideXDS*", "x509/server1_cert.pem", "x509/server1_key.pem", "x509/client_ca_cert.pem") + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + + // Create a directory to hold certs and key files used on the client side. + clientDir, err := createTmpDirWithCerts("testClientSideXDS*", "x509/client1_cert.pem", "x509/client1_key.pem", "x509/server_ca_cert.pem") + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + + // Create certificate providers section of the bootstrap config with entries + // for both the client and server sides. + cpc := map[string]json.RawMessage{ + ServerSideCertProviderInstance: DefaultFileWatcherConfig(path.Join(serverDir, certFile), path.Join(serverDir, keyFile), path.Join(serverDir, rootFile)), + ClientSideCertProviderInstance: DefaultFileWatcherConfig(path.Join(clientDir, certFile), path.Join(clientDir, keyFile), path.Join(clientDir, rootFile)), + } + + // Create the bootstrap configuration. + bs, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []json.RawMessage{[]byte(fmt.Sprintf(`{ + "server_uri": "passthrough:///%s", + "channel_creds": [{"type": "insecure"}] + }`, serverURI))}, + NodeID: nodeID, + CertificateProviders: cpc, + ServerListenerResourceNameTemplate: ServerListenerResourceNameTemplate, + }) + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + return bs +} + +const ( + // Names of files inside tempdir, for certprovider plugin to watch. + certFile = "cert.pem" + keyFile = "key.pem" + rootFile = "ca.pem" +) + +func createTmpFile(src, dst string) error { + data, err := os.ReadFile(src) + if err != nil { + return fmt.Errorf("os.ReadFile(%q) failed: %v", src, err) + } + if err := os.WriteFile(dst, data, os.ModePerm); err != nil { + return fmt.Errorf("os.WriteFile(%q) failed: %v", dst, err) + } + return nil +} + +// createTmpDirWithCerts creates a temporary directory under the system default +// tempDir with the given dirPattern. It also reads from certSrc, keySrc and +// rootSrc files and creates appropriate files under the newly create tempDir. +// Returns the path of the created tempDir if successful, and an error +// otherwise. +func createTmpDirWithCerts(dirPattern, certSrc, keySrc, rootSrc string) (string, error) { + // Create a temp directory. Passing an empty string for the first argument + // uses the system temp directory. + dir, err := os.MkdirTemp("", dirPattern) + if err != nil { + return "", fmt.Errorf("os.MkdirTemp() failed: %v", err) + } + + if err := createTmpFile(testdata.Path(certSrc), path.Join(dir, certFile)); err != nil { + return "", err + } + if err := createTmpFile(testdata.Path(keySrc), path.Join(dir, keyFile)); err != nil { + return "", err + } + if err := createTmpFile(testdata.Path(rootSrc), path.Join(dir, rootFile)); err != nil { + return "", err + } + return dir, nil +} diff --git a/internal/testutils/xds/e2e/logging.go b/internal/testutils/xds/e2e/logging.go index 2a0925a1306..8767ad0f5ed 100644 --- a/internal/testutils/xds/e2e/logging.go +++ b/internal/testutils/xds/e2e/logging.go @@ -18,31 +18,23 @@ package e2e -import ( - "fmt" - - "google.golang.org/grpc/grpclog" -) - -var logger = grpclog.Component("xds-e2e") - // serverLogger implements the Logger interface defined at // envoyproxy/go-control-plane/pkg/log. This is passed to the Snapshot cache. -type serverLogger struct{} +type serverLogger struct { + logger interface { + Logf(format string, args ...any) + } +} func (l serverLogger) Debugf(format string, args ...any) { - msg := fmt.Sprintf(format, args...) - logger.InfoDepth(1, msg) + l.logger.Logf(format, args) } func (l serverLogger) Infof(format string, args ...any) { - msg := fmt.Sprintf(format, args...) - logger.InfoDepth(1, msg) + l.logger.Logf(format, args) } func (l serverLogger) Warnf(format string, args ...any) { - msg := fmt.Sprintf(format, args...) - logger.WarningDepth(1, msg) + l.logger.Logf(format, args) } func (l serverLogger) Errorf(format string, args ...any) { - msg := fmt.Sprintf(format, args...) - logger.ErrorDepth(1, msg) + l.logger.Logf(format, args) } diff --git a/internal/testutils/xds/e2e/server.go b/internal/testutils/xds/e2e/server.go index e3bf0de5572..0ae551fa7cb 100644 --- a/internal/testutils/xds/e2e/server.go +++ b/internal/testutils/xds/e2e/server.go @@ -25,6 +25,7 @@ import ( "net" "reflect" "strconv" + "testing" "github.com/envoyproxy/go-control-plane/pkg/cache/types" "google.golang.org/grpc" @@ -60,6 +61,11 @@ type ManagementServer struct { gs *grpc.Server // gRPC server which exports the ADS service. cache v3cache.SnapshotCache // Resource snapshot. version int // Version of resource snapshot. + + // A logging interface, usually supplied from *testing.T. + logger interface { + Logf(format string, args ...any) + } } // ManagementServerOptions contains options to be passed to the management @@ -120,23 +126,25 @@ type ManagementServerOptions struct { // StartManagementServer initializes a management server which implements the // AggregatedDiscoveryService endpoint. The management server is initialized // with no resources. Tests should call the Update() method to change the -// resource snapshot held by the management server, as required by the test -// logic. When the test is done, it should call the Stop() method to cleanup -// resources allocated by the management server. -func StartManagementServer(opts ManagementServerOptions) (*ManagementServer, error) { +// resource snapshot held by the management server, as per by the test logic. +// +// Registers a cleanup function on t to stop the management server. +func StartManagementServer(t *testing.T, opts ManagementServerOptions) *ManagementServer { + t.Helper() + // Create a snapshot cache. The first parameter to NewSnapshotCache() // controls whether the server should wait for all resources to be // explicitly named in the request before responding to any of them. wait := !opts.AllowResourceSubset - cache := v3cache.NewSnapshotCache(wait, v3cache.IDHash{}, serverLogger{}) - logger.Infof("Created new snapshot cache...") + cache := v3cache.NewSnapshotCache(wait, v3cache.IDHash{}, serverLogger{t}) + t.Logf("Created new snapshot cache...") lis := opts.Listener if lis == nil { var err error lis, err = net.Listen("tcp", "localhost:0") if err != nil { - return nil, fmt.Errorf("listening on local host and port: %v", err) + t.Fatalf("Failed to listen on localhost:0: %v", err) } } @@ -155,7 +163,7 @@ func StartManagementServer(opts ManagementServerOptions) (*ManagementServer, err xs := v3server.NewServer(ctx, cache, callbacks) gs := grpc.NewServer() v3discoverygrpc.RegisterAggregatedDiscoveryServiceServer(gs, xs) - logger.Infof("Registered Aggregated Discovery Service (ADS)...") + t.Logf("Registered Aggregated Discovery Service (ADS)...") mgmtServer := &ManagementServer{ Address: lis.Addr().String(), @@ -164,19 +172,20 @@ func StartManagementServer(opts ManagementServerOptions) (*ManagementServer, err gs: gs, xs: xs, cache: cache, + logger: t, } if opts.SupportLoadReportingService { lrs := fakeserver.NewServer(lis.Addr().String()) v3lrsgrpc.RegisterLoadReportingServiceServer(gs, lrs) mgmtServer.LRSServer = lrs - logger.Infof("Registered Load Reporting Service (LRS)...") + t.Logf("Registered Load Reporting Service (LRS)...") } // Start serving. go gs.Serve(lis) - logger.Infof("xDS management server serving at: %v...", lis.Addr().String()) - - return mgmtServer, nil + t.Logf("xDS management server serving at: %v...", lis.Addr().String()) + t.Cleanup(mgmtServer.Stop) + return mgmtServer } // UpdateOptions wraps parameters to be passed to the Update() method. @@ -217,13 +226,13 @@ func (s *ManagementServer) Update(ctx context.Context, opts UpdateOptions) error return fmt.Errorf("failed to create new resource snapshot: %v", err) } } - logger.Infof("Created new resource snapshot...") + s.logger.Logf("Created new resource snapshot...") // Update the cache with the new resource snapshot. if err := s.cache.SetSnapshot(ctx, opts.NodeID, snapshot); err != nil { return fmt.Errorf("failed to update resource snapshot in management server: %v", err) } - logger.Infof("Updated snapshot cache with resource snapshot...") + s.logger.Logf("Updated snapshot cache with resource snapshot...") return nil } diff --git a/internal/testutils/xds/e2e/setup_management_server.go b/internal/testutils/xds/e2e/setup_management_server.go deleted file mode 100644 index 80fe007191c..00000000000 --- a/internal/testutils/xds/e2e/setup_management_server.go +++ /dev/null @@ -1,112 +0,0 @@ -/* - * - * Copyright 2022 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package e2e - -import ( - "encoding/json" - "fmt" - "path" - "testing" - - "github.com/google/uuid" - "google.golang.org/grpc/internal" - "google.golang.org/grpc/internal/testutils/xds/bootstrap" - "google.golang.org/grpc/resolver" -) - -// SetupManagementServer performs the following: -// - spin up an xDS management server on a local port -// - set up certificates for consumption by the file_watcher plugin -// - creates a bootstrap file in a temporary location -// - creates an xDS resolver using the above bootstrap contents -// -// Returns the following: -// - management server -// - nodeID to be used by the client when connecting to the management server -// - bootstrap contents to be used by the client -// - xDS resolver builder to be used by the client -// - a cleanup function to be invoked at the end of the test -func SetupManagementServer(t *testing.T, opts ManagementServerOptions) (*ManagementServer, string, []byte, resolver.Builder, func()) { - t.Helper() - - // Spin up an xDS management server on a local port. - server, err := StartManagementServer(opts) - if err != nil { - t.Fatalf("Failed to spin up the xDS management server: %v", err) - } - defer func() { - if err != nil { - server.Stop() - } - }() - - nodeID := uuid.New().String() - bootstrapContents, err := DefaultBootstrapContents(nodeID, fmt.Sprintf("passthrough:///%s", server.Address)) - if err != nil { - server.Stop() - t.Fatal(err) - } - var rb resolver.Builder - if newResolver := internal.NewXDSResolverWithConfigForTesting; newResolver != nil { - rb, err = newResolver.(func([]byte) (resolver.Builder, error))(bootstrapContents) - if err != nil { - server.Stop() - t.Fatalf("Failed to create xDS resolver for testing: %v", err) - } - } - - return server, nodeID, bootstrapContents, rb, func() { server.Stop() } -} - -// DefaultBootstrapContents creates a default bootstrap configuration with the -// given node ID and server URI. It also creates certificate provider -// configuration and sets the listener resource name template to be used on the -// server side. -func DefaultBootstrapContents(nodeID, serverURI string) ([]byte, error) { - // Create a directory to hold certs and key files used on the server side. - serverDir, err := createTmpDirWithFiles("testServerSideXDS*", "x509/server1_cert.pem", "x509/server1_key.pem", "x509/client_ca_cert.pem") - if err != nil { - return nil, fmt.Errorf("failed to create bootstrap configuration: %v", err) - } - - // Create a directory to hold certs and key files used on the client side. - clientDir, err := createTmpDirWithFiles("testClientSideXDS*", "x509/client1_cert.pem", "x509/client1_key.pem", "x509/server_ca_cert.pem") - if err != nil { - return nil, fmt.Errorf("failed to create bootstrap configuration: %v", err) - } - - // Create certificate providers section of the bootstrap config with entries - // for both the client and server sides. - cpc := map[string]json.RawMessage{ - ServerSideCertProviderInstance: DefaultFileWatcherConfig(path.Join(serverDir, certFile), path.Join(serverDir, keyFile), path.Join(serverDir, rootFile)), - ClientSideCertProviderInstance: DefaultFileWatcherConfig(path.Join(clientDir, certFile), path.Join(clientDir, keyFile), path.Join(clientDir, rootFile)), - } - - // Create the bootstrap configuration. - bs, err := bootstrap.Contents(bootstrap.Options{ - NodeID: nodeID, - ServerURI: serverURI, - CertificateProviders: cpc, - ServerListenerResourceNameTemplate: ServerListenerResourceNameTemplate, - }) - if err != nil { - return nil, fmt.Errorf("failed to create bootstrap configuration: %v", err) - } - return bs, nil -} diff --git a/internal/testutils/xds_bootsrap.go b/internal/testutils/xds_bootsrap.go new file mode 100644 index 00000000000..9e4fd2c96d2 --- /dev/null +++ b/internal/testutils/xds_bootsrap.go @@ -0,0 +1,53 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package testutils + +import ( + "os" + "testing" + + "google.golang.org/grpc/internal/envconfig" +) + +// CreateBootstrapFileForTesting creates a temporary file with the provided +// bootstrap contents, and updates the bootstrap environment variable to point +// to this file. +// +// Registers a cleanup function on the provided testing.T, that deletes the +// temporary file and resets the bootstrap environment variable. +func CreateBootstrapFileForTesting(t *testing.T, bootstrapContents []byte) { + t.Helper() + + f, err := os.CreateTemp("", "test_xds_bootstrap_*") + if err != nil { + t.Fatalf("Failed to created bootstrap file: %v", err) + } + + if err := os.WriteFile(f.Name(), bootstrapContents, 0644); err != nil { + t.Fatalf("Failed to created bootstrap file: %v", err) + } + t.Logf("Created bootstrap file at %q with contents: %s\n", f.Name(), bootstrapContents) + + origBootstrapFileName := envconfig.XDSBootstrapFileName + envconfig.XDSBootstrapFileName = f.Name() + t.Cleanup(func() { + os.Remove(f.Name()) + envconfig.XDSBootstrapFileName = origBootstrapFileName + }) +} diff --git a/internal/xds/bootstrap/bootstrap.go b/internal/xds/bootstrap/bootstrap.go index 7eaff2c9d36..2b789320c54 100644 --- a/internal/xds/bootstrap/bootstrap.go +++ b/internal/xds/bootstrap/bootstrap.go @@ -219,9 +219,9 @@ func (sc *ServerConfig) String() string { // The following fields correspond 1:1 with the JSON schema for ServerConfig. type serverConfigJSON struct { - ServerURI string `json:"server_uri"` - ChannelCreds []ChannelCreds `json:"channel_creds"` - ServerFeatures []string `json:"server_features"` + ServerURI string `json:"server_uri,omitempty"` + ChannelCreds []ChannelCreds `json:"channel_creds,omitempty"` + ServerFeatures []string `json:"server_features,omitempty"` } // MarshalJSON returns marshaled JSON bytes corresponding to this server config. @@ -571,6 +571,72 @@ func newConfigFromContents(data []byte) (*Config, error) { return config, nil } +// ConfigOptionsForTesting specifies options for creating a new bootstrap +// configuration for testing purposes. +// +// # Testing-Only +type ConfigOptionsForTesting struct { + // Servers is the top-level xDS server configuration + Servers []json.RawMessage + // CertificateProviders is the certificate providers configuration. + CertificateProviders map[string]json.RawMessage + // ServerListenerResourceNameTemplate is the listener resource name template + // to be used on the gRPC server. + ServerListenerResourceNameTemplate string + // ClientDefaultListenerResourceNameTemplate is the default listener + // resource name template to be used on the gRPC client. + ClientDefaultListenerResourceNameTemplate string + // Authorities is a list of non-default authorities. + Authorities map[string]json.RawMessage + // NodeID is the node identifier of the gRPC client/server node in the + // proxyless service mesh. + NodeID string +} + +// NewContentsForTesting creates a new bootstrap configuration from the passed in +// options, for testing purposes. +// +// # Testing-Only +func NewContentsForTesting(opts ConfigOptionsForTesting) ([]byte, error) { + var servers []*ServerConfig + for _, serverCfgJSON := range opts.Servers { + server := &ServerConfig{} + if err := server.UnmarshalJSON(serverCfgJSON); err != nil { + return nil, err + } + servers = append(servers, server) + } + certProviders := make(map[string]certproviderNameAndConfig) + for k, v := range opts.CertificateProviders { + cp := certproviderNameAndConfig{} + if err := json.Unmarshal(v, &cp); err != nil { + return nil, fmt.Errorf("failed to unmarshal certificate provider configuration for %s: %s", k, string(v)) + } + certProviders[k] = cp + } + authorities := make(map[string]*Authority) + for k, v := range opts.Authorities { + a := &Authority{} + if err := json.Unmarshal(v, a); err != nil { + return nil, fmt.Errorf("failed to unmarshal authority configuration for %s: %s", k, string(v)) + } + authorities[k] = a + } + cfgJSON := configJSON{ + XDSServers: servers, + CertificateProviders: certProviders, + ServerListenerResourceNameTemplate: opts.ServerListenerResourceNameTemplate, + ClientDefaultListenerResourceNameTemplate: opts.ClientDefaultListenerResourceNameTemplate, + Authorities: authorities, + Node: node{ID: opts.NodeID}, + } + contents, err := json.Marshal(cfgJSON) + if err != nil { + return nil, fmt.Errorf("failed to marshal bootstrap configuration for provided options %+v: %v", opts, err) + } + return contents, nil +} + // certproviderNameAndConfig is the internal representation of // the`certificate_providers` field in the bootstrap configuration. type certproviderNameAndConfig struct { diff --git a/internal/xds/bootstrap/tlscreds/bundle_ext_test.go b/internal/xds/bootstrap/tlscreds/bundle_ext_test.go index c6cd4024621..c5b5cd94aae 100644 --- a/internal/xds/bootstrap/tlscreds/bundle_ext_test.go +++ b/internal/xds/bootstrap/tlscreds/bundle_ext_test.go @@ -32,7 +32,7 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/internal/grpctest" "google.golang.org/grpc/internal/stubserver" - "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/testutils" "google.golang.org/grpc/internal/xds/bootstrap/tlscreds" testgrpc "google.golang.org/grpc/interop/grpc_testing" testpb "google.golang.org/grpc/interop/grpc_testing" @@ -165,7 +165,7 @@ func (s) TestCaReloading(t *testing.T) { } defer stop() - serverCredentials := grpc.Creds(e2e.CreateServerTLSCredentials(t, tls.NoClientCert)) + serverCredentials := grpc.Creds(testutils.CreateServerTLSCredentials(t, tls.NoClientCert)) server := stubserver.StartTestService(t, nil, serverCredentials) conn, err := grpc.NewClient( @@ -225,7 +225,7 @@ func (s) TestCaReloading(t *testing.T) { } func (s) TestMTLS(t *testing.T) { - s := stubserver.StartTestService(t, nil, grpc.Creds(e2e.CreateServerTLSCredentials(t, tls.RequireAndVerifyClientCert))) + s := stubserver.StartTestService(t, nil, grpc.Creds(testutils.CreateServerTLSCredentials(t, tls.RequireAndVerifyClientCert))) defer s.Stop() cfg := fmt.Sprintf(`{ diff --git a/internal/xds/bootstrap/tlscreds/bundle_test.go b/internal/xds/bootstrap/tlscreds/bundle_test.go index b4bee7f2f51..712e6d06195 100644 --- a/internal/xds/bootstrap/tlscreds/bundle_test.go +++ b/internal/xds/bootstrap/tlscreds/bundle_test.go @@ -30,10 +30,11 @@ import ( "google.golang.org/grpc/credentials/tls/certprovider" "google.golang.org/grpc/internal/grpctest" "google.golang.org/grpc/internal/stubserver" - "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/testdata" + testgrpc "google.golang.org/grpc/interop/grpc_testing" testpb "google.golang.org/grpc/interop/grpc_testing" - "google.golang.org/grpc/testdata" ) type s struct { @@ -53,7 +54,7 @@ func (f failingProvider) KeyMaterial(context.Context) (*certprovider.KeyMaterial func (f failingProvider) Close() {} func (s) TestFailingProvider(t *testing.T) { - s := stubserver.StartTestService(t, nil, grpc.Creds(e2e.CreateServerTLSCredentials(t, tls.RequireAndVerifyClientCert))) + s := stubserver.StartTestService(t, nil, grpc.Creds(testutils.CreateServerTLSCredentials(t, tls.RequireAndVerifyClientCert))) defer s.Stop() cfg := fmt.Sprintf(`{ diff --git a/stats/opentelemetry/csm/observability_test.go b/stats/opentelemetry/csm/observability_test.go index efa97199f53..b86c988010a 100644 --- a/stats/opentelemetry/csm/observability_test.go +++ b/stats/opentelemetry/csm/observability_test.go @@ -32,12 +32,13 @@ import ( "google.golang.org/grpc/encoding/gzip" istats "google.golang.org/grpc/internal/stats" "google.golang.org/grpc/internal/stubserver" - "google.golang.org/grpc/internal/testutils/xds/bootstrap" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" testgrpc "google.golang.org/grpc/interop/grpc_testing" testpb "google.golang.org/grpc/interop/grpc_testing" "google.golang.org/grpc/metadata" "google.golang.org/grpc/stats/opentelemetry" - "google.golang.org/grpc/stats/opentelemetry/internal/testutils" + itestutils "google.golang.org/grpc/stats/opentelemetry/internal/testutils" ) // setupEnv configures the environment for CSM Observability Testing. It sets @@ -46,13 +47,9 @@ import ( // simulate the environment. It registers a cleanup function on the provided t // to restore the environment to it's original state. func setupEnv(t *testing.T, resourceDetectorEmissions map[string]string, nodeID, csmCanonicalServiceName, csmWorkloadName string) { - cleanup, err := bootstrap.CreateFile(bootstrap.Options{ - NodeID: nodeID, - ServerURI: "xds_server_uri", - }) - if err != nil { - t.Fatalf("Failed to create bootstrap: %v", err) - } + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, "xds_server_uri") + testutils.CreateBootstrapFileForTesting(t, bootstrapContents) + oldCSMCanonicalServiceName, csmCanonicalServiceNamePresent := os.LookupEnv("CSM_CANONICAL_SERVICE_NAME") oldCSMWorkloadName, csmWorkloadNamePresent := os.LookupEnv("CSM_WORKLOAD_NAME") os.Setenv("CSM_CANONICAL_SERVICE_NAME", csmCanonicalServiceName) @@ -70,7 +67,6 @@ func setupEnv(t *testing.T, resourceDetectorEmissions map[string]string, nodeID, return &attrSet } t.Cleanup(func() { - cleanup() if csmCanonicalServiceNamePresent { os.Setenv("CSM_CANONICAL_SERVICE_NAME", oldCSMCanonicalServiceName) } else { @@ -134,7 +130,7 @@ func (s) TestCSMPluginOptionUnary(t *testing.T) { // To test the different operations for Unary RPC's from the interceptor // level that can plumb metadata exchange header in. unaryCallFunc func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) - opts testutils.MetricDataOptions + opts itestutils.MetricDataOptions }{ { name: "normal-flow", @@ -143,7 +139,7 @@ func (s) TestCSMPluginOptionUnary(t *testing.T) { Body: make([]byte, len(in.GetPayload().GetBody())), }}, nil }, - opts: testutils.MetricDataOptions{ + opts: itestutils.MetricDataOptions{ CSMLabels: csmLabels, UnaryCompressedMessageSize: float64(57), }, @@ -153,7 +149,7 @@ func (s) TestCSMPluginOptionUnary(t *testing.T) { unaryCallFunc: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { return nil, errors.New("some error") // return an error and no message - this triggers trailers only - no messages or headers sent }, - opts: testutils.MetricDataOptions{ + opts: itestutils.MetricDataOptions{ CSMLabels: csmLabels, UnaryCallFailed: true, }, @@ -167,7 +163,7 @@ func (s) TestCSMPluginOptionUnary(t *testing.T) { Body: make([]byte, len(in.GetPayload().GetBody())), }}, nil }, - opts: testutils.MetricDataOptions{ + opts: itestutils.MetricDataOptions{ CSMLabels: csmLabels, UnaryCompressedMessageSize: float64(57), }, @@ -181,7 +177,7 @@ func (s) TestCSMPluginOptionUnary(t *testing.T) { Body: make([]byte, len(in.GetPayload().GetBody())), }}, nil }, - opts: testutils.MetricDataOptions{ + opts: itestutils.MetricDataOptions{ CSMLabels: csmLabels, UnaryCompressedMessageSize: float64(57), }, @@ -193,7 +189,7 @@ func (s) TestCSMPluginOptionUnary(t *testing.T) { Body: make([]byte, len(in.GetPayload().GetBody())), }}, nil }, - opts: testutils.MetricDataOptions{ + opts: itestutils.MetricDataOptions{ CSMLabels: csmLabels, UnaryCompressedMessageSize: float64(57), }, @@ -247,8 +243,8 @@ func (s) TestCSMPluginOptionUnary(t *testing.T) { opts := test.opts opts.Target = ss.Target - wantMetrics := testutils.MetricDataUnary(opts) - testutils.CompareMetrics(ctx, t, reader, gotMetrics, wantMetrics) + wantMetrics := itestutils.MetricDataUnary(opts) + itestutils.CompareMetrics(ctx, t, reader, gotMetrics, wantMetrics) }) } } @@ -301,7 +297,7 @@ func (s) TestCSMPluginOptionStreaming(t *testing.T) { // To test the different operations for Streaming RPC's from the // interceptor level that can plumb metadata exchange header in. streamingCallFunc func(stream testgrpc.TestService_FullDuplexCallServer) error - opts testutils.MetricDataOptions + opts itestutils.MetricDataOptions }{ { name: "trailers-only", @@ -312,7 +308,7 @@ func (s) TestCSMPluginOptionStreaming(t *testing.T) { } } }, - opts: testutils.MetricDataOptions{ + opts: itestutils.MetricDataOptions{ CSMLabels: csmLabels, }, }, @@ -326,7 +322,7 @@ func (s) TestCSMPluginOptionStreaming(t *testing.T) { } } }, - opts: testutils.MetricDataOptions{ + opts: itestutils.MetricDataOptions{ CSMLabels: csmLabels, }, }, @@ -340,7 +336,7 @@ func (s) TestCSMPluginOptionStreaming(t *testing.T) { } } }, - opts: testutils.MetricDataOptions{ + opts: itestutils.MetricDataOptions{ CSMLabels: csmLabels, }, }, @@ -356,7 +352,7 @@ func (s) TestCSMPluginOptionStreaming(t *testing.T) { } } }, - opts: testutils.MetricDataOptions{ + opts: itestutils.MetricDataOptions{ CSMLabels: csmLabels, StreamingCompressedMessageSize: float64(57), }, @@ -420,8 +416,8 @@ func (s) TestCSMPluginOptionStreaming(t *testing.T) { opts := test.opts opts.Target = ss.Target - wantMetrics := testutils.MetricDataStreaming(opts) - testutils.CompareMetrics(ctx, t, reader, gotMetrics, wantMetrics) + wantMetrics := itestutils.MetricDataStreaming(opts) + itestutils.CompareMetrics(ctx, t, reader, gotMetrics, wantMetrics) }) } } @@ -538,7 +534,7 @@ func (s) TestXDSLabels(t *testing.T) { { Attributes: attribute.NewSet(unaryMethodClientSideEnd...), Count: 1, - Bounds: testutils.DefaultLatencyBounds, + Bounds: itestutils.DefaultLatencyBounds, }, }, Temporality: metricdata.CumulativeTemporality, @@ -553,7 +549,7 @@ func (s) TestXDSLabels(t *testing.T) { { Attributes: attribute.NewSet(unaryMethodClientSideEnd...), Count: 1, - Bounds: testutils.DefaultSizeBounds, + Bounds: itestutils.DefaultSizeBounds, BucketCounts: unaryBucketCounts, Min: unaryExtrema, Max: unaryExtrema, @@ -572,7 +568,7 @@ func (s) TestXDSLabels(t *testing.T) { { Attributes: attribute.NewSet(unaryMethodClientSideEnd...), Count: 1, - Bounds: testutils.DefaultSizeBounds, + Bounds: itestutils.DefaultSizeBounds, BucketCounts: unaryBucketCounts, Min: unaryExtrema, Max: unaryExtrema, @@ -591,7 +587,7 @@ func (s) TestXDSLabels(t *testing.T) { { Attributes: attribute.NewSet(unaryMethodAttr, targetAttr, unaryStatusAttr), Count: 1, - Bounds: testutils.DefaultLatencyBounds, + Bounds: itestutils.DefaultLatencyBounds, }, }, Temporality: metricdata.CumulativeTemporality, @@ -599,7 +595,7 @@ func (s) TestXDSLabels(t *testing.T) { }, } - testutils.CompareMetrics(ctx, t, reader, gotMetrics, wantMetrics) + itestutils.CompareMetrics(ctx, t, reader, gotMetrics, wantMetrics) } // TestObservability tests that Observability global function compiles and runs diff --git a/stats/opentelemetry/csm/pluginoption_test.go b/stats/opentelemetry/csm/pluginoption_test.go index 7bddcfad37f..8588c545360 100644 --- a/stats/opentelemetry/csm/pluginoption_test.go +++ b/stats/opentelemetry/csm/pluginoption_test.go @@ -29,7 +29,8 @@ import ( "google.golang.org/grpc/internal/envconfig" "google.golang.org/grpc/internal/grpctest" - "google.golang.org/grpc/internal/testutils/xds/bootstrap" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" "google.golang.org/grpc/metadata" "github.com/google/go-cmp/cmp" @@ -333,14 +334,8 @@ func (s) TestBootstrap(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - cleanup, err := bootstrap.CreateFile(bootstrap.Options{ - NodeID: test.nodeID, - ServerURI: "xds_server_uri", - }) - if err != nil { - t.Fatalf("failed to create bootstrap: %v", err) - } - defer cleanup() + bootstrapContents := e2e.DefaultBootstrapContents(t, test.nodeID, "xds_server_uri") + testutils.CreateBootstrapFileForTesting(t, bootstrapContents) nodeIDGot := getNodeID() // this should return the node ID plumbed into bootstrap above if nodeIDGot != test.nodeID { t.Fatalf("getNodeID: got %v, want %v", nodeIDGot, test.nodeID) @@ -486,14 +481,8 @@ func (s) TestSetLabels(t *testing.T) { defer os.Unsetenv("CSM_WORKLOAD_NAME") } if test.bootstrapGeneratorPopulated { - cleanup, err := bootstrap.CreateFile(bootstrap.Options{ - NodeID: "projects/12345/networks/mesh:mesh_id/nodes/aaaa-aaaa-aaaa-aaaa", - ServerURI: "xds_server_uri", - }) - if err != nil { - t.Fatalf("failed to create bootstrap: %v", err) - } - defer cleanup() + bootstrapContents := e2e.DefaultBootstrapContents(t, "projects/12345/networks/mesh:mesh_id/nodes/aaaa-aaaa-aaaa-aaaa", "xds_server_uri") + testutils.CreateBootstrapFileForTesting(t, bootstrapContents) } var attributes []attribute.KeyValue for k, v := range test.resourceKeyValues { diff --git a/stats/opentelemetry/go.mod b/stats/opentelemetry/go.mod index a25a9a8298c..81ef34628e3 100644 --- a/stats/opentelemetry/go.mod +++ b/stats/opentelemetry/go.mod @@ -16,8 +16,10 @@ require ( ) require ( + cel.dev/expr v0.15.0 // indirect cloud.google.com/go/compute/metadata v0.3.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.23.0 // indirect + github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect github.com/cncf/xds/go v0.0.0-20240423153145-555b57ec207b // indirect github.com/envoyproxy/go-control-plane v0.12.0 // indirect github.com/envoyproxy/protoc-gen-validate v1.0.4 // indirect @@ -30,5 +32,6 @@ require ( golang.org/x/sync v0.7.0 // indirect golang.org/x/sys v0.21.0 // indirect golang.org/x/text v0.16.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 // indirect ) diff --git a/stats/opentelemetry/go.sum b/stats/opentelemetry/go.sum index af593a7c398..6649316eb96 100644 --- a/stats/opentelemetry/go.sum +++ b/stats/opentelemetry/go.sum @@ -23,8 +23,6 @@ github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= -github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= @@ -51,7 +49,6 @@ golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= -google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98 h1:Z0hjGZePRE0ZBWotvtrwxFNrNE9CUAGtplaDK5NNI/g= google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117 h1:+rdxYoE3E5htTEWIe15GlN6IfvbURM//Jt0mmkmm6ZU= google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117/go.mod h1:OimBR/bc1wPO9iV4NC2bpyjy3VnAwZh5EBPQdtaE5oo= google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 h1:1GBuWVLM/KMVUv1t1En5Gs+gFZCNd360GGb4sSxtrhU= diff --git a/test/xds/xds_client_ack_nack_test.go b/test/xds/xds_client_ack_nack_test.go index ce65bb8e495..9506a1ab8ee 100644 --- a/test/xds/xds_client_ack_nack_test.go +++ b/test/xds/xds_client_ack_nack_test.go @@ -24,12 +24,15 @@ import ( "testing" "github.com/google/go-cmp/cmp" + "github.com/google/uuid" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal" "google.golang.org/grpc/internal/grpcsync" "google.golang.org/grpc/internal/stubserver" "google.golang.org/grpc/internal/testutils" "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/resolver" v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" @@ -83,7 +86,7 @@ func (s) TestClientResourceVersionAfterStreamRestart(t *testing.T) { // Map from stream id to a map of resource type to resource version. ackVersionsMap := make(map[int64]map[string]string) - managementServer, nodeID, _, resolver, cleanup1 := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{ + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ Listener: lis, OnStreamRequest: func(id int64, req *v3discoverypb.DiscoveryRequest) error { // Return early under the following circumstances: @@ -122,7 +125,19 @@ func (s) TestClientResourceVersionAfterStreamRestart(t *testing.T) { streamRestarted.Fire() }, }) - defer cleanup1() + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) + + // Create an xDS resolver with the above bootstrap configuration. + var xdsResolver resolver.Builder + if newResolver := internal.NewXDSResolverWithConfigForTesting; newResolver != nil { + xdsResolver, err = newResolver.(func([]byte) (resolver.Builder, error))(bootstrapContents) + if err != nil { + t.Fatalf("Failed to create xDS resolver for testing: %v", err) + } + } server := stubserver.StartTestService(t, nil) defer server.Stop() @@ -142,7 +157,7 @@ func (s) TestClientResourceVersionAfterStreamRestart(t *testing.T) { } // Create a ClientConn and make a successful RPC. - cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolver)) + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver)) if err != nil { t.Fatalf("failed to dial local test server: %v", err) } diff --git a/test/xds/xds_client_affinity_test.go b/test/xds/xds_client_affinity_test.go index de8dd343e25..a8439e0673d 100644 --- a/test/xds/xds_client_affinity_test.go +++ b/test/xds/xds_client_affinity_test.go @@ -83,8 +83,7 @@ func ringhashCluster(clusterName, edsServiceName string) *v3clusterpb.Cluster { // propagated to pick the ring_hash policy. It doesn't test the affinity // behavior in ring_hash policy. func (s) TestClientSideAffinitySanityCheck(t *testing.T) { - managementServer, nodeID, _, resolver, cleanup1 := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) - defer cleanup1() + managementServer, nodeID, _, xdsResolver := setupManagementServerAndResolver(t) server := stubserver.StartTestService(t, nil) defer server.Stop() @@ -115,7 +114,7 @@ func (s) TestClientSideAffinitySanityCheck(t *testing.T) { } // Create a ClientConn and make a successful RPC. - cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolver)) + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver)) if err != nil { t.Fatalf("failed to dial local test server: %v", err) } diff --git a/test/xds/xds_client_certificate_providers_test.go b/test/xds/xds_client_certificate_providers_test.go index ed5304fb495..5dc7dc9a89f 100644 --- a/test/xds/xds_client_certificate_providers_test.go +++ b/test/xds/xds_client_certificate_providers_test.go @@ -21,6 +21,7 @@ package xds_test import ( "context" "crypto/tls" + "encoding/json" "fmt" "strings" "testing" @@ -34,8 +35,8 @@ import ( "google.golang.org/grpc/internal" "google.golang.org/grpc/internal/stubserver" "google.golang.org/grpc/internal/testutils" - "google.golang.org/grpc/internal/testutils/xds/bootstrap" "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/xds/bootstrap" "google.golang.org/grpc/peer" "google.golang.org/grpc/resolver" "google.golang.org/grpc/status" @@ -59,31 +60,7 @@ import ( // used on the client. func (s) TestClientSideXDS_WithNoCertificateProvidersInBootstrap_Success(t *testing.T) { // Spin up an xDS management server. - mgmtServer, err := e2e.StartManagementServer(e2e.ManagementServerOptions{}) - if err != nil { - t.Fatalf("Failed to start management server: %v", err) - } - defer mgmtServer.Stop() - - // Create bootstrap configuration with no certificate providers. - nodeID := uuid.New().String() - bs, err := bootstrap.Contents(bootstrap.Options{ - NodeID: nodeID, - ServerURI: mgmtServer.Address, - }) - if err != nil { - t.Fatalf("Failed to create bootstrap configuration: %v", err) - } - - // Create an xDS resolver with the above bootstrap configuration. - newResolver := internal.NewXDSResolverWithConfigForTesting - if newResolver == nil { - t.Fatal("internal.NewXDSResolverWithConfigForTesting is unset") - } - resolverBuilder, err := newResolver.(func([]byte) (resolver.Builder, error))(bs) - if err != nil { - t.Fatalf("Failed to create xDS resolver for testing: %v", err) - } + mgmtServer, nodeID, _, resolverBuilder := setupManagementServerAndResolver(t) // Spin up a test backend. server := stubserver.StartTestService(t, nil) @@ -134,31 +111,30 @@ func (s) TestClientSideXDS_WithNoCertificateProvidersInBootstrap_Success(t *test // channel creation does not fail, but it moves to TRANSIENT_FAILURE and // subsequent rpcs fail. func (s) TestClientSideXDS_WithNoCertificateProvidersInBootstrap_Failure(t *testing.T) { - // Spin up an xDS management server. - mgmtServer, err := e2e.StartManagementServer(e2e.ManagementServerOptions{}) - if err != nil { - t.Fatalf("Failed to start management server: %v", err) - } - defer mgmtServer.Stop() + // Start an xDS management server. + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) - // Create bootstrap configuration with no certificate providers. + // Create bootstrap configuration pointing to the above management server, + // with no certificate providers. nodeID := uuid.New().String() - bs, err := bootstrap.Contents(bootstrap.Options{ - NodeID: nodeID, - ServerURI: mgmtServer.Address, + bc, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []json.RawMessage{[]byte(fmt.Sprintf(`{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }`, mgmtServer.Address))}, + NodeID: nodeID, }) if err != nil { t.Fatalf("Failed to create bootstrap configuration: %v", err) } // Create an xDS resolver with the above bootstrap configuration. - newResolver := internal.NewXDSResolverWithConfigForTesting - if newResolver == nil { - t.Fatal("internal.NewXDSResolverWithConfigForTesting is unset") - } - resolverBuilder, err := newResolver.(func([]byte) (resolver.Builder, error))(bs) - if err != nil { - t.Fatalf("Failed to create xDS resolver for testing: %v", err) + var resolverBuilder resolver.Builder + if newResolver := internal.NewXDSResolverWithConfigForTesting; newResolver != nil { + resolverBuilder, err = newResolver.(func([]byte) (resolver.Builder, error))(bc) + if err != nil { + t.Fatalf("Failed to create xDS resolver for testing: %v", err) + } } // Spin up a test backend. @@ -218,17 +194,29 @@ func (s) TestClientSideXDS_WithNoCertificateProvidersInBootstrap_Failure(t *test // The test verifies that RPCs to the first two clusters succeed, while RPCs to // the third cluster fails with an appropriate code and error message. func (s) TestClientSideXDS_WithValidAndInvalidSecurityConfiguration(t *testing.T) { - // Spin up an xDS management server. This uses a bootstrap config with a - // certificate provider instance name e2e.ClientSideCertProviderInstance. - mgmtServer, nodeID, _, resolver, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) - defer cleanup() + // Spin up an xDS management server. + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + + // Create an xDS resolver with the above bootstrap configuration. + var xdsResolver resolver.Builder + if newResolver := internal.NewXDSResolverWithConfigForTesting; newResolver != nil { + var err error + xdsResolver, err = newResolver.(func([]byte) (resolver.Builder, error))(bc) + if err != nil { + t.Fatalf("Failed to create xDS resolver for testing: %v", err) + } + } // Create test backends for all three clusters // backend1 configured with TLS creds, represents cluster1 // backend2 configured with insecure creds, represents cluster2 // backend3 configured with insecure creds, represents cluster3 - creds := e2e.CreateServerTLSCredentials(t, tls.RequireAndVerifyClientCert) - server1 := stubserver.StartTestService(t, nil, grpc.Creds(creds)) + serverCreds := testutils.CreateServerTLSCredentials(t, tls.RequireAndVerifyClientCert) + server1 := stubserver.StartTestService(t, nil, grpc.Creds(serverCreds)) defer server1.Stop() server2 := stubserver.StartTestService(t, nil) defer server2.Stop() @@ -324,13 +312,13 @@ func (s) TestClientSideXDS_WithValidAndInvalidSecurityConfiguration(t *testing.T } // Create client-side xDS credentials with an insecure fallback. - creds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{FallbackCreds: insecure.NewCredentials()}) + clientCreds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{FallbackCreds: insecure.NewCredentials()}) if err != nil { t.Fatal(err) } // Create a ClientConn. - cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(creds), grpc.WithResolvers(resolver)) + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(clientCreds), grpc.WithResolvers(xdsResolver)) if err != nil { t.Fatalf("failed to dial local test server: %v", err) } diff --git a/test/xds/xds_client_custom_lb_test.go b/test/xds/xds_client_custom_lb_test.go index 7638f4f0c0c..d0a5e56f053 100644 --- a/test/xds/xds_client_custom_lb_test.go +++ b/test/xds/xds_client_custom_lb_test.go @@ -221,8 +221,9 @@ func (s) TestWrrLocality(t *testing.T) { } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - managementServer, nodeID, _, r, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) - defer cleanup() + // Start an xDS management server. + managementServer, nodeID, _, xdsResolver := setupManagementServerAndResolver(t) + routeConfigName := "route-" + serviceName clusterName := "cluster-" + serviceName endpointsName := "endpoints-" + serviceName @@ -253,7 +254,7 @@ func (s) TestWrrLocality(t *testing.T) { t.Fatal(err) } - cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r)) + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver)) if err != nil { t.Fatalf("Failed to dial local test server: %v", err) } diff --git a/test/xds/xds_client_federation_test.go b/test/xds/xds_client_federation_test.go index fdc70ec0ce9..68571fe396f 100644 --- a/test/xds/xds_client_federation_test.go +++ b/test/xds/xds_client_federation_test.go @@ -20,6 +20,7 @@ package xds_test import ( "context" + "encoding/json" "fmt" "strings" "testing" @@ -31,8 +32,8 @@ import ( "google.golang.org/grpc/internal" "google.golang.org/grpc/internal/stubserver" "google.golang.org/grpc/internal/testutils" - "google.golang.org/grpc/internal/testutils/xds/bootstrap" "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/xds/bootstrap" "google.golang.org/grpc/resolver" "google.golang.org/grpc/status" @@ -54,28 +55,32 @@ import ( // - EDS: new style, in a different authority func (s) TestClientSideFederation(t *testing.T) { // Start a management server as the default authority. - serverDefaultAuth, err := e2e.StartManagementServer(e2e.ManagementServerOptions{}) - if err != nil { - t.Fatalf("Failed to spin up the xDS management server: %v", err) - } - t.Cleanup(serverDefaultAuth.Stop) + serverDefaultAuth := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) // Start another management server as the other authority. const nonDefaultAuth = "non-default-auth" - serverAnotherAuth, err := e2e.StartManagementServer(e2e.ManagementServerOptions{}) - if err != nil { - t.Fatalf("Failed to spin up the xDS management server: %v", err) - } - t.Cleanup(serverAnotherAuth.Stop) + serverAnotherAuth := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) // Create a bootstrap file in a temporary directory. nodeID := uuid.New().String() - bootstrapContents, err := bootstrap.Contents(bootstrap.Options{ + bootstrapContents, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []json.RawMessage{[]byte(fmt.Sprintf(`{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }`, serverDefaultAuth.Address))}, NodeID: nodeID, - ServerURI: serverDefaultAuth.Address, ServerListenerResourceNameTemplate: e2e.ServerListenerResourceNameTemplate, // Specify the address of the non-default authority. - Authorities: map[string]string{nonDefaultAuth: serverAnotherAuth.Address}, + Authorities: map[string]json.RawMessage{ + nonDefaultAuth: []byte(fmt.Sprintf(`{ + "xds_servers": [ + { + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + } + ] + }`, serverAnotherAuth.Address)), + }, }) if err != nil { t.Fatalf("Failed to create bootstrap file: %v", err) @@ -148,19 +153,28 @@ func (s) TestClientSideFederation(t *testing.T) { func (s) TestClientSideFederationWithOnlyXDSTPStyleLDS(t *testing.T) { // Start a management server as a sophisticated authority. const authority = "traffic-manager.xds.notgoogleapis.com" - mgmtServer, err := e2e.StartManagementServer(e2e.ManagementServerOptions{}) - if err != nil { - t.Fatalf("Failed to spin up the xDS management server: %v", err) - } - t.Cleanup(mgmtServer.Stop) + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) // Create a bootstrap file in a temporary directory. nodeID := uuid.New().String() - bootstrapContents, err := bootstrap.Contents(bootstrap.Options{ - NodeID: nodeID, - ServerURI: mgmtServer.Address, + bootstrapContents, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []json.RawMessage{[]byte(fmt.Sprintf(`{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }`, mgmtServer.Address))}, + NodeID: nodeID, ClientDefaultListenerResourceNameTemplate: fmt.Sprintf("xdstp://%s/envoy.config.listener.v3.Listener/%%s", authority), - Authorities: map[string]string{authority: mgmtServer.Address}, + // Specify the address of the non-default authority. + Authorities: map[string]json.RawMessage{ + authority: []byte(fmt.Sprintf(`{ + "xds_servers": [ + { + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + } + ] + }`, mgmtServer.Address)), + }, }) if err != nil { t.Fatalf("Failed to create bootstrap file: %v", err) @@ -236,8 +250,7 @@ func (s) TestFederation_UnknownAuthorityInDialTarget(t *testing.T) { // server and actually making an RPC ensures that the xDS client is // configured properly, and when we dial with an unknown authority in the // next step, we can be sure that the error we receive is legitimate. - managementServer, nodeID, _, resolver, cleanup1 := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) - defer cleanup1() + managementServer, nodeID, _, xdsResolver := setupManagementServerAndResolver(t) server := stubserver.StartTestService(t, nil) defer server.Stop() @@ -258,7 +271,7 @@ func (s) TestFederation_UnknownAuthorityInDialTarget(t *testing.T) { // Create a ClientConn and make a successful RPC. target := fmt.Sprintf("xds:///%s", serviceName) - cc, err := grpc.NewClient(target, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolver)) + cc, err := grpc.NewClient(target, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver)) if err != nil { t.Fatalf("Dialing target %q: %v", target, err) } @@ -274,7 +287,7 @@ func (s) TestFederation_UnknownAuthorityInDialTarget(t *testing.T) { target = fmt.Sprintf("xds://unknown-authority/%s", serviceName) t.Logf("Dialing target %q with unknown authority which is expected to fail", target) wantErr := fmt.Sprintf("authority \"unknown-authority\" specified in dial target %q is not found in the bootstrap file", target) - _, err = grpc.Dial(target, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolver)) + _, err = grpc.Dial(target, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver)) if err == nil || !strings.Contains(err.Error(), wantErr) { t.Fatalf("grpc.Dial(%q) returned %v, want: %s", target, err, wantErr) } @@ -285,27 +298,7 @@ func (s) TestFederation_UnknownAuthorityInDialTarget(t *testing.T) { // with an authority which is not specified in the bootstrap configuration. The // test verifies that RPCs fail with an appropriate error. func (s) TestFederation_UnknownAuthorityInReceivedResponse(t *testing.T) { - mgmtServer, err := e2e.StartManagementServer(e2e.ManagementServerOptions{}) - if err != nil { - t.Fatalf("Failed to spin up the xDS management server: %v", err) - } - defer mgmtServer.Stop() - - nodeID := uuid.New().String() - bootstrapContents, err := bootstrap.Contents(bootstrap.Options{ - NodeID: nodeID, - ServerURI: mgmtServer.Address, - ServerListenerResourceNameTemplate: e2e.ServerListenerResourceNameTemplate, - }) - if err != nil { - t.Fatal(err) - } - - resolverBuilder := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error)) - resolver, err := resolverBuilder(bootstrapContents) - if err != nil { - t.Fatalf("Creating xDS resolver for testing: %v", err) - } + mgmtServer, nodeID, _, xdsResolver := setupManagementServerAndResolver(t) // LDS is old style name. // RDS is new style, with an unknown authority. @@ -327,7 +320,7 @@ func (s) TestFederation_UnknownAuthorityInReceivedResponse(t *testing.T) { } target := fmt.Sprintf("xds:///%s", serviceName) - cc, err := grpc.NewClient(target, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolver)) + cc, err := grpc.NewClient(target, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver)) if err != nil { t.Fatalf("Dialing target %q: %v", target, err) } diff --git a/test/xds/xds_client_ignore_resource_deletion_test.go b/test/xds/xds_client_ignore_resource_deletion_test.go index 4c15ca4e0a9..ba38f3e0551 100644 --- a/test/xds/xds_client_ignore_resource_deletion_test.go +++ b/test/xds/xds_client_ignore_resource_deletion_test.go @@ -20,6 +20,7 @@ package xds_test import ( "context" + "encoding/json" "fmt" "net" "sync" @@ -33,8 +34,8 @@ import ( "google.golang.org/grpc/internal" "google.golang.org/grpc/internal/stubserver" "google.golang.org/grpc/internal/testutils" - "google.golang.org/grpc/internal/testutils/xds/bootstrap" "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/xds/bootstrap" "google.golang.org/grpc/resolver" "google.golang.org/grpc/xds" @@ -147,7 +148,7 @@ func (s) TestIgnoreResourceDeletionOnClient(t *testing.T) { func testResourceDeletionIgnored(t *testing.T, initialResource func(string) e2e.UpdateOptions, updateResource func(r *e2e.UpdateOptions)) { ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) t.Cleanup(cancel) - mgmtServer := startManagementServer(t) + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) nodeID := uuid.New().String() bs := generateBootstrapContents(t, mgmtServer.Address, true, nodeID) xdsR := xdsResolverBuilder(t, bs) @@ -202,7 +203,7 @@ func testResourceDeletionIgnored(t *testing.T, initialResource func(string) e2e. func testResourceDeletionNotIgnored(t *testing.T, initialResource func(string) e2e.UpdateOptions, updateResource func(r *e2e.UpdateOptions)) { ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout*1000) t.Cleanup(cancel) - mgmtServer := startManagementServer(t) + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) nodeID := uuid.New().String() bs := generateBootstrapContents(t, mgmtServer.Address, false, nodeID) xdsR := xdsResolverBuilder(t, bs) @@ -256,25 +257,27 @@ func testResourceDeletionNotIgnored(t *testing.T, initialResource func(string) e } } -// This helper creates a management server for the test. -func startManagementServer(t *testing.T) *e2e.ManagementServer { - t.Helper() - mgmtServer, err := e2e.StartManagementServer(e2e.ManagementServerOptions{}) - if err != nil { - t.Fatalf("Failed to start management server: %v", err) - } - t.Cleanup(mgmtServer.Stop) - return mgmtServer -} - // This helper generates a custom bootstrap config for the test. func generateBootstrapContents(t *testing.T, serverURI string, ignoreResourceDeletion bool, nodeID string) []byte { t.Helper() - bootstrapContents, err := bootstrap.Contents(bootstrap.Options{ + var serverCfg json.RawMessage + if ignoreResourceDeletion { + serverCfg = []byte(fmt.Sprintf(`{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}], + "server_features": ["ignore_resource_deletion"] + }`, serverURI)) + } else { + serverCfg = []byte(fmt.Sprintf(`{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }`, serverURI)) + + } + bootstrapContents, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []json.RawMessage{serverCfg}, NodeID: nodeID, - ServerURI: serverURI, ServerListenerResourceNameTemplate: e2e.ServerListenerResourceNameTemplate, - IgnoreResourceDeletion: ignoreResourceDeletion, }) if err != nil { t.Fatal(err) @@ -351,7 +354,7 @@ func resourceWithListenerForGRPCServer(t *testing.T, nodeID string) (e2e.UpdateO // case, when the listener resource is deleted on the management server, the gRPC // server should continue to serve RPCs. func (s) TestListenerResourceDeletionOnServerIgnored(t *testing.T) { - mgmtServer := startManagementServer(t) + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) nodeID := uuid.New().String() bs := generateBootstrapContents(t, mgmtServer.Address, true, nodeID) xdsR := xdsResolverBuilder(t, bs) @@ -418,7 +421,7 @@ func (s) TestListenerResourceDeletionOnServerIgnored(t *testing.T) { // which case, when the listener resource is deleted on the management server, the // gRPC server should stop serving RPCs and switch mode to ServingModeNotServing. func (s) TestListenerResourceDeletionOnServerNotIgnored(t *testing.T) { - mgmtServer := startManagementServer(t) + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) nodeID := uuid.New().String() bs := generateBootstrapContents(t, mgmtServer.Address, false, nodeID) xdsR := xdsResolverBuilder(t, bs) diff --git a/test/xds/xds_client_integration_test.go b/test/xds/xds_client_integration_test.go index de253bd31fa..150171dc816 100644 --- a/test/xds/xds_client_integration_test.go +++ b/test/xds/xds_client_integration_test.go @@ -24,12 +24,15 @@ import ( "testing" "time" + "github.com/google/uuid" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal" "google.golang.org/grpc/internal/grpctest" "google.golang.org/grpc/internal/stubserver" "google.golang.org/grpc/internal/testutils" "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/resolver" testgrpc "google.golang.org/grpc/interop/grpc_testing" testpb "google.golang.org/grpc/interop/grpc_testing" @@ -48,9 +51,40 @@ const ( defaultTestShortTimeout = 10 * time.Millisecond // For events expected to *not* happen. ) +// setupManagementServerAndResolver sets up an xDS management server, creates +// bootstrap configuration pointing to that server and creates an xDS resolver +// using that configuration. +// +// Registers a cleanup function on t to stop the management server. +// +// Returns the following: +// - the xDS management server +// - the node ID to use when talking to this management server +// - bootstrap configuration to use (if creating an xDS-enabled gRPC server) +// - xDS resolver builder (if creating an xDS-enabled gRPC client) +func setupManagementServerAndResolver(t *testing.T) (*e2e.ManagementServer, string, []byte, resolver.Builder) { + // Start an xDS management server. + xdsServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, xdsServer.Address) + + // Create an xDS resolver with the above bootstrap configuration. + var r resolver.Builder + var err error + if newResolver := internal.NewXDSResolverWithConfigForTesting; newResolver != nil { + r, err = newResolver.(func([]byte) (resolver.Builder, error))(bc) + if err != nil { + t.Fatalf("Failed to create xDS resolver for testing: %v", err) + } + } + + return xdsServer, nodeID, bc, r +} + func (s) TestClientSideXDS(t *testing.T) { - managementServer, nodeID, _, resolver, cleanup1 := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) - defer cleanup1() + managementServer, nodeID, _, xdsResolver := setupManagementServerAndResolver(t) server := stubserver.StartTestService(t, nil) defer server.Stop() @@ -70,7 +104,7 @@ func (s) TestClientSideXDS(t *testing.T) { } // Create a ClientConn and make a successful RPC. - cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolver)) + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver)) if err != nil { t.Fatalf("failed to dial local test server: %v", err) } diff --git a/test/xds/xds_client_outlier_detection_test.go b/test/xds/xds_client_outlier_detection_test.go index 5ca22a06282..eb91b87b14c 100644 --- a/test/xds/xds_client_outlier_detection_test.go +++ b/test/xds/xds_client_outlier_detection_test.go @@ -50,8 +50,7 @@ import ( // Detection balancer. This test verifies that an RPC is able to proceed // normally with this configuration. func (s) TestOutlierDetection_NoopConfig(t *testing.T) { - managementServer, nodeID, _, resolver, cleanup1 := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) - defer cleanup1() + managementServer, nodeID, _, xdsResolver := setupManagementServerAndResolver(t) server := &stubserver.StubServer{ EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { return &testpb.Empty{}, nil }, @@ -75,7 +74,7 @@ func (s) TestOutlierDetection_NoopConfig(t *testing.T) { } // Create a ClientConn and make a successful RPC. - cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolver)) + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver)) if err != nil { t.Fatalf("failed to dial local test server: %v", err) } @@ -162,8 +161,7 @@ func checkRoundRobinRPCs(ctx context.Context, client testgrpc.TestServiceClient, // the unhealthy upstream is ejected, RPC's should regularly round robin across // all three upstreams. func (s) TestOutlierDetectionWithOutlier(t *testing.T) { - managementServer, nodeID, _, r, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) - defer cleanup() + managementServer, nodeID, _, xdsResolver := setupManagementServerAndResolver(t) // Working backend 1. backend1 := stubserver.StartTestService(t, nil) @@ -204,7 +202,7 @@ func (s) TestOutlierDetectionWithOutlier(t *testing.T) { t.Fatal(err) } - cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r)) + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver)) if err != nil { t.Fatalf("failed to dial local test server: %v", err) } @@ -244,8 +242,7 @@ func (s) TestOutlierDetectionWithOutlier(t *testing.T) { // Detection present in the CDS update, but with SuccessRateEjection unset, and // asserts that Outlier Detection is turned on and ejects upstreams. func (s) TestOutlierDetectionXDSDefaultOn(t *testing.T) { - managementServer, nodeID, _, r, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) - defer cleanup() + managementServer, nodeID, _, xdsResolver := setupManagementServerAndResolver(t) // Working backend 1. backend1 := stubserver.StartTestService(t, nil) @@ -291,7 +288,7 @@ func (s) TestOutlierDetectionXDSDefaultOn(t *testing.T) { t.Fatal(err) } - cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r)) + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver)) if err != nil { t.Fatalf("failed to dial local test server: %v", err) } diff --git a/test/xds/xds_client_retry_test.go b/test/xds/xds_client_retry_test.go index 6f06de045ec..78c1c95d462 100644 --- a/test/xds/xds_client_retry_test.go +++ b/test/xds/xds_client_retry_test.go @@ -41,8 +41,7 @@ func (s) TestClientSideRetry(t *testing.T) { ctr := 0 errs := []codes.Code{codes.ResourceExhausted} - managementServer, nodeID, _, resolver, cleanup1 := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) - defer cleanup1() + managementServer, nodeID, _, xdsResolver := setupManagementServerAndResolver(t) server := stubserver.StartTestService(t, &stubserver.StubServer{ EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { @@ -69,7 +68,7 @@ func (s) TestClientSideRetry(t *testing.T) { } // Create a ClientConn and make a successful RPC. - cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolver)) + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver)) if err != nil { t.Fatalf("failed to dial local test server: %v", err) } diff --git a/test/xds/xds_rls_clusterspecifier_plugin_test.go b/test/xds/xds_rls_clusterspecifier_plugin_test.go index d6322468316..d0eb753a6fc 100644 --- a/test/xds/xds_rls_clusterspecifier_plugin_test.go +++ b/test/xds/xds_rls_clusterspecifier_plugin_test.go @@ -105,8 +105,7 @@ func testRLSinxDS(t *testing.T, lbPolicy e2e.LoadBalancingPolicy) { // Set up all components and configuration necessary - management server, // xDS resolver, fake RLS Server, and xDS configuration which specifies an // RLS Balancer that communicates to this set up fake RLS Server. - managementServer, nodeID, _, resolver, cleanup1 := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) - defer cleanup1() + managementServer, nodeID, _, xdsResolver := setupManagementServerAndResolver(t) server := stubserver.StartTestService(t, nil) defer server.Stop() @@ -142,7 +141,7 @@ func testRLSinxDS(t *testing.T, lbPolicy e2e.LoadBalancingPolicy) { }) // Create a ClientConn and make a successful RPC. - cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolver)) + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver)) if err != nil { t.Fatalf("failed to dial local test server: %v", err) } diff --git a/test/xds/xds_security_config_nack_test.go b/test/xds/xds_security_config_nack_test.go index cf139fa2118..697d135f57c 100644 --- a/test/xds/xds_security_config_nack_test.go +++ b/test/xds/xds_security_config_nack_test.go @@ -26,12 +26,15 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" xdscreds "google.golang.org/grpc/credentials/xds" + "google.golang.org/grpc/internal" "google.golang.org/grpc/internal/stubserver" "google.golang.org/grpc/internal/testutils" "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/resolver" v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3" + "github.com/google/uuid" testgrpc "google.golang.org/grpc/interop/grpc_testing" testpb "google.golang.org/grpc/interop/grpc_testing" ) @@ -144,8 +147,7 @@ func (s) TestUnmarshalListener_WithUpdateValidatorFunc(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - managementServer, nodeID, bootstrapContents, resolver, cleanup1 := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) - defer cleanup1() + managementServer, nodeID, bootstrapContents, xdsResolver := setupManagementServerAndResolver(t) lis, cleanup2 := setupGRPCServer(t, bootstrapContents) defer cleanup2() @@ -188,7 +190,7 @@ func (s) TestUnmarshalListener_WithUpdateValidatorFunc(t *testing.T) { } // Create a ClientConn with the xds scheme and make an RPC. - cc, err := grpc.DialContext(ctx, fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(creds), grpc.WithResolvers(resolver)) + cc, err := grpc.DialContext(ctx, fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(creds), grpc.WithResolvers(xdsResolver)) if err != nil { t.Fatalf("failed to dial local test server: %v", err) } @@ -318,11 +320,22 @@ func (s) TestUnmarshalCluster_WithUpdateValidatorFunc(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - // SetupManagementServer() sets up a bootstrap file with certificate - // provider instance names: `e2e.ServerSideCertProviderInstance` and - // `e2e.ClientSideCertProviderInstance`. - managementServer, nodeID, _, resolver, cleanup1 := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) - defer cleanup1() + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) + + // Create bootstrap configuration pointing to the above management + // server with certificate provider configuration. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) + + // Create an xDS resolver with the above bootstrap configuration. + var xdsResolver resolver.Builder + if newResolver := internal.NewXDSResolverWithConfigForTesting; newResolver != nil { + var err error + xdsResolver, err = newResolver.(func([]byte) (resolver.Builder, error))(bootstrapContents) + if err != nil { + t.Fatalf("Failed to create xDS resolver for testing: %v", err) + } + } server := stubserver.StartTestService(t, nil) defer server.Stop() @@ -344,7 +357,7 @@ func (s) TestUnmarshalCluster_WithUpdateValidatorFunc(t *testing.T) { t.Fatal(err) } - cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolver)) + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver)) if err != nil { t.Fatalf("failed to dial local test server: %v", err) } diff --git a/test/xds/xds_server_certificate_providers_test.go b/test/xds/xds_server_certificate_providers_test.go index fbb37fedfd2..b4305b3a706 100644 --- a/test/xds/xds_server_certificate_providers_test.go +++ b/test/xds/xds_server_certificate_providers_test.go @@ -20,6 +20,7 @@ package xds_test import ( "context" + "encoding/json" "fmt" "net" "strconv" @@ -32,8 +33,8 @@ import ( "google.golang.org/grpc/credentials/insecure" xdscreds "google.golang.org/grpc/credentials/xds" "google.golang.org/grpc/internal/testutils" - "google.golang.org/grpc/internal/testutils/xds/bootstrap" "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/xds/bootstrap" "google.golang.org/grpc/xds" "google.golang.org/protobuf/types/known/wrapperspb" @@ -56,26 +57,11 @@ import ( // credentials are getting used on the server. func (s) TestServerSideXDS_WithNoCertificateProvidersInBootstrap_Success(t *testing.T) { // Spin up an xDS management server. - mgmtServer, err := e2e.StartManagementServer(e2e.ManagementServerOptions{AllowResourceSubset: true}) - if err != nil { - t.Fatalf("Failed to start management server: %v", err) - } - defer mgmtServer.Stop() - - // Create bootstrap configuration with no certificate providers. - nodeID := uuid.New().String() - bs, err := bootstrap.Contents(bootstrap.Options{ - NodeID: nodeID, - ServerURI: mgmtServer.Address, - ServerListenerResourceNameTemplate: e2e.ServerListenerResourceNameTemplate, - }) - if err != nil { - t.Fatalf("Failed to create bootstrap configuration: %v", err) - } + mgmtServer, nodeID, bootstrapContents, _ := setupManagementServerAndResolver(t) // Spin up an xDS-enabled gRPC server that uses xDS credentials with // insecure fallback, and the above bootstrap configuration. - lis, cleanup := setupGRPCServer(t, bs) + lis, cleanup := setupGRPCServer(t, bootstrapContents) defer cleanup() // Create an inbound xDS listener resource for the server side that does not @@ -124,7 +110,7 @@ func (s) TestServerSideXDS_WithNoCertificateProvidersInBootstrap_Failure(t *test // Spin up an xDS management server that pushes on a channel when it // receives a NACK for an LDS response. nackCh := make(chan struct{}, 1) - mgmtServer, err := e2e.StartManagementServer(e2e.ManagementServerOptions{ + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error { if req.GetTypeUrl() != "type.googleapis.com/envoy.config.listener.v3.Listener" { return nil @@ -140,16 +126,15 @@ func (s) TestServerSideXDS_WithNoCertificateProvidersInBootstrap_Failure(t *test }, AllowResourceSubset: true, }) - if err != nil { - t.Fatalf("Failed to start management server: %v", err) - } - defer mgmtServer.Stop() // Create bootstrap configuration with no certificate providers. nodeID := uuid.New().String() - bs, err := bootstrap.Contents(bootstrap.Options{ + bs, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []json.RawMessage{[]byte(fmt.Sprintf(`{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }`, mgmtServer.Address))}, NodeID: nodeID, - ServerURI: mgmtServer.Address, ServerListenerResourceNameTemplate: e2e.ServerListenerResourceNameTemplate, }) if err != nil { @@ -252,7 +237,7 @@ func (s) TestServerSideXDS_WithValidAndInvalidSecurityConfiguration(t *testing.T // Spin up an xDS management server that pushes on a channel when it // receives a NACK for an LDS response. nackCh := make(chan struct{}, 1) - mgmtServer, nodeID, bs, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{ + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error { if req.GetTypeUrl() != "type.googleapis.com/envoy.config.listener.v3.Listener" { return nil @@ -268,7 +253,10 @@ func (s) TestServerSideXDS_WithValidAndInvalidSecurityConfiguration(t *testing.T }, AllowResourceSubset: true, }) - defer cleanup() + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) // Create two local listeners. lis1, err := testutils.LocalTCPListener() @@ -295,7 +283,7 @@ func (s) TestServerSideXDS_WithValidAndInvalidSecurityConfiguration(t *testing.T } } }) - server, err := xds.NewGRPCServer(grpc.Creds(creds), modeChangeOpt, xds.BootstrapContentsForTesting(bs)) + server, err := xds.NewGRPCServer(grpc.Creds(creds), modeChangeOpt, xds.BootstrapContentsForTesting(bootstrapContents)) if err != nil { t.Fatalf("Failed to create an xDS enabled gRPC server: %v", err) } @@ -437,12 +425,12 @@ func (s) TestServerSideXDS_WithValidAndInvalidSecurityConfiguration(t *testing.T Listeners: []*v3listenerpb.Listener{resource1, resource2}, SkipValidation: true, } - if err := mgmtServer.Update(ctx, resources); err != nil { + if err := managementServer.Update(ctx, resources); err != nil { t.Fatal(err) } // Create a client that uses TLS creds and verify RPCs to listener1. - clientCreds := e2e.CreateClientTLSCredentials(t) + clientCreds := testutils.CreateClientTLSCredentials(t) cc1, err := grpc.NewClient(lis1.Addr().String(), grpc.WithTransportCredentials(clientCreds)) if err != nil { t.Fatalf("Failed to dial local test server: %v", err) diff --git a/test/xds/xds_server_integration_test.go b/test/xds/xds_server_integration_test.go index 9e4d7b2fb6b..08aed76137c 100644 --- a/test/xds/xds_server_integration_test.go +++ b/test/xds/xds_server_integration_test.go @@ -26,12 +26,15 @@ import ( "strconv" "testing" + "github.com/google/uuid" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials/insecure" xdscreds "google.golang.org/grpc/credentials/xds" + "google.golang.org/grpc/internal" "google.golang.org/grpc/internal/testutils" "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/resolver" "google.golang.org/grpc/status" "google.golang.org/grpc/xds" @@ -140,8 +143,7 @@ func hostPortFromListener(lis net.Listener) (string, uint32, error) { // the client and the server. This results in both of them using the // configured fallback credentials (which is insecure creds in this case). func (s) TestServerSideXDS_Fallback(t *testing.T) { - managementServer, nodeID, bootstrapContents, resolver, cleanup1 := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) - defer cleanup1() + managementServer, nodeID, bootstrapContents, xdsResolver := setupManagementServerAndResolver(t) lis, cleanup2 := setupGRPCServer(t, bootstrapContents) defer cleanup2() @@ -184,7 +186,7 @@ func (s) TestServerSideXDS_Fallback(t *testing.T) { } // Create a ClientConn with the xds scheme and make a successful RPC. - cc, err := grpc.DialContext(ctx, fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(creds), grpc.WithResolvers(resolver)) + cc, err := grpc.DialContext(ctx, fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(creds), grpc.WithResolvers(xdsResolver)) if err != nil { t.Fatalf("failed to dial local test server: %v", err) } @@ -222,9 +224,7 @@ func (s) TestServerSideXDS_FileWatcherCerts(t *testing.T) { } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - managementServer, nodeID, bootstrapContents, resolver, cleanup1 := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) - defer cleanup1() - + managementServer, nodeID, bootstrapContents, xdsResolver := setupManagementServerAndResolver(t) lis, cleanup2 := setupGRPCServer(t, bootstrapContents) defer cleanup2() @@ -269,7 +269,7 @@ func (s) TestServerSideXDS_FileWatcherCerts(t *testing.T) { } // Create a ClientConn with the xds scheme and make an RPC. - cc, err := grpc.DialContext(ctx, fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(creds), grpc.WithResolvers(resolver)) + cc, err := grpc.DialContext(ctx, fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(creds), grpc.WithResolvers(xdsResolver)) if err != nil { t.Fatalf("failed to dial local test server: %v", err) } @@ -292,8 +292,21 @@ func (s) TestServerSideXDS_FileWatcherCerts(t *testing.T) { // configuration pointing to the use of the file_watcher plugin and we verify // that the same client is now able to successfully make an RPC. func (s) TestServerSideXDS_SecurityConfigChange(t *testing.T) { - managementServer, nodeID, bootstrapContents, resolver, cleanup1 := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) - defer cleanup1() + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) + + // Create an xDS resolver with the above bootstrap configuration. + var xdsResolver resolver.Builder + if newResolver := internal.NewXDSResolverWithConfigForTesting; newResolver != nil { + var err error + xdsResolver, err = newResolver.(func([]byte) (resolver.Builder, error))(bootstrapContents) + if err != nil { + t.Fatalf("Failed to create xDS resolver for testing: %v", err) + } + } lis, cleanup2 := setupGRPCServer(t, bootstrapContents) defer cleanup2() @@ -337,7 +350,7 @@ func (s) TestServerSideXDS_SecurityConfigChange(t *testing.T) { } // Create a ClientConn with the xds scheme and make a successful RPC. - xdsCC, err := grpc.DialContext(ctx, fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(xdsCreds), grpc.WithResolvers(resolver)) + xdsCC, err := grpc.DialContext(ctx, fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(xdsCreds), grpc.WithResolvers(xdsResolver)) if err != nil { t.Fatalf("failed to dial local test server: %v", err) } @@ -350,7 +363,7 @@ func (s) TestServerSideXDS_SecurityConfigChange(t *testing.T) { // Create a ClientConn with TLS creds. This should fail since the server is // using fallback credentials which in this case in insecure creds. - tlsCreds := e2e.CreateClientTLSCredentials(t) + tlsCreds := testutils.CreateClientTLSCredentials(t) tlsCC, err := grpc.DialContext(ctx, lis.Addr().String(), grpc.WithTransportCredentials(tlsCreds)) if err != nil { t.Fatalf("failed to dial local test server: %v", err) diff --git a/test/xds/xds_server_rbac_test.go b/test/xds/xds_server_rbac_test.go index 953d88f96fc..13cce4f2ecf 100644 --- a/test/xds/xds_server_rbac_test.go +++ b/test/xds/xds_server_rbac_test.go @@ -59,8 +59,7 @@ import ( // (NonForwardingAction), and the RPC's matching those routes should proceed as // normal. func (s) TestServerSideXDS_RouteConfiguration(t *testing.T) { - managementServer, nodeID, bootstrapContents, resolver, cleanup1 := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) - defer cleanup1() + managementServer, nodeID, bootstrapContents, xdsResolver := setupManagementServerAndResolver(t) lis, cleanup2 := setupGRPCServer(t, bootstrapContents) defer cleanup2() @@ -245,7 +244,7 @@ func (s) TestServerSideXDS_RouteConfiguration(t *testing.T) { t.Fatal(err) } - cc, err := grpc.DialContext(ctx, fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolver)) + cc, err := grpc.DialContext(ctx, fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver)) if err != nil { t.Fatalf("failed to dial local test server: %v", err) } @@ -627,8 +626,7 @@ func (s) TestRBACHTTPFilter(t *testing.T) { } audit.RegisterLoggerBuilder(lb) - managementServer, nodeID, bootstrapContents, resolver, cleanup1 := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) - defer cleanup1() + managementServer, nodeID, bootstrapContents, xdsResolver := setupManagementServerAndResolver(t) lis, cleanup2 := setupGRPCServer(t, bootstrapContents) defer cleanup2() @@ -655,7 +653,7 @@ func (s) TestRBACHTTPFilter(t *testing.T) { t.Fatal(err) } - cc, err := grpc.DialContext(ctx, fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolver)) + cc, err := grpc.DialContext(ctx, fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver)) if err != nil { t.Fatalf("failed to dial local test server: %v", err) } @@ -804,8 +802,7 @@ func serverListenerWithBadRouteConfiguration(t *testing.T, host string, port uin } func (s) TestRBACToggledOn_WithBadRouteConfiguration(t *testing.T) { - managementServer, nodeID, bootstrapContents, resolver, cleanup1 := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) - defer cleanup1() + managementServer, nodeID, bootstrapContents, xdsResolver := setupManagementServerAndResolver(t) lis, cleanup2 := setupGRPCServer(t, bootstrapContents) defer cleanup2() @@ -838,7 +835,7 @@ func (s) TestRBACToggledOn_WithBadRouteConfiguration(t *testing.T) { t.Fatal(err) } - cc, err := grpc.DialContext(ctx, fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolver)) + cc, err := grpc.DialContext(ctx, fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver)) if err != nil { t.Fatalf("failed to dial local test server: %v", err) } diff --git a/test/xds/xds_server_serving_mode_test.go b/test/xds/xds_server_serving_mode_test.go index 95751560d58..d68436e3459 100644 --- a/test/xds/xds_server_serving_mode_test.go +++ b/test/xds/xds_server_serving_mode_test.go @@ -43,8 +43,7 @@ import ( // change callback is not invoked and client connections to the server are not // recycled. func (s) TestServerSideXDS_RedundantUpdateSuppression(t *testing.T) { - managementServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) - defer cleanup() + managementServer, nodeID, bootstrapContents, _ := setupManagementServerAndResolver(t) creds, err := xdscreds.NewServerCredentials(xdscreds.ServerOptions{FallbackCreds: insecure.NewCredentials()}) if err != nil { @@ -166,8 +165,7 @@ func (s) TestServerSideXDS_RedundantUpdateSuppression(t *testing.T) { // xDS enabled gRPC servers. It verifies that appropriate mode changes happen in // the server, and also verifies behavior of clientConns under these modes. func (s) TestServerSideXDS_ServingModeChanges(t *testing.T) { - managementServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) - defer cleanup() + managementServer, nodeID, bootstrapContents, _ := setupManagementServerAndResolver(t) // Configure xDS credentials to be used on the server-side. creds, err := xdscreds.NewServerCredentials(xdscreds.ServerOptions{ diff --git a/test/xds/xds_server_test.go b/test/xds/xds_server_test.go index f4dcac71d3c..20adfea7ab6 100644 --- a/test/xds/xds_server_test.go +++ b/test/xds/xds_server_test.go @@ -55,8 +55,8 @@ var ( // dynamically, and subsequent RPC's on that connection should start failing // with status code UNAVAILABLE. func (s) TestServeLDSRDS(t *testing.T) { - managementServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) - defer cleanup() + managementServer, nodeID, bootstrapContents, _ := setupManagementServerAndResolver(t) + lis, err := testutils.LocalTCPListener() if err != nil { t.Fatalf("testutils.LocalTCPListener() failed: %v", err) @@ -165,8 +165,7 @@ func waitForFailedRPCWithStatus(ctx context.Context, t *testing.T, cc *grpc.Clie // serving, successfully Accept Connections, and fail at the L7 level with a // certain error message. func (s) TestRDSNack(t *testing.T) { - managementServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) - defer cleanup() + managementServer, nodeID, bootstrapContents, _ := setupManagementServerAndResolver(t) lis, err := testutils.LocalTCPListener() if err != nil { t.Fatalf("testutils.LocalTCPListener() failed: %v", err) @@ -236,8 +235,7 @@ func (s) TestRDSNack(t *testing.T) { // RPC's will match to). This configuration should eventually be represented in // the Server's state, and RPCs should proceed successfully. func (s) TestMultipleUpdatesImmediatelySwitch(t *testing.T) { - managementServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) - defer cleanup() + managementServer, nodeID, bootstrapContents, _ := setupManagementServerAndResolver(t) lis, err := testutils.LocalTCPListener() if err != nil { t.Fatalf("testutils.LocalTCPListener() failed: %v", err) diff --git a/test/xds/xds_telemetry_labels_test.go b/test/xds/xds_telemetry_labels_test.go index d67f260f540..544b6878a6e 100644 --- a/test/xds/xds_telemetry_labels_test.go +++ b/test/xds/xds_telemetry_labels_test.go @@ -49,8 +49,7 @@ const serviceNamespaceValue = "grpc-service-namespace" // handler asserts that subsequent HandleRPC calls from the RPC lifecycle // contain telemetry labels that it can see. func (s) TestTelemetryLabels(t *testing.T) { - managementServer, nodeID, _, resolver, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) - defer cleanup() + managementServer, nodeID, _, xdsResolver := setupManagementServerAndResolver(t) server := stubserver.StartTestService(t, nil) defer server.Stop() @@ -85,7 +84,7 @@ func (s) TestTelemetryLabels(t *testing.T) { t: t, } - cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", xdsServiceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolver), grpc.WithStatsHandler(fsh)) + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", xdsServiceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver), grpc.WithStatsHandler(fsh)) if err != nil { t.Fatalf("failed to create a new client to local test server: %v", err) } diff --git a/xds/csds/csds_e2e_test.go b/xds/csds/csds_e2e_test.go index 97b5fa3a7aa..ffa6a7354bc 100644 --- a/xds/csds/csds_e2e_test.go +++ b/xds/csds/csds_e2e_test.go @@ -33,7 +33,6 @@ import ( "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/internal/grpctest" "google.golang.org/grpc/internal/testutils" - "google.golang.org/grpc/internal/testutils/xds/bootstrap" "google.golang.org/grpc/internal/testutils/xds/e2e" "google.golang.org/grpc/xds/csds" "google.golang.org/grpc/xds/internal/xdsclient" @@ -116,22 +115,12 @@ func (unimplementedEndpointsWatcher) OnResourceDoesNotExist() func (s) TestCSDS(t *testing.T) { // Spin up a xDS management server on a local port. - nodeID := uuid.New().String() - mgmtServer, err := e2e.StartManagementServer(e2e.ManagementServerOptions{}) - if err != nil { - t.Fatal(err) - } - defer mgmtServer.Stop() + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) // Create a bootstrap file in a temporary directory. - bootstrapCleanup, err := bootstrap.CreateFile(bootstrap.Options{ - NodeID: nodeID, - ServerURI: mgmtServer.Address, - }) - if err != nil { - t.Fatal(err) - } - defer bootstrapCleanup() + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + testutils.CreateBootstrapFileForTesting(t, bootstrapContents) // Create an xDS client. This will end up using the same singleton as used // by the CSDS service. @@ -403,14 +392,10 @@ func checkClientStatusResponse(stream v3statuspbgrpc.ClientStatusDiscoveryServic } func (s) TestCSDSNoXDSClient(t *testing.T) { - // Create a bootstrap file in a temporary directory. Since we pass empty - // options, it would end up creating a bootstrap file with an empty - // serverURI which will fail xDS client creation. - bootstrapCleanup, err := bootstrap.CreateFile(bootstrap.Options{}) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { bootstrapCleanup() }) + // Create a bootstrap file in a temporary directory. Since we pass an empty + // bootstrap configuration, it will fail xDS client creation because the + // `server_uri` field is unset. + testutils.CreateBootstrapFileForTesting(t, []byte(``)) // Initialize an gRPC server and register CSDS on it. server := grpc.NewServer() diff --git a/xds/internal/balancer/cdsbalancer/cdsbalancer_security_test.go b/xds/internal/balancer/cdsbalancer/cdsbalancer_security_test.go index 05931951d10..dda56f69cf8 100644 --- a/xds/internal/balancer/cdsbalancer/cdsbalancer_security_test.go +++ b/xds/internal/balancer/cdsbalancer/cdsbalancer_security_test.go @@ -28,6 +28,7 @@ import ( "unsafe" "github.com/google/go-cmp/cmp" + "github.com/google/uuid" "google.golang.org/grpc" "google.golang.org/grpc/attributes" "google.golang.org/grpc/balancer" @@ -41,8 +42,8 @@ import ( xdscredsinternal "google.golang.org/grpc/internal/credentials/xds" "google.golang.org/grpc/internal/stubserver" "google.golang.org/grpc/internal/testutils" - xdsbootstrap "google.golang.org/grpc/internal/testutils/xds/bootstrap" "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/xds/bootstrap" "google.golang.org/grpc/peer" "google.golang.org/grpc/resolver" "google.golang.org/grpc/resolver/manual" @@ -260,12 +261,15 @@ func (s) TestSecurityConfigWithoutXDSCreds(t *testing.T) { registerWrappedCDSPolicyWithNewSubConnOverride(t, handshakeInfoCh) // Spin up an xDS management server. - mgmtServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) - t.Cleanup(cleanup) + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) // Create a grpc channel with insecure creds talking to a test server with // insecure credentials. - cc, serverAddress := setupForSecurityTests(t, bootstrapContents, insecure.NewCredentials(), nil) + cc, serverAddress := setupForSecurityTests(t, bc, insecure.NewCredentials(), nil) // Configure cluster and endpoints resources in the management server. The // cluster resource is configured to return security configuration. @@ -311,12 +315,15 @@ func (s) TestNoSecurityConfigWithXDSCreds(t *testing.T) { registerWrappedCDSPolicyWithNewSubConnOverride(t, handshakeInfoCh) // Spin up an xDS management server. - mgmtServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) - t.Cleanup(cleanup) + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) // Create a grpc channel with xDS creds talking to a test server with // insecure credentials. - cc, serverAddress := setupForSecurityTests(t, bootstrapContents, xdsClientCredsWithInsecureFallback(t), nil) + cc, serverAddress := setupForSecurityTests(t, bc, xdsClientCredsWithInsecureFallback(t), nil) // Configure cluster and endpoints resources in the management server. The // cluster resource is not configured to return any security configuration. @@ -359,15 +366,17 @@ func (s) TestNoSecurityConfigWithXDSCreds(t *testing.T) { // that the cds LB policy puts the channel in TRANSIENT_FAILURE. func (s) TestSecurityConfigNotFoundInBootstrap(t *testing.T) { // Spin up an xDS management server. - mgmtServer, nodeID, _, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) - t.Cleanup(cleanup) - - // Ignore the bootstrap configuration returned by the above call to - // e2e.SetupManagementServer and create a new one that does not have - // ceritificate providers configuration. - bootstrapContents, err := xdsbootstrap.Contents(xdsbootstrap.Options{ + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + // Create bootstrap configuration pointing to the above management server, + // and one that does not have ceritificate providers configuration. + nodeID := uuid.New().String() + bootstrapContents, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []json.RawMessage{[]byte(fmt.Sprintf(`{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }`, mgmtServer.Address))}, NodeID: nodeID, - ServerURI: mgmtServer.Address, ServerListenerResourceNameTemplate: e2e.ServerListenerResourceNameTemplate, }) if err != nil { @@ -418,21 +427,24 @@ func init() { // policy attempts to build a certificate provider. Verifies that the cds LB // policy puts the channel in TRANSIENT_FAILURE. func (s) TestCertproviderStoreError(t *testing.T) { - mgmtServer, nodeID, _, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) - t.Cleanup(cleanup) + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) - // Ignore the bootstrap configuration returned by the above call to - // e2e.SetupManagementServer and create a new one that includes ceritificate - // providers configuration for errCertProviderBuilder. + // Create bootstrap configuration pointing to the above management server + // and one that includes ceritificate providers configuration for + // errCertProviderBuilder. + nodeID := uuid.New().String() providerCfg := json.RawMessage(fmt.Sprintf(`{ "plugin_name": "%s", "config": {} }`, errCertProviderName)) - bootstrapContents, err := xdsbootstrap.Contents(xdsbootstrap.Options{ + bootstrapContents, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []json.RawMessage{[]byte(fmt.Sprintf(`{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }`, mgmtServer.Address))}, NodeID: nodeID, - ServerURI: mgmtServer.Address, - CertificateProviders: map[string]json.RawMessage{e2e.ClientSideCertProviderInstance: providerCfg}, ServerListenerResourceNameTemplate: e2e.ServerListenerResourceNameTemplate, + CertificateProviders: map[string]json.RawMessage{e2e.ClientSideCertProviderInstance: providerCfg}, }) if err != nil { t.Fatalf("Failed to create bootstrap configuration: %v", err) @@ -463,12 +475,16 @@ func (s) TestCertproviderStoreError(t *testing.T) { // the server is secure. func (s) TestGoodSecurityConfig(t *testing.T) { // Spin up an xDS management server. - mgmtServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) - t.Cleanup(cleanup) + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + // Create bootstrap configuration pointing to the above management server + // and one that includes ceritificate providers configuration. + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) // Create a grpc channel with xDS creds talking to a test server with TLS // credentials. - cc, serverAddress := setupForSecurityTests(t, bootstrapContents, xdsClientCredsWithInsecureFallback(t), tlsServerCreds(t)) + cc, serverAddress := setupForSecurityTests(t, bc, xdsClientCredsWithInsecureFallback(t), tlsServerCreds(t)) // Configure cluster and endpoints resources in the management server. The // cluster resource is configured to return security configuration. @@ -502,12 +518,15 @@ func (s) TestGoodSecurityConfig(t *testing.T) { // server is secure. func (s) TestSecurityConfigUpdate_BadToGood(t *testing.T) { // Spin up an xDS management server. - mgmtServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) - t.Cleanup(cleanup) + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) // Create a grpc channel with xDS creds talking to a test server with TLS // credentials. - cc, serverAddress := setupForSecurityTests(t, bootstrapContents, xdsClientCredsWithInsecureFallback(t), tlsServerCreds(t)) + cc, serverAddress := setupForSecurityTests(t, bc, xdsClientCredsWithInsecureFallback(t), tlsServerCreds(t)) // Configure cluster and endpoints resources in the management server. The // cluster resource contains security configuration with a certificate @@ -571,12 +590,15 @@ func (s) TestSecurityConfigUpdate_BadToGood(t *testing.T) { // use of fallback credentials, which in this case is insecure creds. func (s) TestSecurityConfigUpdate_GoodToFallback(t *testing.T) { // Spin up an xDS management server. - mgmtServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) - t.Cleanup(cleanup) + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) // Create a grpc channel with xDS creds talking to a test server with TLS // credentials. - cc, serverAddress := setupForSecurityTests(t, bootstrapContents, xdsClientCredsWithInsecureFallback(t), tlsServerCreds(t)) + cc, serverAddress := setupForSecurityTests(t, bc, xdsClientCredsWithInsecureFallback(t), tlsServerCreds(t)) // Configure cluster and endpoints resources in the management server. The // cluster resource is configured to return security configuration. @@ -647,12 +669,15 @@ func (s) TestSecurityConfigUpdate_GoodToBad(t *testing.T) { _, resolverErrCh, _, _ := registerWrappedClusterResolverPolicy(t) // Spin up an xDS management server. - mgmtServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) - t.Cleanup(cleanup) + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) // Create a grpc channel with xDS creds talking to a test server with TLS // credentials. - cc, serverAddress := setupForSecurityTests(t, bootstrapContents, xdsClientCredsWithInsecureFallback(t), tlsServerCreds(t)) + cc, serverAddress := setupForSecurityTests(t, bc, xdsClientCredsWithInsecureFallback(t), tlsServerCreds(t)) // Configure cluster and endpoints resources in the management server. The // cluster resource is configured to return security configuration. diff --git a/xds/internal/balancer/cdsbalancer/cdsbalancer_test.go b/xds/internal/balancer/cdsbalancer/cdsbalancer_test.go index 82f4fb5f4e0..1cd91995a5a 100644 --- a/xds/internal/balancer/cdsbalancer/cdsbalancer_test.go +++ b/xds/internal/balancer/cdsbalancer/cdsbalancer_test.go @@ -26,6 +26,7 @@ import ( "time" "github.com/google/go-cmp/cmp" + "github.com/google/uuid" "google.golang.org/grpc" "google.golang.org/grpc/balancer" "google.golang.org/grpc/codes" @@ -205,7 +206,7 @@ func setupWithManagementServer(t *testing.T) (*e2e.ManagementServer, string, *gr cdsResourceRequestedCh := make(chan []string, 1) cdsResourceCanceledCh := make(chan struct{}, 1) - mgmtServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{ + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error { if req.GetTypeUrl() == version.V3ClusterURL { switch len(req.GetResourceNames()) { @@ -227,9 +228,12 @@ func setupWithManagementServer(t *testing.T) (*e2e.ManagementServer, string, *gr // at once. AllowResourceSubset: true, }) - t.Cleanup(cleanup) - xdsC, xdsClose, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bootstrapContents}) + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + + xdsC, xdsClose, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bc}) if err != nil { t.Fatalf("Failed to create xDS client: %v", err) } @@ -343,9 +347,13 @@ func (s) TestConfigurationUpdate_Success(t *testing.T) { // the CDS LB policy. Verifies that ErrBadResolverState is returned. func (s) TestConfigurationUpdate_EmptyCluster(t *testing.T) { // Setup a management server and an xDS client to talk to it. - _, _, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) - t.Cleanup(cleanup) - xdsClient, xdsClose, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bootstrapContents}) + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + + xdsClient, xdsClose, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bc}) if err != nil { t.Fatalf("Failed to create xDS client: %v", err) } diff --git a/xds/internal/balancer/clusterimpl/tests/balancer_test.go b/xds/internal/balancer/clusterimpl/tests/balancer_test.go index 86230e132f6..2d3c49e0493 100644 --- a/xds/internal/balancer/clusterimpl/tests/balancer_test.go +++ b/xds/internal/balancer/clusterimpl/tests/balancer_test.go @@ -28,14 +28,17 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal" "google.golang.org/grpc/internal/grpctest" "google.golang.org/grpc/internal/stubserver" "google.golang.org/grpc/internal/testutils" "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/resolver" "google.golang.org/grpc/status" v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + "github.com/google/uuid" testgrpc "google.golang.org/grpc/interop/grpc_testing" testpb "google.golang.org/grpc/interop/grpc_testing" @@ -62,9 +65,22 @@ func Test(t *testing.T) { // created. func (s) TestConfigUpdateWithSameLoadReportingServerConfig(t *testing.T) { // Create an xDS management server that serves ADS and LRS requests. - opts := e2e.ManagementServerOptions{SupportLoadReportingService: true} - mgmtServer, nodeID, _, resolver, mgmtServerCleanup := e2e.SetupManagementServer(t, opts) - defer mgmtServerCleanup() + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{SupportLoadReportingService: true}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + testutils.CreateBootstrapFileForTesting(t, bc) + + // Create an xDS resolver with the above bootstrap configuration. + var resolverBuilder resolver.Builder + var err error + if newResolver := internal.NewXDSResolverWithConfigForTesting; newResolver != nil { + resolverBuilder, err = newResolver.(func([]byte) (resolver.Builder, error))(bc) + if err != nil { + t.Fatalf("Failed to create xDS resolver for testing: %v", err) + } + } // Start a server backend exposing the test service. server := stubserver.StartTestService(t, nil) @@ -92,7 +108,7 @@ func (s) TestConfigUpdateWithSameLoadReportingServerConfig(t *testing.T) { } // Create a ClientConn and make a successful RPC. - cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolver)) + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolverBuilder)) if err != nil { t.Fatalf("failed to dial local test server: %v", err) } diff --git a/xds/internal/balancer/clustermanager/e2e_test/clustermanager_test.go b/xds/internal/balancer/clustermanager/e2e_test/clustermanager_test.go index 443e08b3bf5..ef28c7b3fcf 100644 --- a/xds/internal/balancer/clustermanager/e2e_test/clustermanager_test.go +++ b/xds/internal/balancer/clustermanager/e2e_test/clustermanager_test.go @@ -38,6 +38,7 @@ import ( "google.golang.org/grpc/internal/testutils" "google.golang.org/grpc/internal/testutils/xds/e2e" "google.golang.org/grpc/peer" + "google.golang.org/grpc/resolver" "google.golang.org/grpc/serviceconfig" "google.golang.org/grpc/status" "google.golang.org/protobuf/types/known/structpb" @@ -49,6 +50,7 @@ import ( v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" v3pickfirstpb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/pick_first/v3" + "github.com/google/uuid" testgrpc "google.golang.org/grpc/interop/grpc_testing" testpb "google.golang.org/grpc/interop/grpc_testing" @@ -92,8 +94,22 @@ func makeUnaryCallRPCAndVerifyPeer(ctx context.Context, client testgrpc.TestServ func (s) TestConfigUpdate_ChildPolicyChange(t *testing.T) { // Spin up an xDS management server. - mgmtServer, nodeID, _, resolver, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) - defer cleanup() + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + testutils.CreateBootstrapFileForTesting(t, bc) + + // Create an xDS resolver with the above bootstrap configuration. + var resolverBuilder resolver.Builder + var err error + if newResolver := internal.NewXDSResolverWithConfigForTesting; newResolver != nil { + resolverBuilder, err = newResolver.(func([]byte) (resolver.Builder, error))(bc) + if err != nil { + t.Fatalf("Failed to create xDS resolver for testing: %v", err) + } + } // Configure client side xDS resources on the management server. const ( @@ -162,7 +178,7 @@ func (s) TestConfigUpdate_ChildPolicyChange(t *testing.T) { } // Create a ClientConn. - cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolver)) + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolverBuilder)) if err != nil { t.Fatalf("failed to dial local test server: %v", err) } diff --git a/xds/internal/balancer/clusterresolver/e2e_test/aggregate_cluster_test.go b/xds/internal/balancer/clusterresolver/e2e_test/aggregate_cluster_test.go index 88659318c0c..32e8e3c9cd1 100644 --- a/xds/internal/balancer/clusterresolver/e2e_test/aggregate_cluster_test.go +++ b/xds/internal/balancer/clusterresolver/e2e_test/aggregate_cluster_test.go @@ -27,6 +27,7 @@ import ( "time" "github.com/google/go-cmp/cmp" + "github.com/google/uuid" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/connectivity" @@ -105,7 +106,7 @@ func (s) TestAggregateCluster_WithTwoEDSClusters(t *testing.T) { // Start an xDS management server that pushes the EDS resource names onto a // channel when requested. edsResourceNameCh := make(chan []string, 1) - managementServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{ + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error { if req.GetTypeUrl() != version.V3EndpointsURL { return nil @@ -122,7 +123,10 @@ func (s) TestAggregateCluster_WithTwoEDSClusters(t *testing.T) { }, AllowResourceSubset: true, }) - defer cleanup() + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) // Start two test backends and extract their host and port. The first // backend belongs to EDS cluster "cluster-1", while the second backend @@ -208,8 +212,11 @@ func (s) TestAggregateCluster_WithTwoEDSClusters(t *testing.T) { // are routed to the highest priority EDS cluster. func (s) TestAggregateCluster_WithTwoEDSClusters_PrioritiesChange(t *testing.T) { // Start an xDS management server. - managementServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) - defer cleanup() + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) // Start two test backends and extract their host and port. The first // backend belongs to EDS cluster "cluster-1", while the second backend @@ -302,8 +309,11 @@ func hostAndPortFromAddress(t *testing.T, addr string) (string, uint32) { // make up the LOGICAL_DNS cluster. func (s) TestAggregateCluster_WithOneDNSCluster(t *testing.T) { // Start an xDS management server. - managementServer, nodeID, bootstrapContents, _, cleanup2 := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) - defer cleanup2() + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) // Start a test service backend. server := stubserver.StartTestService(t, nil) @@ -349,8 +359,11 @@ func (s) TestAggregateCluster_WithOneDNSCluster(t *testing.T) { // TRANSIENT_FAILURE. func (s) TestAggregateCluster_WithOneDNSCluster_ParseFailure(t *testing.T) { // Start an xDS management server. - managementServer, nodeID, bootstrapContents, _, cleanup2 := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) - defer cleanup2() + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) // Configure an aggregate cluster pointing to a single LOGICAL_DNS cluster. const dnsClusterName = clusterName + "-dns" @@ -388,8 +401,11 @@ func (s) TestAggregateCluster_WithOneDNSCluster_ParseFailure(t *testing.T) { // made to backends that the new hostname resolves to. func (s) TestAggregateCluster_WithOneDNSCluster_HostnameChange(t *testing.T) { // Start an xDS management server. - managementServer, nodeID, bootstrapContents, _, cleanup1 := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) - defer cleanup1() + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) // Start two test backends and extract their host and port. The first // backend is used initially for the LOGICAL_DNS cluster and an update @@ -468,7 +484,7 @@ func (s) TestAggregateCluster_WithEDSAndDNS(t *testing.T) { // Start an xDS management server that pushes the name of the requested EDS // resource onto a channel. edsResourceCh := make(chan string, 1) - managementServer, nodeID, bootstrapContents, _, cleanup2 := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{ + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error { if req.GetTypeUrl() != version.V3EndpointsURL { return nil @@ -483,7 +499,10 @@ func (s) TestAggregateCluster_WithEDSAndDNS(t *testing.T) { }, AllowResourceSubset: true, }) - defer cleanup2() + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) // Start two test backends and extract their host and port. The first // backend is used for the EDS cluster and the second backend is used for @@ -573,8 +592,11 @@ func (s) TestAggregateCluster_WithEDSAndDNS(t *testing.T) { // the DNS cluster. func (s) TestAggregateCluster_SwitchEDSAndDNS(t *testing.T) { // Start an xDS management server. - managementServer, nodeID, bootstrapContents, _, cleanup2 := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) - defer cleanup2() + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) // Start two test backends and extract their host and port. The first // backend is used for the EDS cluster and the second backend is used for @@ -655,8 +677,11 @@ func (s) TestAggregateCluster_BadEDS_GoodToBadDNS(t *testing.T) { dnsTargetCh, dnsR := setupDNS(t) // Start an xDS management server. - managementServer, nodeID, bootstrapContents, _, cleanup2 := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) - defer cleanup2() + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) // Start two test backends. servers, cleanup3 := startTestServiceBackends(t, 2) @@ -757,8 +782,11 @@ func (s) TestAggregateCluster_BadEDS_GoodToBadDNS(t *testing.T) { // DNS cluster and can make a successful RPC. func (s) TestAggregateCluster_BadEDSFromError_GoodToBadDNS(t *testing.T) { // Start an xDS management server. - managementServer, nodeID, bootstrapContents, _, cleanup2 := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) - defer cleanup2() + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) // Start a test service backend. server := stubserver.StartTestService(t, nil) @@ -813,8 +841,11 @@ func (s) TestAggregateCluster_BadEDSFromError_GoodToBadDNS(t *testing.T) { // back from the LOGICAL_DNS cluster to the EDS cluster. func (s) TestAggregateCluster_BadDNS_GoodEDS(t *testing.T) { // Start an xDS management server. - managementServer, nodeID, bootstrapContents, _, cleanup2 := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) - defer cleanup2() + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) // Start a test service backend. server := stubserver.StartTestService(t, nil) @@ -868,8 +899,11 @@ func (s) TestAggregateCluster_BadDNS_GoodEDS(t *testing.T) { // Discovery Mechanism (from sending an empty address list down). func (s) TestAggregateCluster_BadEDS_BadDNS(t *testing.T) { // Start an xDS management server. - managementServer, nodeID, bootstrapContents, _, cleanup2 := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) - defer cleanup2() + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) // Configure an aggregate cluster pointing to an EDS and LOGICAL_DNS // cluster. Also configure an empty endpoints resource for the EDS cluster @@ -927,8 +961,11 @@ func (s) TestAggregateCluster_BadEDS_BadDNS(t *testing.T) { // cluster. func (s) TestAggregateCluster_NoFallback_EDSNackedWithPreviousGoodUpdate(t *testing.T) { // Start an xDS management server. - mgmtServer, nodeID, bootstrapContents, _, cleanup2 := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) - defer cleanup2() + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) // Start two test backends and extract their host and port. The first // backend is used for the EDS cluster and the second backend is used for @@ -956,7 +993,7 @@ func (s) TestAggregateCluster_NoFallback_EDSNackedWithPreviousGoodUpdate(t *test } ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) defer cancel() - if err := mgmtServer.Update(ctx, resources); err != nil { + if err := managementServer.Update(ctx, resources); err != nil { t.Fatal(err) } @@ -980,7 +1017,7 @@ func (s) TestAggregateCluster_NoFallback_EDSNackedWithPreviousGoodUpdate(t *test // NACKed by the xDS client. Since the cluster_resolver LB policy has a // previously received good EDS resource, it will continue to use that. resources.Endpoints[0].Endpoints[0].LbEndpoints[0].LoadBalancingWeight = &wrapperspb.UInt32Value{Value: 0} - if err := mgmtServer.Update(ctx, resources); err != nil { + if err := managementServer.Update(ctx, resources); err != nil { t.Fatal(err) } @@ -1004,8 +1041,11 @@ func (s) TestAggregateCluster_NoFallback_EDSNackedWithPreviousGoodUpdate(t *test // as though it received an update with no endpoints. func (s) TestAggregateCluster_Fallback_EDSNackedWithoutPreviousGoodUpdate(t *testing.T) { // Start an xDS management server. - mgmtServer, nodeID, bootstrapContents, _, cleanup2 := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) - defer cleanup2() + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) // Start two test backends and extract their host and port. The first // backend is used for the EDS cluster and the second backend is used for @@ -1038,7 +1078,7 @@ func (s) TestAggregateCluster_Fallback_EDSNackedWithoutPreviousGoodUpdate(t *tes resources.Endpoints[0].Endpoints[0].LbEndpoints[0].LoadBalancingWeight = &wrapperspb.UInt32Value{Value: 0} ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) defer cancel() - if err := mgmtServer.Update(ctx, resources); err != nil { + if err := managementServer.Update(ctx, resources); err != nil { t.Fatal(err) } @@ -1065,8 +1105,11 @@ func (s) TestAggregateCluster_Fallback_EDSNackedWithoutPreviousGoodUpdate(t *tes // the LOGICAL_DNS cluster in this case. func (s) TestAggregateCluster_Fallback_EDS_ResourceNotFound(t *testing.T) { // Start an xDS management server. - mgmtServer, nodeID, _, _, cleanup2 := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) - defer cleanup2() + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) // Start a test backend for the LOGICAL_DNS cluster. server := stubserver.StartTestService(t, nil) @@ -1090,17 +1133,13 @@ func (s) TestAggregateCluster_Fallback_EDS_ResourceNotFound(t *testing.T) { } ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) defer cancel() - if err := mgmtServer.Update(ctx, resources); err != nil { + if err := managementServer.Update(ctx, resources); err != nil { t.Fatal(err) } // Create an xDS client talking to the above management server, configured // with a short watch expiry timeout. - bc, err := e2e.DefaultBootstrapContents(nodeID, mgmtServer.Address) - if err != nil { - t.Fatalf("Failed to create bootstrap configuration: %v", err) - } - xdsClient, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bc, WatchExpiryTimeout: defaultTestWatchExpiryTimeout}) + xdsClient, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bootstrapContents, WatchExpiryTimeout: defaultTestWatchExpiryTimeout}) if err != nil { t.Fatalf("Failed to create an xDS client: %v", err) } diff --git a/xds/internal/balancer/clusterresolver/e2e_test/balancer_test.go b/xds/internal/balancer/clusterresolver/e2e_test/balancer_test.go index 7ac6306ccd8..7f8851a93a2 100644 --- a/xds/internal/balancer/clusterresolver/e2e_test/balancer_test.go +++ b/xds/internal/balancer/clusterresolver/e2e_test/balancer_test.go @@ -25,6 +25,7 @@ import ( "time" "github.com/google/go-cmp/cmp" + "github.com/google/uuid" "google.golang.org/grpc" "google.golang.org/grpc/balancer" "google.golang.org/grpc/balancer/roundrobin" @@ -121,7 +122,7 @@ func (s) TestErrorFromParentLB_ConnectionError(t *testing.T) { // Start an xDS management server with the above restartable listener, and // push a channel when the stream is closed. streamClosedCh := make(chan struct{}, 1) - managementServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{ + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ Listener: lis, OnStreamClosed: func(int64, *v3corepb.Node) { select { @@ -130,7 +131,10 @@ func (s) TestErrorFromParentLB_ConnectionError(t *testing.T) { } }, }) - defer cleanup() + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) server := stubserver.StartTestService(t, nil) defer server.Stop() @@ -190,7 +194,7 @@ func (s) TestErrorFromParentLB_ResourceNotFound(t *testing.T) { // resource. edsResourceRequestedCh := make(chan struct{}, 1) edsResourceCanceledCh := make(chan struct{}, 1) - managementServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{ + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error { if req.GetTypeUrl() == version.V3EndpointsURL { switch len(req.GetResourceNames()) { @@ -213,7 +217,10 @@ func (s) TestErrorFromParentLB_ResourceNotFound(t *testing.T) { return nil }, }) - defer cleanup() + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) server := stubserver.StartTestService(t, nil) defer server.Stop() @@ -345,8 +352,11 @@ func (s) TestOutlierDetectionConfigPropagationToChildPolicy(t *testing.T) { }) defer balancer.Register(priorityBuilder) - managementServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) - defer cleanup() + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) server := stubserver.StartTestService(t, nil) defer server.Stop() @@ -377,7 +387,7 @@ func (s) TestOutlierDetectionConfigPropagationToChildPolicy(t *testing.T) { // Create xDS client, configure cds_experimental LB policy with a manual // resolver, and dial the test backends. - _, cleanup = setupAndDial(t, bootstrapContents) + _, cleanup := setupAndDial(t, bootstrapContents) defer cleanup() // The priority configuration generated should have Outlier Detection as a diff --git a/xds/internal/balancer/clusterresolver/e2e_test/eds_impl_test.go b/xds/internal/balancer/clusterresolver/e2e_test/eds_impl_test.go index 207993c2983..fa4730a8b38 100644 --- a/xds/internal/balancer/clusterresolver/e2e_test/eds_impl_test.go +++ b/xds/internal/balancer/clusterresolver/e2e_test/eds_impl_test.go @@ -125,8 +125,11 @@ func clientEndpointsResource(nodeID, edsServiceName string, localities []e2e.Loc // 4. Replace the backend. Test verifies that all RPCs reach the new backend. func (s) TestEDS_OneLocality(t *testing.T) { // Spin up a management server to receive xDS resources from. - managementServer, nodeID, bootstrapContents, _, cleanup1 := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) - defer cleanup1() + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) // Start backend servers which provide an implementation of the TestService. servers, cleanup2 := startTestServiceBackends(t, 3) @@ -247,8 +250,11 @@ func (s) TestEDS_OneLocality(t *testing.T) { // weighted roundrobined across them. func (s) TestEDS_MultipleLocalities(t *testing.T) { // Spin up a management server to receive xDS resources from. - managementServer, nodeID, bootstrapContents, _, cleanup1 := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) - defer cleanup1() + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) // Start backend servers which provide an implementation of the TestService. servers, cleanup2 := startTestServiceBackends(t, 4) @@ -390,8 +396,11 @@ func (s) TestEDS_MultipleLocalities(t *testing.T) { // traffic is routed only to backends deemed capable of receiving traffic. func (s) TestEDS_EndpointsHealth(t *testing.T) { // Spin up a management server to receive xDS resources from. - managementServer, nodeID, bootstrapContents, _, cleanup1 := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) - defer cleanup1() + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) // Start backend servers which provide an implementation of the TestService. servers, cleanup2 := startTestServiceBackends(t, 12) @@ -479,8 +488,11 @@ func (s) TestEDS_EndpointsHealth(t *testing.T) { // removed" error. func (s) TestEDS_EmptyUpdate(t *testing.T) { // Spin up a management server to receive xDS resources from. - managementServer, nodeID, bootstrapContents, _, cleanup1 := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) - defer cleanup1() + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) // Start backend servers which provide an implementation of the TestService. servers, cleanup2 := startTestServiceBackends(t, 4) @@ -576,7 +588,7 @@ func (s) TestEDS_ResourceRemoved(t *testing.T) { // resource. edsResourceRequestedCh := make(chan struct{}, 1) edsResourceCanceledCh := make(chan struct{}, 1) - managementServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{ + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error { if req.GetTypeUrl() == version.V3EndpointsURL { switch len(req.GetResourceNames()) { @@ -599,7 +611,10 @@ func (s) TestEDS_ResourceRemoved(t *testing.T) { return nil }, }) - defer cleanup() + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) server := stubserver.StartTestService(t, nil) defer server.Stop() @@ -653,7 +668,7 @@ func (s) TestEDS_ResourceRemoved(t *testing.T) { // cluster name for the EDS resource. func (s) TestEDS_ClusterResourceDoesNotContainEDSServiceName(t *testing.T) { edsResourceCh := make(chan string, 1) - managementServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{ + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error { if req.GetTypeUrl() != version.V3EndpointsURL { return nil @@ -667,7 +682,10 @@ func (s) TestEDS_ClusterResourceDoesNotContainEDSServiceName(t *testing.T) { return nil }, }) - defer cleanup() + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) server := stubserver.StartTestService(t, nil) defer server.Stop() @@ -727,7 +745,7 @@ func (s) TestEDS_ClusterResourceUpdates(t *testing.T) { // Start an xDS management server that pushes the EDS resource names onto a // channel. edsResourceNameCh := make(chan []string, 1) - managementServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{ + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error { if req.GetTypeUrl() != version.V3EndpointsURL { return nil @@ -744,7 +762,10 @@ func (s) TestEDS_ClusterResourceUpdates(t *testing.T) { }, AllowResourceSubset: true, }) - defer cleanup() + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) // Start two test backends and extract their host and port. The first // backend is used for the EDS resource identified by the eds_service_name, @@ -861,8 +882,11 @@ func (s) TestEDS_ClusterResourceUpdates(t *testing.T) { // priorities removed" error. func (s) TestEDS_BadUpdateWithoutPreviousGoodUpdate(t *testing.T) { // Spin up a management server to receive xDS resources from. - mgmtServer, nodeID, bootstrapContents, _, cleanup1 := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) - defer cleanup1() + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) // Start a backend server that implements the TestService. server := stubserver.StartTestService(t, nil) @@ -880,7 +904,7 @@ func (s) TestEDS_BadUpdateWithoutPreviousGoodUpdate(t *testing.T) { resources.Endpoints[0].Endpoints[0].LbEndpoints[0].LoadBalancingWeight = &wrapperspb.UInt32Value{Value: 0} ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) defer cancel() - if err := mgmtServer.Update(ctx, resources); err != nil { + if err := managementServer.Update(ctx, resources); err != nil { t.Fatal(err) } @@ -931,8 +955,11 @@ func (s) TestEDS_BadUpdateWithoutPreviousGoodUpdate(t *testing.T) { // used and that RPCs are still successful. func (s) TestEDS_BadUpdateWithPreviousGoodUpdate(t *testing.T) { // Spin up a management server to receive xDS resources from. - mgmtServer, nodeID, bootstrapContents, _, cleanup1 := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) - defer cleanup1() + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) // Start a backend server that implements the TestService. server := stubserver.StartTestService(t, nil) @@ -946,7 +973,7 @@ func (s) TestEDS_BadUpdateWithPreviousGoodUpdate(t *testing.T) { }}) ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) defer cancel() - if err := mgmtServer.Update(ctx, resources); err != nil { + if err := managementServer.Update(ctx, resources); err != nil { t.Fatal(err) } @@ -994,7 +1021,7 @@ func (s) TestEDS_BadUpdateWithPreviousGoodUpdate(t *testing.T) { // the xDS client. But since the cluster_resolver LB policy has a previously // received good EDS update, it should continue using it. resources.Endpoints[0].Endpoints[0].LbEndpoints[0].LoadBalancingWeight = &wrapperspb.UInt32Value{Value: 0} - if err := mgmtServer.Update(ctx, resources); err != nil { + if err := managementServer.Update(ctx, resources); err != nil { t.Fatal(err) } @@ -1014,19 +1041,12 @@ func (s) TestEDS_BadUpdateWithPreviousGoodUpdate(t *testing.T) { // fail with "all priorities removed" error. func (s) TestEDS_ResourceNotFound(t *testing.T) { // Spin up a management server to receive xDS resources from. - mgmtServer, err := e2e.StartManagementServer(e2e.ManagementServerOptions{}) - if err != nil { - t.Fatalf("Failed to spin up the xDS management server: %v", err) - } - defer mgmtServer.Stop() + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) // Create an xDS client talking to the above management server, configured // with a short watch expiry timeout. nodeID := uuid.New().String() - bc, err := e2e.DefaultBootstrapContents(nodeID, mgmtServer.Address) - if err != nil { - t.Fatalf("Failed to create bootstrap configuration: %v", err) - } + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) xdsClient, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bc, WatchExpiryTimeout: defaultTestWatchExpiryTimeout}) if err != nil { t.Fatalf("Failed to create an xDS client: %v", err) diff --git a/xds/internal/balancer/ringhash/e2e/ringhash_balancer_test.go b/xds/internal/balancer/ringhash/e2e/ringhash_balancer_test.go index f2835baa50f..1a50741f5a6 100644 --- a/xds/internal/balancer/ringhash/e2e/ringhash_balancer_test.go +++ b/xds/internal/balancer/ringhash/e2e/ringhash_balancer_test.go @@ -28,10 +28,12 @@ import ( "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" + "github.com/google/uuid" "google.golang.org/grpc" "google.golang.org/grpc/backoff" "google.golang.org/grpc/connectivity" "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal" "google.golang.org/grpc/internal/envconfig" "google.golang.org/grpc/internal/grpctest" "google.golang.org/grpc/internal/stubserver" @@ -234,12 +236,39 @@ func makeNonExistentBackends(t *testing.T, num int) []e2e.BackendOptions { return backendOptions } +// setupManagementServerAndResolver sets up an xDS management server, creates +// bootstrap configuration pointing to that server and creates an xDS resolver +// using that configuration. +// +// Registers a cleanup function on t to stop the management server. +// +// Returns the management server, node ID and the xDS resolver builder. +func setupManagementServerAndResolver(t *testing.T) (*e2e.ManagementServer, string, resolver.Builder) { + // Start an xDS management server. + xdsServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, xdsServer.Address) + + // Create an xDS resolver with the above bootstrap configuration. + var r resolver.Builder + var err error + if newResolver := internal.NewXDSResolverWithConfigForTesting; newResolver != nil { + r, err = newResolver.(func([]byte) (resolver.Builder, error))(bc) + if err != nil { + t.Fatalf("Failed to create xDS resolver for testing: %v", err) + } + } + + return xdsServer, nodeID, r +} + // Tests that when an aggregate cluster is configured with ring hash policy, and // the first cluster is in transient failure, all RPCs are sent to the second // cluster using the ring hash policy. func (s) TestRingHash_AggregateClusterFallBackFromRingHashAtStartup(t *testing.T) { - xdsServer, nodeID, _, xdsResolver, stop := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) - defer stop() + xdsServer, nodeID, xdsResolver := setupManagementServerAndResolver(t) servers, stop := startTestServiceBackends(t, 2) defer stop() @@ -288,14 +317,13 @@ func (s) TestRingHash_AggregateClusterFallBackFromRingHashAtStartup(t *testing.T ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) defer cancel() - err := xdsServer.Update(ctx, e2e.UpdateOptions{ + if err := xdsServer.Update(ctx, e2e.UpdateOptions{ NodeID: nodeID, Endpoints: []*v3endpointpb.ClusterLoadAssignment{ep1, ep2}, Clusters: []*v3clusterpb.Cluster{cluster, primaryCluster, secundaryCluster}, Routes: []*v3routepb.RouteConfiguration{route}, Listeners: []*v3listenerpb.Listener{listener}, - }) - if err != nil { + }); err != nil { t.Fatalf("Failed to update xDS resources: %v", err) } @@ -348,8 +376,7 @@ func replaceDNSResolver(t *testing.T) *manual.Resolver { // logical DNS cluster, all RPCs are sent to the second cluster using the ring // hash policy. func (s) TestRingHash_AggregateClusterFallBackFromRingHashToLogicalDnsAtStartup(t *testing.T) { - xdsServer, nodeID, _, xdsResolver, stop := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) - defer stop() + xdsServer, nodeID, xdsResolver := setupManagementServerAndResolver(t) const edsClusterName = "eds_cluster" const logicalDNSClusterName = "logical_dns_cluster" @@ -394,14 +421,13 @@ func (s) TestRingHash_AggregateClusterFallBackFromRingHashToLogicalDnsAtStartup( ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) defer cancel() - err := xdsServer.Update(ctx, e2e.UpdateOptions{ + if err := xdsServer.Update(ctx, e2e.UpdateOptions{ NodeID: nodeID, Endpoints: []*v3endpointpb.ClusterLoadAssignment{endpoints}, Clusters: []*v3clusterpb.Cluster{cluster, edsCluster, logicalDNSCluster}, Routes: []*v3routepb.RouteConfiguration{route}, Listeners: []*v3listenerpb.Listener{listener}, - }) - if err != nil { + }); err != nil { t.Fatalf("Failed to update xDS resources: %v", err) } @@ -429,8 +455,7 @@ func (s) TestRingHash_AggregateClusterFallBackFromRingHashToLogicalDnsAtStartup( // the later recovers from transient failure when its backend becomes available. func (s) TestRingHash_AggregateClusterFallBackFromRingHashToLogicalDnsAtStartupNoFailedRPCs(t *testing.T) { // https://github.com/grpc/grpc/blob/083bbee4805c14ce62e6c9535fe936f68b854c4f/test/cpp/end2end/xds/xds_ring_hash_end2end_test.cc#L225 - xdsServer, nodeID, _, xdsResolver, stop := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) - defer stop() + xdsServer, nodeID, xdsResolver := setupManagementServerAndResolver(t) const edsClusterName = "eds_cluster" const logicalDNSClusterName = "logical_dns_cluster" @@ -475,14 +500,13 @@ func (s) TestRingHash_AggregateClusterFallBackFromRingHashToLogicalDnsAtStartupN ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) defer cancel() - err := xdsServer.Update(ctx, e2e.UpdateOptions{ + if err := xdsServer.Update(ctx, e2e.UpdateOptions{ NodeID: nodeID, Endpoints: []*v3endpointpb.ClusterLoadAssignment{endpoints}, Clusters: []*v3clusterpb.Cluster{cluster, edsCluster, logicalDNSCluster}, Routes: []*v3routepb.RouteConfiguration{route}, Listeners: []*v3listenerpb.Listener{listener}, - }) - if err != nil { + }); err != nil { t.Fatalf("Failed to update xDS resources: %v", err) } @@ -560,8 +584,8 @@ func (s) TestRingHash_ChannelIdHashing(t *testing.T) { backends, stop := startTestServiceBackends(t, 4) defer stop() - xdsServer, nodeID, _, xdsResolver, stop := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) - defer stop() + xdsServer, nodeID, xdsResolver := setupManagementServerAndResolver(t) + const clusterName = "cluster" endpoints := e2e.EndpointResourceWithOptions(e2e.EndpointOptions{ ClusterName: clusterName, @@ -581,14 +605,13 @@ func (s) TestRingHash_ChannelIdHashing(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) defer cancel() - err := xdsServer.Update(ctx, e2e.UpdateOptions{ + if err := xdsServer.Update(ctx, e2e.UpdateOptions{ NodeID: nodeID, Endpoints: []*v3endpointpb.ClusterLoadAssignment{endpoints}, Clusters: []*v3clusterpb.Cluster{cluster}, Routes: []*v3routepb.RouteConfiguration{route}, Listeners: []*v3listenerpb.Listener{listener}, - }) - if err != nil { + }); err != nil { t.Fatalf("Failed to update xDS resources: %v", err) } @@ -640,8 +663,8 @@ func (s) TestRingHash_HeaderHashing(t *testing.T) { t.Fatalf("Failed to split host and port from stubserver: %v", err) } - xdsServer, nodeID, _, xdsResolver, stop := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) - defer stop() + xdsServer, nodeID, xdsResolver := setupManagementServerAndResolver(t) + const clusterName = "cluster" endpoints := e2e.EndpointResourceWithOptions(e2e.EndpointOptions{ ClusterName: clusterName, @@ -662,14 +685,13 @@ func (s) TestRingHash_HeaderHashing(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) defer cancel() - err = xdsServer.Update(ctx, e2e.UpdateOptions{ + if err := xdsServer.Update(ctx, e2e.UpdateOptions{ NodeID: nodeID, Endpoints: []*v3endpointpb.ClusterLoadAssignment{endpoints}, Clusters: []*v3clusterpb.Cluster{cluster}, Routes: []*v3routepb.RouteConfiguration{route}, Listeners: []*v3listenerpb.Listener{listener}, - }) - if err != nil { + }); err != nil { t.Fatalf("Failed to update xDS resources: %v", err) } @@ -705,8 +727,8 @@ func (s) TestRingHash_HeaderHashingWithRegexRewrite(t *testing.T) { t.Fatalf("Failed to split host and port from stubserver: %v", err) } - xdsServer, nodeID, _, xdsResolver, stop := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) - defer stop() + xdsServer, nodeID, xdsResolver := setupManagementServerAndResolver(t) + clusterName := "cluster" endpoints := e2e.EndpointResourceWithOptions(e2e.EndpointOptions{ ClusterName: clusterName, @@ -735,14 +757,13 @@ func (s) TestRingHash_HeaderHashingWithRegexRewrite(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) defer cancel() - err = xdsServer.Update(ctx, e2e.UpdateOptions{ + if err := xdsServer.Update(ctx, e2e.UpdateOptions{ NodeID: nodeID, Endpoints: []*v3endpointpb.ClusterLoadAssignment{endpoints}, Clusters: []*v3clusterpb.Cluster{cluster}, Routes: []*v3routepb.RouteConfiguration{route}, Listeners: []*v3listenerpb.Listener{listener}, - }) - if err != nil { + }); err != nil { t.Fatalf("Failed to update xDS resources: %v", err) } @@ -822,8 +843,8 @@ func (s) TestRingHash_NoHashPolicy(t *testing.T) { defer stop() numRPCs := computeIdealNumberOfRPCs(t, .5, errorTolerance) - xdsServer, nodeID, _, xdsResolver, stop := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) - defer stop() + xdsServer, nodeID, xdsResolver := setupManagementServerAndResolver(t) + const clusterName = "cluster" endpoints := e2e.EndpointResourceWithOptions(e2e.EndpointOptions{ ClusterName: clusterName, @@ -843,14 +864,13 @@ func (s) TestRingHash_NoHashPolicy(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) defer cancel() - err := xdsServer.Update(ctx, e2e.UpdateOptions{ + if err := xdsServer.Update(ctx, e2e.UpdateOptions{ NodeID: nodeID, Endpoints: []*v3endpointpb.ClusterLoadAssignment{endpoints}, Clusters: []*v3clusterpb.Cluster{cluster}, Routes: []*v3routepb.RouteConfiguration{route}, Listeners: []*v3listenerpb.Listener{listener}, - }) - if err != nil { + }); err != nil { t.Fatalf("Failed to update xDS resources: %v", err) } @@ -876,8 +896,9 @@ func (s) TestRingHash_NoHashPolicy(t *testing.T) { func (s) TestRingHash_EndpointWeights(t *testing.T) { backends, stop := startTestServiceBackends(t, 3) defer stop() - xdsServer, nodeID, _, xdsResolver, stop := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) - defer stop() + + xdsServer, nodeID, xdsResolver := setupManagementServerAndResolver(t) + const clusterName = "cluster" backendOpts := []e2e.BackendOptions{ {Port: testutils.ParsePort(t, backends[0].Address)}, @@ -907,14 +928,13 @@ func (s) TestRingHash_EndpointWeights(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) defer cancel() - err := xdsServer.Update(ctx, e2e.UpdateOptions{ + if err := xdsServer.Update(ctx, e2e.UpdateOptions{ NodeID: nodeID, Endpoints: []*v3endpointpb.ClusterLoadAssignment{endpoints}, Clusters: []*v3clusterpb.Cluster{cluster}, Routes: []*v3routepb.RouteConfiguration{route}, Listeners: []*v3listenerpb.Listener{listener}, - }) - if err != nil { + }); err != nil { t.Fatalf("Failed to update xDS resources: %v", err) } diff --git a/xds/internal/httpfilter/fault/fault_test.go b/xds/internal/httpfilter/fault/fault_test.go index 8660daec80b..6aa27a77af5 100644 --- a/xds/internal/httpfilter/fault/fault_test.go +++ b/xds/internal/httpfilter/fault/fault_test.go @@ -38,7 +38,6 @@ import ( "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/internal/grpctest" "google.golang.org/grpc/internal/testutils" - "google.golang.org/grpc/internal/testutils/xds/bootstrap" "google.golang.org/grpc/internal/testutils/xds/e2e" "google.golang.org/grpc/metadata" "google.golang.org/grpc/status" @@ -101,20 +100,11 @@ func (*testService) FullDuplexCall(stream testgrpc.TestService_FullDuplexCallSer func clientSetup(t *testing.T) (*e2e.ManagementServer, string, uint32, func()) { // Spin up a xDS management server on a local port. nodeID := uuid.New().String() - fs, err := e2e.StartManagementServer(e2e.ManagementServerOptions{}) - if err != nil { - t.Fatal(err) - } + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) // Create a bootstrap file in a temporary directory. - bootstrapCleanup, err := bootstrap.CreateFile(bootstrap.Options{ - NodeID: nodeID, - ServerURI: fs.Address, - ServerListenerResourceNameTemplate: "grpc/server", - }) - if err != nil { - t.Fatal(err) - } + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) + testutils.CreateBootstrapFileForTesting(t, bootstrapContents) // Initialize a gRPC server and register the stubServer on it. server := grpc.NewServer() @@ -132,9 +122,7 @@ func clientSetup(t *testing.T) (*e2e.ManagementServer, string, uint32, func()) { } }() - return fs, nodeID, uint32(lis.Addr().(*net.TCPAddr).Port), func() { - fs.Stop() - bootstrapCleanup() + return managementServer, nodeID, uint32(lis.Addr().(*net.TCPAddr).Port), func() { server.Stop() } } diff --git a/xds/internal/resolver/helpers_test.go b/xds/internal/resolver/helpers_test.go index e15982a46eb..698ecf0c9a1 100644 --- a/xds/internal/resolver/helpers_test.go +++ b/xds/internal/resolver/helpers_test.go @@ -32,7 +32,6 @@ import ( "google.golang.org/grpc/internal/grpctest" iresolver "google.golang.org/grpc/internal/resolver" "google.golang.org/grpc/internal/testutils" - xdsbootstrap "google.golang.org/grpc/internal/testutils/xds/bootstrap" "google.golang.org/grpc/internal/testutils/xds/e2e" "google.golang.org/grpc/resolver" "google.golang.org/grpc/serviceconfig" @@ -196,7 +195,7 @@ func setupManagementServerForTest(ctx context.Context, t *testing.T, nodeID stri // Setup the management server to push the requested listener and route // configuration resource names on to separate channels for the test to // inspect. - mgmtServer, err := e2e.StartManagementServer(e2e.ManagementServerOptions{ + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error { switch req.GetTypeUrl() { case version.V3ListenerURL: @@ -222,20 +221,10 @@ func setupManagementServerForTest(ctx context.Context, t *testing.T, nodeID stri }, AllowResourceSubset: true, }) - if err != nil { - t.Fatalf("Failed to start xDS management server: %v", err) - } - t.Cleanup(mgmtServer.Stop) // Create a bootstrap configuration specifying the above management server. - cleanup, err := xdsbootstrap.CreateFile(xdsbootstrap.Options{ - NodeID: nodeID, - ServerURI: mgmtServer.Address, - }) - if err != nil { - t.Fatal(err) - } - t.Cleanup(cleanup) + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + testutils.CreateBootstrapFileForTesting(t, bootstrapContents) return mgmtServer, listenerResourceNamesCh, routeConfigResourceNamesCh } diff --git a/xds/internal/resolver/xds_resolver_test.go b/xds/internal/resolver/xds_resolver_test.go index fb3560d6545..25207334efb 100644 --- a/xds/internal/resolver/xds_resolver_test.go +++ b/xds/internal/resolver/xds_resolver_test.go @@ -20,6 +20,7 @@ package resolver_test import ( "context" + "encoding/json" "fmt" "strings" "testing" @@ -34,8 +35,8 @@ import ( "google.golang.org/grpc/internal/grpcsync" iresolver "google.golang.org/grpc/internal/resolver" "google.golang.org/grpc/internal/testutils" - xdsbootstrap "google.golang.org/grpc/internal/testutils/xds/bootstrap" "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/xds/bootstrap" "google.golang.org/grpc/metadata" "google.golang.org/grpc/resolver" "google.golang.org/grpc/serviceconfig" @@ -85,14 +86,8 @@ func (s) TestResolverBuilder_ClientCreationFails_NoBootstrap(t *testing.T) { // not specified in the bootstrap file. Verifies that the resolver.Build method // fails with the expected error string. func (s) TestResolverBuilder_AuthorityNotDefinedInBootstrap(t *testing.T) { - bootstrapCleanup, err := xdsbootstrap.CreateFile(xdsbootstrap.Options{ - NodeID: "node-id", - ServerURI: "dummy-management-server", - }) - if err != nil { - t.Fatal(err) - } - defer bootstrapCleanup() + contents := e2e.DefaultBootstrapContents(t, "node-id", "dummy-management-server") + testutils.CreateBootstrapFileForTesting(t, contents) builder := resolver.Get(xdsresolver.Scheme) if builder == nil { @@ -165,23 +160,30 @@ func (s) TestResolverResourceName(t *testing.T) { mgmtServer, lisCh, _ := setupManagementServerForTest(ctx, t, nodeID) // Create a bootstrap configuration with test options. - opts := xdsbootstrap.Options{ - ServerURI: mgmtServer.Address, + opts := bootstrap.ConfigOptionsForTesting{ + Servers: []json.RawMessage{[]byte(fmt.Sprintf(`{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }`, mgmtServer.Address))}, ClientDefaultListenerResourceNameTemplate: tt.listenerResourceNameTemplate, + NodeID: nodeID, } if tt.extraAuthority != "" { // In this test, we really don't care about having multiple // management servers. All we need to verify is whether the // resource name matches expectation. - opts.Authorities = map[string]string{ - tt.extraAuthority: mgmtServer.Address, + opts.Authorities = map[string]json.RawMessage{ + tt.extraAuthority: []byte(fmt.Sprintf(`{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }`, mgmtServer.Address)), } } - cleanup, err := xdsbootstrap.CreateFile(opts) + contents, err := bootstrap.NewContentsForTesting(opts) if err != nil { - t.Fatal(err) + t.Fatalf("Failed to create bootstrap configuration: %v", err) } - defer cleanup() + testutils.CreateBootstrapFileForTesting(t, contents) buildResolverForTarget(t, resolver.Target{URL: *testutils.MustParseURL(tt.dialTarget)}) waitForResourceNames(ctx, t, lisCh, tt.wantResourceNames) @@ -200,7 +202,7 @@ func (s) TestResolverWatchCallbackAfterClose(t *testing.T) { // closed. routeConfigResourceNamesCh := make(chan []string, 1) waitForResolverCloseCh := make(chan struct{}) - mgmtServer, err := e2e.StartManagementServer(e2e.ManagementServerOptions{ + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error { if req.GetTypeUrl() == version.V3RouteConfigURL { select { @@ -216,21 +218,11 @@ func (s) TestResolverWatchCallbackAfterClose(t *testing.T) { return nil }, }) - if err != nil { - t.Fatalf("Failed to start xDS management server: %v", err) - } - defer mgmtServer.Stop() // Create a bootstrap configuration specifying the above management server. nodeID := uuid.New().String() - cleanup, err := xdsbootstrap.CreateFile(xdsbootstrap.Options{ - NodeID: nodeID, - ServerURI: mgmtServer.Address, - }) - if err != nil { - t.Fatal(err) - } - defer cleanup() + contents := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + testutils.CreateBootstrapFileForTesting(t, contents) // Configure resources on the management server. listeners := []*v3listenerpb.Listener{e2e.DefaultClientListener(defaultTestServiceName, defaultTestRouteConfigName)} @@ -261,10 +253,7 @@ func (s) TestResolverCloseClosesXDSClient(t *testing.T) { origNewClient := rinternal.NewXDSClient closeCh := make(chan struct{}) rinternal.NewXDSClient = func() (xdsclient.XDSClient, func(), error) { - bc, err := e2e.DefaultBootstrapContents(uuid.New().String(), "dummy-management-server-address") - if err != nil { - t.Fatalf("Failed to create bootstrap configuration: %v", err) - } + bc := e2e.DefaultBootstrapContents(t, uuid.New().String(), "dummy-management-server-address") c, cancel, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bc, WatchExpiryTimeout: defaultTestTimeout}) return c, grpcsync.OnceFunc(func() { close(closeCh) diff --git a/xds/internal/server/rds_handler_test.go b/xds/internal/server/rds_handler_test.go index faaa62cd98e..b159ef48c6a 100644 --- a/xds/internal/server/rds_handler_test.go +++ b/xds/internal/server/rds_handler_test.go @@ -26,6 +26,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" + "github.com/google/uuid" "google.golang.org/grpc/internal/grpctest" "google.golang.org/grpc/internal/testutils/xds/e2e" "google.golang.org/grpc/xds/internal/xdsclient" @@ -77,7 +78,7 @@ func xdsSetupForTests(t *testing.T) (*e2e.ManagementServer, string, chan []strin // Setup the management server to push the requested route configuration // resource names on to a channel for the test to inspect. - mgmtServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{ + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error { switch req.GetTypeUrl() { case version.V3ListenerURL: // Waits on the listener, and route config below... @@ -105,7 +106,10 @@ func xdsSetupForTests(t *testing.T) (*e2e.ManagementServer, string, chan []strin }, AllowResourceSubset: true, }) - t.Cleanup(cleanup) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) xdsC, cancel, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bootstrapContents}) if err != nil { diff --git a/xds/internal/test/e2e/controlplane.go b/xds/internal/test/e2e/controlplane.go index 98030dd448f..aae80a49168 100644 --- a/xds/internal/test/e2e/controlplane.go +++ b/xds/internal/test/e2e/controlplane.go @@ -18,10 +18,9 @@ package e2e import ( - "fmt" + "testing" "github.com/google/uuid" - "google.golang.org/grpc/internal/testutils/xds/bootstrap" "google.golang.org/grpc/internal/testutils/xds/e2e" ) @@ -31,31 +30,16 @@ type controlPlane struct { bootstrapContent string } -func newControlPlane() (*controlPlane, error) { +func newControlPlane(t *testing.T) (*controlPlane, error) { // Spin up an xDS management server on a local port. - server, err := e2e.StartManagementServer(e2e.ManagementServerOptions{}) - if err != nil { - return nil, fmt.Errorf("failed to spin up the xDS management server: %v", err) - } + server := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) nodeID := uuid.New().String() - bootstrapContentBytes, err := bootstrap.Contents(bootstrap.Options{ - NodeID: nodeID, - ServerURI: server.Address, - ServerListenerResourceNameTemplate: e2e.ServerListenerResourceNameTemplate, - }) - if err != nil { - server.Stop() - return nil, fmt.Errorf("failed to create bootstrap file: %v", err) - } + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, server.Address) return &controlPlane{ server: server, nodeID: nodeID, - bootstrapContent: string(bootstrapContentBytes), + bootstrapContent: string(bootstrapContents), }, nil } - -func (cp *controlPlane) stop() { - cp.server.Stop() -} diff --git a/xds/internal/test/e2e/e2e_test.go b/xds/internal/test/e2e/e2e_test.go index be8af2b0a26..dbbf6d0864b 100644 --- a/xds/internal/test/e2e/e2e_test.go +++ b/xds/internal/test/e2e/e2e_test.go @@ -59,11 +59,10 @@ func setup(t *testing.T, opts testOpts) (*controlPlane, *client, []*server) { backendCount = opts.backendCount } - cp, err := newControlPlane() + cp, err := newControlPlane(t) if err != nil { t.Fatalf("failed to start control-plane: %v", err) } - t.Cleanup(cp.stop) var clientLog bytes.Buffer c, err := newClient(fmt.Sprintf("xds:///%s", opts.testName), *clientPath, cp.bootstrapContent, &clientLog, opts.clientFlags...) diff --git a/xds/internal/xdsclient/authority_test.go b/xds/internal/xdsclient/authority_test.go index 83954a0987c..9db4284d57c 100644 --- a/xds/internal/xdsclient/authority_test.go +++ b/xds/internal/xdsclient/authority_test.go @@ -59,20 +59,14 @@ func init() { func setupTest(ctx context.Context, t *testing.T, opts e2e.ManagementServerOptions, watchExpiryTimeout time.Duration) (*authority, *e2e.ManagementServer, string) { t.Helper() nodeID := uuid.New().String() - ms, err := e2e.StartManagementServer(opts) - if err != nil { - t.Fatalf("Failed to spin up the xDS management server: %q", err) - } + managementServer := e2e.StartManagementServer(t, opts) - contents, err := e2e.DefaultBootstrapContents(nodeID, ms.Address) - if err != nil { - t.Fatalf("Failed to create bootstrap configuration: %v", err) - } + contents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) config, err := bootstrap.NewConfigFromContents(contents) if err != nil { t.Fatalf("Failed to build bootstrap configuration: %v", err) } - serverCfg, err := bootstrap.ServerConfigForTesting(bootstrap.ServerConfigTestingOptions{URI: ms.Address}) + serverCfg, err := bootstrap.ServerConfigForTesting(bootstrap.ServerConfigTestingOptions{URI: managementServer.Address}) if err != nil { t.Fatalf("Failed to create server config for testing: %v", err) } @@ -87,7 +81,7 @@ func setupTest(ctx context.Context, t *testing.T, opts e2e.ManagementServerOptio if err != nil { t.Fatalf("Failed to create authority: %q", err) } - return a, ms, nodeID + return a, managementServer, nodeID } // This tests verifies watch and timer state for the scenario where a watch for @@ -97,7 +91,6 @@ func (s) TestTimerAndWatchStateOnSendCallback(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) defer cancel() a, ms, nodeID := setupTest(ctx, t, emptyServerOpts, defaultTestTimeout) - defer ms.Stop() defer a.close() rn := "xdsclient-test-lds-resource" @@ -205,7 +198,6 @@ func (s) TestWatchResourceTimerCanRestartOnIgnoredADSRecvError(t *testing.T) { } a, ms, nodeID := setupTest(ctx, t, serverOpt, defaultTestTimeout) - defer ms.Stop() defer a.close() nameA := "xdsclient-test-lds-resourceA" diff --git a/xds/internal/xdsclient/loadreport_test.go b/xds/internal/xdsclient/loadreport_test.go index 342594a93f8..4d095b64be8 100644 --- a/xds/internal/xdsclient/loadreport_test.go +++ b/xds/internal/xdsclient/loadreport_test.go @@ -48,10 +48,7 @@ func (s) TestLRSClient(t *testing.T) { if err != nil { t.Fatalf("Failed to create server config for testing: %v", err) } - bc, err := e2e.DefaultBootstrapContents(nodeID, fs1.Address) - if err != nil { - t.Fatalf("Failed to create bootstrap configuration: %v", err) - } + bc := e2e.DefaultBootstrapContents(t, nodeID, fs1.Address) xdsC, close, err := NewForTesting(OptionsForTesting{Contents: bc, WatchExpiryTimeout: defaultTestWatchExpiryTimeout}) if err != nil { t.Fatalf("Failed to create an xDS client: %v", err) diff --git a/xds/internal/xdsclient/singleton_test.go b/xds/internal/xdsclient/singleton_test.go index 1875ea118d0..9890528030b 100644 --- a/xds/internal/xdsclient/singleton_test.go +++ b/xds/internal/xdsclient/singleton_test.go @@ -24,7 +24,7 @@ import ( "github.com/google/uuid" "google.golang.org/grpc/internal/testutils" - "google.golang.org/grpc/internal/testutils/xds/bootstrap" + "google.golang.org/grpc/internal/testutils/xds/e2e" ) // Test that multiple New() returns the same Client. And only when the last @@ -33,14 +33,8 @@ func (s) TestClientNewSingleton(t *testing.T) { // Create a bootstrap configuration, place it in a file in the temp // directory, and set the bootstrap env vars to point to it. nodeID := uuid.New().String() - cleanup, err := bootstrap.CreateFile(bootstrap.Options{ - NodeID: nodeID, - ServerURI: "non-existent-server-address", - }) - if err != nil { - t.Fatal(err) - } - defer cleanup() + contents := e2e.DefaultBootstrapContents(t, nodeID, "non-existent-server-address") + testutils.CreateBootstrapFileForTesting(t, contents) // Override the singleton creation hook to get notified. origSingletonClientImplCreateHook := singletonClientImplCreateHook diff --git a/xds/internal/xdsclient/tests/authority_test.go b/xds/internal/xdsclient/tests/authority_test.go index 24be1d4c275..aa46e57888b 100644 --- a/xds/internal/xdsclient/tests/authority_test.go +++ b/xds/internal/xdsclient/tests/authority_test.go @@ -20,13 +20,15 @@ package xdsclient_test import ( "context" + "encoding/json" + "fmt" "testing" "time" "github.com/google/uuid" "google.golang.org/grpc/internal/testutils" - testbootstrap "google.golang.org/grpc/internal/testutils/xds/bootstrap" "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/xds/bootstrap" xdstestutils "google.golang.org/grpc/xds/internal/testutils" "google.golang.org/grpc/xds/internal/xdsclient" "google.golang.org/grpc/xds/internal/xdsclient/xdsresource" @@ -71,30 +73,29 @@ func setupForAuthorityTests(ctx context.Context, t *testing.T, idleTimeout time. lisNonDefault := testutils.NewListenerWrapper(t, nil) // Start a management server to act as the default authority. - defaultAuthorityServer, err := e2e.StartManagementServer(e2e.ManagementServerOptions{Listener: lisDefault}) - if err != nil { - t.Fatalf("Failed to spin up the xDS management server: %v", err) - } - t.Cleanup(func() { defaultAuthorityServer.Stop() }) + defaultAuthorityServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: lisDefault}) // Start a management server to act as the non-default authority. - nonDefaultAuthorityServer, err := e2e.StartManagementServer(e2e.ManagementServerOptions{Listener: lisNonDefault}) - if err != nil { - t.Fatalf("Failed to spin up the xDS management server: %v", err) - } - t.Cleanup(func() { nonDefaultAuthorityServer.Stop() }) + nonDefaultAuthorityServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: lisNonDefault}) // Create a bootstrap configuration with two non-default authorities which // have empty server configs, and therefore end up using the default server // config, which points to the above management server. nodeID := uuid.New().String() - bootstrapContents, err := testbootstrap.Contents(testbootstrap.Options{ - NodeID: nodeID, - ServerURI: defaultAuthorityServer.Address, - Authorities: map[string]string{ - testAuthority1: "", - testAuthority2: "", - testAuthority3: nonDefaultAuthorityServer.Address, + bootstrapContents, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []json.RawMessage{[]byte(fmt.Sprintf(`{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }`, defaultAuthorityServer.Address))}, + NodeID: nodeID, + Authorities: map[string]json.RawMessage{ + testAuthority1: []byte(`{}`), + testAuthority2: []byte(`{}`), + testAuthority3: []byte(fmt.Sprintf(`{ + "xds_servers": [{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }]}`, nonDefaultAuthorityServer.Address)), }, }) if err != nil { diff --git a/xds/internal/xdsclient/tests/cds_watchers_test.go b/xds/internal/xdsclient/tests/cds_watchers_test.go index c7780889519..fe362e66eea 100644 --- a/xds/internal/xdsclient/tests/cds_watchers_test.go +++ b/xds/internal/xdsclient/tests/cds_watchers_test.go @@ -20,6 +20,7 @@ package xdsclient_test import ( "context" + "encoding/json" "fmt" "strings" "testing" @@ -31,6 +32,7 @@ import ( "google.golang.org/grpc/internal/grpcsync" "google.golang.org/grpc/internal/testutils" "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/xds/bootstrap" "google.golang.org/grpc/xds/internal/xdsclient" "google.golang.org/grpc/xds/internal/xdsclient/xdsresource" @@ -174,11 +176,31 @@ func (s) TestCDSWatch(t *testing.T) { for _, test := range tests { t.Run(test.desc, func(t *testing.T) { - mgmtServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) - defer cleanup() + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + bc, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []json.RawMessage{[]byte(fmt.Sprintf(`{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }`, mgmtServer.Address))}, + NodeID: nodeID, + Authorities: map[string]json.RawMessage{ + // Xdstp resource names used in this test do not specify an + // authority. These will end up looking up an entry with the + // empty key in the authorities map. Having an entry with an + // empty key and empty configuration, results in these + // resources also using the top-level configuration. + "": []byte(`{}`), + }, + }) + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + testutils.CreateBootstrapFileForTesting(t, bc) // Create an xDS client with the above bootstrap contents. - client, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bootstrapContents}) + client, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bc}) if err != nil { t.Fatalf("Failed to create xDS client: %v", err) } @@ -301,11 +323,31 @@ func (s) TestCDSWatch_TwoWatchesForSameResourceName(t *testing.T) { for _, test := range tests { t.Run(test.desc, func(t *testing.T) { - mgmtServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) - defer cleanup() + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + bc, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []json.RawMessage{[]byte(fmt.Sprintf(`{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }`, mgmtServer.Address))}, + NodeID: nodeID, + Authorities: map[string]json.RawMessage{ + // Xdstp resource names used in this test do not specify an + // authority. These will end up looking up an entry with the + // empty key in the authorities map. Having an entry with an + // empty key and empty configuration, results in these + // resources also using the top-level configuration. + "": []byte(`{}`), + }, + }) + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + testutils.CreateBootstrapFileForTesting(t, bc) // Create an xDS client with the above bootstrap contents. - client, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bootstrapContents}) + client, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bc}) if err != nil { t.Fatalf("Failed to create xDS client: %v", err) } @@ -381,11 +423,31 @@ func (s) TestCDSWatch_TwoWatchesForSameResourceName(t *testing.T) { // the management server containing both resources results in the invocation of // all watch callbacks. func (s) TestCDSWatch_ThreeWatchesForDifferentResourceNames(t *testing.T) { - mgmtServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) - defer cleanup() + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + authority := makeAuthorityName(t.Name()) + bc, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []json.RawMessage{[]byte(fmt.Sprintf(`{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }`, mgmtServer.Address))}, + NodeID: nodeID, + Authorities: map[string]json.RawMessage{ + // Xdstp style resource names used in this test use a slash removed + // version of t.Name as their authority, and the empty config + // results in the top-level xds server configuration being used for + // this authority. + authority: []byte(`{}`), + }, + }) + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + testutils.CreateBootstrapFileForTesting(t, bc) // Create an xDS client with the above bootstrap contents. - client, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bootstrapContents}) + client, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bc}) if err != nil { t.Fatalf("Failed to create xDS client: %v", err) } @@ -402,6 +464,7 @@ func (s) TestCDSWatch_ThreeWatchesForDifferentResourceNames(t *testing.T) { // Register the third watch for a different cluster resource, and push the // received updates onto a channel. + cdsNameNewStyle := makeNewStyleCDSName(authority) cw3 := newClusterWatcher() cdsCancel3 := xdsresource.WatchCluster(client, cdsNameNewStyle, cw3) defer cdsCancel3() @@ -455,7 +518,7 @@ func (s) TestCDSWatch_ResourceCaching(t *testing.T) { firstAckReceived := grpcsync.NewEvent() secondRequestReceived := grpcsync.NewEvent() - mgmtServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{ + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ OnStreamRequest: func(id int64, req *v3discoverypb.DiscoveryRequest) error { // The first request has an empty version string. if !firstRequestReceived && req.GetVersionInfo() == "" { @@ -472,10 +535,13 @@ func (s) TestCDSWatch_ResourceCaching(t *testing.T) { return nil }, }) - defer cleanup() + + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + testutils.CreateBootstrapFileForTesting(t, bc) // Create an xDS client with the above bootstrap contents. - client, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bootstrapContents}) + client, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bc}) if err != nil { t.Fatalf("Failed to create xDS client: %v", err) } @@ -539,16 +605,12 @@ func (s) TestCDSWatch_ResourceCaching(t *testing.T) { // verifies that the watch callback is invoked with an error once the // watchExpiryTimer fires. func (s) TestCDSWatch_ExpiryTimerFiresBeforeResponse(t *testing.T) { - mgmtServer, err := e2e.StartManagementServer(e2e.ManagementServerOptions{}) - if err != nil { - t.Fatalf("Failed to spin up the xDS management server: %v", err) - } - defer mgmtServer.Stop() + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + testutils.CreateBootstrapFileForTesting(t, bc) - bc, err := e2e.DefaultBootstrapContents("", mgmtServer.Address) - if err != nil { - t.Fatalf("Failed to create bootstrap configuration: %v", err) - } client, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bc, WatchExpiryTimeout: defaultTestWatchExpiryTimeout}) if err != nil { t.Fatalf("Failed to create an xDS client: %v", err) @@ -578,18 +640,13 @@ func (s) TestCDSWatch_ExpiryTimerFiresBeforeResponse(t *testing.T) { // verifies that the behavior associated with the expiry timer (i.e, callback // invocation with error) does not take place. func (s) TestCDSWatch_ValidResponseCancelsExpiryTimerBehavior(t *testing.T) { - mgmtServer, err := e2e.StartManagementServer(e2e.ManagementServerOptions{}) - if err != nil { - t.Fatalf("Failed to spin up the xDS management server: %v", err) - } - defer mgmtServer.Stop() + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) // Create an xDS client talking to the above management server. nodeID := uuid.New().String() - bc, err := e2e.DefaultBootstrapContents(nodeID, mgmtServer.Address) - if err != nil { - t.Fatalf("Failed to create bootstrap configuration: %v", err) - } + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + testutils.CreateBootstrapFileForTesting(t, bc) + client, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bc, WatchExpiryTimeout: defaultTestWatchExpiryTimeout}) if err != nil { t.Fatalf("Failed to create an xDS client: %v", err) @@ -645,12 +702,31 @@ func (s) TestCDSWatch_ValidResponseCancelsExpiryTimerBehavior(t *testing.T) { // callback associated with that resource. It should not result in the // invocation of the watch callback associated with the deleted resource. func (s) TestCDSWatch_ResourceRemoved(t *testing.T) { - t.Skip("Disabled; see https://github.com/grpc/grpc-go/issues/6781") - mgmtServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) - defer cleanup() + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + authority := makeAuthorityName(t.Name()) + bc, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []json.RawMessage{[]byte(fmt.Sprintf(`{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }`, mgmtServer.Address))}, + NodeID: nodeID, + Authorities: map[string]json.RawMessage{ + // Xdstp style resource names used in this test use a slash removed + // version of t.Name as their authority, and the empty config + // results in the top-level xds server configuration being used for + // this authority. + authority: []byte(`{}`), + }, + }) + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + testutils.CreateBootstrapFileForTesting(t, bc) // Create an xDS client with the above bootstrap contents. - client, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bootstrapContents}) + client, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bc}) if err != nil { t.Fatalf("Failed to create xDS client: %v", err) } @@ -662,13 +738,15 @@ func (s) TestCDSWatch_ResourceRemoved(t *testing.T) { cw1 := newClusterWatcher() cdsCancel1 := xdsresource.WatchCluster(client, resourceName1, cw1) defer cdsCancel1() - resourceName2 := cdsNameNewStyle + + resourceName2 := makeNewStyleCDSName(authority) cw2 := newClusterWatcher() - cdsCancel2 := xdsresource.WatchCluster(client, resourceName1, cw2) + cdsCancel2 := xdsresource.WatchCluster(client, resourceName2, cw2) defer cdsCancel2() // Configure the management server to return two cluster resources, // corresponding to the registered watches. + edsNameNewStyle := makeNewStyleEDSName(authority) resources := e2e.UpdateOptions{ NodeID: nodeID, Clusters: []*v3clusterpb.Cluster{ @@ -750,11 +828,14 @@ func (s) TestCDSWatch_ResourceRemoved(t *testing.T) { // server is NACK'ed by the xdsclient. The test verifies that the error is // propagated to the watcher. func (s) TestCDSWatch_NACKError(t *testing.T) { - mgmtServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) - defer cleanup() + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + testutils.CreateBootstrapFileForTesting(t, bc) // Create an xDS client with the above bootstrap contents. - client, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bootstrapContents}) + client, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bc}) if err != nil { t.Fatalf("Failed to create xDS client: %v", err) } @@ -796,11 +877,31 @@ func (s) TestCDSWatch_NACKError(t *testing.T) { // to the valid resource receive the update, while watchers corresponding to the // invalid resource receive an error. func (s) TestCDSWatch_PartialValid(t *testing.T) { - mgmtServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) - defer cleanup() + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + authority := makeAuthorityName(t.Name()) + bc, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []json.RawMessage{[]byte(fmt.Sprintf(`{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }`, mgmtServer.Address))}, + NodeID: nodeID, + Authorities: map[string]json.RawMessage{ + // Xdstp style resource names used in this test use a slash removed + // version of t.Name as their authority, and the empty config + // results in the top-level xds server configuration being used for + // this authority. + authority: []byte(`{}`), + }, + }) + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + testutils.CreateBootstrapFileForTesting(t, bc) // Create an xDS client with the above bootstrap contents. - client, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bootstrapContents}) + client, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bc}) if err != nil { t.Fatalf("Failed to create xDS client: %v", err) } @@ -813,7 +914,7 @@ func (s) TestCDSWatch_PartialValid(t *testing.T) { cw1 := newClusterWatcher() cdsCancel1 := xdsresource.WatchCluster(client, badResourceName, cw1) defer cdsCancel1() - goodResourceName := cdsNameNewStyle + goodResourceName := makeNewStyleCDSName(authority) cw2 := newClusterWatcher() cdsCancel2 := xdsresource.WatchCluster(client, goodResourceName, cw2) defer cdsCancel2() @@ -865,11 +966,31 @@ func (s) TestCDSWatch_PartialValid(t *testing.T) { // expected to wait for the watch timeout to expire before concluding that the // resource does not exist on the server func (s) TestCDSWatch_PartialResponse(t *testing.T) { - mgmtServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) - defer cleanup() + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + authority := makeAuthorityName(t.Name()) + bc, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []json.RawMessage{[]byte(fmt.Sprintf(`{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }`, mgmtServer.Address))}, + NodeID: nodeID, + Authorities: map[string]json.RawMessage{ + // Xdstp style resource names used in this test use a slash removed + // version of t.Name as their authority, and the empty config + // results in the top-level xds server configuration being used for + // this authority. + authority: []byte(`{}`), + }, + }) + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + testutils.CreateBootstrapFileForTesting(t, bc) // Create an xDS client with the above bootstrap contents. - client, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bootstrapContents}) + client, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bc}) if err != nil { t.Fatalf("Failed to create xDS client: %v", err) } @@ -881,7 +1002,7 @@ func (s) TestCDSWatch_PartialResponse(t *testing.T) { cw1 := newClusterWatcher() cdsCancel1 := xdsresource.WatchCluster(client, resourceName1, cw1) defer cdsCancel1() - resourceName2 := cdsNameNewStyle + resourceName2 := makeNewStyleCDSName(authority) cw2 := newClusterWatcher() cdsCancel2 := xdsresource.WatchCluster(client, resourceName2, cw2) defer cdsCancel2() diff --git a/xds/internal/xdsclient/tests/dump_test.go b/xds/internal/xdsclient/tests/dump_test.go index 3b93b11e9e5..3f5a7f3c46c 100644 --- a/xds/internal/xdsclient/tests/dump_test.go +++ b/xds/internal/xdsclient/tests/dump_test.go @@ -34,6 +34,7 @@ import ( v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" v3statuspb "github.com/envoyproxy/go-control-plane/envoy/service/status/v3" + "github.com/google/uuid" ) func (s) TestDumpResources(t *testing.T) { @@ -70,11 +71,14 @@ func (s) TestDumpResources(t *testing.T) { } // Spin up an xDS management server on a local port. - mgmtServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) - defer cleanup() + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + testutils.CreateBootstrapFileForTesting(t, bc) // Create an xDS client with the above bootstrap contents. - client, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bootstrapContents}) + client, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bc}) if err != nil { t.Fatalf("Failed to create xDS client: %v", err) } diff --git a/xds/internal/xdsclient/tests/eds_watchers_test.go b/xds/internal/xdsclient/tests/eds_watchers_test.go index 48179137436..15f06d66b25 100644 --- a/xds/internal/xdsclient/tests/eds_watchers_test.go +++ b/xds/internal/xdsclient/tests/eds_watchers_test.go @@ -20,6 +20,7 @@ package xdsclient_test import ( "context" + "encoding/json" "fmt" "strings" "testing" @@ -31,6 +32,7 @@ import ( "google.golang.org/grpc/internal/grpcsync" "google.golang.org/grpc/internal/testutils" "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/xds/bootstrap" "google.golang.org/grpc/xds/internal" "google.golang.org/grpc/xds/internal/xdsclient" "google.golang.org/grpc/xds/internal/xdsclient/xdsresource" @@ -205,11 +207,31 @@ func (s) TestEDSWatch(t *testing.T) { for _, test := range tests { t.Run(test.desc, func(t *testing.T) { - mgmtServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) - defer cleanup() + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + bc, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []json.RawMessage{[]byte(fmt.Sprintf(`{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }`, mgmtServer.Address))}, + NodeID: nodeID, + Authorities: map[string]json.RawMessage{ + // Xdstp resource names used in this test do not specify an + // authority. These will end up looking up an entry with the + // empty key in the authorities map. Having an entry with an + // empty key and empty configuration, results in these + // resources also using the top-level configuration. + "": []byte(`{}`), + }, + }) + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + testutils.CreateBootstrapFileForTesting(t, bc) // Create an xDS client with the above bootstrap contents. - client, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bootstrapContents}) + client, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bc}) if err != nil { t.Fatalf("Failed to create xDS client: %v", err) } @@ -372,11 +394,31 @@ func (s) TestEDSWatch_TwoWatchesForSameResourceName(t *testing.T) { for _, test := range tests { t.Run(test.desc, func(t *testing.T) { - mgmtServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) - defer cleanup() + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + bc, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []json.RawMessage{[]byte(fmt.Sprintf(`{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }`, mgmtServer.Address))}, + NodeID: nodeID, + Authorities: map[string]json.RawMessage{ + // Xdstp resource names used in this test do not specify an + // authority. These will end up looking up an entry with the + // empty key in the authorities map. Having an entry with an + // empty key and empty configuration, results in these + // resources also using the top-level configuration. + "": []byte(`{}`), + }, + }) + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + testutils.CreateBootstrapFileForTesting(t, bc) // Create an xDS client with the above bootstrap contents. - client, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bootstrapContents}) + client, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bc}) if err != nil { t.Fatalf("Failed to create xDS client: %v", err) } @@ -453,11 +495,31 @@ func (s) TestEDSWatch_TwoWatchesForSameResourceName(t *testing.T) { // // The test is run with both old and new style names. func (s) TestEDSWatch_ThreeWatchesForDifferentResourceNames(t *testing.T) { - mgmtServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) - defer cleanup() + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + authority := makeAuthorityName(t.Name()) + bc, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []json.RawMessage{[]byte(fmt.Sprintf(`{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }`, mgmtServer.Address))}, + NodeID: nodeID, + Authorities: map[string]json.RawMessage{ + // Xdstp style resource names used in this test use a slash removed + // version of t.Name as their authority, and the empty config + // results in the top-level xds server configuration being used for + // this authority. + authority: []byte(`{}`), + }, + }) + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + testutils.CreateBootstrapFileForTesting(t, bc) // Create an xDS client with the above bootstrap contents. - client, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bootstrapContents}) + client, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bc}) if err != nil { t.Fatalf("Failed to create xDS client: %v", err) } @@ -473,6 +535,7 @@ func (s) TestEDSWatch_ThreeWatchesForDifferentResourceNames(t *testing.T) { defer edsCancel2() // Register the third watch for a different endpoint resource. + edsNameNewStyle := makeNewStyleEDSName(authority) ew3 := newEndpointsWatcher() edsCancel3 := xdsresource.WatchEndpoints(client, edsNameNewStyle, ew3) defer edsCancel3() @@ -532,7 +595,7 @@ func (s) TestEDSWatch_ResourceCaching(t *testing.T) { firstAckReceived := grpcsync.NewEvent() secondRequestReceived := grpcsync.NewEvent() - mgmtServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{ + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ OnStreamRequest: func(id int64, req *v3discoverypb.DiscoveryRequest) error { // The first request has an empty version string. if !firstRequestReceived && req.GetVersionInfo() == "" { @@ -549,10 +612,13 @@ func (s) TestEDSWatch_ResourceCaching(t *testing.T) { return nil }, }) - defer cleanup() + + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + testutils.CreateBootstrapFileForTesting(t, bc) // Create an xDS client with the above bootstrap contents. - client, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bootstrapContents}) + client, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bc}) if err != nil { t.Fatalf("Failed to create xDS client: %v", err) } @@ -627,16 +693,12 @@ func (s) TestEDSWatch_ResourceCaching(t *testing.T) { // verifies that the watch callback is invoked with an error once the // watchExpiryTimer fires. func (s) TestEDSWatch_ExpiryTimerFiresBeforeResponse(t *testing.T) { - mgmtServer, err := e2e.StartManagementServer(e2e.ManagementServerOptions{}) - if err != nil { - t.Fatalf("Failed to spin up the xDS management server: %v", err) - } - defer mgmtServer.Stop() + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + testutils.CreateBootstrapFileForTesting(t, bc) - bc, err := e2e.DefaultBootstrapContents("", mgmtServer.Address) - if err != nil { - t.Fatalf("Failed to create bootstrap configuration: %v", err) - } client, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bc, WatchExpiryTimeout: defaultTestWatchExpiryTimeout}) if err != nil { t.Fatalf("Failed to create an xDS client: %v", err) @@ -666,18 +728,14 @@ func (s) TestEDSWatch_ExpiryTimerFiresBeforeResponse(t *testing.T) { // verifies that the behavior associated with the expiry timer (i.e, callback // invocation with error) does not take place. func (s) TestEDSWatch_ValidResponseCancelsExpiryTimerBehavior(t *testing.T) { - mgmtServer, err := e2e.StartManagementServer(e2e.ManagementServerOptions{}) - if err != nil { - t.Fatalf("Failed to spin up the xDS management server: %v", err) - } - defer mgmtServer.Stop() + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) // Create an xDS client talking to the above management server. nodeID := uuid.New().String() - bc, err := e2e.DefaultBootstrapContents(nodeID, mgmtServer.Address) - if err != nil { - t.Fatalf("Failed to create bootstrap configuration: %v", err) - } + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + testutils.CreateBootstrapFileForTesting(t, bc) + + // Create an xDS client talking to the above management server. client, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bc, WatchExpiryTimeout: defaultTestWatchExpiryTimeout}) if err != nil { t.Fatalf("Failed to create an xDS client: %v", err) @@ -736,11 +794,14 @@ func (s) TestEDSWatch_ValidResponseCancelsExpiryTimerBehavior(t *testing.T) { // server is NACK'ed by the xdsclient. The test verifies that the error is // propagated to the watcher. func (s) TestEDSWatch_NACKError(t *testing.T) { - mgmtServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) - defer cleanup() + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + testutils.CreateBootstrapFileForTesting(t, bc) // Create an xDS client with the above bootstrap contents. - client, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bootstrapContents}) + client, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bc}) if err != nil { t.Fatalf("Failed to create xDS client: %v", err) } @@ -782,11 +843,31 @@ func (s) TestEDSWatch_NACKError(t *testing.T) { // to the valid resource receive the update, while watchers corresponding to the // invalid resource receive an error. func (s) TestEDSWatch_PartialValid(t *testing.T) { - mgmtServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) - defer cleanup() + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + authority := makeAuthorityName(t.Name()) + bc, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []json.RawMessage{[]byte(fmt.Sprintf(`{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }`, mgmtServer.Address))}, + NodeID: nodeID, + Authorities: map[string]json.RawMessage{ + // Xdstp style resource names used in this test use a slash removed + // version of t.Name as their authority, and the empty config + // results in the top-level xds server configuration being used for + // this authority. + authority: []byte(`{}`), + }, + }) + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + testutils.CreateBootstrapFileForTesting(t, bc) // Create an xDS client with the above bootstrap contents. - client, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bootstrapContents}) + client, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bc}) if err != nil { t.Fatalf("Failed to create xDS client: %v", err) } @@ -795,11 +876,11 @@ func (s) TestEDSWatch_PartialValid(t *testing.T) { // Register two watches for two endpoint resources. The first watch is // expected to receive an error because the received resource is NACKed. // The second watch is expected to get a good update. - badResourceName := rdsName + badResourceName := edsName ew1 := newEndpointsWatcher() edsCancel1 := xdsresource.WatchEndpoints(client, badResourceName, ew1) defer edsCancel1() - goodResourceName := ldsNameNewStyle + goodResourceName := makeNewStyleEDSName(authority) ew2 := newEndpointsWatcher() edsCancel2 := xdsresource.WatchEndpoints(client, goodResourceName, ew2) defer edsCancel2() diff --git a/xds/internal/xdsclient/tests/federation_watchers_test.go b/xds/internal/xdsclient/tests/federation_watchers_test.go index bd55dcae0b3..bac3e5281ea 100644 --- a/xds/internal/xdsclient/tests/federation_watchers_test.go +++ b/xds/internal/xdsclient/tests/federation_watchers_test.go @@ -19,12 +19,13 @@ package xdsclient_test import ( "context" + "encoding/json" "fmt" "testing" "github.com/google/uuid" - "google.golang.org/grpc/internal/testutils/xds/bootstrap" "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/xds/bootstrap" "google.golang.org/grpc/xds/internal" "google.golang.org/grpc/xds/internal/xdsclient" "google.golang.org/grpc/xds/internal/xdsclient/xdsresource" @@ -44,29 +45,28 @@ const testNonDefaultAuthority = "non-default-authority" // nodeID to use, and the xDS client. func setupForFederationWatchersTest(t *testing.T) (*e2e.ManagementServer, string, xdsclient.XDSClient) { // Start a management server as the default authority. - serverDefaultAuthority, err := e2e.StartManagementServer(e2e.ManagementServerOptions{}) - if err != nil { - t.Fatalf("Failed to spin up the xDS management server: %v", err) - } - t.Cleanup(serverDefaultAuthority.Stop) + serverDefaultAuthority := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) // Start another management server as the other authority. - serverNonDefaultAuthority, err := e2e.StartManagementServer(e2e.ManagementServerOptions{}) - if err != nil { - t.Fatalf("Failed to spin up the xDS management server: %v", err) - } - t.Cleanup(serverNonDefaultAuthority.Stop) + serverNonDefaultAuthority := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) nodeID := uuid.New().String() - bootstrapContents, err := bootstrap.Contents(bootstrap.Options{ - NodeID: nodeID, - ServerURI: serverDefaultAuthority.Address, - ServerListenerResourceNameTemplate: e2e.ServerListenerResourceNameTemplate, - // Specify the address of the non-default authority. - Authorities: map[string]string{testNonDefaultAuthority: serverNonDefaultAuthority.Address}, + bootstrapContents, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []json.RawMessage{[]byte(fmt.Sprintf(`{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }`, serverDefaultAuthority.Address))}, + NodeID: nodeID, + Authorities: map[string]json.RawMessage{ + testNonDefaultAuthority: []byte(fmt.Sprintf(`{ + "xds_servers": [{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }]}`, serverNonDefaultAuthority.Address)), + }, }) if err != nil { - t.Fatalf("Failed to create bootstrap file: %v", err) + t.Fatalf("Failed to create bootstrap configuration: %v", err) } // Create an xDS client with the above bootstrap contents. client, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bootstrapContents}) diff --git a/xds/internal/xdsclient/tests/helpers_test.go b/xds/internal/xdsclient/tests/helpers_test.go new file mode 100644 index 00000000000..1718cc8f5b4 --- /dev/null +++ b/xds/internal/xdsclient/tests/helpers_test.go @@ -0,0 +1,73 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xdsclient_test + +import ( + "fmt" + "strings" + "testing" + "time" + + "google.golang.org/grpc/internal/grpctest" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +const ( + defaultTestWatchExpiryTimeout = 500 * time.Millisecond + defaultTestIdleAuthorityTimeout = 50 * time.Millisecond + defaultTestTimeout = 5 * time.Second + defaultTestShortTimeout = 10 * time.Millisecond // For events expected to *not* happen. + + ldsName = "xdsclient-test-lds-resource" + rdsName = "xdsclient-test-rds-resource" + cdsName = "xdsclient-test-cds-resource" + edsName = "xdsclient-test-eds-resource" + ldsNameNewStyle = "xdstp:///envoy.config.listener.v3.Listener/xdsclient-test-lds-resource" + rdsNameNewStyle = "xdstp:///envoy.config.route.v3.RouteConfiguration/xdsclient-test-rds-resource" + cdsNameNewStyle = "xdstp:///envoy.config.cluster.v3.Cluster/xdsclient-test-cds-resource" + edsNameNewStyle = "xdstp:///envoy.config.endpoint.v3.ClusterLoadAssignment/xdsclient-test-eds-resource" +) + +func makeAuthorityName(name string) string { + segs := strings.Split(name, "/") + return strings.Join(segs, "") +} + +func makeNewStyleLDSName(authority string) string { + return fmt.Sprintf("xdstp://%s/envoy.config.listener.v3.Listener/xdsclient-test-lds-resource", authority) +} + +func makeNewStyleRDSName(authority string) string { + return fmt.Sprintf("xdstp://%s/envoy.config.route.v3.RouteConfiguration/xdsclient-test-rds-resource", authority) +} + +func makeNewStyleCDSName(authority string) string { + return fmt.Sprintf("xdstp://%s/envoy.config.cluster.v3.Cluster/xdsclient-test-cds-resource", authority) +} + +func makeNewStyleEDSName(authority string) string { + return fmt.Sprintf("xdstp://%s/envoy.config.endpoint.v3.ClusterLoadAssignment/xdsclient-test-eds-resource", authority) +} diff --git a/xds/internal/xdsclient/tests/lds_watchers_test.go b/xds/internal/xdsclient/tests/lds_watchers_test.go index 2c276d37319..bf18d91aac7 100644 --- a/xds/internal/xdsclient/tests/lds_watchers_test.go +++ b/xds/internal/xdsclient/tests/lds_watchers_test.go @@ -20,6 +20,7 @@ package xdsclient_test import ( "context" + "encoding/json" "fmt" "strings" "testing" @@ -30,9 +31,9 @@ import ( "github.com/google/go-cmp/cmp/cmpopts" "github.com/google/uuid" "google.golang.org/grpc/internal/grpcsync" - "google.golang.org/grpc/internal/grpctest" "google.golang.org/grpc/internal/testutils" "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/xds/bootstrap" "google.golang.org/grpc/xds/internal/xdsclient" "google.golang.org/grpc/xds/internal/xdsclient/xdsresource" @@ -45,30 +46,6 @@ import ( _ "google.golang.org/grpc/xds/internal/httpfilter/router" // Register the router filter. ) -type s struct { - grpctest.Tester -} - -func Test(t *testing.T) { - grpctest.RunSubTests(t, s{}) -} - -const ( - defaultTestWatchExpiryTimeout = 500 * time.Millisecond - defaultTestIdleAuthorityTimeout = 50 * time.Millisecond - defaultTestTimeout = 5 * time.Second - defaultTestShortTimeout = 10 * time.Millisecond // For events expected to *not* happen. - - ldsName = "xdsclient-test-lds-resource" - rdsName = "xdsclient-test-rds-resource" - cdsName = "xdsclient-test-cds-resource" - edsName = "xdsclient-test-eds-resource" - ldsNameNewStyle = "xdstp:///envoy.config.listener.v3.Listener/xdsclient-test-lds-resource" - rdsNameNewStyle = "xdstp:///envoy.config.route.v3.RouteConfiguration/xdsclient-test-rds-resource" - cdsNameNewStyle = "xdstp:///envoy.config.cluster.v3.Cluster/xdsclient-test-cds-resource" - edsNameNewStyle = "xdstp:///envoy.config.endpoint.v3.ClusterLoadAssignment/xdsclient-test-eds-resource" -) - type noopListenerWatcher struct{} func (noopListenerWatcher) OnUpdate(update *xdsresource.ListenerResourceData) {} @@ -219,11 +196,31 @@ func (s) TestLDSWatch(t *testing.T) { for _, test := range tests { t.Run(test.desc, func(t *testing.T) { - mgmtServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) - defer cleanup() + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + bc, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []json.RawMessage{[]byte(fmt.Sprintf(`{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }`, mgmtServer.Address))}, + NodeID: nodeID, + Authorities: map[string]json.RawMessage{ + // Xdstp resource names used in this test do not specify an + // authority. These will end up looking up an entry with the + // empty key in the authorities map. Having an entry with an + // empty key and empty configuration, results in these + // resources also using the top-level configuration. + "": []byte(`{}`), + }, + }) + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + testutils.CreateBootstrapFileForTesting(t, bc) // Create an xDS client with the above bootstrap contents. - client, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bootstrapContents}) + client, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bc}) if err != nil { t.Fatalf("Failed to create xDS client: %v", err) } @@ -346,11 +343,31 @@ func (s) TestLDSWatch_TwoWatchesForSameResourceName(t *testing.T) { for _, test := range tests { t.Run(test.desc, func(t *testing.T) { - mgmtServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) - defer cleanup() + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + bc, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []json.RawMessage{[]byte(fmt.Sprintf(`{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }`, mgmtServer.Address))}, + NodeID: nodeID, + Authorities: map[string]json.RawMessage{ + // Xdstp resource names used in this test do not specify an + // authority. These will end up looking up an entry with the + // empty key in the authorities map. Having an entry with an + // empty key and empty configuration, results in these + // resources also using the top-level configuration. + "": []byte(`{}`), + }, + }) + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + testutils.CreateBootstrapFileForTesting(t, bc) // Create an xDS client with the above bootstrap contents. - client, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bootstrapContents}) + client, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bc}) if err != nil { t.Fatalf("Failed to create xDS client: %v", err) } @@ -427,11 +444,31 @@ func (s) TestLDSWatch_TwoWatchesForSameResourceName(t *testing.T) { // // The test is run with both old and new style names. func (s) TestLDSWatch_ThreeWatchesForDifferentResourceNames(t *testing.T) { - mgmtServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) - defer cleanup() + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + authority := makeAuthorityName(t.Name()) + bc, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []json.RawMessage{[]byte(fmt.Sprintf(`{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }`, mgmtServer.Address))}, + NodeID: nodeID, + Authorities: map[string]json.RawMessage{ + // Xdstp style resource names used in this test use a slash removed + // version of t.Name as their authority, and the empty config + // results in the top-level xds server configuration being used for + // this authority. + authority: []byte(`{}`), + }, + }) + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + testutils.CreateBootstrapFileForTesting(t, bc) // Create an xDS client with the above bootstrap contents. - client, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bootstrapContents}) + client, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bc}) if err != nil { t.Fatalf("Failed to create xDS client: %v", err) } @@ -447,6 +484,7 @@ func (s) TestLDSWatch_ThreeWatchesForDifferentResourceNames(t *testing.T) { defer ldsCancel2() // Register the third watch for a different listener resource. + ldsNameNewStyle := makeNewStyleLDSName(authority) lw3 := newListenerWatcher() ldsCancel3 := xdsresource.WatchListener(client, ldsNameNewStyle, lw3) defer ldsCancel3() @@ -496,7 +534,7 @@ func (s) TestLDSWatch_ResourceCaching(t *testing.T) { firstAckReceived := grpcsync.NewEvent() secondRequestReceived := grpcsync.NewEvent() - mgmtServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{ + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ OnStreamRequest: func(id int64, req *v3discoverypb.DiscoveryRequest) error { // The first request has an empty version string. if !firstRequestReceived && req.GetVersionInfo() == "" { @@ -513,10 +551,13 @@ func (s) TestLDSWatch_ResourceCaching(t *testing.T) { return nil }, }) - defer cleanup() + + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + testutils.CreateBootstrapFileForTesting(t, bc) // Create an xDS client with the above bootstrap contents. - client, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bootstrapContents}) + client, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bc}) if err != nil { t.Fatalf("Failed to create xDS client: %v", err) } @@ -580,17 +621,12 @@ func (s) TestLDSWatch_ResourceCaching(t *testing.T) { // verifies that the watch callback is invoked with an error once the // watchExpiryTimer fires. func (s) TestLDSWatch_ExpiryTimerFiresBeforeResponse(t *testing.T) { - mgmtServer, err := e2e.StartManagementServer(e2e.ManagementServerOptions{}) - if err != nil { - t.Fatalf("Failed to spin up the xDS management server: %v", err) - } - defer mgmtServer.Stop() + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) nodeID := uuid.New().String() - bc, err := e2e.DefaultBootstrapContents(nodeID, mgmtServer.Address) - if err != nil { - t.Fatalf("Failed to create bootstrap configuration: %v", err) - } + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + testutils.CreateBootstrapFileForTesting(t, bc) + client, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bc, WatchExpiryTimeout: defaultTestWatchExpiryTimeout}) if err != nil { t.Fatalf("Failed to create an xDS client: %v", err) @@ -620,18 +656,13 @@ func (s) TestLDSWatch_ExpiryTimerFiresBeforeResponse(t *testing.T) { // verifies that the behavior associated with the expiry timer (i.e, callback // invocation with error) does not take place. func (s) TestLDSWatch_ValidResponseCancelsExpiryTimerBehavior(t *testing.T) { - mgmtServer, err := e2e.StartManagementServer(e2e.ManagementServerOptions{}) - if err != nil { - t.Fatalf("Failed to spin up the xDS management server: %v", err) - } - defer mgmtServer.Stop() + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) - // Create an xDS client talking to the above management server. nodeID := uuid.New().String() - bc, err := e2e.DefaultBootstrapContents(nodeID, mgmtServer.Address) - if err != nil { - t.Fatalf("Failed to create bootstrap configuration: %v", err) - } + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + testutils.CreateBootstrapFileForTesting(t, bc) + + // Create an xDS client talking to the above management server. client, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bc, WatchExpiryTimeout: defaultTestWatchExpiryTimeout}) if err != nil { t.Fatalf("Failed to create an xDS client: %v", err) @@ -688,11 +719,31 @@ func (s) TestLDSWatch_ValidResponseCancelsExpiryTimerBehavior(t *testing.T) { // // The test is run with both old and new style names. func (s) TestLDSWatch_ResourceRemoved(t *testing.T) { - mgmtServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) - defer cleanup() + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + authority := makeAuthorityName(t.Name()) + bc, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []json.RawMessage{[]byte(fmt.Sprintf(`{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }`, mgmtServer.Address))}, + NodeID: nodeID, + Authorities: map[string]json.RawMessage{ + // Xdstp style resource names used in this test use a slash removed + // version of t.Name as their authority, and the empty config + // results in the top-level xds server configuration being used for + // this authority. + authority: []byte(`{}`), + }, + }) + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + testutils.CreateBootstrapFileForTesting(t, bc) // Create an xDS client with the above bootstrap contents. - client, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bootstrapContents}) + client, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bc}) if err != nil { t.Fatalf("Failed to create xDS client: %v", err) } @@ -705,7 +756,7 @@ func (s) TestLDSWatch_ResourceRemoved(t *testing.T) { ldsCancel1 := xdsresource.WatchListener(client, resourceName1, lw1) defer ldsCancel1() - resourceName2 := ldsNameNewStyle + resourceName2 := makeNewStyleLDSName(authority) lw2 := newListenerWatcher() ldsCancel2 := xdsresource.WatchListener(client, resourceName2, lw2) defer ldsCancel2() @@ -791,11 +842,14 @@ func (s) TestLDSWatch_ResourceRemoved(t *testing.T) { // server is NACK'ed by the xdsclient. The test verifies that the error is // propagated to the watcher. func (s) TestLDSWatch_NACKError(t *testing.T) { - mgmtServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) - defer cleanup() + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + testutils.CreateBootstrapFileForTesting(t, bc) // Create an xDS client with the above bootstrap contents. - client, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bootstrapContents}) + client, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bc}) if err != nil { t.Fatalf("Failed to create xDS client: %v", err) } @@ -837,11 +891,31 @@ func (s) TestLDSWatch_NACKError(t *testing.T) { // to the valid resource receive the update, while watchers corresponding to the // invalid resource receive an error. func (s) TestLDSWatch_PartialValid(t *testing.T) { - mgmtServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) - defer cleanup() + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + authority := makeAuthorityName(t.Name()) + bc, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []json.RawMessage{[]byte(fmt.Sprintf(`{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }`, mgmtServer.Address))}, + NodeID: nodeID, + Authorities: map[string]json.RawMessage{ + // Xdstp style resource names used in this test use a slash removed + // version of t.Name as their authority, and the empty config + // results in the top-level xds server configuration being used for + // this authority. + authority: []byte(`{}`), + }, + }) + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + testutils.CreateBootstrapFileForTesting(t, bc) // Create an xDS client with the above bootstrap contents. - client, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bootstrapContents}) + client, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bc}) if err != nil { t.Fatalf("Failed to create xDS client: %v", err) } @@ -856,7 +930,7 @@ func (s) TestLDSWatch_PartialValid(t *testing.T) { lw1 := newListenerWatcher() ldsCancel1 := xdsresource.WatchListener(client, badResourceName, lw1) defer ldsCancel1() - goodResourceName := ldsNameNewStyle + goodResourceName := makeNewStyleLDSName(authority) lw2 := newListenerWatcher() ldsCancel2 := xdsresource.WatchListener(client, goodResourceName, lw2) defer ldsCancel2() @@ -907,11 +981,31 @@ func (s) TestLDSWatch_PartialValid(t *testing.T) { // expected to wait for the watch timeout to expire before concluding that the // resource does not exist on the server func (s) TestLDSWatch_PartialResponse(t *testing.T) { - mgmtServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) - defer cleanup() + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + authority := makeAuthorityName(t.Name()) + bc, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []json.RawMessage{[]byte(fmt.Sprintf(`{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }`, mgmtServer.Address))}, + NodeID: nodeID, + Authorities: map[string]json.RawMessage{ + // Xdstp style resource names used in this test use a slash removed + // version of t.Name as their authority, and the empty config + // results in the top-level xds server configuration being used for + // this authority. + authority: []byte(`{}`), + }, + }) + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + testutils.CreateBootstrapFileForTesting(t, bc) // Create an xDS client with the above bootstrap contents. - client, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bootstrapContents}) + client, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bc}) if err != nil { t.Fatalf("Failed to create xDS client: %v", err) } @@ -924,7 +1018,7 @@ func (s) TestLDSWatch_PartialResponse(t *testing.T) { ldsCancel1 := xdsresource.WatchListener(client, resourceName1, lw1) defer ldsCancel1() - resourceName2 := ldsNameNewStyle + resourceName2 := makeNewStyleLDSName(authority) lw2 := newListenerWatcher() ldsCancel2 := xdsresource.WatchListener(client, resourceName2, lw2) defer ldsCancel2() diff --git a/xds/internal/xdsclient/tests/misc_watchers_test.go b/xds/internal/xdsclient/tests/misc_watchers_test.go index 511856486c4..1b11c786790 100644 --- a/xds/internal/xdsclient/tests/misc_watchers_test.go +++ b/xds/internal/xdsclient/tests/misc_watchers_test.go @@ -20,14 +20,15 @@ package xdsclient_test import ( "context" + "encoding/json" "fmt" "testing" "github.com/google/uuid" "google.golang.org/grpc/internal/testutils" - "google.golang.org/grpc/internal/testutils/xds/bootstrap" "google.golang.org/grpc/internal/testutils/xds/e2e" "google.golang.org/grpc/internal/testutils/xds/fakeserver" + "google.golang.org/grpc/internal/xds/bootstrap" "google.golang.org/grpc/xds/internal" xdstestutils "google.golang.org/grpc/xds/internal/testutils" "google.golang.org/grpc/xds/internal/xdsclient" @@ -98,17 +99,39 @@ func (rw *testRouteConfigWatcher) cancel() { func (s) TestWatchCallAnotherWatch(t *testing.T) { // Start an xDS management server and set the option to allow it to respond // to requests which only specify a subset of the configured resources. - mgmtServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) - defer cleanup() + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) + + nodeID := uuid.New().String() + authority := makeAuthorityName(t.Name()) + bs, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []json.RawMessage{[]byte(fmt.Sprintf(`{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }`, mgmtServer.Address))}, + NodeID: nodeID, + Authorities: map[string]json.RawMessage{ + // Xdstp style resource names used in this test use a slash removed + // version of t.Name as their authority, and the empty config + // results in the top-level xds server configuration being used for + // this authority. + authority: []byte(`{}`), + }, + }) + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + testutils.CreateBootstrapFileForTesting(t, bs) // Create an xDS client with the above bootstrap contents. - client, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bootstrapContents}) + client, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bs}) if err != nil { t.Fatalf("Failed to create xDS client: %v", err) } defer close() // Configure the management server to respond with route config resources. + ldsNameNewStyle := makeNewStyleLDSName(authority) + rdsNameNewStyle := makeNewStyleRDSName(authority) resources := e2e.UpdateOptions{ NodeID: nodeID, Routes: []*v3routepb.RouteConfiguration{ @@ -204,13 +227,7 @@ func (s) TestNodeProtoSentOnlyInFirstRequest(t *testing.T) { // Create a bootstrap file in a temporary directory. nodeID := uuid.New().String() - bootstrapContents, err := bootstrap.Contents(bootstrap.Options{ - NodeID: nodeID, - ServerURI: mgmtServer.Address, - }) - if err != nil { - t.Fatalf("Failed to create bootstrap file: %v", err) - } + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) // Create an xDS client with the above bootstrap contents. client, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bootstrapContents}) diff --git a/xds/internal/xdsclient/tests/rds_watchers_test.go b/xds/internal/xdsclient/tests/rds_watchers_test.go index 3c29e7da8c2..9da3c5efd98 100644 --- a/xds/internal/xdsclient/tests/rds_watchers_test.go +++ b/xds/internal/xdsclient/tests/rds_watchers_test.go @@ -20,6 +20,7 @@ package xdsclient_test import ( "context" + "encoding/json" "fmt" "strings" "testing" @@ -31,6 +32,7 @@ import ( "google.golang.org/grpc/internal/grpcsync" "google.golang.org/grpc/internal/testutils" "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/xds/bootstrap" "google.golang.org/grpc/xds/internal/xdsclient" "google.golang.org/grpc/xds/internal/xdsclient/xdsresource" "google.golang.org/protobuf/types/known/wrapperspb" @@ -207,11 +209,31 @@ func (s) TestRDSWatch(t *testing.T) { for _, test := range tests { t.Run(test.desc, func(t *testing.T) { - mgmtServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) - defer cleanup() + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + bc, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []json.RawMessage{[]byte(fmt.Sprintf(`{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }`, mgmtServer.Address))}, + NodeID: nodeID, + Authorities: map[string]json.RawMessage{ + // Xdstp resource names used in this test do not specify an + // authority. These will end up looking up an entry with the + // empty key in the authorities map. Having an entry with an + // empty key and empty configuration, results in these + // resources also using the top-level configuration. + "": []byte(`{}`), + }, + }) + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + testutils.CreateBootstrapFileForTesting(t, bc) // Create an xDS client with the above bootstrap contents. - client, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bootstrapContents}) + client, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bc}) if err != nil { t.Fatalf("Failed to create xDS client: %v", err) } @@ -374,11 +396,31 @@ func (s) TestRDSWatch_TwoWatchesForSameResourceName(t *testing.T) { for _, test := range tests { t.Run(test.desc, func(t *testing.T) { - mgmtServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) - defer cleanup() + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + bc, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []json.RawMessage{[]byte(fmt.Sprintf(`{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }`, mgmtServer.Address))}, + NodeID: nodeID, + Authorities: map[string]json.RawMessage{ + // Xdstp resource names used in this test do not specify an + // authority. These will end up looking up an entry with the + // empty key in the authorities map. Having an entry with an + // empty key and empty configuration, results in these + // resources also using the top-level configuration. + "": []byte(`{}`), + }, + }) + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + testutils.CreateBootstrapFileForTesting(t, bc) // Create an xDS client with the above bootstrap contents. - client, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bootstrapContents}) + client, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bc}) if err != nil { t.Fatalf("Failed to create xDS client: %v", err) } @@ -455,11 +497,31 @@ func (s) TestRDSWatch_TwoWatchesForSameResourceName(t *testing.T) { // // The test is run with both old and new style names. func (s) TestRDSWatch_ThreeWatchesForDifferentResourceNames(t *testing.T) { - mgmtServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) - defer cleanup() + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + authority := makeAuthorityName(t.Name()) + bc, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []json.RawMessage{[]byte(fmt.Sprintf(`{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }`, mgmtServer.Address))}, + NodeID: nodeID, + Authorities: map[string]json.RawMessage{ + // Xdstp style resource names used in this test use a slash removed + // version of t.Name as their authority, and the empty config + // results in the top-level xds server configuration being used for + // this authority. + authority: []byte(`{}`), + }, + }) + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + testutils.CreateBootstrapFileForTesting(t, bc) // Create an xDS client with the above bootstrap contents. - client, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bootstrapContents}) + client, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bc}) if err != nil { t.Fatalf("Failed to create xDS client: %v", err) } @@ -475,6 +537,7 @@ func (s) TestRDSWatch_ThreeWatchesForDifferentResourceNames(t *testing.T) { defer rdsCancel2() // Register the third watch for a different route configuration resource. + rdsNameNewStyle := makeNewStyleRDSName(authority) rw3 := newRouteConfigWatcher() rdsCancel3 := xdsresource.WatchRouteConfig(client, rdsNameNewStyle, rw3) defer rdsCancel3() @@ -534,7 +597,7 @@ func (s) TestRDSWatch_ResourceCaching(t *testing.T) { firstAckReceived := grpcsync.NewEvent() secondRequestReceived := grpcsync.NewEvent() - mgmtServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{ + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ OnStreamRequest: func(id int64, req *v3discoverypb.DiscoveryRequest) error { // The first request has an empty version string. if !firstRequestReceived && req.GetVersionInfo() == "" { @@ -551,10 +614,13 @@ func (s) TestRDSWatch_ResourceCaching(t *testing.T) { return nil }, }) - defer cleanup() + + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + testutils.CreateBootstrapFileForTesting(t, bc) // Create an xDS client with the above bootstrap contents. - client, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bootstrapContents}) + client, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bc}) if err != nil { t.Fatalf("Failed to create xDS client: %v", err) } @@ -628,18 +694,13 @@ func (s) TestRDSWatch_ResourceCaching(t *testing.T) { // verifies that the watch callback is invoked with an error once the // watchExpiryTimer fires. func (s) TestRDSWatch_ExpiryTimerFiresBeforeResponse(t *testing.T) { - mgmtServer, err := e2e.StartManagementServer(e2e.ManagementServerOptions{}) - if err != nil { - t.Fatalf("Failed to spin up the xDS management server: %v", err) - } - defer mgmtServer.Stop() + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) - // Create an xDS client talking to the above management server. nodeID := uuid.New().String() - bc, err := e2e.DefaultBootstrapContents(nodeID, mgmtServer.Address) - if err != nil { - t.Fatalf("Failed to create bootstrap configuration: %v", err) - } + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + testutils.CreateBootstrapFileForTesting(t, bc) + + // Create an xDS client talking to the above management server. client, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bc, WatchExpiryTimeout: defaultTestWatchExpiryTimeout}) if err != nil { t.Fatalf("Failed to create an xDS client: %v", err) @@ -669,18 +730,13 @@ func (s) TestRDSWatch_ExpiryTimerFiresBeforeResponse(t *testing.T) { // verifies that the behavior associated with the expiry timer (i.e, callback // invocation with error) does not take place. func (s) TestRDSWatch_ValidResponseCancelsExpiryTimerBehavior(t *testing.T) { - mgmtServer, err := e2e.StartManagementServer(e2e.ManagementServerOptions{}) - if err != nil { - t.Fatalf("Failed to spin up the xDS management server: %v", err) - } - defer mgmtServer.Stop() + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) - // Create an xDS client talking to the above management server. nodeID := uuid.New().String() - bc, err := e2e.DefaultBootstrapContents(nodeID, mgmtServer.Address) - if err != nil { - t.Fatalf("Failed to create bootstrap configuration: %v", err) - } + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + testutils.CreateBootstrapFileForTesting(t, bc) + + // Create an xDS client talking to the above management server. client, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bc, WatchExpiryTimeout: defaultTestWatchExpiryTimeout}) if err != nil { t.Fatalf("Failed to create an xDS client: %v", err) @@ -739,11 +795,14 @@ func (s) TestRDSWatch_ValidResponseCancelsExpiryTimerBehavior(t *testing.T) { // server is NACK'ed by the xdsclient. The test verifies that the error is // propagated to the watcher. func (s) TestRDSWatch_NACKError(t *testing.T) { - mgmtServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) - defer cleanup() + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + testutils.CreateBootstrapFileForTesting(t, bc) // Create an xDS client with the above bootstrap contents. - client, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bootstrapContents}) + client, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bc}) if err != nil { t.Fatalf("Failed to create xDS client: %v", err) } @@ -785,11 +844,31 @@ func (s) TestRDSWatch_NACKError(t *testing.T) { // to the valid resource receive the update, while watchers corresponding to the // invalid resource receive an error. func (s) TestRDSWatch_PartialValid(t *testing.T) { - mgmtServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) - defer cleanup() + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + authority := makeAuthorityName(t.Name()) + bc, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []json.RawMessage{[]byte(fmt.Sprintf(`{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }`, mgmtServer.Address))}, + NodeID: nodeID, + Authorities: map[string]json.RawMessage{ + // Xdstp style resource names used in this test use a slash removed + // version of t.Name as their authority, and the empty config + // results in the top-level xds server configuration being used for + // this authority. + authority: []byte(`{}`), + }, + }) + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + testutils.CreateBootstrapFileForTesting(t, bc) // Create an xDS client with the above bootstrap contents. - client, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bootstrapContents}) + client, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bc}) if err != nil { t.Fatalf("Failed to create xDS client: %v", err) } @@ -804,7 +883,7 @@ func (s) TestRDSWatch_PartialValid(t *testing.T) { rw1 := newRouteConfigWatcher() rdsCancel1 := xdsresource.WatchRouteConfig(client, badResourceName, rw1) defer rdsCancel1() - goodResourceName := rdsNameNewStyle + goodResourceName := makeNewStyleRDSName(authority) rw2 := newRouteConfigWatcher() rdsCancel2 := xdsresource.WatchRouteConfig(client, goodResourceName, rw2) defer rdsCancel2() diff --git a/xds/internal/xdsclient/tests/resource_update_test.go b/xds/internal/xdsclient/tests/resource_update_test.go index 5ac0ab42c33..94b519b6f3d 100644 --- a/xds/internal/xdsclient/tests/resource_update_test.go +++ b/xds/internal/xdsclient/tests/resource_update_test.go @@ -283,10 +283,7 @@ func (s) TestHandleListenerResponseFromManagementServer(t *testing.T) { // Create an xDS client talking to the above management server. nodeID := uuid.New().String() - bc, err := e2e.DefaultBootstrapContents(nodeID, mgmtServer.Address) - if err != nil { - t.Fatalf("Failed to create bootstrap configuration: %v", err) - } + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) client, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bc, WatchExpiryTimeout: defaultTestWatchExpiryTimeout}) if err != nil { t.Fatalf("Failed to create an xDS client: %v", err) @@ -561,10 +558,7 @@ func (s) TestHandleRouteConfigResponseFromManagementServer(t *testing.T) { // Create an xDS client talking to the above management server. nodeID := uuid.New().String() - bc, err := e2e.DefaultBootstrapContents(nodeID, mgmtServer.Address) - if err != nil { - t.Fatalf("Failed to create bootstrap configuration: %v", err) - } + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) client, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bc, WatchExpiryTimeout: defaultTestWatchExpiryTimeout}) if err != nil { t.Fatalf("Failed to create an xDS client: %v", err) @@ -800,10 +794,7 @@ func (s) TestHandleClusterResponseFromManagementServer(t *testing.T) { // Create an xDS client talking to the above management server. nodeID := uuid.New().String() - bc, err := e2e.DefaultBootstrapContents(nodeID, mgmtServer.Address) - if err != nil { - t.Fatalf("Failed to create bootstrap configuration: %v", err) - } + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) client, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bc, WatchExpiryTimeout: defaultTestWatchExpiryTimeout}) if err != nil { t.Fatalf("Failed to create an xDS client: %v", err) @@ -867,7 +858,7 @@ func (s) TestHandleClusterResponseFromManagementServer(t *testing.T) { // server at that point, hence we do it here before verifying the // received update. if test.wantErr == "" { - serverCfg, err := bootstrap.ServerConfigForTesting(bootstrap.ServerConfigTestingOptions{URI: mgmtServer.Address}) + serverCfg, err := bootstrap.ServerConfigForTesting(bootstrap.ServerConfigTestingOptions{URI: fmt.Sprintf("passthrough:///%s", mgmtServer.Address)}) if err != nil { t.Fatalf("Failed to create server config for testing: %v", err) } @@ -1151,10 +1142,7 @@ func (s) TestHandleEndpointsResponseFromManagementServer(t *testing.T) { // Create an xDS client talking to the above management server. nodeID := uuid.New().String() - bc, err := e2e.DefaultBootstrapContents(nodeID, mgmtServer.Address) - if err != nil { - t.Fatalf("Failed to create bootstrap configuration: %v", err) - } + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) client, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{Contents: bc, WatchExpiryTimeout: defaultTestWatchExpiryTimeout}) if err != nil { t.Fatalf("Failed to create an xDS client: %v", err) diff --git a/xds/internal/xdsclient/transport/transport_ack_nack_test.go b/xds/internal/xdsclient/transport/transport_ack_nack_test.go index b59ac0f799a..582c7fa73dc 100644 --- a/xds/internal/xdsclient/transport/transport_ack_nack_test.go +++ b/xds/internal/xdsclient/transport/transport_ack_nack_test.go @@ -85,7 +85,7 @@ func (s) TestSimpleAckAndNack(t *testing.T) { // the test goroutine to verify ack version and nonce. streamRequestCh := make(chan *v3discoverypb.DiscoveryRequest, 1) streamResponseCh := make(chan *v3discoverypb.DiscoveryResponse, 1) - mgmtServer, err := e2e.StartManagementServer(e2e.ManagementServerOptions{ + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error { select { case streamRequestCh <- req: @@ -100,11 +100,6 @@ func (s) TestSimpleAckAndNack(t *testing.T) { } }, }) - if err != nil { - t.Fatalf("Failed to start xDS management server: %v", err) - } - defer mgmtServer.Stop() - t.Logf("Started xDS management server on %s", mgmtServer.Address) // Configure the management server with appropriate resources. apiListener := &v3listenerpb.ApiListener{ @@ -269,7 +264,7 @@ func (s) TestInvalidFirstResponse(t *testing.T) { // the test goroutine to verify ack version and nonce. streamRequestCh := make(chan *v3discoverypb.DiscoveryRequest, 1) streamResponseCh := make(chan *v3discoverypb.DiscoveryResponse, 1) - mgmtServer, err := e2e.StartManagementServer(e2e.ManagementServerOptions{ + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error { select { case streamRequestCh <- req: @@ -284,11 +279,6 @@ func (s) TestInvalidFirstResponse(t *testing.T) { } }, }) - if err != nil { - t.Fatalf("Failed to start xDS management server: %v", err) - } - defer mgmtServer.Stop() - t.Logf("Started xDS management server on %s", mgmtServer.Address) // Configure the management server with appropriate resources. apiListener := &v3listenerpb.ApiListener{ @@ -397,7 +387,7 @@ func (s) TestResourceIsNotRequestedAnymore(t *testing.T) { // the test goroutine to verify ack version and nonce. streamRequestCh := make(chan *v3discoverypb.DiscoveryRequest, 1) streamResponseCh := make(chan *v3discoverypb.DiscoveryResponse, 1) - mgmtServer, err := e2e.StartManagementServer(e2e.ManagementServerOptions{ + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error { select { case streamRequestCh <- req: @@ -412,11 +402,6 @@ func (s) TestResourceIsNotRequestedAnymore(t *testing.T) { } }, }) - if err != nil { - t.Fatalf("Failed to start xDS management server: %v", err) - } - defer mgmtServer.Stop() - t.Logf("Started xDS management server on %s", mgmtServer.Address) // Configure the management server with appropriate resources. apiListener := &v3listenerpb.ApiListener{ diff --git a/xds/internal/xdsclient/transport/transport_backoff_test.go b/xds/internal/xdsclient/transport/transport_backoff_test.go index 4f6283f887d..0a028ca8dca 100644 --- a/xds/internal/xdsclient/transport/transport_backoff_test.go +++ b/xds/internal/xdsclient/transport/transport_backoff_test.go @@ -60,7 +60,7 @@ func (s) TestTransport_BackoffAfterStreamFailure(t *testing.T) { // Create an xDS management server listening on a local port. streamErr := errors.New("ADS stream error") - mgmtServer, err := e2e.StartManagementServer(e2e.ManagementServerOptions{ + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ // Push on a channel whenever the stream is closed. OnStreamClosed: func(int64, *v3corepb.Node) { select { @@ -80,11 +80,6 @@ func (s) TestTransport_BackoffAfterStreamFailure(t *testing.T) { return streamErr }, }) - if err != nil { - t.Fatalf("Failed to start xDS management server: %v", err) - } - defer mgmtServer.Stop() - t.Logf("Started xDS management server on %s", mgmtServer.Address) // Override the backoff implementation to push on a channel that is read by // the test goroutine. @@ -202,7 +197,7 @@ func (s) TestTransport_RetriesAfterBrokenStream(t *testing.T) { t.Fatalf("Failed to create a local listener for the xDS management server: %v", err) } lis := testutils.NewRestartableListener(l) - mgmtServer, err := e2e.StartManagementServer(e2e.ManagementServerOptions{ + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ Listener: lis, // Push the received request on to a channel for the test goroutine to // verify that it matches expectations. @@ -223,11 +218,6 @@ func (s) TestTransport_RetriesAfterBrokenStream(t *testing.T) { } }, }) - if err != nil { - t.Fatalf("Failed to start xDS management server: %v", err) - } - defer mgmtServer.Stop() - t.Logf("Started xDS management server on %s", lis.Addr().String()) // Configure the management server with appropriate resources. apiListener := &v3listenerpb.ApiListener{ @@ -374,7 +364,7 @@ func (s) TestTransport_ResourceRequestedBeforeStreamCreation(t *testing.T) { lis := testutils.NewRestartableListener(l) streamErr := errors.New("ADS stream error") - mgmtServer, err := e2e.StartManagementServer(e2e.ManagementServerOptions{ + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ Listener: lis, // Return an error everytime a request is sent on the stream. This @@ -388,11 +378,6 @@ func (s) TestTransport_ResourceRequestedBeforeStreamCreation(t *testing.T) { return streamErr }, }) - if err != nil { - t.Fatalf("Failed to start xDS management server: %v", err) - } - defer mgmtServer.Stop() - t.Logf("Started xDS management server on %s", lis.Addr().String()) // Bring down the management server before creating the transport. This // allows us to test the case where SendRequest() is called when there is no diff --git a/xds/server_ext_test.go b/xds/server_ext_test.go index de4791f6654..67dee039c3d 100644 --- a/xds/server_ext_test.go +++ b/xds/server_ext_test.go @@ -46,6 +46,7 @@ import ( v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + "github.com/google/uuid" testgrpc "google.golang.org/grpc/interop/grpc_testing" testpb "google.golang.org/grpc/interop/grpc_testing" ) @@ -105,8 +106,12 @@ func hostPortFromListener(lis net.Listener) (string, uint32, error) { // ready (through an explicit LDS Resource Not Found), previously running RPC's // should be gracefully closed and still work, and new RPC's should fail. func (s) TestServingModeChanges(t *testing.T) { - managementServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) - defer cleanup() + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) + lis, err := testutils.LocalTCPListener() if err != nil { t.Fatalf("testutils.LocalTCPListener() failed: %v", err) @@ -229,8 +234,12 @@ func (s) TestServingModeChanges(t *testing.T) { // error, the server should move to serving, successfully Accept Connections, // and fail at the L7 level with resource not found specified. func (s) TestResourceNotFoundRDS(t *testing.T) { - managementServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) - defer cleanup() + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) + lis, err := testutils.LocalTCPListener() if err != nil { t.Fatalf("testutils.LocalTCPListener() failed: %v", err) diff --git a/xds/server_test.go b/xds/server_test.go index d06236b1905..bd32bdf254d 100644 --- a/xds/server_test.go +++ b/xds/server_test.go @@ -40,8 +40,8 @@ import ( "google.golang.org/grpc/credentials/xds" "google.golang.org/grpc/internal/grpctest" "google.golang.org/grpc/internal/testutils" - "google.golang.org/grpc/internal/testutils/xds/bootstrap" "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/xds/bootstrap" "google.golang.org/grpc/xds/internal/xdsclient" "google.golang.org/grpc/xds/internal/xdsclient/xdsresource/version" @@ -102,12 +102,7 @@ func newFakeGRPCServer() *fakeGRPCServer { } func generateBootstrapContents(t *testing.T, nodeID, serverURI string) []byte { - t.Helper() - - bs, err := e2e.DefaultBootstrapContents(nodeID, serverURI) - if err != nil { - t.Fatal(err) - } + bs := e2e.DefaultBootstrapContents(t, nodeID, serverURI) return bs } @@ -198,15 +193,18 @@ func (s) TestNewServer_Failure(t *testing.T) { serverOpts: []grpc.ServerOption{ grpc.Creds(xdsCreds), func() grpc.ServerOption { - bs, err := bootstrap.Contents(bootstrap.Options{ - NodeID: uuid.New().String(), - ServerURI: nonExistentManagementServer, + bs, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []json.RawMessage{[]byte(fmt.Sprintf(`{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }`, nonExistentManagementServer))}, + NodeID: uuid.New().String(), CertificateProviders: map[string]json.RawMessage{ "cert-provider-instance": json.RawMessage("{}"), }, }) if err != nil { - t.Errorf("Failed to create bootstrap configuration: %v", err) + t.Fatalf("Failed to create bootstrap configuration: %v", err) } return BootstrapContentsForTesting(bs) }(), @@ -356,7 +354,7 @@ func (s) TestServeSuccess(t *testing.T) { // Setup an xDS management server that pushes on a channel when an LDS // request is received by it. ldsRequestCh := make(chan []string, 1) - mgmtServer, nodeID, bootstrapContents, _, cancel := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{ + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ OnStreamRequest: func(id int64, req *v3discoverypb.DiscoveryRequest) error { if req.GetTypeUrl() == version.V3ListenerURL { select { @@ -367,7 +365,10 @@ func (s) TestServeSuccess(t *testing.T) { return nil }, }) - defer cancel() + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) // Override the function to create the underlying grpc.Server to allow the // test to verify that Serve() is called on the underlying server. @@ -488,19 +489,18 @@ func (s) TestNewServer_ClientCreationFailure(t *testing.T) { // server is not configured with xDS credentials. Verifies that the security // config received as part of a Listener update is not acted upon. func (s) TestHandleListenerUpdate_NoXDSCreds(t *testing.T) { - mgmtServer, err := e2e.StartManagementServer(e2e.ManagementServerOptions{}) - if err != nil { - t.Fatalf("Failed to start xDS management server: %v", err) - } - defer mgmtServer.Stop() + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) // Generate bootstrap configuration pointing to the above management server // with certificate provider configuration pointing to fake certificate // providers. nodeID := uuid.NewString() - bootstrapContents, err := bootstrap.Contents(bootstrap.Options{ - NodeID: nodeID, - ServerURI: mgmtServer.Address, + bootstrapContents, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []json.RawMessage{[]byte(fmt.Sprintf(`{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }`, mgmtServer.Address))}, + NodeID: nodeID, CertificateProviders: map[string]json.RawMessage{ e2e.ServerSideCertProviderInstance: fakeProvider1Config, e2e.ClientSideCertProviderInstance: fakeProvider2Config, @@ -571,7 +571,7 @@ func (s) TestHandleListenerUpdate_ErrorUpdate(t *testing.T) { // Setup an xDS management server that pushes on a channel when an LDS // request is received by it. ldsRequestCh := make(chan []string, 1) - mgmtServer, nodeID, _, _, cancel := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{ + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ OnStreamRequest: func(id int64, req *v3discoverypb.DiscoveryRequest) error { if req.GetTypeUrl() == version.V3ListenerURL { select { @@ -582,14 +582,17 @@ func (s) TestHandleListenerUpdate_ErrorUpdate(t *testing.T) { return nil }, }) - defer cancel() // Generate bootstrap configuration pointing to the above management server // with certificate provider configuration pointing to fake certificate // providers. - bootstrapContents, err := bootstrap.Contents(bootstrap.Options{ - NodeID: nodeID, - ServerURI: mgmtServer.Address, + nodeID := uuid.New().String() + bootstrapContents, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []json.RawMessage{[]byte(fmt.Sprintf(`{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }`, mgmtServer.Address))}, + NodeID: nodeID, CertificateProviders: map[string]json.RawMessage{ e2e.ServerSideCertProviderInstance: fakeProvider1Config, e2e.ClientSideCertProviderInstance: fakeProvider2Config,